diff --git a/CDB/LOFAR_ConfigDb.json b/CDB/LOFAR_ConfigDb.json
index 8aa444d302dc352578fa57c2aff9c0c5688289df..73fb4d5731ec78b84d094939e5db94b301c56f04 100644
--- a/CDB/LOFAR_ConfigDb.json
+++ b/CDB/LOFAR_ConfigDb.json
@@ -1,392 +1,426 @@
{
- "servers": {
+ "servers": {
+ "StationManager": {
+ "STAT": {
"StationManager": {
- "STAT": {
- "StationManager": {
- "STAT/StationManager/1": {
- "properties": {
- "Station_Name": [
- "DevStation"
- ],
- "Station_Number":[
- "999"
- ]
- }
- }
- }
+ "STAT/StationManager/1": {
+ "properties": {
+ "Station_Name": [
+ "DevStation"
+ ],
+ "Station_Number": [
+ "999"
+ ]
}
- },
+ }
+ }
+ }
+ },
+ "Docker": {
+ "STAT": {
"Docker": {
- "STAT": {
- "Docker": {
- "STAT/Docker/1": {}
- }
- }
- },
+ "STAT/Docker/1": {}
+ }
+ }
+ },
+ "Calibration": {
+ "STAT": {
+ "Calibration": {
+ "STAT/Calibration/1": {}
+ }
+ }
+ },
+ "Configuration": {
+ "STAT": {
"Configuration": {
- "STAT": {
- "Configuration": {
- "STAT/Configuration/1": {}
- }
- }
- },
+ "STAT/Configuration/1": {}
+ }
+ }
+ },
+ "Observation": {
+ "STAT": {
"Observation": {
- "STAT": {
- "Observation": {
- }
- }
- },
+ }
+ }
+ },
+ "ObservationControl": {
+ "STAT": {
"ObservationControl": {
- "STAT": {
- "ObservationControl": {
- "STAT/ObservationControl/1": {}
- }
- }
- },
+ "STAT/ObservationControl/1": {}
+ }
+ }
+ },
+ "AntennaField": {
+ "STAT": {
"AntennaField": {
- "STAT": {
- "AntennaField": {
- "STAT/AntennaField/HBA": {
- "properties": {
- "RECV_devices": ["STAT/RECV/1"]
- }
- }
- }
+ "STAT/AntennaField/HBA": {
+ "properties": {
+ "RECV_devices": [
+ "STAT/RECV/1"
+ ]
}
- },
+ }
+ }
+ }
+ },
+ "PSOC": {
+ "STAT": {
"PSOC": {
- "STAT": {
- "PSOC": {
- "STAT/PSOC/1": {
- "properties": {
- "SNMP_host": ["127.0.0.1"],
- "SNMP_community": ["public"],
- "SNMP_mib_dir": ["devices/mibs/PowerNet-MIB.mib"],
- "SNMP_timeout": ["10.0"],
- "SNMP_version": ["1"],
- "PSOC_sockets": [
- "socket_1",
- "socket_2",
- "socket_3",
- "socket_4",
- "socket_5",
- "socket_6",
- "socket_7",
- "socket_8"
- ]
- }
- }
- }
+ "STAT/PSOC/1": {
+ "properties": {
+ "SNMP_host": [
+ "127.0.0.1"
+ ],
+ "SNMP_community": [
+ "public"
+ ],
+ "SNMP_mib_dir": [
+ "devices/mibs/PowerNet-MIB.mib"
+ ],
+ "SNMP_timeout": [
+ "10.0"
+ ],
+ "SNMP_version": [
+ "1"
+ ],
+ "PSOC_sockets": [
+ "socket_1",
+ "socket_2",
+ "socket_3",
+ "socket_4",
+ "socket_5",
+ "socket_6",
+ "socket_7",
+ "socket_8"
+ ]
}
- },
+ }
+ }
+ }
+ },
+ "PCON": {
+ "STAT": {
"PCON": {
- "STAT": {
- "PCON": {
- "STAT/PCON/1": {
- "properties": {
- "SNMP_host": ["127.0.0.1"],
- "SNMP_community": ["public"],
- "SNMP_mib_dir": ["devices/mibs/ACC-MIB.mib"],
- "SNMP_timeout": ["10.0"],
- "SNMP_version": ["1"]
- }
- }
- }
+ "STAT/PCON/1": {
+ "properties": {
+ "SNMP_host": [
+ "127.0.0.1"
+ ],
+ "SNMP_community": [
+ "public"
+ ],
+ "SNMP_mib_dir": [
+ "devices/mibs/ACC-MIB.mib"
+ ],
+ "SNMP_timeout": [
+ "10.0"
+ ],
+ "SNMP_version": [
+ "1"
+ ]
}
- },
+ }
+ }
+ }
+ },
+ "TemperatureManager": {
+ "STAT": {
"TemperatureManager": {
- "STAT": {
- "TemperatureManager": {
- "STAT/TemperatureManager/1": {
- "properties": {
- "Alarm_Error_List": [
- "APSCT, APSCT_TEMP_error_R",
- "APSPU, APSPU_TEMP_error_R",
- "UNB2, UNB2_TEMP_error_R",
- "RECV, RECV_TEMP_error_R"
- ],
- "Shutdown_Device_List":[
- "STAT/SDP/1", "STAT/UNB2/1", "STAT/RECV/1", "STAT/APSCT/1", "STAT/CCD/1", "STAT/APSPU/1"
- ]
- }
- }
- }
+ "STAT/TemperatureManager/1": {
+ "properties": {
+ "Alarm_Error_List": [
+ "APSCT, APSCT_TEMP_error_R",
+ "APSPU, APSPU_TEMP_error_R",
+ "UNB2, UNB2_TEMP_error_R",
+ "RECV, RECV_TEMP_error_R"
+ ],
+ "Shutdown_Device_List": [
+ "STAT/SDP/1",
+ "STAT/UNB2/1",
+ "STAT/RECV/1",
+ "STAT/APSCT/1",
+ "STAT/CCD/1",
+ "STAT/APSPU/1"
+ ]
}
- },
+ }
+ }
+ }
+ },
+ "TileBeam": {
+ "STAT": {
"TileBeam": {
- "STAT": {
- "TileBeam": {
- "STAT/TileBeam/HBA": {
- }
- }
- }
- },
+ "STAT/TileBeam/HBA": {
+ }
+ }
+ }
+ },
+ "Beamlet": {
+ "STAT": {
"Beamlet": {
- "STAT": {
- "Beamlet": {
- "STAT/Beamlet/1": {
- "properties": {
- "FPGA_beamlet_output_hdr_eth_source_mac_RW_default": [
- "00:22:86:08:00:00",
- "00:22:86:08:00:01",
- "00:22:86:08:00:02",
- "00:22:86:08:00:03",
- "00:22:86:08:01:00",
- "00:22:86:08:01:01",
- "00:22:86:08:01:02",
- "00:22:86:08:01:03",
- "00:22:86:08:02:00",
- "00:22:86:08:02:01",
- "00:22:86:08:02:02",
- "00:22:86:08:02:03",
- "00:22:86:08:03:00",
- "00:22:86:08:03:01",
- "00:22:86:08:03:02",
- "00:22:86:08:03:03"
- ],
- "FPGA_beamlet_output_hdr_ip_source_address_RW_default": [
- "192.168.0.1",
- "192.168.0.2",
- "192.168.0.3",
- "192.168.0.4",
- "192.168.1.1",
- "192.168.1.2",
- "192.168.1.3",
- "192.168.1.4",
- "192.168.2.1",
- "192.168.2.2",
- "192.168.2.3",
- "192.168.2.4",
- "192.168.3.1",
- "192.168.3.2",
- "192.168.3.3",
- "192.168.3.4"
- ],
- "FPGA_beamlet_output_hdr_udp_source_port_RW_default": [
- "53248",
- "53249",
- "53250",
- "53251",
- "53252",
- "53253",
- "53254",
- "53255",
- "53256",
- "53257",
- "53258",
- "53259",
- "53260",
- "53261",
- "53262",
- "53263"
- ],
- "FPGA_beamlet_output_hdr_udp_destination_port_RW_default": [
- "10001",
- "10001",
- "10001",
- "10001",
- "10001",
- "10001",
- "10001",
- "10001",
- "10001",
- "10001",
- "10001",
- "10001",
- "10001",
- "10001",
- "10001",
- "10001"
- ]
- }
- }
- }
+ "STAT/Beamlet/1": {
+ "properties": {
+ "FPGA_beamlet_output_hdr_eth_source_mac_RW_default": [
+ "00:22:86:08:00:00",
+ "00:22:86:08:00:01",
+ "00:22:86:08:00:02",
+ "00:22:86:08:00:03",
+ "00:22:86:08:01:00",
+ "00:22:86:08:01:01",
+ "00:22:86:08:01:02",
+ "00:22:86:08:01:03",
+ "00:22:86:08:02:00",
+ "00:22:86:08:02:01",
+ "00:22:86:08:02:02",
+ "00:22:86:08:02:03",
+ "00:22:86:08:03:00",
+ "00:22:86:08:03:01",
+ "00:22:86:08:03:02",
+ "00:22:86:08:03:03"
+ ],
+ "FPGA_beamlet_output_hdr_ip_source_address_RW_default": [
+ "192.168.0.1",
+ "192.168.0.2",
+ "192.168.0.3",
+ "192.168.0.4",
+ "192.168.1.1",
+ "192.168.1.2",
+ "192.168.1.3",
+ "192.168.1.4",
+ "192.168.2.1",
+ "192.168.2.2",
+ "192.168.2.3",
+ "192.168.2.4",
+ "192.168.3.1",
+ "192.168.3.2",
+ "192.168.3.3",
+ "192.168.3.4"
+ ],
+ "FPGA_beamlet_output_hdr_udp_source_port_RW_default": [
+ "53248",
+ "53249",
+ "53250",
+ "53251",
+ "53252",
+ "53253",
+ "53254",
+ "53255",
+ "53256",
+ "53257",
+ "53258",
+ "53259",
+ "53260",
+ "53261",
+ "53262",
+ "53263"
+ ],
+ "FPGA_beamlet_output_hdr_udp_destination_port_RW_default": [
+ "10001",
+ "10001",
+ "10001",
+ "10001",
+ "10001",
+ "10001",
+ "10001",
+ "10001",
+ "10001",
+ "10001",
+ "10001",
+ "10001",
+ "10001",
+ "10001",
+ "10001",
+ "10001"
+ ]
}
- },
+ }
+ }
+ }
+ },
+ "DigitalBeam": {
+ "STAT": {
"DigitalBeam": {
- "STAT": {
- "DigitalBeam": {
- "STAT/DigitalBeam/HBA": {
- "properties": {
- "Beam_tracking_interval": [
- "1.0"
- ],
- "Beam_tracking_preparation_period": [
- "0.5"
- ]
- }
- }
- }
+ "STAT/DigitalBeam/HBA": {
+ "properties": {
+ "Beam_tracking_interval": [
+ "1.0"
+ ],
+ "Beam_tracking_preparation_period": [
+ "0.5"
+ ]
}
- },
+ }
+ }
+ }
+ },
+ "Boot": {
+ "STAT": {
"Boot": {
- "STAT": {
- "Boot": {
- "STAT/Boot/1": {}
- }
- }
- },
+ "STAT/Boot/1": {}
+ }
+ }
+ },
+ "APSCT": {
+ "STAT": {
"APSCT": {
- "STAT": {
- "APSCT": {
- "STAT/APSCT/1": {
- "properties": {
- }
- }
- }
+ "STAT/APSCT/1": {
+ "properties": {
}
- },
+ }
+ }
+ }
+ },
+ "CCD": {
+ "STAT": {
"CCD": {
- "STAT": {
- "CCD": {
- "STAT/CCD/1": {
- "properties": {
- }
- }
- }
+ "STAT/CCD/1": {
+ "properties": {
}
- },
+ }
+ }
+ }
+ },
+ "APSPU": {
+ "STAT": {
"APSPU": {
- "STAT": {
- "APSPU": {
- "STAT/APSPU/1": {
- "properties": {
- }
- }
- }
+ "STAT/APSPU/1": {
+ "properties": {
}
- },
+ }
+ }
+ }
+ },
+ "RECV": {
+ "STAT": {
"RECV": {
- "STAT": {
- "RECV": {
- "STAT/RECV/1": {
- "properties": {
- }
- }
- }
+ "STAT/RECV/1": {
+ "properties": {
}
- },
+ }
+ }
+ }
+ },
+ "SDP": {
+ "STAT": {
"SDP": {
- "STAT": {
- "SDP": {
- "STAT/SDP/1": {
- "properties": {
- }
- }
- }
+ "STAT/SDP/1": {
+ "properties": {
}
- },
+ }
+ }
+ }
+ },
+ "BST": {
+ "STAT": {
"BST": {
- "STAT": {
- "BST": {
- "STAT/BST/1": {
- "properties": {
- "Statistics_Client_UDP_Port": [
- "5003"
- ],
- "Statistics_Client_TCP_Port": [
- "5103"
- ],
- "FPGA_bst_offload_hdr_udp_destination_port_RW_default": [
- "5003",
- "5003",
- "5003",
- "5003",
- "5003",
- "5003",
- "5003",
- "5003",
- "5003",
- "5003",
- "5003",
- "5003",
- "5003",
- "5003",
- "5003",
- "5003"
- ]
- }
- }
- }
+ "STAT/BST/1": {
+ "properties": {
+ "Statistics_Client_UDP_Port": [
+ "5003"
+ ],
+ "Statistics_Client_TCP_Port": [
+ "5103"
+ ],
+ "FPGA_bst_offload_hdr_udp_destination_port_RW_default": [
+ "5003",
+ "5003",
+ "5003",
+ "5003",
+ "5003",
+ "5003",
+ "5003",
+ "5003",
+ "5003",
+ "5003",
+ "5003",
+ "5003",
+ "5003",
+ "5003",
+ "5003",
+ "5003"
+ ]
}
- },
+ }
+ }
+ }
+ },
+ "SST": {
+ "STAT": {
"SST": {
- "STAT": {
- "SST": {
- "STAT/SST/1": {
- "properties": {
- "Statistics_Client_UDP_Port": [
- "5001"
- ],
- "Statistics_Client_TCP_Port": [
- "5101"
- ],
- "FPGA_sst_offload_hdr_udp_destination_port_RW_default": [
- "5001",
- "5001",
- "5001",
- "5001",
- "5001",
- "5001",
- "5001",
- "5001",
- "5001",
- "5001",
- "5001",
- "5001",
- "5001",
- "5001",
- "5001",
- "5001"
- ]
- }
- }
- }
+ "STAT/SST/1": {
+ "properties": {
+ "Statistics_Client_UDP_Port": [
+ "5001"
+ ],
+ "Statistics_Client_TCP_Port": [
+ "5101"
+ ],
+ "FPGA_sst_offload_hdr_udp_destination_port_RW_default": [
+ "5001",
+ "5001",
+ "5001",
+ "5001",
+ "5001",
+ "5001",
+ "5001",
+ "5001",
+ "5001",
+ "5001",
+ "5001",
+ "5001",
+ "5001",
+ "5001",
+ "5001",
+ "5001"
+ ]
}
- },
+ }
+ }
+ }
+ },
+ "XST": {
+ "STAT": {
"XST": {
- "STAT": {
- "XST": {
- "STAT/XST/1": {
- "properties": {
- "Statistics_Client_UDP_Port": [
- "5002"
- ],
- "Statistics_Client_TCP_Port": [
- "5102"
- ],
- "FPGA_xst_offload_hdr_udp_destination_port_RW_default": [
- "5002",
- "5002",
- "5002",
- "5002",
- "5002",
- "5002",
- "5002",
- "5002",
- "5002",
- "5002",
- "5002",
- "5002",
- "5002",
- "5002",
- "5002",
- "5002"
- ]
- }
- }
- }
+ "STAT/XST/1": {
+ "properties": {
+ "Statistics_Client_UDP_Port": [
+ "5002"
+ ],
+ "Statistics_Client_TCP_Port": [
+ "5102"
+ ],
+ "FPGA_xst_offload_hdr_udp_destination_port_RW_default": [
+ "5002",
+ "5002",
+ "5002",
+ "5002",
+ "5002",
+ "5002",
+ "5002",
+ "5002",
+ "5002",
+ "5002",
+ "5002",
+ "5002",
+ "5002",
+ "5002",
+ "5002",
+ "5002"
+ ]
}
- },
+ }
+ }
+ }
+ },
+ "UNB2": {
+ "STAT": {
"UNB2": {
- "STAT": {
- "UNB2": {
- "STAT/UNB2/1": {
- "properties": {
- }
- }
- }
+ "STAT/UNB2/1": {
+ "properties": {
}
+ }
}
+ }
}
+ }
}
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..992ce93adc8faea96632dd15043d3dc1b7337697
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,11 @@
+# forward everything to docker-compose subdirectory
+$(firstword $(MAKECMDGOALS)):
+ $(MAKE) -C docker-compose $(MAKECMDGOALS)
+
+docker-compose:
+ $(MAKE) -C $@ $(MAKECMDGOALS)
+
+.PHONY: docker-compose tox
+
+%::
+ cd .
diff --git a/docker-compose/.env b/docker-compose/.env
index 7de2eff68b7741790da2a3cabe1a98139b20202f..f410ee249419741fa9bc99eb5a67e2cb0717d5e4 100644
--- a/docker-compose/.env
+++ b/docker-compose/.env
@@ -16,4 +16,7 @@ TANGO_STARTER_VERSION=2021-05-28
MYSQL_ROOT_PASSWORD=secret
MYSQL_PASSWORD=tango
+MINIO_ROOT_USER=minioadmin
+MINIO_ROOT_PASSWORD=minioadmin
+
TEST_MODULE=default
diff --git a/docker-compose/Makefile b/docker-compose/Makefile
index 34decf3d511dc1ee8d49c383d0821d6473499630..8a691278ef38891feec7569a8d37b313b6c758fc 100644
--- a/docker-compose/Makefile
+++ b/docker-compose/Makefile
@@ -5,6 +5,11 @@
MAKEPATH := $(abspath $(lastword $(MAKEFILE_LIST)))
BASEDIR := $(notdir $(patsubst %/,%,$(dir $(MAKEPATH))))
+ifeq (, $(shell which docker-compose))
+ DOCKER_COMPOSE = docker compose
+endif
+DOCKER_COMPOSE ?= docker-compose
+
DOCKER_COMPOSE_ENV_FILE := $(abspath .env)
COMPOSE_FILES := $(wildcard *.yml)
COMPOSE_FILE_ARGS := --env-file $(DOCKER_COMPOSE_ENV_FILE) $(foreach yml,$(COMPOSE_FILES),-f $(yml))
@@ -164,33 +169,33 @@ DOCKER_COMPOSE_ARGS := DISPLAY=$(DISPLAY) \
.DEFAULT_GOAL := help
pull: ## pull the images from the Docker hub
- $(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) pull
+ $(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) pull
base: context ## Build base lofar device image
- $(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) build --progress=plain lofar-device-base
+ $(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) build --progress=plain lofar-device-base
base-nocache: context ## Rebuild base lofar device image
- $(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) build --no-cache --progress=plain lofar-device-base
+ $(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) build --no-cache --progress=plain lofar-device-base
build: base ## build images
# docker-compose does not support build dependencies, so manage those here
- $(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) build --parallel --progress=plain $(SERVICE)
+ $(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) build --parallel --progress=plain $(SERVICE)
build-nocache: base-nocache ## rebuild images from scratch
# docker-compose does not support build dependencies, so manage those here
- $(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) build --no-cache --progress=plain $(SERVICE)
+ $(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) build --no-cache --progress=plain $(SERVICE)
up: base minimal ## start the base TANGO system and prepare requested services
- $(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) up --no-start --no-recreate $(SERVICE)
+ $(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) up --no-start --no-recreate $(SERVICE)
run: base minimal ## run a service using arguments and delete it afterwards
- $(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) run -T --no-deps --rm $(SERVICE) $(SERVICE_ARGS)
+ $(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) run -T --no-deps --rm $(SERVICE) $(SERVICE_ARGS)
integration: minimal ## run a service using arguments and delete it afterwards
- $(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) run -T --no-deps --rm integration-test $(INTEGRATION_ARGS)
+ $(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) run -T --no-deps --rm integration-test $(INTEGRATION_ARGS)
down: ## stop all services and tear down the system
- $(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) down
+ $(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) down
ifneq ($(NETWORK_MODE),host)
docker network inspect $(NETWORK_MODE) &> /dev/null && ([ $$? -eq 0 ] && docker network rm $(NETWORK_MODE)) || true
docker network inspect 9000-$(NETWORK_MODE) &> /dev/null && ([ $$? -eq 0 ] && docker network rm 9000-$(NETWORK_MODE)) || true
@@ -202,7 +207,7 @@ ifneq ($(NETWORK_MODE),host)
docker network inspect 9000-$(NETWORK_MODE) &> /dev/null || ([ $$? -ne 0 ] && docker network create 9000-$(NETWORK_MODE) -o com.docker.network.driver.mtu=9000)
endif
- $(DOCKER_COMPOSE_ARGS) docker-compose -f tango.yml -f networks.yml up --no-recreate -d
+ $(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) -f tango.yml -f networks.yml up --no-recreate -d
context: ## Move the necessary files to create minimal docker context
@mkdir -p tmp
@@ -216,24 +221,24 @@ bootstrap: pull build # first start, initialise from scratch
start: up ## start a service (usage: make start <servicename>)
if [ $(UNAME_S) = Linux ]; then touch ~/.Xauthority; chmod a+r ~/.Xauthority; fi
- $(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) start $(SERVICE)
+ $(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) start $(SERVICE)
stop: ## stop a service (usage: make stop <servicename>)
- $(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) stop $(SERVICE)
- $(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) rm -f $(SERVICE)
+ $(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) stop $(SERVICE)
+ $(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) rm -f $(SERVICE)
restart: ## restart a service (usage: make restart <servicename>)
make stop $(SERVICE) # cannot use dependencies, as that would allow start and stop to run in parallel..
make start $(SERVICE)
attach: ## attach a service to an existing Tango network
- $(DOCKER_COMPOSE_ARGS) docker-compose $(ATTACH_COMPOSE_FILE_ARGS) up --no-recreate -d $(SERVICE)
+ $(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(ATTACH_COMPOSE_FILE_ARGS) up --no-recreate -d $(SERVICE)
TIME := 0
await: ## Await every container with total max timeout of 300, do not reset timeout
time=$(TIME); \
for i in $(SERVICE); do \
- current_service=$$($(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) ps -q $${i}); \
+ current_service=$$($(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) ps -q $${i}); \
if [ -z "$${current_service}" ]; then \
continue; \
fi; \
@@ -254,14 +259,14 @@ await: ## Await every container with total max timeout of 300, do not reset tim
done
status: ## show the container status
- $(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) ps
+ $(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) ps
images: ## show the container images
- $(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) images
+ $(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) images
clean: down ## clear all TANGO database entries, and all containers
docker volume rm $(BASEDIR)_tangodb
- $(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) rm -f
+ $(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) rm -f
help: ## show this help.
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
diff --git a/docker-compose/calibration-tables/Dockerfile b/docker-compose/calibration-tables/Dockerfile
deleted file mode 100644
index e7ce3d858a362c221401627b722ffef2e5fd9abe..0000000000000000000000000000000000000000
--- a/docker-compose/calibration-tables/Dockerfile
+++ /dev/null
@@ -1,2 +0,0 @@
-FROM nginx
-COPY caltables /usr/share/nginx/html
diff --git a/docker-compose/calibration-tables/README.md b/docker-compose/calibration-tables/README.md
deleted file mode 100644
index 10261e80cbacf9081284a2ca4e950d3300a8b639..0000000000000000000000000000000000000000
--- a/docker-compose/calibration-tables/README.md
+++ /dev/null
@@ -1,3 +0,0 @@
-# Calibration Tables
-
-Service to serve HDF5 calibration table files over HTTP. Used by antennafield device.
diff --git a/docker-compose/device-antennafield.yml b/docker-compose/device-antennafield.yml
index 69ccb78c0f84b967ba0b0dc952e6d433621739fc..5c38de7f3326b7f8a3069766394f4f890d2170f3 100644
--- a/docker-compose/device-antennafield.yml
+++ b/docker-compose/device-antennafield.yml
@@ -34,9 +34,6 @@ services:
- "host.docker.internal:host-gateway"
volumes:
- ..:/opt/lofar/tango:rw
- - type: bind
- source: ./calibration-tables/caltables
- target: /opt/calibration-tables
environment:
- TANGO_HOST=${TANGO_HOST}
- TANGO_ZMQ_EVENT_PORT=5815
diff --git a/docker-compose/device-calibration.yml b/docker-compose/device-calibration.yml
new file mode 100644
index 0000000000000000000000000000000000000000..2be7c73bdcfdba4560f7ab7f169d8b9dccc110ca
--- /dev/null
+++ b/docker-compose/device-calibration.yml
@@ -0,0 +1,58 @@
+# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: Apache-2.0
+#
+# Docker compose file that launches an interactive iTango session.
+#
+# Connect to the interactive session with 'docker attach itango'.
+# Disconnect with the Docker deattach sequence: <CTRL>+<P> <CTRL>+<Q>
+#
+# Defines:
+# - itango: iTango interactive session
+#
+# Requires:
+# - lofar-device-base.yml
+#
+version: '2.1'
+
+services:
+ device-calibration:
+ image: lofar-device-base
+ container_name: device-calibration
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "100m"
+ max-file: "10"
+ networks:
+ - control
+ ports:
+ - "5724:5724" # unique port for this DS
+ - "5824:5824" # ZeroMQ event port
+ - "5924:5924" # ZeroMQ heartbeat port
+ extra_hosts:
+ - "host.docker.internal:host-gateway"
+ volumes:
+ - ..:/opt/lofar/tango:rw
+ environment:
+ - TANGO_HOST=${TANGO_HOST}
+ - TANGO_ZMQ_EVENT_PORT=5824
+ - TANGO_ZMQ_HEARTBEAT_PORT=5924
+ - MINIO_ROOT_USER=${MINIO_ROOT_USER}
+ - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD}
+ healthcheck:
+ test: l2ss-health STAT/Calibration/1
+ interval: 1m
+ timeout: 30s
+ retries: 3
+ start_period: 30s
+ working_dir: /opt/lofar/tango
+ entrypoint:
+ - bin/start-ds.sh
+ # configure CORBA to _listen_ on 0:port, but tell others we're _reachable_ through ${HOSTNAME}:port, since CORBA
+ # can't know about our Docker port forwarding
+ - l2ss-calibration Calibration STAT -v -ORBendPoint giop:tcp:0:5724 -ORBendPointPublish giop:tcp:${HOSTNAME}:5724
+ restart: unless-stopped
+ stop_signal: SIGINT # request a graceful shutdown of Tango
+ stop_grace_period: 2s
+ depends_on:
+ - object-storage
diff --git a/docker-compose/lofar-device-base/Dockerfile b/docker-compose/lofar-device-base/Dockerfile
index 42254ec6f31dfc7e4844826e208556a45edba233..7b49a6cfcd2db4b786d7f5698267a672c6790a01 100644
--- a/docker-compose/lofar-device-base/Dockerfile
+++ b/docker-compose/lofar-device-base/Dockerfile
@@ -27,4 +27,3 @@ COPY lofar-device-base/casarc /home/tango/.casarc
ENV TANGO_LOG_PATH=/var/log/tango
RUN sudo mkdir -p /var/log/tango && sudo chmod a+rwx /var/log/tango
-RUN sudo mkdir -p /opt/calibration-tables && sudo chmod a+rwx /opt/calibration-tables
diff --git a/docker-compose/object-storage.yml b/docker-compose/object-storage.yml
new file mode 100644
index 0000000000000000000000000000000000000000..5ba417b415080e287500149896f727216a66b95c
--- /dev/null
+++ b/docker-compose/object-storage.yml
@@ -0,0 +1,48 @@
+# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: Apache-2.0
+#
+# Docker compose file that launches Minio as a S3 compatible object storage. A UI is available via http
+#
+# Connect by surfing to http://localhost:9001/
+# View logs through 'docker logs -f -t object-storage'
+
+version: '2.1'
+
+services:
+ object-storage:
+ image: minio/minio
+ container_name: object-storage
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "100m"
+ max-file: "10"
+ networks:
+ - control
+ volumes:
+ - object-storage:/data:rw
+ environment:
+ - MINIO_ROOT_USER=${MINIO_ROOT_USER}
+ - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD}
+ ports:
+ - "9000:9000"
+ - "9001:9001"
+ command: server --console-address ":9001" /data
+ restart: unless-stopped
+
+ init-object-storage:
+ image: minio/mc
+ networks:
+ - control
+ depends_on:
+ - object-storage
+ volumes:
+ - ..:/opt/lofar/tango:rw
+ entrypoint: ''
+ command: >
+ sh -c "mc alias set object-storage http://object-storage:9000 $MINIO_ROOT_USER $MINIO_ROOT_PASSWORD
+ mc mb --with-versioning object-storage/caltables
+ mc cp --recursive /opt/lofar/tango/docker-compose/object-storage/caltables/ object-storage/caltables/"
+
+volumes:
+ object-storage:
diff --git a/docker-compose/calibration-tables/caltables/CalTable-DevStation-HBA-150MHz.h5 b/docker-compose/object-storage/caltables/DevStation/CalTable-DevStation-HBA-150MHz.h5
similarity index 94%
rename from docker-compose/calibration-tables/caltables/CalTable-DevStation-HBA-150MHz.h5
rename to docker-compose/object-storage/caltables/DevStation/CalTable-DevStation-HBA-150MHz.h5
index f2beb4b078b240d5dc6d52804999707d98b5524e..0cb1af7b0cc5b0517a00de2b41a294643138aed7 100644
Binary files a/docker-compose/calibration-tables/caltables/CalTable-DevStation-HBA-150MHz.h5 and b/docker-compose/object-storage/caltables/DevStation/CalTable-DevStation-HBA-150MHz.h5 differ
diff --git a/docker-compose/calibration-tables/caltables/CalTable-DevStation-HBA-200MHz.h5 b/docker-compose/object-storage/caltables/DevStation/CalTable-DevStation-HBA-200MHz.h5
similarity index 92%
rename from docker-compose/calibration-tables/caltables/CalTable-DevStation-HBA-200MHz.h5
rename to docker-compose/object-storage/caltables/DevStation/CalTable-DevStation-HBA-200MHz.h5
index d560a6f71dd2c935bd993ad947503e864e129bc6..8af326794446c05ce57a1b548146346dc3b8fae3 100644
Binary files a/docker-compose/calibration-tables/caltables/CalTable-DevStation-HBA-200MHz.h5 and b/docker-compose/object-storage/caltables/DevStation/CalTable-DevStation-HBA-200MHz.h5 differ
diff --git a/docker-compose/calibration-tables/caltables/CalTable-DevStation-HBA-250MHz.h5 b/docker-compose/object-storage/caltables/DevStation/CalTable-DevStation-HBA-250MHz.h5
similarity index 94%
rename from docker-compose/calibration-tables/caltables/CalTable-DevStation-HBA-250MHz.h5
rename to docker-compose/object-storage/caltables/DevStation/CalTable-DevStation-HBA-250MHz.h5
index 868cf35e8677d1070f048e4b6150d1a41291033e..af5980a9fbabf0f5aa9ef3cd55e89fea3cbeea2f 100644
Binary files a/docker-compose/calibration-tables/caltables/CalTable-DevStation-HBA-250MHz.h5 and b/docker-compose/object-storage/caltables/DevStation/CalTable-DevStation-HBA-250MHz.h5 differ
diff --git a/docker-compose/calibration-tables/caltables/CalTable-DevStation-LBA-50MHz.h5 b/docker-compose/object-storage/caltables/DevStation/CalTable-DevStation-LBA-50MHz.h5
similarity index 94%
rename from docker-compose/calibration-tables/caltables/CalTable-DevStation-LBA-50MHz.h5
rename to docker-compose/object-storage/caltables/DevStation/CalTable-DevStation-LBA-50MHz.h5
index 6637b2019a02e3d02335b5393b908bc20146fa44..baef10d555c1b0101146cca088bcac9c37365d67 100644
Binary files a/docker-compose/calibration-tables/caltables/CalTable-DevStation-LBA-50MHz.h5 and b/docker-compose/object-storage/caltables/DevStation/CalTable-DevStation-LBA-50MHz.h5 differ
diff --git a/sbin/run_integration_test.sh b/sbin/run_integration_test.sh
index 2a814ffaaeafaa7da9d39cf84c7f496e0a2f7ca0..9267f4692b12e174d7b5b9e631304e8a8030d727 100755
--- a/sbin/run_integration_test.sh
+++ b/sbin/run_integration_test.sh
@@ -1,6 +1,8 @@
#!/bin/bash -e
-# Copyright (C) 2022 ASTRON (Netherlands Institute for Radio Astronomy)
+#
+# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy)
# SPDX-License-Identifier: Apache-2.0
+#
# Usage function explains how parameters are parsed
function usage {
@@ -74,7 +76,7 @@ echo '/usr/local/bin/wait-for-it.sh ${TANGO_HOST} --strict --timeout=300 -- true
# Devices list is used to explitly word split when supplied to commands, must
# disable shellcheck SC2086 for each case.
-DEVICES=(device-station-manager device-boot device-apsct device-ccd device-apspu device-sdp device-recv device-bst device-sst device-unb2 device-xst device-beamlet device-digitalbeam device-tilebeam device-psoc device-pcon device-antennafield device-temperature-manager device-observation device-observation-control device-configuration)
+DEVICES=(device-station-manager device-boot device-apsct device-ccd device-apspu device-sdp device-recv device-bst device-sst device-unb2 device-xst device-beamlet device-digitalbeam device-tilebeam device-psoc device-pcon device-antennafield device-temperature-manager device-observation device-observation-control device-configuration device-calibration)
SIMULATORS=(sdptr-sim recv-sim unb2-sim apsct-sim apspu-sim ccd-sim)
@@ -87,11 +89,12 @@ make build logstash integration-test http-json-schemas
# Start and stop sequence
make stop http-json-schemas
+make stop object-storage init-object-storage
make stop "${DEVICES[@]}" "${SIMULATORS[@]}"
make stop device-docker # this one does not test well in docker-in-docker
make stop logstash
-make start logstash http-json-schemas
+make start logstash http-json-schemas object-storage init-object-storage
# Update the dsconfig
# Do not remove `bash`, otherwise statement ignored by gitlab ci shell!
diff --git a/sbin/tag_and_push_docker_image.sh b/sbin/tag_and_push_docker_image.sh
index 4ae48590ba149268bc3670999bdbc1cd00ded147..7d398b0757c60494b10b1581922d0c17d1eaf4e3 100755
--- a/sbin/tag_and_push_docker_image.sh
+++ b/sbin/tag_and_push_docker_image.sh
@@ -1,5 +1,5 @@
#!/bin/bash -e
-# Copyright (C) 2022 ASTRON (Netherlands Institute for Radio Astronomy)
+# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy)
# SPDX-License-Identifier: Apache-2.0
function usage {
diff --git a/tangostationcontrol/requirements.txt b/tangostationcontrol/requirements.txt
index 7c6af9b857826f93a7240f428a412609565f9fc1..86f4505ebed4e03c60bbc2fb6f317f30bf6f3edf 100644
--- a/tangostationcontrol/requirements.txt
+++ b/tangostationcontrol/requirements.txt
@@ -17,3 +17,4 @@ python-casacore >= 3.3.1 # LGPLv3
etrs-itrs@git+https://github.com/brentjens/etrs-itrs # Apache 2
lofarantpos >= 0.5.0 # Apache 2
python-geohash >= 0.8.5 # Apache 2 / MIT
+minio >= 7.1.14 # Apache 2
diff --git a/tangostationcontrol/setup.cfg b/tangostationcontrol/setup.cfg
index 5ccd4a8ac711b068f986fd11c257ab5c7eb29380..2b87dbffc9d46a051aa3245e1561f535732c0342 100644
--- a/tangostationcontrol/setup.cfg
+++ b/tangostationcontrol/setup.cfg
@@ -54,6 +54,7 @@ console_scripts =
l2ss-xst = tangostationcontrol.devices.sdp.xst:main
l2ss-temperaturemanager = tangostationcontrol.devices.temperature_manager:main
l2ss-configuration = tangostationcontrol.devices.configuration:main
+ l2ss-calibration = tangostationcontrol.devices.calibration:main
# The following entry points should eventually be removed / replaced
l2ss-hardware-device-template = tangostationcontrol.examples.HW_device_template:main
diff --git a/tangostationcontrol/tangostationcontrol/common/__init__.py b/tangostationcontrol/tangostationcontrol/common/__init__.py
index a212ea51d7847a8f4e0340d00932f723c51f2857..bbdd80eaa2ac676116d0ae5a10856f970e3378b0 100644
--- a/tangostationcontrol/tangostationcontrol/common/__init__.py
+++ b/tangostationcontrol/tangostationcontrol/common/__init__.py
@@ -1,8 +1,6 @@
-# Copyright (C) 2022 ASTRON (Netherlands Institute for Radio Astronomy)
-# SPDX-License-Identifier: Apache-2.0
+# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: Apache-2.0
from .observation_controller import ObservationController
-__all__ = [
- "ObservationController",
-]
+__all__ = ["ObservationController"]
diff --git a/tangostationcontrol/tangostationcontrol/common/calibration.py b/tangostationcontrol/tangostationcontrol/common/calibration.py
index 970781c70a8ba0e5470725fe6bce5bf498babec7..e949179e7396a134f9d59608774db011ff31543c 100644
--- a/tangostationcontrol/tangostationcontrol/common/calibration.py
+++ b/tangostationcontrol/tangostationcontrol/common/calibration.py
@@ -1,17 +1,159 @@
-# Copyright (C) 2022 ASTRON (Netherlands Institute for Radio Astronomy)
-# SPDX-License-Identifier: Apache-2.0
+# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: Apache-2.0
+import logging
+import os
+import tempfile
+from typing import Dict
+from urllib.parse import urlparse
import numpy
-from lofar_station_client.file_access import read_hdf5
+from lofar_station_client.file_access import member, attribute, read_hdf5
+from minio import Minio
+from tango import DeviceProxy
-from tangostationcontrol.common.calibration_table import (
- CalibrationTable as Hdf5CalibrationTable,
-)
from tangostationcontrol.common.constants import (
- N_pol,
N_subbands,
+ N_pol,
+ N_pn,
+ S_pn,
)
+logger = logging.getLogger()
+
+
+class CalibrationData:
+ x: numpy.ndarray = member()
+ y: numpy.ndarray = member()
+
+
+class CalibrationTable:
+ observation_station: str = attribute()
+ observation_station_version: str = attribute()
+ observation_mode: str = attribute()
+ observation_source: str = attribute()
+ observation_date: str = attribute()
+ calibration_version: int = attribute()
+ calibration_name: str = attribute()
+ calibration_date: str = attribute()
+ antennas: Dict[str, CalibrationData] = member()
+
+
+class CalibrationManager:
+ @property
+ def url(self):
+ return self._url
+
+ @url.setter
+ def url(self, new_url):
+ self._url = new_url
+
+ def __init__(self, url: str, station_name: str):
+ self._url = url
+ self._station_name = station_name
+ self._tmp_dir = tempfile.TemporaryDirectory(ignore_cleanup_errors=True)
+ self.bucket_name = "caltabules"
+ self.prefix = self._station_name
+ self._init_minio()
+ self.sync_calibration_tables()
+
+ def _init_minio(self):
+ result = urlparse(self._url)
+ bucket_name, prefix, *_ = result.path[1:].split("/", 1) + [""]
+ if len(prefix) > 0:
+ self.prefix = "/".join([prefix.rstrip("/"), self._station_name])
+ if len(bucket_name) > 0:
+ self.bucket_name = bucket_name
+
+ self._storage = Minio(
+ result.netloc,
+ access_key=os.getenv("MINIO_ROOT_USER"),
+ secret_key=os.getenv("MINIO_ROOT_PASSWORD"),
+ secure=result.scheme == "https",
+ )
+
+ def sync_calibration_tables(self):
+ logger.debug(
+ f"Sync calibration tables from bucket {self.bucket_name} "
+ f"with prefix {self.prefix}/"
+ )
+ objects = self._storage.list_objects(self.bucket_name, prefix=f"{self.prefix}/")
+ for obj in objects:
+ filename = os.path.basename(obj.object_name)
+ self._storage.fget_object(
+ self.bucket_name,
+ obj.object_name,
+ os.path.join(self._tmp_dir.name, filename),
+ )
+
+ @staticmethod
+ def _band_to_reference_frequency(is_hba, rcu_band):
+ if is_hba:
+ match rcu_band:
+ case 1:
+ return "200"
+ case 2:
+ return "150"
+ case 4:
+ return "250"
+ return "50"
+
+ def calibrate_subband_weights(self, antenna_field: DeviceProxy, sdp: DeviceProxy):
+ # -----------------------------------------------------------
+ # Compute calibration of subband weights for the remaining
+ # delay and loss corrections.
+ # -----------------------------------------------------------
+
+ # Mapping [antenna] -> [fpga][input]
+ antenna_to_sdp_mapping = antenna_field.Antenna_to_SDP_Mapping_R
+
+ # read-modify-write on [fpga][(input, polarisation)]
+ fpga_subband_weights = sdp.FPGA_subband_weights_RW.reshape(
+ (N_pn, S_pn, N_subbands)
+ )
+
+ antennafield_name = antenna_field.name().split("/")[2]
+ rcu_bands = antenna_field.RCU_band_select_RW
+ antenna_names = antenna_field.Antenna_Names_R
+
+ is_hba = antenna_field.Antenna_Type_R != "LBA"
+
+ for antenna_nr, rcu_band in enumerate(rcu_bands):
+ fpga_nr, input_nr = antenna_to_sdp_mapping[antenna_nr]
+
+ if input_nr == -1:
+ # skip unconnected antennas
+ continue
+
+ calibration_filename = os.path.join(
+ self._tmp_dir.name,
+ f"CalTable-{self._station_name}-{antennafield_name}"
+ f"-{self._band_to_reference_frequency(is_hba, rcu_band)}MHz.h5",
+ )
+ with read_hdf5(calibration_filename, CalibrationTable) as table:
+ # Retrieve data and convert them in the correct Tango attr shape
+ if (
+ table.observation_station.casefold()
+ != self._station_name.casefold()
+ ):
+ logger.error(
+ f"Expected calibration table for {self._station_name}, "
+ f"but got {table.observation_station}"
+ )
+
+ # set weights
+ fpga_subband_weights[fpga_nr, input_nr * N_pol + 0] = table.antennas[
+ antenna_names[antenna_nr]
+ ].x
+
+ fpga_subband_weights[fpga_nr, input_nr * N_pol + 1] = table.antennas[
+ antenna_names[antenna_nr]
+ ].y
+
+ # TODO(L2SS-1312): This should use atomic_read_modify_write
+ sdp.FPGA_subband_weights_RW = fpga_subband_weights.reshape(
+ N_pn, S_pn * N_subbands
+ )
+
def delay_compensation(delays_seconds: numpy.ndarray, clock: int):
"""Return the delay compensation required to line up
@@ -48,7 +190,7 @@ def delay_compensation(delays_seconds: numpy.ndarray, clock: int):
signal_delays_subsample_seconds = delays_seconds - signal_delays_samples / clock
input_delays_subsample_seconds = -signal_delays_subsample_seconds
- return (input_delays_samples, input_delays_subsample_seconds)
+ return input_delays_samples, input_delays_subsample_seconds
def dB_to_factor(dB: numpy.ndarray) -> numpy.ndarray:
@@ -56,6 +198,63 @@ def dB_to_factor(dB: numpy.ndarray) -> numpy.ndarray:
return 10 ** (dB / 10)
+# SDP input delay calibration
+def calibrate_input_samples_delay(antenna_field: DeviceProxy, sdp: DeviceProxy):
+ # Mapping [antenna] -> [fpga][input]
+ antenna_to_sdp_mapping = antenna_field.Antenna_to_SDP_Mapping_R
+
+ # -----------------------------------------------------------
+ # Set coarse delay compensation by delaying the samples.
+ # -----------------------------------------------------------
+
+ # The delay to apply, in samples [antenna]
+ # Correct for signal delays in the cables
+ signal_delay_seconds = antenna_field.Antenna_Cables_Delay_R
+
+ # compute the required compensation
+ clock = sdp.clock_RW
+ input_samples_delay, _ = delay_compensation(signal_delay_seconds, clock)
+
+ # read-modify-write on [fpga][(input, polarisation)]
+ fpga_signal_input_samples_delay = sdp.FPGA_signal_input_samples_delay_RW
+
+ for antenna_nr, (fpga_nr, input_nr) in enumerate(antenna_to_sdp_mapping):
+ if input_nr == -1:
+ # skip unconnected antennas
+ continue
+
+ # set for X polarisation
+ fpga_signal_input_samples_delay[
+ fpga_nr, input_nr * N_pol + 0
+ ] = input_samples_delay[antenna_nr]
+ # set for Y polarisation
+ fpga_signal_input_samples_delay[
+ fpga_nr, input_nr * N_pol + 1
+ ] = input_samples_delay[antenna_nr]
+
+ # TODO(L2SS-1312): This should use atomic_read_modify_write
+ sdp.FPGA_signal_input_samples_delay_RW = fpga_signal_input_samples_delay
+
+
+# RCU calibration
+
+
+def calibrate_RCU_attenuator_dB(antenna_field: DeviceProxy):
+ # -----------------------------------------------------------
+ # Set signal-input attenuation to compensate for
+ # differences in cable length.
+ # -----------------------------------------------------------
+
+ # Correct for signal loss in the cables
+ signal_delay_loss = (
+ antenna_field.Antenna_Cables_Loss_R - antenna_field.Field_Attenuation_R
+ )
+
+ # return coarse attenuation to apply
+ rcu_attenuator_db, _ = loss_compensation(signal_delay_loss)
+ antenna_field.RCU_attenuator_dB_RW = rcu_attenuator_db
+
+
def loss_compensation(losses_dB: numpy.ndarray):
"""Return the attenuation required to line up
signals that are dampened by "lossed_dB" decibel.
@@ -89,42 +288,3 @@ def loss_compensation(losses_dB: numpy.ndarray):
input_attenuation_remainder_factor = dB_to_factor(input_attenuation_remainder_dB)
return (input_attenuation_integer_dB, input_attenuation_remainder_factor)
-
-
-class CalibrationTable:
- """A class to represent calibration tables, and to retrieve calibration data"""
-
- def __init__(self, filename: str):
- # open the file and fill the calibration with actual data
- with read_hdf5(filename, Hdf5CalibrationTable) as table:
- self.table = table
-
- @staticmethod
- def complex_to_float(caltable: numpy.ndarray) -> numpy.ndarray:
- """Numpy array conversion from
- (antenna_list, N_pol, N_subbands),dtype=complex128 to
- (antenna_list, N_pol, N_subbands, VALUES_PER_COMPLEX),dtype=float64
- """
- caltable_float = caltable.view(dtype=numpy.float64)
- return caltable_float.reshape(caltable.shape + (2,))
-
- def get_antenna_data(self, antenna_list: list) -> numpy.ndarray:
- """
- Returns a multi-dimensional array of the calibration data.
- The shape is (antenna_list, N_pol, N_subbands), type is numpy.complex128
- """
- data = []
-
- for antenna in antenna_list:
- if antenna in self.table._data_reader.data.keys():
- # load the calibration for this antenna
- x_data = self.table._data_reader.data[antenna]["x"]
- y_data = self.table._data_reader.data[antenna]["y"]
- x = numpy.array(x_data, dtype=numpy.complex128)
- y = numpy.array(y_data, dtype=numpy.complex128)
- pol = numpy.array([x, y])
- data.append(pol)
- else:
- # append data with default 1.0 value
- data.append([[1.0] * N_subbands] * N_pol)
- return numpy.array(data, dtype=numpy.complex128)
diff --git a/tangostationcontrol/tangostationcontrol/common/calibration_table.py b/tangostationcontrol/tangostationcontrol/common/calibration_table.py
deleted file mode 100644
index 19d4f809c32291502c50a4274b96fbe61f0dd0fa..0000000000000000000000000000000000000000
--- a/tangostationcontrol/tangostationcontrol/common/calibration_table.py
+++ /dev/null
@@ -1,24 +0,0 @@
-# Copyright (C) 2022 ASTRON (Netherlands Institute for Radio Astronomy)
-# SPDX-License-Identifier: Apache-2.0
-
-from typing import Dict
-
-from lofar_station_client.file_access import member, attribute
-from numpy import ndarray
-
-
-class CalibrationData:
- x: ndarray = member()
- y: ndarray = member()
-
-
-class CalibrationTable:
- observation_station: str = attribute()
- observation_station_version: str = attribute()
- observation_mode: str = attribute()
- observation_source: str = attribute()
- observation_date: str = attribute()
- calibration_version: int = attribute()
- calibration_name: str = attribute()
- calibration_date: str = attribute()
- antennas: Dict[str, CalibrationData] = member()
diff --git a/tangostationcontrol/tangostationcontrol/devices/__init__.py b/tangostationcontrol/tangostationcontrol/devices/__init__.py
index 68ddd5cdc3efaa38e853aef337c08beb99c50c4c..c92b615444d854a6e87370b16cf733a5859a07e7 100644
--- a/tangostationcontrol/tangostationcontrol/devices/__init__.py
+++ b/tangostationcontrol/tangostationcontrol/devices/__init__.py
@@ -1,2 +1,2 @@
-# Copyright (C) 2022 ASTRON (Netherlands Institute for Radio Astronomy)
-# SPDX-License-Identifier: Apache-2.0
+# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: Apache-2.0
diff --git a/tangostationcontrol/tangostationcontrol/devices/antennafield.py b/tangostationcontrol/tangostationcontrol/devices/antennafield.py
index d2a2baa771839b71e47710891357a224274ed718..0a2213a6880fcdd519a41c256d134470ade1e26c 100644
--- a/tangostationcontrol/tangostationcontrol/devices/antennafield.py
+++ b/tangostationcontrol/tangostationcontrol/devices/antennafield.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2022 ASTRON (Netherlands Institute for Radio Astronomy)
+# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy)
# SPDX-License-Identifier: Apache-2.0
""" AntennaField Device Server for LOFAR2.0
@@ -28,8 +28,6 @@ from tangostationcontrol.beam.geo import GEO_to_GEOHASH
from tangostationcontrol.beam.geo import ITRF_to_GEO
from tangostationcontrol.beam.hba_tile import HBATAntennaOffsets
from tangostationcontrol.common.cables import cable_types
-from tangostationcontrol.common.calibration import delay_compensation
-from tangostationcontrol.common.calibration import loss_compensation
from tangostationcontrol.common.constants import (
N_elements,
MAX_ANTENNA,
@@ -39,14 +37,10 @@ from tangostationcontrol.common.constants import (
N_rcu,
N_rcu_inp,
N_pn,
- S_pn,
A_pn,
- N_subbands,
- VALUES_PER_COMPLEX,
)
from tangostationcontrol.common.entrypoint import entry
from tangostationcontrol.common.frequency_bands import bands, Band
-from tangostationcontrol.common.calibration import CalibrationTable
from tangostationcontrol.common.lofar_logging import (
device_logging_to_python,
log_exceptions,
@@ -55,13 +49,8 @@ from tangostationcontrol.common.states import DEFAULT_COMMAND_STATES
from tangostationcontrol.common.type_checking import type_not_sequence
from tangostationcontrol.devices.device_decorators import fault_on_error, only_in_states
from tangostationcontrol.devices.interfaces.lofar_device import LOFARDevice
-from tangostationcontrol.devices.sdp.common import (
- real_imag_to_weights,
-)
-from tangostationcontrol.devices.sdp.sdp import SDP
logger = logging.getLogger()
-CALIBRATION_ROOT_DIR = "/opt/calibration-tables"
__all__ = ["AntennaField", "AntennaToRecvMapper", "AntennaToSdpMapper", "main"]
@@ -299,7 +288,7 @@ class AntennaField(LOFARDevice):
)
SDP_device = device_property(
- dtype=str,
+ dtype="DevString",
doc="Which SDP device is processing this AntennaField.",
mandatory=False,
default_value="STAT/SDP/1",
@@ -373,6 +362,26 @@ class AntennaField(LOFARDevice):
max_dim_y=MAX_ANTENNA,
)
+ @attribute(
+ access=AttrWriteType.READ,
+ dtype=str,
+ )
+ def SDP_device_R(self):
+ return self.SDP_device
+
+ @attribute(access=AttrWriteType.READ, dtype="DevFloat")
+ def Field_Attenuation_R(self):
+ return self.Field_Attenuation
+
+ @attribute(
+ access=AttrWriteType.READ,
+ dtype=((numpy.int32,),),
+ max_dim_x=N_pol,
+ max_dim_y=MAX_ANTENNA,
+ )
+ def Control_to_RECV_mapping_R(self):
+ return numpy.array(self.Control_to_RECV_mapping).reshape(-1, 2)
+
Frequency_Band_RW = attribute(
doc="The selected frequency band of each antenna.",
dtype=(str,),
@@ -402,65 +411,6 @@ class AntennaField(LOFARDevice):
unit="dB",
)
- # ----- Calibration information
-
- Calibration_SDP_Signal_Input_Samples_Delay_R = attribute(
- doc="Number of samples that each antenna signal should be delayed to line "
- "up. To be applied on sdp.FPGA_signal_input_samples_delay_RW.",
- dtype=(numpy.uint32,),
- max_dim_x=MAX_ANTENNA,
- unit="samples",
- )
- Calibration_RCU_Attenuation_dB_R = attribute(
- doc="Amount of dB with which each antenna signal must be adjusted to line "
- "up. To be applied on recv.RCU_attenuator_dB_RW.",
- dtype=(numpy.uint32,),
- max_dim_x=MAX_ANTENNA,
- unit="dB",
- )
- Calibration_SDP_Fine_Calibration_Default_R = attribute(
- doc="Computed calibration values for the fine calibration of each "
- "antenna. Each antenna is represented by a (delay, phase_offset, "
- "amplitude_scaling) triplet.",
- dtype=((numpy.float64,),),
- max_dim_y=MAX_ANTENNA * N_pol,
- max_dim_x=3,
- )
- Calibration_SDP_Subband_Weights_R = attribute(
- doc="Calibration values for the rows in sdp.FPGA_subband_weights_RW "
- "relevant for our antennas. Each subband of each polarisation of "
- "each antenna is represented by a real_imag number (real, imag). "
- "Returns the measured values from "
- "Calibration_SDP_Subband_Weights_XXXMHz if available, and values "
- "computed from Calibration_SDP_Fine_Calibration_Default_R otherwise.",
- dtype=((numpy.float64,),),
- max_dim_y=MAX_ANTENNA * N_pol,
- max_dim_x=N_subbands * VALUES_PER_COMPLEX,
- )
-
- # calibration loading
-
- Calibration_SDP_Subband_Weights_50MHz_R = attribute(
- dtype=((numpy.float64,),),
- max_dim_y=MAX_ANTENNA,
- max_dim_x=(N_pol * N_subbands * VALUES_PER_COMPLEX),
- )
- Calibration_SDP_Subband_Weights_150MHz_R = attribute(
- dtype=((numpy.float64,),),
- max_dim_y=MAX_ANTENNA,
- max_dim_x=(N_pol * N_subbands * VALUES_PER_COMPLEX),
- )
- Calibration_SDP_Subband_Weights_200MHz_R = attribute(
- dtype=((numpy.float64,),),
- max_dim_y=MAX_ANTENNA,
- max_dim_x=(N_pol * N_subbands * VALUES_PER_COMPLEX),
- )
- Calibration_SDP_Subband_Weights_250MHz_R = attribute(
- dtype=((numpy.float64,),),
- max_dim_y=MAX_ANTENNA,
- max_dim_x=(N_pol * N_subbands * VALUES_PER_COMPLEX),
- )
-
# ----- Quality and usage information
Antenna_Quality_R = attribute(
@@ -804,7 +754,7 @@ class AntennaField(LOFARDevice):
): # NB: "value[0] in bands" holds at this point
raise ValueError(
f"All frequency bands must use the same clock. \
- These do not: {val} and {value[0,0]}."
+ These do not: {val} and {value[0, 0]}."
)
# apply settings on RECV
@@ -837,63 +787,6 @@ class AntennaField(LOFARDevice):
sdp_nyquist_zone, self.sdp_proxy, "nyquist_zone_RW", None, numpy.uint32
)
- # frequencies changed, so we need to recalibrate
- self.calibrate_recv()
- self.calibrate_sdp()
-
- def _create_calibration_table_filename(self, reference_frequency: str) -> str:
- """Create a valid calibration table filename given a mode"""
- antennafield_name = self.get_name().split("/")[2]
- return (
- f"{CALIBRATION_ROOT_DIR}/"
- + f"CalTable-{self.station}-{antennafield_name}-{reference_frequency}MHz.h5"
- )
-
- def _get_calibration_table(self, reference_frequency: str) -> numpy.ndarray:
- """Returns a calibration table of shape
- (MAX_ANTENNA, N_pol, N_subbands, VALUES_PER_COMPLEX)"""
- filename = self._create_calibration_table_filename(reference_frequency)
- # Initialise table from relative HDF5 file
- try:
- calibration_data = CalibrationTable(filename)
- # Retrieve data and convert them in the correct Tango attr shape
- caltable = CalibrationTable.complex_to_float(
- calibration_data.get_antenna_data(self.Antenna_Names)
- )
- except FileNotFoundError:
- caltable = numpy.full(
- (
- self.read_attribute("nr_antennas_R"),
- N_pol * N_subbands * VALUES_PER_COMPLEX,
- ),
- 1.0,
- )
- return caltable
-
- def read_Calibration_SDP_Subband_Weights_50MHz_R(self):
- subband_weights = self._get_calibration_table(reference_frequency="50")
- return numpy.reshape(
- subband_weights, (self.read_attribute("nr_antennas_R"), -1)
- )
-
- def read_Calibration_SDP_Subband_Weights_150MHz_R(self):
- subband_weights = self._get_calibration_table(reference_frequency="150")
- return numpy.reshape(
- subband_weights, (self.read_attribute("nr_antennas_R"), -1)
- )
-
- def read_Calibration_SDP_Subband_Weights_200MHz_R(self):
- subband_weights = self._get_calibration_table(reference_frequency="200")
- return numpy.reshape(
- subband_weights, (self.read_attribute("nr_antennas_R"), -1)
- )
-
- def read_Calibration_SDP_Subband_Weights_250MHz_R(self):
- subband_weights = self._get_calibration_table(reference_frequency="250")
- return numpy.reshape(
- subband_weights, (self.read_attribute("nr_antennas_R"), -1)
- )
-
def read_Antenna_Cables_R(self):
return self.Antenna_Cables
@@ -921,150 +814,6 @@ class AntennaField(LOFARDevice):
]
)
- def read_Calibration_SDP_Signal_Input_Samples_Delay_R(self):
- # Correct for signal delays in the cables
- signal_delay_seconds = self.read_attribute("Antenna_Cables_Delay_R")
-
- # compute the required compensation
- clock = self.sdp_proxy.clock_RW
- input_delay_samples, _ = delay_compensation(signal_delay_seconds, clock)
-
- # return the delay to apply (in samples)
- return input_delay_samples
-
- def read_Calibration_SDP_Fine_Calibration_Default_R(self):
- def repeat_per_pol(arr):
- # repeat values twice, and restore the shape (with the inner dimension
- # being twice the size now)
- return numpy.dstack((arr, arr)).reshape(
- arr.shape[0] * N_pol, *arr.shape[1:]
- )
-
- # ----- Delay
-
- # correct for signal delays in the cables (equal for both polarisations)
- signal_delay_seconds = repeat_per_pol(
- self.read_attribute("Antenna_Cables_Delay_R")
- )
-
- # compute the required compensation
- clock = self.sdp_proxy.clock_RW
- _, input_delay_subsample_seconds = delay_compensation(
- signal_delay_seconds, clock
- )
-
- # ----- Phase offsets
-
- # we don't have any
- phase_offsets = repeat_per_pol(
- numpy.zeros((self.read_attribute("nr_antennas_R"),), dtype=numpy.float64)
- )
-
- # ----- Amplitude
-
- # correct for signal loss in the cables
- signal_delay_loss = repeat_per_pol(
- self.read_attribute("Antenna_Cables_Loss_R") - self.Field_Attenuation
- )
-
- # return fine scaling to apply
- _, input_attenuation_remaining_factor = loss_compensation(signal_delay_loss)
-
- # Return as (delay, phase_offset, amplitude) triplet per polarisation
- return numpy.stack(
- (
- input_delay_subsample_seconds,
- phase_offsets,
- input_attenuation_remaining_factor,
- ),
- axis=1,
- )
-
- def _rcu_band_to_calibration_table(self) -> dict:
- """
- Returns the SDP subband weights to apply per RCU band.
- """
- nr_antennas = self.read_attribute("nr_antennas_R")
-
- # construct selector for the right calibration table
- if self.Antenna_Type == "LBA":
- rcu_band_to_caltable = {
- 1: self.read_attribute(
- "Calibration_SDP_Subband_Weights_50MHz_R"
- ).tolist(),
- 2: self.read_attribute(
- "Calibration_SDP_Subband_Weights_50MHz_R"
- ).tolist(),
- }
- else: # HBA
- rcu_band_to_caltable = {
- 2: self.read_attribute(
- "Calibration_SDP_Subband_Weights_150MHz_R"
- ).tolist(),
- 1: self.read_attribute(
- "Calibration_SDP_Subband_Weights_200MHz_R"
- ).tolist(),
- 4: self.read_attribute(
- "Calibration_SDP_Subband_Weights_250MHz_R"
- ).tolist(),
- }
-
- # reshape them into their actual form
- for band, caltable in rcu_band_to_caltable.items():
- rcu_band_to_caltable[band] = numpy.array(caltable).reshape(
- nr_antennas, N_pol, N_subbands, VALUES_PER_COMPLEX
- )
-
- return rcu_band_to_caltable
-
- def read_Calibration_SDP_Subband_Weights_R(self):
- # obtain the calibration tables and the RCU bands they depend on
- rcu_bands = self.read_attribute("RCU_band_select_RW")
- rcu_band_to_caltable = self._rcu_band_to_calibration_table()
-
- # antenna mapping onto RECV
- control_to_recv_mapping = numpy.array(self.Control_to_RECV_mapping).reshape(
- -1, 2
- )
- recvs = control_to_recv_mapping[:, 0] # first column is RECV device number
-
- # antenna mapping onto SDP
- antenna_to_sdp_mapping = self.read_attribute("Antenna_to_SDP_Mapping_R")
-
- # construct the subband weights based on the rcu_band of each antenna,
- # combining the relevant tables.
- nr_antennas = self.read_attribute("nr_antennas_R")
- subband_weights = numpy.zeros(
- (nr_antennas, N_pol, N_subbands, VALUES_PER_COMPLEX), dtype=numpy.float64
- )
- for antenna_nr, rcu_band in enumerate(rcu_bands):
- # Skip antennas not connected to RECV. These do not have a valid RCU band
- # selected.
- if recvs[antenna_nr] == 0:
- continue
-
- # Skip antennas not connected to SDP. They must retain a weight of 0.
- if antenna_to_sdp_mapping[antenna_nr, 1] == -1:
- continue
-
- subband_weights[antenna_nr, :, :, :] = rcu_band_to_caltable[rcu_band][
- antenna_nr, :, :, :
- ]
-
- return subband_weights.reshape(
- nr_antennas * N_pol, N_subbands * VALUES_PER_COMPLEX
- )
-
- def read_Calibration_RCU_Attenuation_dB_R(self):
- # Correct for signal loss in the cables
- signal_delay_loss = (
- self.read_attribute("Antenna_Cables_Loss_R") - self.Field_Attenuation
- )
-
- # return coarse attenuation to apply
- input_attenuation_integer_db, _ = loss_compensation(signal_delay_loss)
- return input_attenuation_integer_db
-
def read_Antenna_Use_R(self):
return self.Antenna_Use
@@ -1322,105 +1071,6 @@ class AntennaField(LOFARDevice):
antenna_type, self.sdp_proxy, "antenna_type_RW", None, str
)
- @command()
- def calibrate_recv(self):
- """Calibrate RECV for our antennas.
-
- Run whenever the following changes:
- sdp.clock_RW
- antennafield.RCU_band_select_RW
- """
-
- # -----------------------------------------------------------
- # Set signal-input attenuation to compensate for
- # differences in cable length.
- # -----------------------------------------------------------
-
- rcu_attenuator_db = self.read_attribute("Calibration_RCU_Attenuation_dB_R")
- self.proxy.write_attribute("RCU_attenuator_dB_RW", rcu_attenuator_db)
-
- @command()
- def calibrate_sdp(self):
- """Calibrate SDP for our antennas.
-
- Run whenever the following changes:
- sdp.clock_RW
- antennafield.RCU_band_select_RW
- """
-
- # Mapping [antenna] -> [fpga][input]
- antenna_to_sdp_mapping = self.read_attribute("Antenna_to_SDP_Mapping_R")
-
- # -----------------------------------------------------------
- # Set coarse delay compensation by delaying the samples.
- # -----------------------------------------------------------
-
- # The delay to apply, in samples [antenna]
- input_samples_delay = self.read_attribute(
- "Calibration_SDP_Signal_Input_Samples_Delay_R"
- )
-
- # read-modify-write on [fpga][(input, polarisation)]
- fpga_signal_input_samples_delay = numpy.full((N_pn, A_pn * N_pol), None)
-
- for antenna_nr, (fpga_nr, input_nr) in enumerate(antenna_to_sdp_mapping):
- if input_nr == -1:
- # skip unconnected antennas
- continue
-
- # set for X polarisation
- fpga_signal_input_samples_delay[
- fpga_nr, input_nr * N_pol + 0
- ] = input_samples_delay[antenna_nr]
- # set for Y polarisation
- fpga_signal_input_samples_delay[
- fpga_nr, input_nr * N_pol + 1
- ] = input_samples_delay[antenna_nr]
-
- self.atomic_read_modify_write_attribute(
- fpga_signal_input_samples_delay,
- self.sdp_proxy,
- "FPGA_signal_input_samples_delay_RW",
- None,
- numpy.uint32,
- )
-
- # -----------------------------------------------------------
- # Compute calibration of subband weights for the remaining
- # delay and loss corrections.
- # -----------------------------------------------------------
-
- # obtain caltable
- caltable = self.read_attribute("Calibration_SDP_Subband_Weights_R")
-
- # read-modify-write on [fpga][(input, polarisation)]
- fpga_subband_weights = numpy.full((N_pn, S_pn, N_subbands), None)
-
- for antenna_nr, (fpga_nr, input_nr) in enumerate(antenna_to_sdp_mapping):
- if input_nr == -1:
- # skip unconnected antennas
- continue
-
- # set weights
- fpga_subband_weights[
- fpga_nr, input_nr * N_pol + 0, :
- ] = real_imag_to_weights(
- caltable[antenna_nr * N_pol + 0, :], SDP.SUBBAND_UNIT_WEIGHT
- )
- fpga_subband_weights[
- fpga_nr, input_nr * N_pol + 1, :
- ] = real_imag_to_weights(
- caltable[antenna_nr * N_pol + 1, :], SDP.SUBBAND_UNIT_WEIGHT
- )
-
- self.atomic_read_modify_write_attribute(
- fpga_subband_weights.reshape(N_pn, S_pn * N_subbands),
- self.sdp_proxy,
- "FPGA_subband_weights_RW",
- None,
- numpy.uint32,
- )
-
@command(dtype_in=DevVarFloatArray, dtype_out=DevVarLongArray)
def calculate_HBAT_bf_delay_steps(self, delays: numpy.ndarray):
num_tiles = self.read_nr_antennas_R()
diff --git a/tangostationcontrol/tangostationcontrol/devices/boot.py b/tangostationcontrol/tangostationcontrol/devices/boot.py
index 6bc05990be595c2d3dcd3f63d1500d80a4136bf2..5e4d79d4fcbec0c8dedd5a93ce0f4dd34aca89d3 100644
--- a/tangostationcontrol/tangostationcontrol/devices/boot.py
+++ b/tangostationcontrol/tangostationcontrol/devices/boot.py
@@ -1,5 +1,5 @@
-# Copyright (C) 2022 ASTRON (Netherlands Institute for Radio Astronomy)
-# SPDX-License-Identifier: Apache-2.0
+# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: Apache-2.0
""" Boot Device Server for LOFAR2.0
@@ -17,6 +17,7 @@ from tango import AttrWriteType, DeviceProxy, DevState, DevSource
from tango import DebugIt
from tango.server import command
from tango.server import device_property, attribute
+
from tangostationcontrol.common.entrypoint import entry
from tangostationcontrol.common.lofar_logging import (
device_logging_to_python,
@@ -148,13 +149,16 @@ class DevicesInitialiser(object):
# reset initialisation parameters
self.progress = 0
+ logger.debug(self.devices.keys())
# restart devices in order
for num_restarted_devices, device in enumerate(self.devices.keys(), 1):
# allow resuming by skipping already initialised devices
if self.device_initialised[device]:
+ logger.debug(f"{device} already initialised")
continue
if self.is_available(device):
+ logger.debug(f"{device} available")
if (
self.reboot
or self.devices[device].state() not in OPERATIONAL_STATES
@@ -246,16 +250,22 @@ class Boot(LOFARDevice):
dtype="DevVarStringArray",
mandatory=False,
default_value=[
- "STAT/Docker/1", # Docker controls the device containers, so it goes before anything else
- "STAT/Configuration/1", # Configuration device loads and update station configuration
- "STAT/PSOC/1", # PSOC boot early to detect power delivery failure as fast as possible
- "STAT/PCON/1", # PCON boot early because it is responsible for power delivery.
- "STAT/APSPU/1", # APS Power Units control other hardware we want to initialise
+ "STAT/Docker/1",
+ # Docker controls the device containers, so it goes before anything else
+ "STAT/Configuration/1",
+ # Configuration device loads and update station configuration
+ "STAT/PSOC/1",
+ # PSOC boot early to detect power delivery failure as fast as possible
+ "STAT/PCON/1",
+ # PCON boot early because it is responsible for power delivery.
+ "STAT/APSPU/1",
+ # APS Power Units control other hardware we want to initialise
"STAT/APSCT/1",
"STAT/CCD/1",
"STAT/RECV/1", # RCUs are input for SDP, so initialise them first
"STAT/UNB2/1", # Uniboards host SDP, so initialise them first
- "STAT/SDP/1", # SDP controls the mask for SST/XST/BST/Beamlet, so initialise it first
+ "STAT/SDP/1",
+ # SDP controls the mask for SST/XST/BST/Beamlet, so initialise it first
"STAT/BST/1",
"STAT/SST/1",
"STAT/XST/1",
@@ -267,6 +277,8 @@ class Boot(LOFARDevice):
"STAT/DigitalBeam/HBA", # Accessed SDP and Beamlet
# "STAT/DigitalBeam/LBA", # Accessed SDP and Beamlet
"STAT/TemperatureManager/1",
+ "STAT/Calibration/1",
+ # Calibration device loads and update station calibration
],
)
diff --git a/tangostationcontrol/tangostationcontrol/devices/calibration.py b/tangostationcontrol/tangostationcontrol/devices/calibration.py
new file mode 100644
index 0000000000000000000000000000000000000000..d5894f836baa37dbeaf0411a8cb6edcb673e234f
--- /dev/null
+++ b/tangostationcontrol/tangostationcontrol/devices/calibration.py
@@ -0,0 +1,212 @@
+# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: Apache-2.0
+import logging
+
+from tango import DeviceProxy, EventType, Database, AttributeProxy
+from tango.server import device_property, command, attribute
+
+from tangostationcontrol.common.calibration import (
+ CalibrationManager,
+ calibrate_RCU_attenuator_dB,
+ calibrate_input_samples_delay,
+)
+from tangostationcontrol.common.entrypoint import entry
+from tangostationcontrol.common.lofar_logging import (
+ device_logging_to_python,
+ log_exceptions,
+)
+from tangostationcontrol.common.states import DEFAULT_COMMAND_STATES
+from tangostationcontrol.devices.antennafield import AntennaField
+from tangostationcontrol.devices.device_decorators import only_in_states
+from tangostationcontrol.devices.interfaces.lofar_device import LOFARDevice
+from tangostationcontrol.devices.sdp.sdp import SDP
+
+logger = logging.getLogger()
+__all__ = ["Calibration", "main"]
+
+
+@device_logging_to_python()
+class Calibration(LOFARDevice):
+ """Manages the calibration of antenna field, SDP and RECV devices."""
+
+ def __init__(self, cl, name):
+ super().__init__(cl, name)
+ self._calibration_manager: CalibrationManager = None
+ self.event_subscriptions: list = []
+ self.sdp_proxies: dict = {}
+ self.ant_proxies: dict = {}
+ self.station_name: AttributeProxy = None
+
+ @log_exceptions()
+ def _frequency_band_changed_event(self, event):
+ """Trigger on external changes in frequency settings."""
+
+ if event.err:
+ # little we can do here. note that errors are also
+ # thrown if the device we subscribed to is offline
+ return
+
+ logger.info(
+ f"Received attribute change event from {event.device}: "
+ f"{event.attr_value.name} := {event.attr_value.value}"
+ )
+
+ if self.dev_state() not in DEFAULT_COMMAND_STATES:
+ logger.warning("Device not active. Ignore freq band changed event")
+ return
+
+ # frequencies changed, so we need to recalibrate
+ self.calibrate_recv(event.device)
+ self.calibrate_sdp(event.device)
+
+ @log_exceptions()
+ def _clock_changed_event(self, event):
+ """Trigger on external changes in frequency settings."""
+ if event.err:
+ # little we can do here. note that errors are also
+ # thrown if the device we subscribed to is offline
+ return
+
+ logger.info(
+ f"Received attribute change event from {event.device}: "
+ f"{event.attr_value.name} := {event.attr_value.value}"
+ )
+
+ if self.dev_state() not in DEFAULT_COMMAND_STATES:
+ logger.warning("Device not active. Ignore clock changed event")
+ return
+
+ for k, ant in self.ant_proxies.items():
+ if ant.SDP_device_R.casefold() == str(event.device).casefold():
+ logger.info(f"Re-calibrate antenna field {k}")
+ self.calibrate_sdp(k)
+ self.calibrate_recv(k)
+
+ Calibration_Table_Base_URL = device_property(
+ doc="Base URL of the calibration tables",
+ dtype="DevString",
+ mandatory=False,
+ update_db=True,
+ default_value="http://object-storage:9000/caltables",
+ )
+
+ @attribute(dtype=(str,), max_dim_x=20)
+ def AntennaFields_Monitored_R(self):
+ return list(self.ant_proxies.keys())
+
+ @attribute(dtype=(str,), max_dim_x=20)
+ def SDPs_Monitored_R(self):
+ return list(self.sdp_proxies.keys())
+
+ @command
+ def download_calibration_tables(self):
+ self._calibration_manager.sync_calibration_tables()
+ for ant in self.ant_proxies.keys():
+ self.calibrate_recv(ant)
+ self.calibrate_sdp(ant)
+
+ @command(dtype_in=str)
+ @only_in_states(DEFAULT_COMMAND_STATES)
+ def calibrate_recv(self, device: str):
+ """Calibrate RECV for our antennas.
+
+ Run whenever the following changes:
+ sdp.clock_RW
+ antennafield.RCU_band_select_RW
+ """
+
+ # -----------------------------------------------------------
+ # Set signal-input attenuation to compensate for
+ # differences in cable length.
+ # -----------------------------------------------------------
+
+ calibrate_RCU_attenuator_dB(self.ant_proxies[device])
+
+ @command(dtype_in=str)
+ @only_in_states(DEFAULT_COMMAND_STATES)
+ def calibrate_sdp(self, device: str):
+ """Calibrate SDP for our antennas.
+
+ Run whenever the following changes:
+ sdp.clock_RW
+ antennafield.RCU_band_select_RW
+ """
+
+ ant_proxy = self.ant_proxies[device]
+ sdp_device = str(ant_proxy.SDP_device_R)
+ sdp_proxy = self.sdp_proxies[sdp_device]
+
+ calibrate_input_samples_delay(ant_proxy, sdp_proxy)
+
+ self._calibration_manager.calibrate_subband_weights(ant_proxy, sdp_proxy)
+
+ # --------
+ # Overloaded functions
+ # --------
+
+ def configure_for_initialise(self):
+ super().configure_for_initialise()
+
+ self.station_name = AttributeProxy("STAT/StationManager/1/station_name_R")
+ self._calibration_manager = CalibrationManager(
+ self.Calibration_Table_Base_URL, self.station_name.read().value
+ )
+
+ db = Database()
+ devices = db.get_device_exported_for_class(AntennaField.__name__)
+ for d in devices:
+ logger.debug("found antenna field device " + str(d))
+ self.ant_proxies = {d: DeviceProxy(d) for d in devices}
+
+ devices = db.get_device_exported_for_class(SDP.__name__)
+ for d in devices:
+ logger.debug("found SDP device " + str(d))
+ self.sdp_proxies = {d: DeviceProxy(d) for d in devices}
+
+ # subscribe to events to notice setting changes in SDP that determine the
+ # input frequency
+ for prx in self.ant_proxies.values():
+ self.event_subscriptions.append(
+ (
+ prx,
+ prx.subscribe_event(
+ "Frequency_Band_RW",
+ EventType.CHANGE_EVENT,
+ self._frequency_band_changed_event,
+ stateless=True,
+ ),
+ )
+ )
+ for prx in self.sdp_proxies.values():
+ self.event_subscriptions.append(
+ (
+ prx,
+ prx.subscribe_event(
+ "clock_RW",
+ EventType.CHANGE_EVENT,
+ self._clock_changed_event,
+ stateless=True,
+ ),
+ )
+ )
+
+ def configure_for_off(self):
+ super().configure_for_off()
+
+ # unsubscribe from all events
+ subscriptions = self.event_subscriptions
+ self.event_subscriptions = []
+ for prx, s in subscriptions:
+ prx.unsubscribe_event(s)
+
+
+# ----------
+# Run server
+# ----------
+def main(**kwargs):
+ """Main function of the Calibration module."""
+ return entry(Calibration, **kwargs)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_antennafield.py b/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_antennafield.py
index fafcc330804084ab4d187d1c3aef8f8428a1c468..d9f95b6a2b7cf2c3fd8af230a932e3494004ef3f 100644
--- a/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_antennafield.py
+++ b/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_antennafield.py
@@ -1,9 +1,11 @@
-# Copyright (C) 2022 ASTRON (Netherlands Institute for Radio Astronomy)
-# SPDX-License-Identifier: Apache-2.0
+# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: Apache-2.0
-import numpy
import time
+
+import numpy
from tango import DevState
+
from tangostationcontrol.common.constants import (
N_elements,
MAX_ANTENNA,
@@ -14,68 +16,14 @@ from tangostationcontrol.common.constants import (
CLK_200_MHZ,
N_pn,
S_pn,
- N_subbands,
- VALUES_PER_COMPLEX,
)
from tangostationcontrol.common.frequency_bands import bands
from tangostationcontrol.devices.antennafield import AntennaQuality, AntennaUse
from tangostationcontrol.integration_test.device_proxy import TestDeviceProxy
-
from .base import AbstractTestBases
class TestAntennaFieldDevice(AbstractTestBases.TestDeviceBase):
- HBA_ANTENNA_NAMES = [
- "H0",
- "H1",
- "H2",
- "H3",
- "H4",
- "H5",
- "H6",
- "H7",
- "H8",
- "H9",
- "H10",
- "H11",
- "H12",
- "H13",
- "H14",
- "H15",
- "H16",
- "H17",
- "H18",
- "H19",
- "H20",
- "H21",
- "H22",
- "H23",
- "H24",
- "H25",
- "H26",
- "H27",
- "H28",
- "H29",
- "H30",
- "H31",
- "H32",
- "H33",
- "H34",
- "H35",
- "H36",
- "H37",
- "H38",
- "H39",
- "H40",
- "H41",
- "H42",
- "H43",
- "H44",
- "H45",
- "H46",
- "H47",
- ]
-
def setUp(self):
self.stationmanager_proxy = self.setup_stationmanager_proxy()
super().setUp("STAT/AntennaField/HBA")
@@ -452,107 +400,3 @@ class TestAntennaFieldDevice(AbstractTestBases.TestDeviceBase):
antennafield_proxy.read_attribute("Frequency_Band_RW").value[:2],
err_msg=f"{band.name}",
)
-
- def test_calibrate_recv(self):
- calibration_properties = {
- "Antenna_Type": ["LBA"],
- "Antenna_Cables": ["50m", "80m"] * (DEFAULT_N_HBA_TILES // 2),
- "Control_to_RECV_mapping":
- # [1, 0, 1, 1, 1, 2, 1, x ... 1, 47]
- numpy.array([[1, x] for x in range(0, DEFAULT_N_HBA_TILES)]).flatten(),
- }
-
- antennafield_proxy = self.proxy
- antennafield_proxy.off()
- antennafield_proxy.put_property(calibration_properties)
- antennafield_proxy.boot()
-
- # calibrate
- antennafield_proxy.calibrate_recv()
-
- # check the results
- rcu_attenuator_db = antennafield_proxy.RCU_attenuator_dB_RW
-
- # values should be the same for the same cable length
- self.assertEqual(
- 1,
- len(set(rcu_attenuator_db[0::2])),
- msg=f"rcu_attenuator_db={rcu_attenuator_db}",
- )
- self.assertEqual(
- 1,
- len(set(rcu_attenuator_db[1::2])),
- msg=f"rcu_attenuator_db={rcu_attenuator_db}",
- )
- # value should be larger for the shorter cable, as those signals need damping
- self.assertGreater(rcu_attenuator_db[0], rcu_attenuator_db[1])
- # longest cable should require no damping
- self.assertEqual(0, rcu_attenuator_db[1])
-
- def test_calibrate_sdp(self):
- calibration_properties = {
- "Antenna_Type": ["HBA"],
- "Antenna_Names": self.HBA_ANTENNA_NAMES,
- "Antenna_Cables": ["50m", "80m"] * (DEFAULT_N_HBA_TILES // 2),
- "Antenna_to_SDP_Mapping": [0, 1, 0, 0]
- + [-1, -1] * (DEFAULT_N_HBA_TILES - 2),
- "Control_to_RECV_mapping":
- # [1, 0, 1, 1, 1, 2, 1, x ... 1, 47]
- numpy.array([[1, x] for x in range(0, DEFAULT_N_HBA_TILES)]).flatten(),
- }
-
- antennafield_proxy = self.proxy
- antennafield_proxy.off()
- antennafield_proxy.put_property(calibration_properties)
- antennafield_proxy.boot()
-
- # calibrate
- antennafield_proxy.calibrate_sdp()
-
- # check the results
- # antenna #0 is on FPGA 0, input 2 and 3,
- # antenna #1 is on FPGA 0, input 0 and 1
- signal_input_samples_delay = self.sdp_proxy.FPGA_signal_input_samples_delay_RW
-
- # delays should be equal for both polarisations
- self.assertEqual(
- signal_input_samples_delay[0, 0], signal_input_samples_delay[0, 1]
- )
- self.assertEqual(
- signal_input_samples_delay[0, 2], signal_input_samples_delay[0, 3]
- )
-
- # antenna #0 is shorter, so should have a greater delay
- self.assertGreater(
- signal_input_samples_delay[0, 2],
- signal_input_samples_delay[0, 0],
- msg=f"{signal_input_samples_delay}",
- )
- # antenna #1 is longest, so should have delay 0
- self.assertEqual(0, signal_input_samples_delay[0, 0])
-
- def test_calibration_table(self):
- """Test whether calibration table are correctly retrieved and reshaped"""
- calibration_properties = {
- "Antenna_Type": ["HBA"],
- "Antenna_Names": self.HBA_ANTENNA_NAMES,
- "Control_to_RECV_mapping": [1, 1, 1, 0]
- + [-1, -1] * (DEFAULT_N_HBA_TILES - 2),
- "Antenna_to_SDP_Mapping": [0, 1, 0, 0]
- + [-1, -1] * (DEFAULT_N_HBA_TILES - 2),
- }
-
- antennafield_proxy = self.proxy
- antennafield_proxy.off()
- antennafield_proxy.put_property(calibration_properties)
- antennafield_proxy.warm_boot()
-
- calibration_table = antennafield_proxy.Calibration_SDP_Subband_Weights_250MHz_R
-
- # test whether the shape is correct
- shape = calibration_table.shape
- self.assertEqual(
- shape,
- (antennafield_proxy.nr_antennas_R, N_pol * N_subbands * VALUES_PER_COMPLEX),
- f"Wrong shape, got {shape}",
- )
diff --git a/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_calibration.py b/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_calibration.py
new file mode 100644
index 0000000000000000000000000000000000000000..8757756475fd811231781fe8c32452c9f9252065
--- /dev/null
+++ b/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_calibration.py
@@ -0,0 +1,229 @@
+# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: Apache-2.0
+
+import numpy
+from tango import DevState
+
+from tangostationcontrol.common.constants import (
+ N_rcu,
+ N_rcu_inp,
+ DEFAULT_N_HBA_TILES,
+ CLK_200_MHZ,
+)
+from tangostationcontrol.integration_test.device_proxy import TestDeviceProxy
+from .base import AbstractTestBases
+
+
+class TestCalibrationDevice(AbstractTestBases.TestDeviceBase):
+ HBA_ANTENNA_NAMES = [
+ "H0",
+ "H1",
+ "H2",
+ "H3",
+ "H4",
+ "H5",
+ "H6",
+ "H7",
+ "H8",
+ "H9",
+ "H10",
+ "H11",
+ "H12",
+ "H13",
+ "H14",
+ "H15",
+ "H16",
+ "H17",
+ "H18",
+ "H19",
+ "H20",
+ "H21",
+ "H22",
+ "H23",
+ "H24",
+ "H25",
+ "H26",
+ "H27",
+ "H28",
+ "H29",
+ "H30",
+ "H31",
+ "H32",
+ "H33",
+ "H34",
+ "H35",
+ "H36",
+ "H37",
+ "H38",
+ "H39",
+ "H40",
+ "H41",
+ "H42",
+ "H43",
+ "H44",
+ "H45",
+ "H46",
+ "H47",
+ ]
+
+ def setUp(self):
+ self.stationmanager_proxy = self.setup_stationmanager_proxy()
+ super().setUp("STAT/Calibration/1")
+ self.antennafield_proxy = self.setup_proxy("STAT/AntennaField/HBA")
+ self.antennafield_proxy.put_property(
+ {
+ "RECV_devices": ["STAT/RECV/1"],
+ "Power_to_RECV_mapping": [1, 1, 1, 0]
+ + [-1] * ((DEFAULT_N_HBA_TILES * 2) - 4),
+ }
+ )
+ self.recv_proxy = self.setup_recv_proxy()
+ self.sdp_proxy = self.setup_sdp_proxy()
+
+ self.addCleanup(self.shutdown_recv)
+ self.addCleanup(self.shutdown_sdp)
+
+ # configure the frequencies, which allows access
+ # to the calibration attributes and commands
+ self.sdp_proxy.clock_RW = CLK_200_MHZ
+ self.recv_proxy.RCU_band_select_RW = [[1] * N_rcu_inp] * N_rcu
+
+ def restore_antennafield(self):
+ self.proxy.put_property(
+ {
+ "RECV_devices": ["STAT/RECV/1"],
+ "Power_to_RECV_mapping": [-1, -1] * DEFAULT_N_HBA_TILES,
+ "Control_to_RECV_mapping": [-1, -1] * DEFAULT_N_HBA_TILES,
+ }
+ )
+
+ @staticmethod
+ def shutdown_recv():
+ recv_proxy = TestDeviceProxy("STAT/RECV/1")
+ recv_proxy.off()
+
+ @staticmethod
+ def shutdown_sdp():
+ sdp_proxy = TestDeviceProxy("STAT/SDP/1")
+ sdp_proxy.off()
+
+ @staticmethod
+ def shutdown(device: str):
+ def off():
+ proxy = TestDeviceProxy(device)
+ proxy.off()
+
+ return off
+
+ def setup_recv_proxy(self):
+ # setup RECV
+ recv_proxy = TestDeviceProxy("STAT/RECV/1")
+ recv_proxy.off()
+ recv_proxy.warm_boot()
+ recv_proxy.set_defaults()
+ return recv_proxy
+
+ def setup_sdp_proxy(self):
+ # setup SDP
+ sdp_proxy = TestDeviceProxy("STAT/SDP/1")
+ sdp_proxy.off()
+ sdp_proxy.warm_boot()
+ return sdp_proxy
+
+ def setup_proxy(self, dev: str):
+ # setup SDP
+ proxy = TestDeviceProxy(dev)
+ proxy.off()
+ proxy.warm_boot()
+ self.addCleanup(self.shutdown(dev))
+ return proxy
+
+ def setup_stationmanager_proxy(self):
+ """Setup StationManager"""
+ stationmanager_proxy = TestDeviceProxy("STAT/StationManager/1")
+ stationmanager_proxy.off()
+ stationmanager_proxy.boot()
+ self.assertEqual(stationmanager_proxy.state(), DevState.ON)
+ return stationmanager_proxy
+
+ def test_calibrate_recv(self):
+ calibration_properties = {
+ "Antenna_Type": ["LBA"],
+ "Antenna_Cables": ["50m", "80m"] * (DEFAULT_N_HBA_TILES // 2),
+ "Control_to_RECV_mapping":
+ # [1, 0, 1, 1, 1, 2, 1, x ... 1, 47]
+ numpy.array([[1, x] for x in range(0, DEFAULT_N_HBA_TILES)]).flatten(),
+ }
+
+ self.antennafield_proxy = self.setup_proxy("STAT/AntennaField/HBA")
+ self.antennafield_proxy.off()
+ self.antennafield_proxy.put_property(calibration_properties)
+ self.antennafield_proxy.boot()
+
+ self.proxy.boot()
+
+ # calibrate
+ self.proxy.calibrate_recv("STAT/AntennaField/HBA")
+
+ # check the results
+ rcu_attenuator_db = self.antennafield_proxy.RCU_attenuator_dB_RW
+
+ # values should be the same for the same cable length
+ self.assertEqual(
+ 1,
+ len(set(rcu_attenuator_db[0::2])),
+ msg=f"rcu_attenuator_db={rcu_attenuator_db}",
+ )
+ self.assertEqual(
+ 1,
+ len(set(rcu_attenuator_db[1::2])),
+ msg=f"rcu_attenuator_db={rcu_attenuator_db}",
+ )
+ # value should be larger for the shorter cable, as those signals need damping
+ self.assertGreater(rcu_attenuator_db[0], rcu_attenuator_db[1])
+ # longest cable should require no damping
+ self.assertEqual(0, rcu_attenuator_db[1])
+
+ def test_calibrate_sdp(self):
+ calibration_properties = {
+ "Antenna_Type": ["HBA"],
+ "Antenna_Names": self.HBA_ANTENNA_NAMES,
+ "Antenna_Cables": ["50m", "80m"] * (DEFAULT_N_HBA_TILES // 2),
+ "Antenna_to_SDP_Mapping": [0, 1, 0, 0]
+ + [-1, -1] * (DEFAULT_N_HBA_TILES - 2),
+ "Control_to_RECV_mapping":
+ # [1, 0, 1, 1, 1, 2, 1, x ... 1, 47]
+ numpy.array([[1, x] for x in range(0, DEFAULT_N_HBA_TILES)]).flatten(),
+ }
+
+ self.antennafield_proxy = self.setup_proxy("STAT/AntennaField/HBA")
+ self.antennafield_proxy.off()
+ self.antennafield_proxy.put_property(calibration_properties)
+ self.antennafield_proxy.boot()
+
+ self.proxy.boot()
+
+ # calibrate
+ self.proxy.calibrate_sdp("STAT/AntennaField/HBA")
+
+ # check the results
+ # antenna #0 is on FPGA 0, input 2 and 3,
+ # antenna #1 is on FPGA 0, input 0 and 1
+ signal_input_samples_delay = self.sdp_proxy.FPGA_signal_input_samples_delay_RW
+
+ # delays should be equal for both polarisations
+ self.assertEqual(
+ signal_input_samples_delay[0, 0], signal_input_samples_delay[0, 1]
+ )
+ self.assertEqual(
+ signal_input_samples_delay[0, 2], signal_input_samples_delay[0, 3]
+ )
+
+ # antenna #0 is shorter, so should have a greater delay
+ self.assertGreater(
+ signal_input_samples_delay[0, 2],
+ signal_input_samples_delay[0, 0],
+ msg=f"{signal_input_samples_delay}",
+ )
+ # antenna #1 is longest, so should have delay 0
+ self.assertEqual(0, signal_input_samples_delay[0, 0])
diff --git a/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_digitalbeam.py b/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_digitalbeam.py
index ed7fd69821bdbd9d54edca82821f7a08a5f1d9e2..8225f89c246a7b797df076c80ebd2f8483c2580f 100644
--- a/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_digitalbeam.py
+++ b/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_digitalbeam.py
@@ -1,8 +1,9 @@
-# Copyright (C) 2022 ASTRON (Netherlands Institute for Radio Astronomy)
-# SPDX-License-Identifier: Apache-2.0
+# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: Apache-2.0
-import time
import logging
+import time
+
import numpy
import timeout_decorator
@@ -16,7 +17,6 @@ from tangostationcontrol.common.constants import (
)
from tangostationcontrol.devices.antennafield import AntennaQuality, AntennaUse
from tangostationcontrol.integration_test.device_proxy import TestDeviceProxy
-
from .base import AbstractTestBases
logger = logging.getLogger()
@@ -88,6 +88,105 @@ class TestDeviceDigitalBeam(AbstractTestBases.TestDeviceBase):
"Control_to_RECV_mapping": numpy.array(control_mapping).flatten(),
"Antenna_Quality": antenna_qualities,
"Antenna_Use": antenna_use,
+ "Antenna_Cables": ["50m", "80m"] * (DEFAULT_N_HBA_TILES // 2),
+ "Antenna_to_SDP_Mapping": [
+ "0",
+ "0",
+ "0",
+ "1",
+ "0",
+ "2",
+ "0",
+ "3",
+ "0",
+ "4",
+ "0",
+ "5",
+ "1",
+ "0",
+ "1",
+ "1",
+ "1",
+ "2",
+ "1",
+ "3",
+ "1",
+ "4",
+ "1",
+ "5",
+ "2",
+ "0",
+ "2",
+ "1",
+ "2",
+ "2",
+ "2",
+ "3",
+ "2",
+ "4",
+ "2",
+ "5",
+ "3",
+ "0",
+ "3",
+ "1",
+ "3",
+ "2",
+ "3",
+ "3",
+ "3",
+ "4",
+ "3",
+ "5",
+ "4",
+ "0",
+ "4",
+ "1",
+ "4",
+ "2",
+ "4",
+ "3",
+ "4",
+ "4",
+ "4",
+ "5",
+ "5",
+ "0",
+ "5",
+ "1",
+ "5",
+ "2",
+ "5",
+ "3",
+ "5",
+ "4",
+ "5",
+ "5",
+ "6",
+ "0",
+ "6",
+ "1",
+ "6",
+ "2",
+ "6",
+ "3",
+ "6",
+ "4",
+ "6",
+ "5",
+ "7",
+ "0",
+ "7",
+ "1",
+ "7",
+ "2",
+ "7",
+ "3",
+ "7",
+ "4",
+ "7",
+ "5",
+ ],
}
)
antennafield_proxy.off()
diff --git a/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_observation.py b/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_observation.py
index 00257ead770400ae9e0d09fe27b4e5401d53243b..d16455a3558400dd1d774f8c7bf1bf3cfe160b05 100644
--- a/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_observation.py
+++ b/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_observation.py
@@ -1,11 +1,12 @@
-# Copyright (C) 2022 ASTRON (Netherlands Institute for Radio Astronomy)
-# SPDX-License-Identifier: Apache-2.0
+# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: Apache-2.0
from datetime import datetime
from json import loads
import numpy
from tango import DevState, DevFailed
+
from tangostationcontrol.common.constants import (
N_beamlets_ctrl,
MAX_ANTENNA,
@@ -14,7 +15,6 @@ from tangostationcontrol.common.constants import (
from tangostationcontrol.devices.antennafield import AntennaQuality, AntennaUse
from tangostationcontrol.integration_test.device_proxy import TestDeviceProxy
from tangostationcontrol.test.devices.test_observation_base import TestObservationBase
-
from .base import AbstractTestBases
@@ -152,6 +152,7 @@ class TestDeviceObservation(AbstractTestBases.TestDeviceBase):
antenna_use = numpy.array([AntennaUse.AUTO] * MAX_ANTENNA)
antennafield_proxy.put_property(
{
+ "Antenna_Type": ["LBA"],
"RECV_devices": ["STAT/RECV/1"],
"Control_to_RECV_mapping": numpy.array(control_mapping).flatten(),
"Antenna_to_SDP_Mapping": self.ANTENNA_TO_SDP_MAPPING,
diff --git a/tangostationcontrol/test/common/test_calibration.py b/tangostationcontrol/test/common/test_calibration.py
index 44b87271f5ff535e2fc17b747d446d7be727f86a..9a71bdd131b55eb146b6043eec9ea82750dd1fda 100644
--- a/tangostationcontrol/test/common/test_calibration.py
+++ b/tangostationcontrol/test/common/test_calibration.py
@@ -1,17 +1,129 @@
-# Copyright (C) 2022 ASTRON (Netherlands Institute for Radio Astronomy)
-# SPDX-License-Identifier: Apache-2.0
+# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: Apache-2.0
+import os
+from os import path
+from unittest.mock import patch, Mock, call, PropertyMock
import numpy
+from numpy.testing import assert_array_equal
from tangostationcontrol.common.calibration import (
delay_compensation,
loss_compensation,
dB_to_factor,
+ CalibrationManager,
+ CalibrationTable,
)
-
+from tangostationcontrol.common.constants import S_pn, N_subbands, N_pn
+from tangostationcontrol.devices.sdp.sdp import SDP
from test import base
+class MockMinio:
+ def __init__(self, **kwargs):
+ self.args = kwargs
+
+
+@patch("tangostationcontrol.common.calibration.Minio")
+@patch.dict(
+ os.environ,
+ {"MINIO_ROOT_USER": "my_user", "MINIO_ROOT_PASSWORD": "my_passwd"},
+ clear=True,
+)
+class TestCalibrationManager(base.TestCase):
+ def test_sync_calibration_tables(self, minio):
+ minio.return_value.list_objects.return_value = [
+ Mock(object_name="/unittest-station/file1.h5"),
+ Mock(object_name="/unittest-station/file2.h5"),
+ Mock(object_name="/unittest-station/file3.h5"),
+ ]
+ sut = CalibrationManager(
+ "http://server:1234/test_bucket/test_prefix", "unittest-station"
+ )
+ minio.has_call_with(
+ "server:1234", access_key="my_user", secret_key="my_passwd", secure=False
+ )
+ minio.return_value.list_objects.has_call_with(
+ "test_bucket", prefix="test_prefix/unittest-station/"
+ )
+ minio.return_value.fget_object.assert_has_calls(
+ [
+ call(
+ "test_bucket",
+ "/unittest-station/file1.h5",
+ path.join(sut._tmp_dir.name, "file1.h5"),
+ ),
+ call(
+ "test_bucket",
+ "/unittest-station/file2.h5",
+ path.join(sut._tmp_dir.name, "file2.h5"),
+ ),
+ call(
+ "test_bucket",
+ "/unittest-station/file3.h5",
+ path.join(sut._tmp_dir.name, "file3.h5"),
+ ),
+ ]
+ )
+
+ @patch("tangostationcontrol.common.calibration.read_hdf5")
+ def test_calibrate_subband_weights(self, hdf_reader, _):
+ antenna_field_mock = Mock(
+ Antenna_to_SDP_Mapping_R=numpy.array(
+ [[1, 1], [1, 2]], dtype=numpy.int32
+ ).reshape(-1, 2),
+ Antenna_Names_R=[f"T{n + 1}" for n in range(2)],
+ Antenna_Type_R="HBA",
+ RCU_band_select_RW=numpy.array([1, 2]),
+ **{"name.return_value": "Stat/AntennaField/TEST"},
+ )
+ subband_weights = numpy.array(
+ [[SDP.SUBBAND_UNIT_WEIGHT] * S_pn * N_subbands] * N_pn
+ )
+
+ def subband_weights_side_effect(new_value=None):
+ nonlocal subband_weights
+ if new_value is not None:
+ subband_weights = new_value
+ return subband_weights
+
+ sdp_mock = Mock()
+ subband_property_mock = PropertyMock(side_effect=subband_weights_side_effect)
+ type(sdp_mock).FPGA_subband_weights_RW = subband_property_mock
+ caltable_mock = Mock(
+ observation_station="unittest-station",
+ antennas={
+ "T1": Mock(x=numpy.arange(0, 512), y=numpy.arange(512, 1024)),
+ "T2": Mock(x=numpy.arange(1024, 1536), y=numpy.arange(1536, 2048)),
+ },
+ )
+ hdf_reader.return_value.__enter__.return_value = caltable_mock
+
+ sut = CalibrationManager("http://server:1234", "unittest-station")
+ sut.calibrate_subband_weights(antenna_field_mock, sdp_mock)
+
+ hdf_reader.assert_has_calls(
+ [
+ call(
+ f"{sut._tmp_dir.name}/CalTable-unittest-station-TEST-200MHz.h5",
+ CalibrationTable,
+ ),
+ call().__enter__(),
+ call().__exit__(None, None, None),
+ call(
+ f"{sut._tmp_dir.name}/CalTable-unittest-station-TEST-150MHz.h5",
+ CalibrationTable,
+ ),
+ call().__enter__(),
+ call().__exit__(None, None, None),
+ ]
+ )
+ assert_array_equal(subband_weights[1, 1024:1536], numpy.arange(0, 512))
+ assert_array_equal(subband_weights[1, 1536:2048], numpy.arange(512, 1024))
+ assert_array_equal(subband_weights[1, 2048:2560], numpy.arange(1024, 1536))
+ assert_array_equal(subband_weights[1, 2560:3072], numpy.arange(1536, 2048))
+
+
class TestCalibration(base.TestCase):
def test_dB_to_factor(self):
# Throw some known values at it