diff --git a/.gitignore b/.gitignore
index cfd4dc461a50e0a01b60ca0f88152e9ca9a2d787..f777364050f38eddf7f7867a0326b5dd3199074c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -28,3 +28,4 @@ tangostationcontrol/docs/build
**/.eggs
docker-compose/alerta-web/alerta-secrets.json
+docker-compose/tmp
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 0ea0331e30bc29d9361e4189752478a6dd9b9057..fcecc7c684ba15b7c8437b5c38b92276d70aaf6f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -550,8 +550,10 @@ unit_test:
path: tangostationcontrol/coverage.xml
paths:
- tangostationcontrol/cover/*
+ - tangostationcontrol/.coverage
integration_test_docker:
stage: integration-tests
+ allow_failure: true
image: docker:latest
tags:
- privileged
@@ -559,6 +561,8 @@ integration_test_docker:
- name: docker:dind
variables:
DOCKER_TLS_CERTDIR: "/certs"
+ needs:
+ - unit_test
artifacts:
when: always
paths:
@@ -572,6 +576,7 @@ integration_test_docker:
tag="$CI_COMMIT_REF_SLUG"
echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag"
fi
+ - apk update
- apk add --update make bash docker-compose
- apk add --update bind-tools
- apk add --update postgresql14-client gzip
diff --git a/CDB/stations/DTS_Outside_ConfigDb.json b/CDB/stations/DTS_Outside_ConfigDb.json
index 7e6c49fa6f8f1631cf5955914baca1ecfb4e04e0..258ac6a1c2436ecd25c3f7cd9993277895658f6c 100644
--- a/CDB/stations/DTS_Outside_ConfigDb.json
+++ b/CDB/stations/DTS_Outside_ConfigDb.json
@@ -18,25 +18,25 @@
"Initialise_Hardware": [
"True"
],
- "Device_Names" : [
+ "Device_Names": [
"STAT/Docker/1",
- "STAT/PSOC/1",
- "STAT/PCON/1",
- "STAT/APSPU/1",
- "STAT/APSCT/1",
- "STAT/CCD/1",
- "STAT/RECV/1",
- "STAT/UNB2/1",
- "STAT/SDP/1",
- "STAT/BST/1",
- "STAT/SST/1",
- "STAT/XST/1",
- "STAT/Beamlet/1",
- "STAT/AntennaField/1",
- "STAT/TileBeam/2",
- "STAT/DigitalBeam/1",
- "STAT/TemperatureManager/1"
- ]
+ "STAT/PSOC/1",
+ "STAT/PCON/1",
+ "STAT/APSPU/1",
+ "STAT/APSCT/1",
+ "STAT/CCD/1",
+ "STAT/RECV/1",
+ "STAT/UNB2/1",
+ "STAT/SDP/1",
+ "STAT/BST/1",
+ "STAT/SST/1",
+ "STAT/XST/1",
+ "STAT/Beamlet/1",
+ "STAT/AntennaField/1",
+ "STAT/TileBeam/2",
+ "STAT/DigitalBeam/1",
+ "STAT/TemperatureManager/1"
+ ]
}
}
}
@@ -112,19 +112,6 @@
}
}
},
- "DigitalBeam": {
- "STAT": {
- "DigitalBeam": {
- "STAT/DigitalBeam/1": {
- "properties": {
- "Tracking_enabled_RW_default": [
- "False"
- ]
- }
- }
- }
- }
- },
"Beamlet": {
"STAT": {
"Beamlet": {
@@ -185,32 +172,39 @@
"AntennaField": {
"STAT/AntennaField/2": {
"properties": {
- "RECV_devices": [
- "STAT/RECV/1"
- ],
+ "RECV_devices": [
+ "STAT/RECV/1"
+ ],
+ "Antenna_Names": [
+ "C0",
+ "C1",
+ "C2",
+ "C3",
+ "C4"
+ ],
"HBAT_Control_to_RECV_mapping": [
- "1", "27",
- "0", "-1",
- "0", "-1",
- "1", "28",
- "1", "29"
- ],
+ "1","27",
+ "0","-1",
+ "0","-1",
+ "1","28",
+ "1","29"
+ ],
"HBAT_Power_to_RECV_mapping": [
- "1", "24",
- "0", "-1",
- "0", "-1",
- "1", "25",
- "1", "26"
- ],
+ "1","24",
+ "0","-1",
+ "0","-1",
+ "1","25",
+ "1","26"
+ ],
"Antenna_Field_Reference_ETRS": [
- "3839371.416", "430339.901", "5057958.886"
+ "3839371.416","430339.901","5057958.886"
],
"HBAT_reference_ETRS": [
- "3839371.416", "430339.901", "5057958.886",
- "3839368.919", "430335.979", "5057961.1",
- "3839365.645", "430339.299", "5057963.288",
- "3839368.142", "430343.221", "5057961.074",
- "3839374.094", "430299.513", "5057960.017"
+ "3839371.416","430339.901","5057958.886",
+ "3839368.919","430335.979","5057961.1",
+ "3839365.645","430339.299","5057963.288",
+ "3839368.142","430343.221","5057961.074",
+ "3839374.094","430299.513","5057960.017"
],
"HBAT_PQR_rotation_angles_deg": [
"45.73",
@@ -220,52 +214,65 @@
"54.40"
],
"HBAT_PQR_to_ETRS_rotation_matrix": [
- "-0.11660087", "-0.79095632", "0.60065992",
- " 0.99317077", "-0.09529842", "0.06730545",
- " 0.00400627", " 0.60440575", "0.79666658"
+ "-0.11660087","-0.79095632","0.60065992",
+ " 0.99317077","-0.09529842","0.06730545",
+ " 0.00400627"," 0.60440575","0.79666658"
]
}
},
"STAT/AntennaField/1": {
"properties": {
- "RECV_devices": [
- "STAT/RECV/1"
- ],
+ "RECV_devices": [
+ "STAT/RECV/1"
+ ],
+ "Antenna_Names": [
+ "LBA1",
+ "LBA2",
+ "LBA3",
+ "LBA4",
+ "LBA5",
+ "LBA6",
+ "LBA7",
+ "LBA8",
+ "LBA9"
+ ],
"HBAT_Control_to_RECV_mapping": [
- "1", "1",
- "1", "3",
- "1", "5",
- "1", "7",
- "1", "9",
- "1", "11",
- "1", "13",
- "1", "15",
- "1", "17"
- ],
+ "1","1",
+ "1","3",
+ "1","5",
+ "1","7",
+ "1","9",
+ "1","11",
+ "1","13",
+ "1","15",
+ "1","17"
+ ],
"HBAT_Power_to_RECV_mapping": [
- "1", "1",
- "1", "3",
- "1", "5",
- "1", "7",
- "1", "9",
- "1", "11",
- "1", "13",
- "1", "15",
- "1", "17"
- ],
+ "1","1",
+ "1","3",
+ "1","5",
+ "1","7",
+ "1","9",
+ "1","11",
+ "1","13",
+ "1","15",
+ "1","17"
+ ],
"Antenna_Field_Reference_ETRS": [
- "3839358.189", "430354.482", "5057967.804"
+ "3839358.189",
+ "430354.482",
+ "5057967.804"
],
"HBAT_reference_ETRS": [
- "3839358.189", "430354.482", "5057967.804",
- "3839359.127", "430348.074", "5057967.607",
- "3839360.084", "430341.872", "5057967.379",
- "3839361.024", "430335.466", "5057967.180",
- "3839361.721", "430329.482", "5057967.130",
- "3839362.786", "430323.311", "5057966.818",
- "3839363.705", "430317.204", "5057966.611",
- "3839364.563", "430311.056", "5057966.454",
- "3839365.497", "430304.996", "5057966.232"
+ "3839358.189","430354.482","5057967.804",
+ "3839359.127","430348.074","5057967.607",
+ "3839360.084","430341.872","5057967.379",
+ "3839361.024","430335.466","5057967.180",
+ "3839361.721","430329.482","5057967.130",
+ "3839362.786","430323.311","5057966.818",
+ "3839363.705","430317.204","5057966.611",
+ "3839364.563","430311.056","5057966.454",
+ "3839365.497","430304.996","5057966.232"
]
}
}
@@ -283,7 +290,7 @@
"Input_to_Antenna_Mapping": [
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
- "0", "3", "4", "-1", "-1", "-1",
+ "0", "3", "4", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
@@ -306,8 +313,8 @@
"STAT/AntennaField/1"
],
"Input_to_Antenna_Mapping": [
- "0", "1", "2", "3", "4", "5",
- "6", "7", "8", "-1", "-1", "-1",
+ "0", "1", "2", "3", "4", "5",
+ "6", "7", "8", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
@@ -322,6 +329,9 @@
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1"
+ ],
+ "Tracking_enabled_RW_default": [
+ "False"
]
}
}
@@ -352,9 +362,9 @@
"SDP": {
"STAT/SDP/1": {
"properties": {
- "AntennaType" : [
- "LBA"
- ],
+ "AntennaType": [
+ "LBA"
+ ],
"OPC_Server_Name": [
"10.99.0.250"
],
@@ -575,7 +585,9 @@
"PSOC": {
"STAT/PSOC/1": {
"properties": {
- "SNMP_host": ["10.87.2.145"],
+ "SNMP_host": [
+ "10.87.2.145"
+ ],
"PSOC_sockets": [
"ccd_socket",
"sdptr0_socket",
@@ -596,7 +608,9 @@
"PCON": {
"STAT/PCON/1": {
"properties": {
- "SNMP_host": ["10.151.225.5"]
+ "SNMP_host": [
+ "10.151.225.5"
+ ]
}
}
}
diff --git a/README.md b/README.md
index b8f08504fbcd221ac8a0ed3e4e6c98869b242f11..c72d120f54c46d5bf9eba273863b4572de9595ff 100644
--- a/README.md
+++ b/README.md
@@ -17,7 +17,7 @@ Station Control software related to Tango devices.
* [Attribute wrapper documentation](tangostationcontrol/tangostationcontrol/clients/README.md)
* [Archiver documentation](tangostationcontrol/tangostationcontrol/toolkit/README.md)
* [Adding a new tango device](tangostationcontrol/tangostationcontrol/devices/README.md)
- * [HDF5 statistics](tangostationcontrol/tangostationcontrol/statistics_writer/README.md)
+ * [HDF5 statistics](tangostationcontrol/tangostationcontrol/statistics/README.md)
* [Unit tests](tangostationcontrol/tangostationcontrol/test/README.md)
* [Integration tests](tangostationcontrol/tangostationcontrol/integration_test/README.md)
diff --git a/bin/start-ds.sh b/bin/start-ds.sh
index 8a5baba501cf7c48dd53a7d3b92196874020d3d6..66389714f17770339ee645129adc9dcb26fa21d7 100755
--- a/bin/start-ds.sh
+++ b/bin/start-ds.sh
@@ -1,5 +1,7 @@
#!/bin/bash
+set -e
+
# Serves as entrypoint script for docker containers
if [[ ! -d "/opt/lofar/tango" ]]; then
@@ -31,8 +33,11 @@ if [[ $TANGOSTATIONCONTROL ]]; then
exit 2
else
# Install the package, exit 1 if it fails
- cd tangostationcontrol || exit 1
- pip install --upgrade --force-reinstall ./
+ # pip install ./ will _NOT_ install dependencies in requirements.txt!
+ rm -rf /tmp/tangostationcontrol
+ cp -R /opt/lofar/tango/tangostationcontrol /tmp/
+ cd /tmp/tangostationcontrol || exit 1
+ pip -vvv install --upgrade --force-reinstall ./
fi
# Return to the stored the directory, this preserves the working_dir argument in
diff --git a/docker-compose/Makefile b/docker-compose/Makefile
index e0689cc7ceedc874798ecf20ce32de7cd69ca6d7..7606e94abd344a9c255463664bf2f55d8e556b93 100644
--- a/docker-compose/Makefile
+++ b/docker-compose/Makefile
@@ -150,20 +150,21 @@ DOCKER_COMPOSE_ARGS := DISPLAY=$(DISPLAY) \
CONTAINER_NAME_PREFIX=$(CONTAINER_NAME_PREFIX) \
COMPOSE_IGNORE_ORPHANS=true \
CONTAINER_EXECUTION_UID=$(shell id -u) \
- DOCKER_GID=$(DOCKER_GID)
+ DOCKER_GID=$(DOCKER_GID) \
+ TEST_MODULE=$(INTEGRATION_MODULE)
-.PHONY: up down minimal run integration start stop restart build build-nocache status clean pull help
+.PHONY: up down minimal context run integration start stop restart build build-nocache status clean pull help
.DEFAULT_GOAL := help
pull: ## pull the images from the Docker hub
$(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) pull
-build: ## rebuild images
+build: context ## rebuild images
# docker-compose does not support build dependencies, so manage those here
$(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) build --parallel --progress=plain $(SERVICE)
-build-nocache: ## rebuild images from scratch
+build-nocache: context ## rebuild images from scratch
# docker-compose does not support build dependencies, so manage those here
$(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) build --no-cache --progress=plain $(SERVICE)
@@ -174,7 +175,7 @@ run: minimal ## run a service using arguments and delete it afterwards
$(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) run --no-deps --rm $(SERVICE) $(SERVICE_ARGS)
integration: minimal ## run a service using arguments and delete it afterwards
- TEST_MODULE=$(INTEGRATION_MODULE) $(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) run --no-deps --rm integration-test $(INTEGRATION_ARGS)
+ $(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) run --no-deps --rm integration-test $(INTEGRATION_ARGS)
down: ## stop all services and tear down the system
$(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) down
@@ -183,13 +184,18 @@ ifneq ($(NETWORK_MODE),host)
docker network inspect 9000-$(NETWORK_MODE) &> /dev/null && ([ $$? -eq 0 ] && docker network rm 9000-$(NETWORK_MODE)) || true
endif
-minimal: ## start the base TANGO system
+minimal: context ## start the base TANGO system
ifneq ($(NETWORK_MODE),host)
docker network inspect $(NETWORK_MODE) &> /dev/null || ([ $$? -ne 0 ] && docker network create $(NETWORK_MODE))
docker network inspect 9000-$(NETWORK_MODE) &> /dev/null || ([ $$? -ne 0 ] && docker network create 9000-$(NETWORK_MODE) -o com.docker.network.driver.mtu=9000)
endif
+
$(DOCKER_COMPOSE_ARGS) docker-compose -f tango.yml -f networks.yml up --no-recreate -d
+context: ## Move the necessary files to create minimal docker context
+ @mkdir -p tmp
+ @cp ../tangostationcontrol/requirements.txt tmp/
+
bootstrap: pull build # first start, initialise from scratch
$(MAKE) start elk-configure-host # configure host kernel for elk container
$(MAKE) start dsconfig # boot up containers to load configurations
diff --git a/docker-compose/device-antennafield.yml b/docker-compose/device-antennafield.yml
index d33dacac0139a2a2946f3c19f3cdc2979cf76e30..0e16043f1170a937d0fe16e3e2853801c3b1339b 100644
--- a/docker-compose/device-antennafield.yml
+++ b/docker-compose/device-antennafield.yml
@@ -19,8 +19,8 @@ services:
# build explicitly, as docker-compose does not understand a local image
# being shared among services.
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-antennafield
diff --git a/docker-compose/device-apsct.yml b/docker-compose/device-apsct.yml
index cb43adf7b800498d4448fdcc2ef99cbf577e11d5..8addefff2bc1ead3510e835fad51187b40e4a996 100644
--- a/docker-compose/device-apsct.yml
+++ b/docker-compose/device-apsct.yml
@@ -18,8 +18,8 @@ services:
# build explicitly, as docker-compose does not understand a local image
# being shared among services.
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-apsct
diff --git a/docker-compose/device-apspu.yml b/docker-compose/device-apspu.yml
index 6613b210971f7514c0822cd82f5185af550f4e2a..55a2d5a9a5d95a9c0e1617bb4732a1d96ab26a20 100644
--- a/docker-compose/device-apspu.yml
+++ b/docker-compose/device-apspu.yml
@@ -18,8 +18,8 @@ services:
# build explicitly, as docker-compose does not understand a local image
# being shared among services.
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-apspu
diff --git a/docker-compose/device-beamlet.yml b/docker-compose/device-beamlet.yml
index 6d067c1544bb805578cfa53ba486a39313238b75..4e80eba8f4033efe31749c51c95002204a0136f4 100644
--- a/docker-compose/device-beamlet.yml
+++ b/docker-compose/device-beamlet.yml
@@ -18,8 +18,8 @@ services:
# build explicitly, as docker-compose does not understand a local image
# being shared among services.
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-beamlet
diff --git a/docker-compose/device-boot.yml b/docker-compose/device-boot.yml
index cbeb916536845b47aa0ded5dba2900fa1fa5d7f5..4d4108d9ffb189e05b8696039d68b2f1be20de2a 100644
--- a/docker-compose/device-boot.yml
+++ b/docker-compose/device-boot.yml
@@ -17,8 +17,8 @@ services:
# build explicitly, as docker-compose does not understand a local image
# being shared among services.
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-boot
diff --git a/docker-compose/device-bst.yml b/docker-compose/device-bst.yml
index c06514b4f3a738ad45ffb847d5b261e515b58a6a..92522527c2071e9e94b9561ab010717c37c63c07 100644
--- a/docker-compose/device-bst.yml
+++ b/docker-compose/device-bst.yml
@@ -18,8 +18,8 @@ services:
# build explicitly, as docker-compose does not understand a local image
# being shared among services.
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-bst
diff --git a/docker-compose/device-digitalbeam.yml b/docker-compose/device-digitalbeam.yml
index 32847ca8d6ce995baeee599ea20cb1c0919b9084..c5b45e9baaef63e15f44b4a539cf93a73244bde3 100644
--- a/docker-compose/device-digitalbeam.yml
+++ b/docker-compose/device-digitalbeam.yml
@@ -18,8 +18,8 @@ services:
# build explicitly, as docker-compose does not understand a local image
# being shared among services.
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-digitalbeam
diff --git a/docker-compose/device-docker.yml b/docker-compose/device-docker.yml
index 2be9467ea6d63fbf08fc30e954de74a585bbe6a3..db33c9aed034fa02fbafe2bc7ab4b66535fe0f33 100644
--- a/docker-compose/device-docker.yml
+++ b/docker-compose/device-docker.yml
@@ -18,8 +18,8 @@ services:
# build explicitly, as docker-compose does not understand a local image
# being shared among services.
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-docker
diff --git a/docker-compose/device-observation-control.yml b/docker-compose/device-observation-control.yml
index 197d192a54ca10360439f41cfd6aeb0adeca70e3..7025b8b3ab38cbb1290971c1907729b8aa09ca0f 100644
--- a/docker-compose/device-observation-control.yml
+++ b/docker-compose/device-observation-control.yml
@@ -17,8 +17,8 @@ services:
# build explicitly, as docker-compose does not understand a local image
# being shared among services.
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-observation-control
diff --git a/docker-compose/device-observation.yml b/docker-compose/device-observation.yml
index ee8f3a1653b447965af9891258de1e8642242e60..3379e41e1887d670b734a1f75f8b942ca51d17df 100644
--- a/docker-compose/device-observation.yml
+++ b/docker-compose/device-observation.yml
@@ -16,8 +16,8 @@ services:
# build explicitly, as docker-compose does not understand a local image
# being shared among services.
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-observation
diff --git a/docker-compose/device-pcon.yml b/docker-compose/device-pcon.yml
index 40ebb3f58605ab09ad402d3e4b5bf80bc201d95e..17fad681d96fbe9cb1b84168144b3668ce4f96f7 100644
--- a/docker-compose/device-pcon.yml
+++ b/docker-compose/device-pcon.yml
@@ -13,8 +13,8 @@ services:
# build explicitly, as docker-compose does not understand a local image
# being shared among services.
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-pcon
diff --git a/docker-compose/device-psoc.yml b/docker-compose/device-psoc.yml
index bc251d2de28a132953259de924b1cc8ee557175e..6d6578e6aa6aa3b44b34f16d6cd1f3373f45217b 100644
--- a/docker-compose/device-psoc.yml
+++ b/docker-compose/device-psoc.yml
@@ -13,8 +13,8 @@ services:
# build explicitly, as docker-compose does not understand a local image
# being shared among services.
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-psoc
diff --git a/docker-compose/device-recv.yml b/docker-compose/device-recv.yml
index 10126cae3612ad12ca6e77c1b191b2879091f1d4..3c79a0a149528557a0d3ca3aa087773538942207 100644
--- a/docker-compose/device-recv.yml
+++ b/docker-compose/device-recv.yml
@@ -18,8 +18,8 @@ services:
# build explicitly, as docker-compose does not understand a local image
# being shared among services.
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-recv
diff --git a/docker-compose/device-sdp.yml b/docker-compose/device-sdp.yml
index 2e9c77312d96b800f06e3c85e6773613a0af758b..144630c883d741c166c6f1a1c48f9e8eda5ab096 100644
--- a/docker-compose/device-sdp.yml
+++ b/docker-compose/device-sdp.yml
@@ -18,8 +18,8 @@ services:
# build explicitly, as docker-compose does not understand a local image
# being shared among services.
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-sdp
diff --git a/docker-compose/device-sst.yml b/docker-compose/device-sst.yml
index 54c221f3b6c051078d55009ec583d0ddab8dd46b..e6b0edb75008791f365d4ec8281c35a314935ca3 100644
--- a/docker-compose/device-sst.yml
+++ b/docker-compose/device-sst.yml
@@ -18,8 +18,8 @@ services:
# build explicitly, as docker-compose does not understand a local image
# being shared among services.
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-sst
diff --git a/docker-compose/device-temperature-manager.yml b/docker-compose/device-temperature-manager.yml
index 4729bd631b87508b12be14b24a9c4e1bbbccf98b..d1b20359bf0f827b99450edd93cf4687ac263532 100644
--- a/docker-compose/device-temperature-manager.yml
+++ b/docker-compose/device-temperature-manager.yml
@@ -13,8 +13,8 @@ services:
# build explicitly, as docker-compose does not understand a local image
# being shared among services.
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-temperature-manager
diff --git a/docker-compose/device-tilebeam.yml b/docker-compose/device-tilebeam.yml
index 20e44c90c112c3ce7a4f3c7c09682c897c032a30..7919beded3237fecc98cddc64b734747e3187304 100644
--- a/docker-compose/device-tilebeam.yml
+++ b/docker-compose/device-tilebeam.yml
@@ -13,8 +13,8 @@ services:
# build explicitly, as docker-compose does not understand a local image
# being shared among services.
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-tilebeam
diff --git a/docker-compose/device-unb2.yml b/docker-compose/device-unb2.yml
index 9a7505f0beb6b6f3f7448a402e812a86ad2c033d..2c05d6e66b887b903d17278e3252cd8f9ea70493 100644
--- a/docker-compose/device-unb2.yml
+++ b/docker-compose/device-unb2.yml
@@ -18,8 +18,8 @@ services:
# build explicitly, as docker-compose does not understand a local image
# being shared among services.
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-unb2
diff --git a/docker-compose/device-xst.yml b/docker-compose/device-xst.yml
index 1ed7a1fe3892c900b14c9c275099816fb45b90a3..6f49e17f6389ff510736543d7cb42aed4ea104b9 100644
--- a/docker-compose/device-xst.yml
+++ b/docker-compose/device-xst.yml
@@ -18,8 +18,8 @@ services:
# build explicitly, as docker-compose does not understand a local image
# being shared among services.
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-xst
diff --git a/docker-compose/integration-test.yml b/docker-compose/integration-test.yml
index 346bbbcc5b179c2066c08a0064335c23957fb902..6e7e5407d422afd1989ef7a127d1f54217307cc3 100644
--- a/docker-compose/integration-test.yml
+++ b/docker-compose/integration-test.yml
@@ -9,8 +9,8 @@ version: '2.1'
services:
integration-test:
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}integration-test
diff --git a/docker-compose/lofar-device-base.yml b/docker-compose/lofar-device-base.yml
index 34f3f1c92c06fadd45b634a2a93bae994a54edeb..8f31696433aaee502eea2af83f15c54119b22ec7 100644
--- a/docker-compose/lofar-device-base.yml
+++ b/docker-compose/lofar-device-base.yml
@@ -17,8 +17,8 @@ services:
lofar-device-base:
image: lofar-device-base
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}lofar-device-base
diff --git a/docker-compose/lofar-device-base/Dockerfile b/docker-compose/lofar-device-base/Dockerfile
index 39b9d652f61d4afb3903b0ea98e3df98f51d2b8a..becc95d0833408a8e4a04f074bff9ed0b0b2b6da 100644
--- a/docker-compose/lofar-device-base/Dockerfile
+++ b/docker-compose/lofar-device-base/Dockerfile
@@ -4,20 +4,23 @@ FROM ${SOURCE_IMAGE}
RUN sudo apt-get update && sudo apt-get install -y git g++ gcc shellcheck graphviz python3-dev && sudo apt-get clean
-COPY docker-compose/lofar-device-base/lofar-requirements.txt /lofar-requirements.txt
+COPY lofar-device-base/lofar-requirements.txt /lofar-requirements.txt
RUN sudo pip3 install -r /lofar-requirements.txt
-COPY tangostationcontrol/requirements.txt /tangostationcontrol-requirements.txt
+# Manually install all requirements from the .txt as part of the base image
+# This reduces runtime overhead as well as preventing issues around dependency
+# installation for development builds (pip install ./ ignores requirements.txt)
+COPY tmp/requirements.txt /tangostationcontrol-requirements.txt
RUN sudo pip3 install -r /tangostationcontrol-requirements.txt
# install and use ephimerides and geodetic ("measures") tables for casacore.
# we install a _stub_ since the tables need to be deployed explicitly from within the software.
RUN sudo mkdir -p /opt/IERS && sudo chmod a+rwx /opt/IERS
ARG IERS_DIRNAME=IERS-1970-01-01T00:00:00-stub
-COPY docker-compose/lofar-device-base/WSRT_Measures_stub /opt/IERS/${IERS_DIRNAME}
+COPY lofar-device-base/WSRT_Measures_stub /opt/IERS/${IERS_DIRNAME}
RUN ln -sfT /opt/IERS/${IERS_DIRNAME} /opt/IERS/current
-COPY docker-compose/lofar-device-base/casarc /home/tango/.casarc
+COPY lofar-device-base/casarc /home/tango/.casarc
ENV TANGO_LOG_PATH=/var/log/tango
RUN sudo mkdir -p /var/log/tango && sudo chmod a+rwx /var/log/tango
diff --git a/sbin/run_integration_test.sh b/sbin/run_integration_test.sh
index b910bae025cee9c225d4a52301082d546c2a49d4..f1ee76c7606c485f5cf68ecb92bf9965390bd192 100755
--- a/sbin/run_integration_test.sh
+++ b/sbin/run_integration_test.sh
@@ -24,12 +24,16 @@ function integration_test {
IFS=" " read -r -a restarts <<< "${2}"
IFS=" " read -r -a configs <<< "${3}"
for config in "${configs[@]}"; do
+ echo "Updating config ${config} ..."
bash "${LOFAR20_DIR}"/sbin/update_ConfigDb.sh "${config}"
done
if [ ! -z "${2+x}" ]; then
+ # shellcheck disable=SC2145
+ echo "make restart ${restarts[@]} ..."
make restart "${restarts[@]}"
fi
sleep 5
+ echo "make integration ${1} ..."
make integration "${1}"
}
diff --git a/tangostationcontrol/requirements.txt b/tangostationcontrol/requirements.txt
index 21a4422bbb65100a0e3b9d3c10136b2f3345e9bb..b252910091df6eaca22673b2931b1ab73405c2f0 100644
--- a/tangostationcontrol/requirements.txt
+++ b/tangostationcontrol/requirements.txt
@@ -2,13 +2,14 @@
# order of appearance. Changing the order has an impact on the overall
# integration process, which may cause wedges in the gate later.
-lofar-station-client@git+https://git.astron.nl/lofar2.0/lofar-station-client@0.3.0
+lofar-station-client@git+https://git.astron.nl/lofar2.0/lofar-station-client@0.6.0
asyncua >= 0.9.90 # LGPLv3
PyMySQL[rsa] >= 1.0.2 # MIT
psycopg2-binary >= 2.9.2 # LGPL
sqlalchemy >= 1.4.26 # MIT
pysnmp >= 0.1.7 # BSD
h5py >= 3.1.0 # BSD
+jsonschema >= 3.2.0 # MIT
psutil >= 5.8.0 # BSD
docker >= 5.0.3 # Apache 2
python-logstash-async >= 2.3.0 # MIT
diff --git a/tangostationcontrol/setup.cfg b/tangostationcontrol/setup.cfg
index 29b26f98acbd83850ac9712dd6717a1af3a3e562..11c8c2d5a19bc410bbe76e36eb77f1027ea50c1b 100644
--- a/tangostationcontrol/setup.cfg
+++ b/tangostationcontrol/setup.cfg
@@ -51,8 +51,8 @@ console_scripts =
l2ss-sdp = tangostationcontrol.devices.sdp.sdp:main
l2ss-bst = tangostationcontrol.devices.sdp.bst:main
l2ss-sst = tangostationcontrol.devices.sdp.sst:main
- l2ss-statistics-reader = tangostationcontrol.statistics_writer.statistics_reader:main
- l2ss-statistics-writer = tangostationcontrol.statistics.writer:main
+ l2ss-statistics-reader = tangostationcontrol.statistics.reader:main
+ l2ss-statistics-writer = tangostationcontrol.statistics.writer.entry:main
l2ss-unb2 = tangostationcontrol.devices.unb2:main
l2ss-xst = tangostationcontrol.devices.sdp.xst:main
l2ss-temperature-manager = tangostationcontrol.devices.temperature_manager:main
diff --git a/tangostationcontrol/tangostationcontrol/devices/antennafield.py b/tangostationcontrol/tangostationcontrol/devices/antennafield.py
index 4c39a6bdea1db44f52c87f2fd3a4e6add71e3d99..c7d43b9226de469e877cf6bfdc4c728a8afcfd0e 100644
--- a/tangostationcontrol/tangostationcontrol/devices/antennafield.py
+++ b/tangostationcontrol/tangostationcontrol/devices/antennafield.py
@@ -85,6 +85,14 @@ class AntennaField(lofar_device):
calculated, as well as the geohash.
"""
+ # ----- Antenna names
+
+ Antenna_Names = device_property(
+ doc="Name of each antenna",
+ dtype='DevVarStringArray',
+ mandatory=False
+ )
+
# ----- Antenna states
Antenna_Quality = device_property(
@@ -195,6 +203,8 @@ class AntennaField(lofar_device):
default_value = []
)
+ Antenna_Names_R = attribute(access=AttrWriteType.READ,
+ dtype=(str,), max_dim_x=MAX_NUMBER_OF_HBAT)
Antenna_Quality_R = attribute(access=AttrWriteType.READ,
dtype=(numpy.uint16,), max_dim_x=MAX_NUMBER_OF_HBAT)
Antenna_Use_R = attribute(access=AttrWriteType.READ,
@@ -247,6 +257,9 @@ class AntennaField(lofar_device):
doc='Number of HBAT in this field',
dtype=numpy.int32)
+ def read_Antenna_Names_R(self):
+ return self.Antenna_Names
+
def read_Antenna_Use_R(self):
return self.Antenna_Use
diff --git a/tangostationcontrol/tangostationcontrol/integration_test/default/statistics/test_writer_sst.py b/tangostationcontrol/tangostationcontrol/integration_test/default/statistics/test_writer_sst.py
index 88382175385a01e3493984fd1a53d026a19bbb02..d8932263df5079790e14cdd988f73c3224e8c53a 100644
--- a/tangostationcontrol/tangostationcontrol/integration_test/default/statistics/test_writer_sst.py
+++ b/tangostationcontrol/tangostationcontrol/integration_test/default/statistics/test_writer_sst.py
@@ -9,9 +9,10 @@
from tangostationcontrol.integration_test.base import BaseIntegrationTestCase
from tangostationcontrol.integration_test.device_proxy import TestDeviceProxy
+
from tangostationcontrol.statistics.collector import StationSSTCollector
-from tangostationcontrol.statistics_writer import statistics_reader
-from tangostationcontrol.statistics import writer
+from tangostationcontrol.statistics import reader
+from tangostationcontrol.statistics.writer import entry
import sys
from os.path import dirname, isfile, join
@@ -19,56 +20,84 @@ from tempfile import TemporaryDirectory
from unittest import mock
from tango import DevState
+import numpy
class TestStatisticsWriterSST(BaseIntegrationTestCase):
+ RECV_PROXY_STRING = "STAT/RECV/1"
+
def setUp(self):
self.recv_proxy = self.setup_recv_proxy()
return super().setUp()
-
- def setup_recv_proxy(self):
+
+ @staticmethod
+ def setup_recv_proxy():
# setup RECV
- recv_proxy = TestDeviceProxy("STAT/RECV/1")
+ recv_proxy = TestDeviceProxy(TestStatisticsWriterSST.RECV_PROXY_STRING)
recv_proxy.off()
recv_proxy.warm_boot()
recv_proxy.set_defaults()
return recv_proxy
def test_retrieve_data_from_RECV(self):
- recv_proxy = self.setup_recv_proxy()
- self.assertEqual(DevState.ON, recv_proxy.state())
- self.assertIsNotNone(recv_proxy.RCU_Attenuator_dB_R)
- self.assertIsNotNone(recv_proxy.RCU_Band_Select_R)
- self.assertIsNotNone(recv_proxy.RCU_DTH_on_R)
+ self.assertEqual(DevState.ON, self.recv_proxy.state())
+ self.assertIsNotNone(self.recv_proxy.RCU_Attenuator_dB_R)
+ self.assertIsNotNone(self.recv_proxy.RCU_Band_Select_R)
+ self.assertIsNotNone(self.recv_proxy.RCU_DTH_on_R)
def test_insert_tango_SST_statistics(self):
- self.setup_recv_proxy()
self.assertEqual(DevState.ON, self.recv_proxy.state())
- collector = StationSSTCollector()
+ collector = StationSSTCollector(device=self.recv_proxy)
# Test attribute values retrieval
- collector.parse_device_attributes(self.recv_proxy)
- self.assertListEqual(collector.parameters["rcu_attenuator_dB"].tolist(), self.recv_proxy.rcu_attenuator_dB_r.tolist())
- self.assertListEqual(collector.parameters["rcu_band_select"].tolist(), self.recv_proxy.rcu_band_select_r.tolist())
- self.assertListEqual(collector.parameters["rcu_dth_on"].tolist(), self.recv_proxy.rcu_dth_on_r.tolist())
+ collector.parse_device_attributes()
+ numpy.testing.assert_equal(
+ collector.parameters["rcu_attenuator_dB"].flatten(),
+ self.recv_proxy.rcu_attenuator_dB_r
+ )
+ numpy.testing.assert_equal(
+ collector.parameters["rcu_band_select"].flatten(),
+ self.recv_proxy.rcu_band_select_r.tolist()
+ )
+ numpy.testing.assert_equal(
+ collector.parameters["rcu_dth_on"].flatten(),
+ self.recv_proxy.rcu_dth_on_r.tolist()
+ )
with TemporaryDirectory() as tmpdir:
- new_sys_argv = [sys.argv[0], "--mode", "SST", "--file", join(dirname(dirname(dirname(dirname(__file__)))), "test/statistics", "SDP_SST_statistics_packets.bin"), "--output_dir", tmpdir]
- with mock.patch.object(writer.sys, 'argv', new_sys_argv):
+ new_sys_argv = [
+ sys.argv[0],
+ "--mode", "SST",
+ "--file", join(
+ dirname(dirname(dirname(dirname(__file__)))),
+ "test/statistics", "SDP_SST_statistics_packets.bin"
+ ),
+ ""
+ "--output_dir", tmpdir
+ ]
+ with mock.patch.object(entry.sys, 'argv', new_sys_argv):
with self.assertRaises(SystemExit):
- writer.main()
+ entry.main()
# check if file was written
self.assertTrue(isfile(f"{tmpdir}/SST_2021-09-20-12-17-40.h5"))
# test statistics reader
- new_sys_argv = [sys.argv[0], "--files", f"{tmpdir}/SST_2021-09-20-12-17-40.h5", "--start_time", "2021-09-20#07:40:08.937+00:00", "--end_time", "2021-10-04#07:50:08.937+00:00"]
- with mock.patch.object(statistics_reader.sys, 'argv', new_sys_argv):
- stat_parser = statistics_reader.setup_stat_parser()
+ new_sys_argv = [
+ sys.argv[0],
+ "--files", f"{tmpdir}/SST_2021-09-20-12-17-40.h5",
+ "--start_time", "2021-09-20#07:40:08.937+00:00",
+ "--end_time", "2021-10-04#07:50:08.937+00:00"
+ ]
+ with mock.patch.object(reader.sys, 'argv', new_sys_argv):
+ stat_parser = reader.setup_stat_parser()
SSTstatistics = stat_parser.list_statistics()
self.assertIsNotNone(SSTstatistics)
- stat = stat_parser.get_statistic('2021-09-20T12:17:40.000+00:00') # same as stat_parser.statistics[0]
+ # same as stat_parser.statistics[0]
+ stat = stat_parser.get_statistic(
+ '2021-09-20T12:17:40.000+00:00'
+ )
self.assertIsNotNone(stat)
self.assertEqual(121, stat.data_id_signal_input_index)
# Test RECV attributes
@@ -77,23 +106,40 @@ class TestStatisticsWriterSST(BaseIntegrationTestCase):
self.assertListEqual(stat.rcu_dth_on.tolist(), [False] * 96)
def test_no_tango_SST_statistics(self):
-
with TemporaryDirectory() as tmpdir:
- new_sys_argv = [sys.argv[0], "--mode", "SST", "--no-tango", "--file", join(dirname(dirname(dirname(dirname(__file__)))), "test/statistics", "SDP_SST_statistics_packets.bin"), "--output_dir", tmpdir]
- with mock.patch.object(writer.sys, 'argv', new_sys_argv):
+ new_sys_argv = [
+ sys.argv[0],
+ "--mode", "SST",
+ "--no-tango",
+ "--file", join(
+ dirname(dirname(dirname(dirname(__file__)))),
+ "test/statistics", "SDP_SST_statistics_packets.bin"
+ ),
+ "--output_dir", tmpdir
+ ]
+
+ with mock.patch.object(entry.sys, 'argv', new_sys_argv):
with self.assertRaises(SystemExit):
- writer.main()
+ entry.main()
# check if file was written
self.assertTrue(isfile(f"{tmpdir}/SST_2021-09-20-12-17-40.h5"))
# test statistics reader
- new_sys_argv = [sys.argv[0], "--files", f"{tmpdir}/SST_2021-09-20-12-17-40.h5", "--start_time", "2021-09-20#07:40:08.937+00:00", "--end_time", "2021-10-04#07:50:08.937+00:00"]
- with mock.patch.object(statistics_reader.sys, 'argv', new_sys_argv):
- stat_parser = statistics_reader.setup_stat_parser()
+ new_sys_argv = [
+ sys.argv[0],
+ "--files", f"{tmpdir}/SST_2021-09-20-12-17-40.h5",
+ "--start_time", "2021-09-20#07:40:08.937+00:00",
+ "--end_time", "2021-10-04#07:50:08.937+00:00"
+ ]
+ with mock.patch.object(reader.sys, 'argv', new_sys_argv):
+ stat_parser = reader.setup_stat_parser()
SSTstatistics = stat_parser.list_statistics()
self.assertIsNotNone(SSTstatistics)
- stat = stat_parser.get_statistic('2021-09-20T12:17:40.000+00:00') # same as stat_parser.statistics[0]
+ # same as stat_parser.statistics[0]
+ stat = stat_parser.get_statistic(
+ '2021-09-20T12:17:40.000+00:00'
+ )
self.assertIsNotNone(stat)
self.assertEqual(121, stat.data_id_signal_input_index)
# Test RECV attributes
@@ -101,27 +147,39 @@ class TestStatisticsWriterSST(BaseIntegrationTestCase):
self.assertEqual(stat.rcu_band_select.tolist(), None)
self.assertEqual(stat.rcu_dth_on.tolist(), None)
- def test_SST_statistics_with_device_in_off(self):
+ def test_SST_statistics_with_device_in_off(self):
self.setup_recv_proxy()
self.recv_proxy.Off()
self.assertEqual(DevState.OFF, self.recv_proxy.state())
with TemporaryDirectory() as tmpdir:
- new_sys_argv = [sys.argv[0], "--mode", "SST", "--file", join(dirname(dirname(dirname(dirname(__file__)))), "test/statistics", "SDP_SST_statistics_packets.bin"), "--output_dir", tmpdir]
- with mock.patch.object(writer.sys, 'argv', new_sys_argv):
+ new_sys_argv = [
+ sys.argv[0],
+ "--mode", "SST",
+ "--file", join(
+ dirname(dirname(dirname(dirname(__file__)))),
+ "test/statistics", "SDP_SST_statistics_packets.bin"
+ ), "--output_dir", tmpdir
+ ]
+ with mock.patch.object(entry.sys, 'argv', new_sys_argv):
with self.assertRaises(SystemExit):
- writer.main()
+ entry.main()
# check if file was written
self.assertTrue(isfile(f"{tmpdir}/SST_2021-09-20-12-17-40.h5"))
# test statistics reader
- new_sys_argv = [sys.argv[0], "--files", f"{tmpdir}/SST_2021-09-20-12-17-40.h5", "--start_time", "2021-09-20#07:40:08.937+00:00", "--end_time", "2021-10-04#07:50:08.937+00:00"]
- with mock.patch.object(statistics_reader.sys, 'argv', new_sys_argv):
- stat_parser = statistics_reader.setup_stat_parser()
+ new_sys_argv = [
+ sys.argv[0],
+ "--files", f"{tmpdir}/SST_2021-09-20-12-17-40.h5",
+ "--start_time", "2021-09-20#07:40:08.937+00:00",
+ "--end_time", "2021-10-04#07:50:08.937+00:00"
+ ]
+ with mock.patch.object(reader.sys, 'argv', new_sys_argv):
+ stat_parser = reader.setup_stat_parser()
SSTstatistics = stat_parser.list_statistics()
self.assertIsNotNone(SSTstatistics)
- stat = stat_parser.get_statistic('2021-09-20T12:17:40.000+00:00') # same as stat_parser.statistics[0]
+ stat = stat_parser.get_statistic('2021-09-20T12:17:40.000+00:00') # same as stat_parser.statistics[0]
self.assertIsNotNone(stat)
self.assertEqual(121, stat.data_id_signal_input_index)
# Test RECV attributes
diff --git a/tangostationcontrol/tangostationcontrol/statistics_writer/README.md b/tangostationcontrol/tangostationcontrol/statistics/README.md
similarity index 70%
rename from tangostationcontrol/tangostationcontrol/statistics_writer/README.md
rename to tangostationcontrol/tangostationcontrol/statistics/README.md
index dc9e285ea47039351d98cb5b8c82d104b85e11bd..ad2a9d3e894dcc2d16c04060487832cc62db8b78 100644
--- a/tangostationcontrol/tangostationcontrol/statistics_writer/README.md
+++ b/tangostationcontrol/tangostationcontrol/statistics/README.md
@@ -1,9 +1,12 @@
# TCP to HDF5 statistics writer
-The TCP to HDF5 statistics writer can be started with `statistics_writer.py` This script imports the receiver script and `hdf5_writer.py`. `receiver.py` only takes care of receiving packets.
-`hdf5_writer.py` takes the receive function from the receiver and uses it to obtain packets.
-Any function that can deliver statistics packets can be used by this code.
-`hdf5_writer.py` takes care of processing the packets it receives, filling statistics matrices
-and writing those matrices (as well as a bunch of metadata) to hdf5.
+
+The TCP to HDF5 statistics writer can be started with `statistics_writer.py`
+This script imports the receiver script and `HDF5Writer.py`. `receiver.py` only
+takes care of receiving packets. `HDF5Writer.py` takes the receive function from
+the receiver and uses it to obtain packets. Any function that can deliver
+statistics packets can be used by this code. `HDF5Writer.py` takes care of
+processing the packets it receives, filling statistics matrices and writing
+those matrices (as well as a bunch of metadata) to hdf5.
### TCP Statistics writer
@@ -21,14 +24,19 @@ This script can be called with the following arguments:
-d --decimation Configure the writer to only store one every n samples. Saves storage space (default: 1 (everything))
-r --reconnect Set the writer to keep trying to reconnect whenever connection is lost. (default: off)
```
-An example call could be: `l2ss-statistics-writer statistics_writer.py --host localhost --port 1234 --mode XST --debug`
-This starts the script up to listen on localhost:1234 for XST's with debug mode on.
+An example call could be:
-## HFD5 structure
-Statistics packets are collected by the StatisticsCollector in to a matrix. Once the matrix is done or a newer
-timestamp arrives this matrix along with the header of first packet header, nof_payload_errors and nof_valid_payloads.
-The file will be named after the mode it is in and the timestamp of the statistics packets. For example: `SST_1970-01-01-00-00-00.h5`.
+```l2ss-statistics-writer --host localhost --port 1234--mode XST --debug```
+
+This starts the script up to listen on localhost:1234 for XSTs with debug mode
+on.
+## HFD5 structure
+Statistics packets are collected by the StatisticsCollector in to a matrix. Once
+the matrix is done or a newer timestamp arrives this matrix along with the
+header of first packet header, nof_payload_errors and nof_valid_payloads. The
+file will be named after the mode it is in and the timestamp of the statistics
+packets. For example: `SST_1970-01-01-00-00-00.h5`.
```
File
@@ -47,8 +55,11 @@ File
```
### reader
-There is a statistics reader that is capable of parsing multiple HDF5 statistics files in to
-a more easily usable format. It also allows for filtering between certain timestamps.
+
+There is a statistics reader that is capable of parsing multiple HDF5 statistics
+files in to a more easily usable format. It also allows for filtering between
+certain timestamps.
+
`statistics_reader.py` takes the following arguments:
`--files list of files to parse`
`--end_time highest timestamp to process in isoformat`
diff --git a/tangostationcontrol/tangostationcontrol/statistics/collector.py b/tangostationcontrol/tangostationcontrol/statistics/collector.py
index 7befd038d108ba3c147126082447a8e813838e42..0f68d1adb2ab2679a665ce5d1f20ae434a6efa13 100644
--- a/tangostationcontrol/tangostationcontrol/statistics/collector.py
+++ b/tangostationcontrol/tangostationcontrol/statistics/collector.py
@@ -19,37 +19,51 @@ from tango import DevState
logger = logging.getLogger()
-class StationStatisticsCollectorInterface(abc.ABC):
+class DeviceCollectorInterface(abc.ABC):
+ """Small interface for deviceproxy enabled collectors"""
+
+ def __init__(self, device: DeviceProxy = None):
+ self.device = device
@abc.abstractmethod
- def parse_device_attributes(self, device: DeviceProxy):
+ def parse_device_attributes(self):
"""Update information based on device attributes"""
raise NotImplementedError
-class StationSSTCollector(StationStatisticsCollectorInterface, SSTCollector):
+class StationSSTCollector(DeviceCollectorInterface, SSTCollector):
+
+ def __init__(self, device: DeviceProxy = None):
+ """Manually combine the constructors with appropriate arguments"""
+
+ DeviceCollectorInterface.__init__(self, device=device)
+ SSTCollector.__init__(self)
- def parse_packet(self, packet, obj):
- super(StationSSTCollector, self).parse_packet(packet, obj)
+ def _parse_packet(self, packet):
+ super()._parse_packet(packet)
# add tango values to packet
- self.parse_device_attributes(obj)
+ self.parse_device_attributes()
- def parse_device_attributes(self, device: DeviceProxy):
+ def parse_device_attributes(self):
# If Tango connection has been disabled, set explicitly to None,
# because otherwise default_values are inserted
- if device is None or device.state() != DevState.ON:
+ if not self.device or self.device.state() != DevState.ON:
self.parameters["rcu_attenuator_dB"] = None
self.parameters["rcu_band_select"] = None
self.parameters["rcu_dth_on"] = None
else:
try:
- self.parameters["rcu_attenuator_dB"] = device.RCU_Attenuator_dB_R
- self.parameters["rcu_band_select"] = device.RCU_Band_Select_R
- self.parameters["rcu_dth_on"] = device.RCU_DTH_on_R
+ self.parameters[
+ "rcu_attenuator_dB"
+ ] = self.device.RCU_Attenuator_dB_R
+ self.parameters[
+ "rcu_band_select"
+ ] = self.device.RCU_Band_Select_R
+ self.parameters["rcu_dth_on"] = self.device.RCU_DTH_on_R
except DevFailed as e:
- logger.warning(f"Device {device.name()} not responding.")
+ logger.warning("Device: %s not responding.", self.device.name())
self.parameters["rcu_attenuator_dB"] = None
self.parameters["rcu_band_select"] = None
self.parameters["rcu_dth_on"] = None
diff --git a/tangostationcontrol/tangostationcontrol/statistics/packet.py b/tangostationcontrol/tangostationcontrol/statistics/packet.py
deleted file mode 100644
index 87383704daf73fc77e2a34d7d329c6e0246725e2..0000000000000000000000000000000000000000
--- a/tangostationcontrol/tangostationcontrol/statistics/packet.py
+++ /dev/null
@@ -1,88 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# This file is part of the LOFAR 2.0 Station Software
-#
-#
-#
-# Distributed under the terms of the APACHE license.
-# See LICENSE.txt for more info.
-
-import sys
-import pprint
-
-from lofar_station_client.statistics.packet import SDPPacket
-from lofar_station_client.statistics.packet import PACKET_CLASS_FOR_MARKER
-
-
-def read_packet(read_func) -> SDPPacket:
- """Read a packet using the given read function, with signature
-
- ```read_func(num_bytes: int) -> bytes```
-
- and return it. The packet type is sensed from the data and
- the correct subclass of SDPPacket is returned.
-
- If read_func() returns None, this function will as well.
- """
-
- # read just the marker
- marker = read_func(1)
- if not marker:
- return None
-
- # read the packet header based on type
- packetClass = PACKET_CLASS_FOR_MARKER[marker]
-
- # read the rest of the header
- header = read_func(packetClass.HEADER_SIZE - len(marker))
- if not header:
- return None
-
- header = marker + header
-
- # parse the packet header size
- packet = packetClass(header)
-
- # read the payload
- payload_size = packet.expected_size() - len(header)
- payload = read_func(payload_size)
-
- if not payload:
- return None
-
- # return full packet
- return packetClass(header + payload)
-
-
-def main(args=None, **kwargs):
- # parse one packet from stdin
-
- # packet counter
- nr = 0
-
- # byte offset in the stream
- offset = 0
-
- while True:
- # read the packet from input
- packet = read_packet(sys.stdin.buffer.read)
-
- if not packet:
- break
-
- # print header
- print(
- f"# Packet {nr} of class {packet.__class__.__name__} starting at "
- f"offset {offset} with length {packet.size()}"
- )
- pprint.pprint(packet.header())
-
- # increment counters
- nr += 1
- offset += packet.size()
-
-
-# this file is very useful to have stand alone to parse raw packet files, so make it
-# work as such
-if __name__ == "__main__":
- main(sys.argv)
diff --git a/tangostationcontrol/tangostationcontrol/statistics_writer/statistics_reader.py b/tangostationcontrol/tangostationcontrol/statistics/reader.py
similarity index 99%
rename from tangostationcontrol/tangostationcontrol/statistics_writer/statistics_reader.py
rename to tangostationcontrol/tangostationcontrol/statistics/reader.py
index 54755e6d67ed60cb3c4c5d5eb2d1c3bddb2f3536..cb9370fd206440de37f31bdbaae7218f71470726 100644
--- a/tangostationcontrol/tangostationcontrol/statistics_writer/statistics_reader.py
+++ b/tangostationcontrol/tangostationcontrol/statistics/reader.py
@@ -164,7 +164,6 @@ class statistics_data:
"data_id_subband_index", "data_id_first_baseline", "data_id_beamlet_index", "nof_valid_payloads", "nof_payload_errors", "values",
"rcu_attenuator_dB", "rcu_band_select", "rcu_dth_on")
-
def __init__(self, file, group_key):
# get all the general header info
@@ -218,6 +217,7 @@ class statistics_data:
self.nof_payload_errors = numpy.array(file.get(f"{group_key}/nof_payload_errors"))
self.values = numpy.array(file.get(f"{group_key}/values"))
+
def parse_arguments():
"""
This function parses the input arguments.
@@ -243,6 +243,7 @@ def parse_arguments():
return files, end_time, start_time
+
def setup_stat_parser():
"""
This function takes care of setting up the statistics parser with the end_time, start_time and files arguments
@@ -267,6 +268,7 @@ def setup_stat_parser():
return stat_parser
+
def main():
# set up everything
diff --git a/tangostationcontrol/tangostationcontrol/statistics_writer/receiver.py b/tangostationcontrol/tangostationcontrol/statistics/receiver.py
similarity index 99%
rename from tangostationcontrol/tangostationcontrol/statistics_writer/receiver.py
rename to tangostationcontrol/tangostationcontrol/statistics/receiver.py
index 233b819c5173fcfefc010f9025563ddcaef3e65c..b9c823c2a822cd4aee4a02bf6e921246e7dec471 100644
--- a/tangostationcontrol/tangostationcontrol/statistics_writer/receiver.py
+++ b/tangostationcontrol/tangostationcontrol/statistics/receiver.py
@@ -12,6 +12,7 @@ import socket
from lofar_station_client.statistics.packet import StatisticsPacket
+
class receiver:
""" Reads data from a file descriptor. """
@@ -57,6 +58,7 @@ class receiver:
return data
+
class tcp_receiver(receiver):
def __init__(self, HOST, PORT):
self.host = HOST
@@ -79,7 +81,6 @@ class tcp_receiver(receiver):
return True
-
class file_receiver(receiver):
def __init__(self, filename):
self.filename = filename
diff --git a/tangostationcontrol/tangostationcontrol/statistics_writer/__init__.py b/tangostationcontrol/tangostationcontrol/statistics/udp_dev/__init__.py
similarity index 100%
rename from tangostationcontrol/tangostationcontrol/statistics_writer/__init__.py
rename to tangostationcontrol/tangostationcontrol/statistics/udp_dev/__init__.py
diff --git a/tangostationcontrol/tangostationcontrol/statistics_writer/udp_dev/udp_client.py b/tangostationcontrol/tangostationcontrol/statistics/udp_dev/udp_client.py
similarity index 98%
rename from tangostationcontrol/tangostationcontrol/statistics_writer/udp_dev/udp_client.py
rename to tangostationcontrol/tangostationcontrol/statistics/udp_dev/udp_client.py
index ea5a644f10985257452b4d93249cea36c37708e9..98a85d62d1d3e610ed747b49f5034eb1dc3afeec 100644
--- a/tangostationcontrol/tangostationcontrol/statistics_writer/udp_dev/udp_client.py
+++ b/tangostationcontrol/tangostationcontrol/statistics/udp_dev/udp_client.py
@@ -4,6 +4,7 @@ import netifaces as ni
from datetime import datetime
import time
+
class UDP_Client:
def __init__(self, server_ip:str, server_port:int):
@@ -40,6 +41,7 @@ class UDP_Client:
# close the socket
s.close()
+
if __name__ == '__main__':
if len(sys.argv) == 3:
@@ -56,7 +58,3 @@ if __name__ == '__main__':
client = UDP_Client(server_ip,server_port)
client.run()
-
-
-
-
\ No newline at end of file
diff --git a/tangostationcontrol/tangostationcontrol/statistics_writer/udp_dev/udp_server.py b/tangostationcontrol/tangostationcontrol/statistics/udp_dev/udp_server.py
similarity index 99%
rename from tangostationcontrol/tangostationcontrol/statistics_writer/udp_dev/udp_server.py
rename to tangostationcontrol/tangostationcontrol/statistics/udp_dev/udp_server.py
index 35ccb6bd92975bcfbd2ea877e5e5b38c3962b0c5..aab1365dfa661b0bef5ab280f1fe0f32ebbb493a 100644
--- a/tangostationcontrol/tangostationcontrol/statistics_writer/udp_dev/udp_server.py
+++ b/tangostationcontrol/tangostationcontrol/statistics/udp_dev/udp_server.py
@@ -43,6 +43,7 @@ class UDP_Server:
def get_recv_data(self):
return self.recv_data
+
if __name__ == '__main__':
local_ip = ni.ifaddresses('eth0')[ni.AF_INET][0]['addr']
server = UDP_Server(local_ip,5600)
diff --git a/tangostationcontrol/tangostationcontrol/statistics_writer/udp_dev/udp_write_manager.py b/tangostationcontrol/tangostationcontrol/statistics/udp_dev/udp_write_manager.py
similarity index 100%
rename from tangostationcontrol/tangostationcontrol/statistics_writer/udp_dev/udp_write_manager.py
rename to tangostationcontrol/tangostationcontrol/statistics/udp_dev/udp_write_manager.py
diff --git a/tangostationcontrol/tangostationcontrol/statistics_writer/test/__init__.py b/tangostationcontrol/tangostationcontrol/statistics/writer/__init__.py
similarity index 100%
rename from tangostationcontrol/tangostationcontrol/statistics_writer/test/__init__.py
rename to tangostationcontrol/tangostationcontrol/statistics/writer/__init__.py
diff --git a/tangostationcontrol/tangostationcontrol/statistics/writer.py b/tangostationcontrol/tangostationcontrol/statistics/writer/entry.py
similarity index 79%
rename from tangostationcontrol/tangostationcontrol/statistics/writer.py
rename to tangostationcontrol/tangostationcontrol/statistics/writer/entry.py
index 23d3869959ccad60d0fa2d9c098aef259b6b9f88..0b3c60484c893d76918e0e926cd1a9c8114974d8 100644
--- a/tangostationcontrol/tangostationcontrol/statistics/writer.py
+++ b/tangostationcontrol/tangostationcontrol/statistics/writer/entry.py
@@ -12,11 +12,18 @@ import time
import sys
from tango import DeviceProxy
-from tangostationcontrol.statistics_writer.receiver import tcp_receiver, file_receiver
-from tangostationcontrol.statistics_writer.hdf5_writer import sst_hdf5_writer, parallel_xst_hdf5_writer, bst_hdf5_writer
+from tangostationcontrol.statistics.receiver import file_receiver
+from tangostationcontrol.statistics.receiver import tcp_receiver
+from tangostationcontrol.statistics.writer.hdf5 import BstHdf5Writer
+from tangostationcontrol.statistics.writer.hdf5 import SstHdf5Writer
+from tangostationcontrol.statistics.writer.hdf5 import ParallelXstHdf5Writer
import logging
-logging.basicConfig(level=logging.INFO, format = '%(asctime)s:%(levelname)s: %(message)s')
+
+logging.basicConfig(
+ level=logging.INFO,
+ format='%(asctime)s:%(levelname)s: %(message)s'
+)
logger = logging.getLogger("statistics_writer")
@@ -77,24 +84,39 @@ def _create_receiver(filename, host, port):
sys.exit(1)
-def _create_writer(mode, interval, output_dir, decimation):
+def _create_writer(
+ mode, interval, output_dir, decimation, device: DeviceProxy = None
+):
"""Create the writer"""
if mode == "XST":
- return parallel_xst_hdf5_writer(new_file_time_interval=interval, file_location=output_dir, decimation_factor=decimation)
+ return ParallelXstHdf5Writer(
+ new_file_time_interval=interval,
+ file_location=output_dir,
+ decimation_factor=decimation,
+ )
elif mode == "SST":
- return sst_hdf5_writer(new_file_time_interval=interval, file_location=output_dir, decimation_factor=decimation)
+ return SstHdf5Writer(
+ new_file_time_interval=interval,
+ file_location=output_dir,
+ decimation_factor=decimation,
+ device=device
+ )
elif mode == "BST":
- return bst_hdf5_writer(new_file_time_interval=interval, file_location=output_dir, decimation_factor=decimation)
+ return BstHdf5Writer(
+ new_file_time_interval=interval,
+ file_location=output_dir,
+ decimation_factor=decimation,
+ )
else:
- logger.fatal(f"Invalid mode: {mode}")
+ logger.fatal("Invalid mode: %s", mode)
sys.exit(1)
-def _start_loop(receiver, writer, reconnect, filename, device):
+def _start_loop(receiver, writer, reconnect, filename):
"""Main loop"""
try:
while True:
- _receive_packets(receiver, writer, reconnect, filename, device)
+ _receive_packets(receiver, writer, reconnect, filename)
except KeyboardInterrupt:
# user abort, don't complain
logger.warning("Received keyboard interrupt. Stopping.")
@@ -102,10 +124,10 @@ def _start_loop(receiver, writer, reconnect, filename, device):
writer.close_writer()
-def _receive_packets(receiver, writer, reconnect, filename, device):
+def _receive_packets(receiver, writer, reconnect, filename):
try:
packet = receiver.get_packet()
- writer.next_packet(packet, device)
+ writer.next_packet(packet)
except EOFError:
if reconnect and not filename:
logger.warning("Connection lost, attempting to reconnect")
@@ -168,7 +190,8 @@ def main():
receiver = _create_receiver(filename, host, port)
# create the writer
- writer = _create_writer(mode, interval, output_dir, decimation)
+ writer = _create_writer(mode, interval, output_dir, decimation, device)
# start looping
- _start_loop(receiver, writer, reconnect, filename, device)
+ _start_loop(receiver, writer, reconnect, filename)
+
diff --git a/tangostationcontrol/tangostationcontrol/statistics_writer/hdf5_writer.py b/tangostationcontrol/tangostationcontrol/statistics/writer/hdf5.py
similarity index 50%
rename from tangostationcontrol/tangostationcontrol/statistics_writer/hdf5_writer.py
rename to tangostationcontrol/tangostationcontrol/statistics/writer/hdf5.py
index 846621208075c644d00eb8260f4d70b4cd4fd122..4ac9dd9d451ae77a93b0f5f0d848d4de138cfd8c 100644
--- a/tangostationcontrol/tangostationcontrol/statistics_writer/hdf5_writer.py
+++ b/tangostationcontrol/tangostationcontrol/statistics/writer/hdf5.py
@@ -20,21 +20,29 @@ from lofar_station_client.statistics.collector import BSTCollector
from lofar_station_client.statistics.collector import XSTCollector
from tangostationcontrol.statistics.collector import StationSSTCollector
-from lofar_station_client.statistics.packet import SSTPacket, XSTPacket, BSTPacket
+from lofar_station_client.statistics.packet import BSTPacket
+from lofar_station_client.statistics.packet import SSTPacket
+from lofar_station_client.statistics.packet import XSTPacket
+from tango import DeviceProxy
logger = logging.getLogger("statistics_writer")
-__all__ = ["hdf5_writer", "parallel_xst_hdf5_writer", "xst_hdf5_writer", "sst_hdf5_writer", "bst_hdf5_writer"]
+__all__ = [
+ "HDF5Writer", "ParallelXstHdf5Writer", "XstHdf5Writer",
+ "SstHdf5Writer", "BstHdf5Writer",
+]
-class hdf5_writer(ABC):
-
+class HDF5Writer(ABC):
SST_MODE = "SST"
XST_MODE = "XST"
BST_MODE = "BST"
- def __init__(self, new_file_time_interval, file_location, statistics_mode, decimation_factor):
+ def __init__(
+ self, new_file_time_interval: int, file_location, statistics_mode,
+ decimation_factor, device: DeviceProxy = None
+ ):
# all variables that deal with the matrix that's currently being decoded
self.current_matrix = None
@@ -44,7 +52,8 @@ class hdf5_writer(ABC):
self.statistics_counter = 0
# the header of the first packet of a new matrix is written as metadata.
- # Assumes all subsequent headers of the same matrix are identical (minus index)
+ # Assumes all subsequent headers of the same matrix are identical
+ # (minus index)
self.statistics_header = None
# file handing
@@ -54,25 +63,33 @@ class hdf5_writer(ABC):
self.last_file_time = datetime.min.replace(tzinfo=pytz.UTC)
self.file = None
- # parameters that are configured depending on the mode the statistics writer is in (SST,XST,BST)
+ # parameters that are configured depending on the mode the statistics
+ # writer is in (SST,XST,BST)
self.mode = statistics_mode.upper()
+ # Set device if any, defaults to None
+ self.device = device
+
@abstractmethod
- def decoder(self):
+ def decoder(self, packet):
pass
@abstractmethod
def new_collector(self):
pass
- def next_packet(self, packet, device):
+ def next_packet(self, packet):
"""
- All statistics packets come with a timestamp of the time they were measured. All the values will be spread across multiple packets.
- As long as the timestamp is the same they belong in the same matrix. This code handles collecting the matrix from those multiple
- packets as well as storing matrices and starting new ones
- The code receives new packets and checks the statistics timestamp of them. If the timestamp is higher than the current timestamp
- it will close the current matrix, store it and start a new one.
+ All statistics packets come with a timestamp of the time they were
+ measured. All the values will be spread across multiple packets.
+ As long as the timestamp is the same they belong in the same matrix.
+ This code handles collecting the matrix from those multiple packets as
+ well as storing matrices and starting new ones
+
+ The code receives new packets and checks the statistics timestamp of
+ them. If the timestamp is higher than the current timestamp it will
+ close the current matrix, store it and start a new one.
"""
# process the packet
@@ -84,54 +101,68 @@ class hdf5_writer(ABC):
# grab the timestamp
statistics_timestamp = statistics_packet.timestamp()
- # ignore packets with no timestamp, as they indicate FPGA processing was disabled
- # and are useless anyway.
+ # ignore packets with no timestamp, as they indicate FPGA processing was
+ # disabled and are useless anyway.
if statistics_packet.block_serial_number == 0:
- logger.warning(f"Received statistics with no timestamp. Packet dropped.")
+ logger.warning(
+ "Received statistics with no timestamp. Packet dropped."
+ )
return
- # check if te statistics timestamp is unexpectedly older than the current one
+ # check if te statistics timestamp is unexpectedly older than the
+ # current one
if statistics_timestamp < self.current_timestamp:
- logger.warning(f"Received statistics with earlier timestamp than is currently being processed ({statistics_timestamp}). Packet dropped.")
+ logger.warning(
+ "Received statistics with earlier timestamp than is currently"
+ "being processed (%s). Packet dropped.",
+ statistics_timestamp
+ )
return
- # if this statistics packet has a new timestamp it means we need to start a new matrix
+ # if this statistics packet has a new timestamp it means we need to
+ # start a new matrix
if statistics_timestamp > self.current_timestamp:
self.start_new_matrix(statistics_timestamp)
self.current_timestamp = statistics_timestamp
- self.process_packet(packet, device)
+ self.process_packet(packet)
def start_new_matrix(self, timestamp):
"""
is called when a statistics packet with a newer timestamp is received.
- Writes the matrix to the hdf5 file
- Creates a new hdf5 file if needed
- updates current timestamp and statistics matrix collector
+ Writes the matrix to the hdf5 file, creates a new hdf5 file if needed
+ and updates current timestamp and statistics matrix collector.
"""
+ # always increment
+ self.statistics_counter += 1
+
# only write the specified fraction of statistics, skip the rest
if self.statistics_counter % self.decimation_factor != 0:
- logger.debug(f"Skipping statistic with timestamp: {timestamp}. Only writing 1/{self.decimation_factor} statistics")
-
- # increment even though its skipped
- self.statistics_counter += 1
+ logger.debug(
+ "Skipping statistic with timestamp: %s. Only writing"
+ "1/%d statistics", timestamp,
+ self.decimation_factor
+ )
return
- # received new statistic, so increment counter
- self.statistics_counter += 1
-
- logger.debug(f"starting new matrix with timestamp: {timestamp}")
+ logger.debug("starting new matrix with timestamp: %s", timestamp)
# write the finished (and checks if its the first matrix)
if self.current_matrix is not None:
try:
self.write_matrix()
except Exception as e:
- time = self.current_timestamp.strftime("%Y-%m-%d-%H-%M-%S-%f")[:-3]
- logger.exception(f"Exception while attempting to write matrix to HDF5. Matrix: {time} dropped")
-
- # only start a new file if its time AND we are done with the previous matrix.
+ time = self.current_timestamp.strftime(
+ "%Y-%m-%d-%H-%M-%S-%f"
+ )[:-3]
+ logger.exception(
+ "Exception while attempting to write matrix to HDF5."
+ "Matrix: %s dropped", time
+ )
+
+ # only start a new file if its time AND we are done with the previous
+ # matrix.
if timestamp >= self.new_file_time_interval + self.last_file_time:
self.start_new_hdf5(timestamp)
@@ -140,20 +171,29 @@ class hdf5_writer(ABC):
self.statistics_header = None
def write_matrix(self):
+ """Writes the finished matrix to the hdf5 file"""
+
logger.debug("writing matrix to file")
- """
- Writes the finished matrix to the hdf5 file
- """
# create the new hdf5 group based on the timestamp of packets
- current_group = self.file.create_group("{}_{}".format(self.mode, self.current_timestamp.isoformat(timespec="milliseconds")))
+ current_group = self.file.create_group(
+ "{}_{}".format(self.mode, self.current_timestamp.isoformat(
+ timespec="milliseconds"
+ ))
+ )
# store the statistics values for the current group
self.write_values_matrix(current_group)
# might be optional, but they're easy to add.
- current_group.create_dataset(name="nof_payload_errors", data=self.current_matrix.parameters["nof_payload_errors"])
- current_group.create_dataset(name="nof_valid_payloads", data=self.current_matrix.parameters["nof_valid_payloads"])
+ current_group.create_dataset(
+ name="nof_payload_errors",
+ data=self.current_matrix.parameters["nof_payload_errors"]
+ )
+ current_group.create_dataset(
+ name="nof_valid_payloads",
+ data=self.current_matrix.parameters["nof_valid_payloads"]
+ )
# get the statistics header
header = self.statistics_header
@@ -163,10 +203,13 @@ class hdf5_writer(ABC):
return
# can't store datetime objects, convert to string instead
- header["timestamp"] = header["timestamp"].isoformat(timespec="milliseconds")
+ header["timestamp"] = header["timestamp"].isoformat(
+ timespec="milliseconds"
+ )
- # Stores the header of the packet received for this matrix as a list of atttributes
- for k,v in header.items():
+ # Stores the header of the packet received for this matrix as a list of
+ # attributes
+ for k, v in header.items():
if type(v) == dict:
for subk, subv in v.items():
current_group.attrs[f"{k}_{subk}"] = subv
@@ -181,7 +224,7 @@ class hdf5_writer(ABC):
time_str = str(timestamp.strftime("%Y-%m-%d-%H-%M-%S"))
return f"{self.file_location}/{self.mode}_{time_str}{suffix}"
- def process_packet(self, packet, device):
+ def process_packet(self, packet):
"""
Adds the newly received statistics packet to the statistics matrix
"""
@@ -189,7 +232,7 @@ class hdf5_writer(ABC):
if self.statistics_counter % self.decimation_factor != 0:
return
- self.current_matrix.process_packet(packet, device)
+ self.current_matrix.process_packet(packet)
def start_new_hdf5(self, timestamp):
@@ -197,7 +240,11 @@ class hdf5_writer(ABC):
try:
self.file.close()
except Exception as e:
- logger.exception(f"Error while attempting to close hdf5 file to disk. file {self.file} likely empty, please verify integrity.")
+ logger.exception(
+ "Error while attempting to close hdf5 file to disk. file"
+ "%s likely empty, please verify integrity.",
+ self.file,
+ )
filename = self.next_filename(timestamp)
logger.info(f"creating new file: {filename}")
@@ -211,9 +258,8 @@ class hdf5_writer(ABC):
self.last_file_time = timestamp
def close_writer(self):
- """
- Function that can be used to stop the writer without data loss.
- """
+ """Function that can be used to stop the writer without data loss."""
+
logger.debug("closing hdf5 file")
if self.file is not None:
if self.current_matrix is not None:
@@ -224,35 +270,84 @@ class hdf5_writer(ABC):
finally:
filename = str(self.file)
self.file.close()
- logger.debug(f"{filename} closed")
- logger.debug(f"Received a total of {self.statistics_counter} statistics while running. With {int(self.statistics_counter/self.decimation_factor)} written to disk ")
-
-
-class sst_hdf5_writer(hdf5_writer):
- def __init__(self, new_file_time_interval, file_location, decimation_factor):
- super().__init__(new_file_time_interval, file_location, hdf5_writer.SST_MODE, decimation_factor)
+ logger.debug("%s closed", filename)
+ logger.debug(
+ "Received a total of %d statistics while running. With "
+ "%d written to disk",
+ self.statistics_counter,
+ int(self.statistics_counter / self.decimation_factor)
+ )
+
+
+class SstHdf5Writer(HDF5Writer):
+ def __init__(
+ self,
+ new_file_time_interval,
+ file_location,
+ decimation_factor,
+ device: DeviceProxy = None,
+ ):
+ super().__init__(
+ new_file_time_interval,
+ file_location,
+ HDF5Writer.SST_MODE,
+ decimation_factor,
+ device=device
+ )
def decoder(self, packet):
return SSTPacket(packet)
def new_collector(self):
- return StationSSTCollector()
+ return StationSSTCollector(self.device)
def write_values_matrix(self, current_group):
# store the SST values
- current_group.create_dataset(name="values", data=self.current_matrix.parameters["sst_values"].astype(numpy.float32), compression="gzip")
- try:
- current_group.create_dataset(name="rcu_attenuator_dB", data=self.current_matrix.parameters["rcu_attenuator_dB"].astype(numpy.int64), compression="gzip")
- current_group.create_dataset(name="rcu_band_select", data=self.current_matrix.parameters["rcu_band_select"].astype(numpy.int64), compression="gzip")
- current_group.create_dataset(name="rcu_dth_on", data=self.current_matrix.parameters["rcu_dth_on"].astype(numpy.bool_), compression="gzip")
+ current_group.create_dataset(
+ name="values",
+ data=self.current_matrix.parameters[
+ "sst_values"
+ ].astype(numpy.float32),
+ compression="gzip",
+ )
+ try:
+ current_group.create_dataset(
+ name="rcu_attenuator_dB",
+ data=self.current_matrix.parameters["rcu_attenuator_dB"].astype(
+ numpy.int64
+ ),
+ compression="gzip",
+ )
+ current_group.create_dataset(
+ name="rcu_band_select",
+ data=self.current_matrix.parameters["rcu_band_select"].astype(
+ numpy.int64
+ ),
+ compression="gzip",
+ )
+ current_group.create_dataset(
+ name="rcu_dth_on",
+ data=self.current_matrix.parameters["rcu_dth_on"].astype(
+ numpy.bool_
+ ),
+ compression="gzip",
+ )
except AttributeError as e:
logger.warning("Device values not written.")
except Exception as e:
raise Exception from e
-class bst_hdf5_writer(hdf5_writer):
- def __init__(self, new_file_time_interval, file_location, decimation_factor):
- super().__init__(new_file_time_interval, file_location, hdf5_writer.BST_MODE, decimation_factor)
+
+class BstHdf5Writer(HDF5Writer):
+ def __init__(
+ self, new_file_time_interval, file_location, decimation_factor
+ ):
+ super().__init__(
+ new_file_time_interval,
+ file_location,
+ HDF5Writer.BST_MODE,
+ decimation_factor,
+ )
def decoder(self, packet):
return BSTPacket(packet)
@@ -262,12 +357,29 @@ class bst_hdf5_writer(hdf5_writer):
def write_values_matrix(self, current_group):
# store the BST values
- current_group.create_dataset(name="values", data=self.current_matrix.parameters["bst_values"].astype(numpy.float32), compression="gzip")
-
-
-class xst_hdf5_writer(hdf5_writer):
- def __init__(self, new_file_time_interval, file_location, decimation_factor, subband_index):
- super().__init__(new_file_time_interval, file_location, hdf5_writer.XST_MODE, decimation_factor)
+ current_group.create_dataset(
+ name="values",
+ data=self.current_matrix.parameters["bst_values"].astype(
+ numpy.float32
+ ),
+ compression="gzip",
+ )
+
+
+class XstHdf5Writer(HDF5Writer):
+ def __init__(
+ self,
+ new_file_time_interval,
+ file_location,
+ decimation_factor,
+ subband_index,
+ ):
+ super().__init__(
+ new_file_time_interval,
+ file_location,
+ HDF5Writer.XST_MODE,
+ decimation_factor,
+ )
self.subband_index = subband_index
def decoder(self, packet):
@@ -276,37 +388,50 @@ class xst_hdf5_writer(hdf5_writer):
def new_collector(self):
return XSTCollector()
- def next_filename(self, timestamp):
+ def next_filename(self, timestamp, suffix=".h5"):
time_str = str(timestamp.strftime("%Y-%m-%d-%H-%M-%S"))
- return f"{self.file_location}/{self.mode}_SB{self.subband_index}_{time_str}.h5"
+ return (
+ f"{self.file_location}/{self.mode}_SB{self.subband_index}_"
+ f"{time_str}{suffix}"
+ )
def write_values_matrix(self, current_group):
- # requires a function call to transform the xst_blocks in to the right structure
+ # requires a function call to transform the xst_blocks in to the right
+ # structure
#
- # since we have a dedicated writer per subband_index, they all end up in slot 0
- # in their writer, so we only need to store the first set of xst_values.
- current_group.create_dataset(name="values", data=self.current_matrix.xst_values([0])[0].astype(numpy.cfloat), compression="gzip")
-
-
-class parallel_xst_hdf5_writer:
- """ Writes multiple subbands in parallel. Each subband gets written to its own HDF5 file(s). """
-
- def __init__(self, new_file_time_interval, file_location, decimation_factor):
- # maintain a dedicated hdf5_writer per subband
+ # since we have a dedicated writer per subband_index, they all end up in
+ # slot 0 in their writer, so we only need to store the first set of
+ # xst_values.
+ current_group.create_dataset(
+ name="values", data=self.current_matrix.xst_values([0])[0].astype(
+ numpy.cfloat
+ ),
+ compression="gzip",
+ )
+
+
+class ParallelXstHdf5Writer:
+ """Writes multiple subbands in parallel. Each subband to separate file."""
+
+ def __init__(
+ self, new_file_time_interval, file_location, decimation_factor
+ ):
+ # maintain a dedicated HDF5Writer per subband
self.writers = {}
# function to create a new writer, to avoid having to store
# all the init parameters just for this purpose.
def new_writer(subband):
- return xst_hdf5_writer(
- new_file_time_interval,
- file_location,
- decimation_factor,
- subband)
+ return XstHdf5Writer(
+ new_file_time_interval,
+ file_location,
+ decimation_factor,
+ subband,
+ )
self.new_writer = new_writer
- def next_packet(self, packet, device=None):
+ def next_packet(self, packet):
# decode to get subband of this packet
fields = XSTPacket(packet)
subband = fields.subband_index
@@ -316,11 +441,10 @@ class parallel_xst_hdf5_writer:
self.writers[subband] = self.new_writer(subband)
# demux packet to the correct writer
- self.writers[subband].next_packet(packet, device)
+ self.writers[subband].next_packet(packet)
def close_writer(self):
for writer in self.writers.values():
writer.close_writer()
self.writers = {}
-
diff --git a/tangostationcontrol/tangostationcontrol/statistics_writer/SST_2021-10-04-07-36-52.h5 b/tangostationcontrol/tangostationcontrol/statistics_writer/SST_2021-10-04-07-36-52.h5
deleted file mode 100644
index 26179fc59a2fb032bb35d779676befd4ebe26356..0000000000000000000000000000000000000000
Binary files a/tangostationcontrol/tangostationcontrol/statistics_writer/SST_2021-10-04-07-36-52.h5 and /dev/null differ
diff --git a/tangostationcontrol/tangostationcontrol/statistics_writer/test/SST_10m_test_1.h5 b/tangostationcontrol/tangostationcontrol/statistics_writer/test/SST_10m_test_1.h5
deleted file mode 100644
index 2d04a526e1ef73d7bd636e3b564192d95e49cef5..0000000000000000000000000000000000000000
Binary files a/tangostationcontrol/tangostationcontrol/statistics_writer/test/SST_10m_test_1.h5 and /dev/null differ
diff --git a/tangostationcontrol/tangostationcontrol/statistics_writer/test/SST_10m_test_2.h5 b/tangostationcontrol/tangostationcontrol/statistics_writer/test/SST_10m_test_2.h5
deleted file mode 100644
index 45fd32d831508f8d632c6f1778d4d9bb73059294..0000000000000000000000000000000000000000
Binary files a/tangostationcontrol/tangostationcontrol/statistics_writer/test/SST_10m_test_2.h5 and /dev/null differ
diff --git a/tangostationcontrol/tangostationcontrol/statistics_writer/test/SST_10m_test_3.h5 b/tangostationcontrol/tangostationcontrol/statistics_writer/test/SST_10m_test_3.h5
deleted file mode 100644
index 5c971e8e2cea131d6c9ba8b7e6b1d645f205f276..0000000000000000000000000000000000000000
Binary files a/tangostationcontrol/tangostationcontrol/statistics_writer/test/SST_10m_test_3.h5 and /dev/null differ
diff --git a/tangostationcontrol/tangostationcontrol/statistics_writer/test/devices_test_SDP_SST_statistics_packets.bin b/tangostationcontrol/tangostationcontrol/statistics_writer/test/devices_test_SDP_SST_statistics_packets.bin
deleted file mode 100644
index e94347b86a0a03b940eb84980ec8f6d3b6d4e2d7..0000000000000000000000000000000000000000
Binary files a/tangostationcontrol/tangostationcontrol/statistics_writer/test/devices_test_SDP_SST_statistics_packets.bin and /dev/null differ
diff --git a/tangostationcontrol/tangostationcontrol/statistics_writer/test/test_server.py b/tangostationcontrol/tangostationcontrol/statistics_writer/test/test_server.py
deleted file mode 100644
index 74101b93de2e83824c70e4630e8560ae24b28fa8..0000000000000000000000000000000000000000
--- a/tangostationcontrol/tangostationcontrol/statistics_writer/test/test_server.py
+++ /dev/null
@@ -1,55 +0,0 @@
-import socket
-import time
-
-import argparse
-
-import logging
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger("statistics_test_server")
-logger.setLevel(logging.DEBUG)
-
-parser = argparse.ArgumentParser(description='Select what hostname to use and what port to use')
-parser.add_argument('--port', type=int, help='port to use', default=65433)
-parser.add_argument('--host', help='host to use', default='127.0.0.1')
-parser.add_argument('--file', help='file to use as data', default='devices_test_SDP_SST_statistics_packets.bin')
-parser.add_argument('--interval', type=int, help='ime between sending entire files content', default=1)
-
-args = parser.parse_args()
-HOST = args.host
-PORT = args.port
-FILE = args.file
-INTERVAL = args.interval
-
-
-while True:
- try:
- f = open(FILE, "rb")
- data = f.read()
- except Exception as e:
- logger.error(f"File not found, are you sure: '{FILE}' is a valid path, Exception: {e}")
- exit()
-
- try:
- with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
- logger.debug(f"Starting TCP test server on {HOST} {PORT}")
- logger.debug("To interrupt the script, press Ctrl-C twice within a second")
-
- s.bind((HOST, PORT))
- s.listen()
- conn, addr = s.accept()
-
- with conn:
- logger.debug(f'Connected by: {addr}')
-
- while True:
- time.sleep(INTERVAL)
- conn.sendall(data)
-
- except KeyboardInterrupt:
- logger.info("Received keyboard interrupt. Stopping.")
- exit()
- except Exception as e:
- logger.warning(f"Exception occurred: {e}")
-
- # just do 2 interrupt within a second to quit the program
- time.sleep(1)
diff --git a/tangostationcontrol/tangostationcontrol/statistics_writer/udp_dev/__init__.py b/tangostationcontrol/tangostationcontrol/statistics_writer/udp_dev/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/tangostationcontrol/tangostationcontrol/test/devices/test_antennafield_device.py b/tangostationcontrol/tangostationcontrol/test/devices/test_antennafield_device.py
index 96eb83300bec3352b1c0572653f38bf96a2e77d9..c567f476b21f05aa1806b454092c5a1660849ade 100644
--- a/tangostationcontrol/tangostationcontrol/test/devices/test_antennafield_device.py
+++ b/tangostationcontrol/tangostationcontrol/test/devices/test_antennafield_device.py
@@ -389,3 +389,11 @@ class TestAntennafieldDevice(device_base.DeviceTestCase):
antenna_properties = {'Antenna_Quality': antenna_qualities, 'Antenna_Use': antenna_use}
with DeviceTestContext(antennafield.AntennaField, properties={**self.AT_PROPERTIES, **antenna_properties}, process=True) as proxy:
numpy.testing.assert_equal(numpy.array([True] + [False] * 95), proxy.Antenna_Usage_Mask_R)
+
+ def test_read_Antenna_Names(self):
+ """ Verify if Antenna_Names_R is correctly retrieved """
+ antenna_names = ["C0","C1","C2","C3","C4"]
+ antenna_properties = {'Antenna_Names' : antenna_names}
+ with DeviceTestContext(antennafield.AntennaField, properties={**self.AT_PROPERTIES, **antenna_properties}, process=True) as proxy:
+ for i in range(len(antenna_names)):
+ self.assertTrue(proxy.Antenna_Names_R[i]==f"C{i}")
diff --git a/tangostationcontrol/tangostationcontrol/test/statistics/test_writer.py b/tangostationcontrol/tangostationcontrol/test/statistics/test_writer.py
index 56d6d85e0d83c6b9ac1e35e981ade193ab90d317..9b73607fafa546aa0cc0db63c1569aebdff99114 100644
--- a/tangostationcontrol/tangostationcontrol/test/statistics/test_writer.py
+++ b/tangostationcontrol/tangostationcontrol/test/statistics/test_writer.py
@@ -7,13 +7,14 @@
# Distributed under the terms of the APACHE license.
# See LICENSE.txt for more info.
-from tangostationcontrol.test import base
-from tangostationcontrol.statistics import writer
import sys
from os.path import dirname, isfile
from tempfile import TemporaryDirectory
from unittest import mock
+from tangostationcontrol.test import base
+from tangostationcontrol.statistics.writer import entry
+
class TestStatisticsWriter(base.TestCase):
@@ -21,20 +22,30 @@ class TestStatisticsWriter(base.TestCase):
def test_xst(self):
with TemporaryDirectory() as tmpdir:
- new_sys_argv = [sys.argv[0], "--mode", "XST", "--file", dirname(__file__) + "/SDP_XST_statistics_packets.bin", "--output_dir", tmpdir]
- with mock.patch.object(writer.sys, 'argv', new_sys_argv):
+ new_sys_argv = [
+ sys.argv[0],
+ "--mode", "XST",
+ "--file", dirname(__file__) + "/SDP_XST_statistics_packets.bin",
+ "--output_dir", tmpdir
+ ]
+ with mock.patch.object(entry.sys, 'argv', new_sys_argv):
with self.assertRaises(SystemExit):
- writer.main()
+ entry.main()
# check if file was written
self.assertTrue(isfile(f"{tmpdir}/XST_SB102_2021-09-13-13-21-32.h5"))
def test_xst_multiple_subbands(self):
with TemporaryDirectory() as tmpdir:
- new_sys_argv = [sys.argv[0], "--mode", "XST", "--file", dirname(__file__) + "/SDP_XST_statistics_packets_multiple_subbands.bin", "--output_dir", tmpdir]
- with mock.patch.object(writer.sys, 'argv', new_sys_argv):
+ new_sys_argv = [
+ sys.argv[0],
+ "--mode", "XST",
+ "--file", dirname(__file__) + "/SDP_XST_statistics_packets_multiple_subbands.bin",
+ "--output_dir", tmpdir
+ ]
+ with mock.patch.object(entry.sys, 'argv', new_sys_argv):
with self.assertRaises(SystemExit):
- writer.main()
+ entry.main()
# check if files were written
self.assertTrue(isfile(f"{tmpdir}/XST_SB102_2021-09-13-13-21-32.h5"))
@@ -42,10 +53,15 @@ class TestStatisticsWriter(base.TestCase):
def test_bst(self):
with TemporaryDirectory() as tmpdir:
- new_sys_argv = [sys.argv[0], "--mode", "BST", "--file", dirname(__file__) + "/SDP_BST_statistics_packets.bin", "--output_dir", tmpdir]
- with mock.patch.object(writer.sys, 'argv', new_sys_argv):
+ new_sys_argv = [
+ sys.argv[0],
+ "--mode", "BST",
+ "--file", dirname(__file__) + "/SDP_BST_statistics_packets.bin",
+ "--output_dir", tmpdir
+ ]
+ with mock.patch.object(entry.sys, 'argv', new_sys_argv):
with self.assertRaises(SystemExit):
- writer.main()
+ entry.main()
# check if file was written
self.assertTrue(isfile(f"{tmpdir}/BST_2022-05-20-11-08-44.h5"))
diff --git a/tangostationcontrol/tox.ini b/tangostationcontrol/tox.ini
index 46de7e82971df8e253affc1f8c7b6f47cd409f9f..9bd8b804bcfabb04d4caf630c926fdcbe30ef1f9 100644
--- a/tangostationcontrol/tox.ini
+++ b/tangostationcontrol/tox.ini
@@ -29,10 +29,19 @@ commands = {envpython} -m stestr run {posargs}
[testenv:integration]
; Warning running integration tests will make changes to your docker system!
; These tests should only be run by the integration-test docker container.
+allowlist_externals = echo
passenv = TANGO_HOST
-setenv = TESTS_DIR=./tangostationcontrol/integration_test/{env:TEST_MODULE:default}
+setenv =
+ VIRTUAL_ENV={envdir}
+ TESTS_DIR=./tangostationcontrol/integration_test/{env:TEST_MODULE:default}
+ PYTHON={envpython} -m coverage run --source tangostationcontrol --parallel-mode
commands =
+ echo "Integration test directory configured for{env:TESTS_DIR} ({env:TEST_MODULE})"
{envpython} -m stestr run --serial {posargs}
+ {envpython} -m coverage combine
+ {envpython} -m coverage html --omit='*test*' -d cover
+ {envpython} -m coverage xml -o coverage.xml
+ {envpython} -m coverage report --omit='*test*'
[testenv:cover]
; stestr does not natively support generating coverage reports use