Merge branch 'master' of https://git.astron.nl/lofar2.0/tango into L2SS-905-support-2-dts-subracks

.gitignore

@@ -28,3 +28,4 @@ tangostationcontrol/docs/build
 **/.eggs
 
 docker-compose/alerta-web/alerta-secrets.json
+docker-compose/tmp

.gitlab-ci.yml

@@ -526,8 +526,10 @@ unit_test:
         path: tangostationcontrol/coverage.xml
     paths:
       - tangostationcontrol/cover/*
+      - tangostationcontrol/.coverage
 integration_test_docker:
   stage: integration-tests
+  allow_failure: true
   image: docker:latest
   tags:
     - privileged
@@ -535,6 +537,8 @@ integration_test_docker:
     - name: docker:dind
   variables:
     DOCKER_TLS_CERTDIR: "/certs"
+  needs:
+    - unit_test
   artifacts:
     when: always
     paths:
@@ -548,6 +552,7 @@ integration_test_docker:
         tag="$CI_COMMIT_REF_SLUG"
         echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag"
       fi
+    - apk update
     - apk add --update make bash docker-compose
     - apk add --update bind-tools
     - apk add --update postgresql14-client gzip

CDB/stations/DTS_Outside_ConfigDb.json

@@ -122,19 +122,6 @@
                 }
             }
         },
-        "DigitalBeam": {
-            "STAT": {
-                "DigitalBeam": {
-                    "STAT/DigitalBeam/1": {
-                        "properties": {
-                            "Tracking_enabled_RW_default": [
-                                "False"
-                            ]
-                        }
-                    }
-                }
-            }
-        },
         "Beamlet": {
             "STAT": {
                 "Beamlet": {
@@ -198,6 +185,13 @@
                             "RECV_devices": [
                                 "STAT/RECV/1"
                             ],
+                            "Antenna_Names": [
+                                "C0",
+                                "C1",
+                                "C2",
+                                "C3",
+                                "C4"
+                            ],
                             "HBAT_Control_to_RECV_mapping": [
                                 "1","27",
                                 "0","-1",
@@ -241,6 +235,17 @@
                             "RECV_devices": [
                                 "STAT/RECV/1"
                             ],
+                            "Antenna_Names": [
+                                "LBA1",
+                                "LBA2",
+                                "LBA3",
+                                "LBA4",
+                                "LBA5",
+                                "LBA6",
+                                "LBA7",
+                                "LBA8",
+                                "LBA9"
+                            ],
                             "HBAT_Control_to_RECV_mapping": [
                                 "1","1",
                                 "1","3",
@@ -264,7 +269,9 @@
                                 "1","17"
                             ],
                             "Antenna_Field_Reference_ETRS": [
-				"3839358.189", "430354.482", "5057967.804"
+                                "3839358.189",
+                                "430354.482",
+                                "5057967.804"
                             ],
                             "HBAT_reference_ETRS": [
                                 "3839358.189","430354.482","5057967.804",
@@ -332,6 +339,9 @@
                                 "-1", "-1", "-1", "-1", "-1", "-1",
                                 "-1", "-1", "-1", "-1", "-1", "-1",
                                 "-1", "-1", "-1", "-1", "-1", "-1"
+                            ],
+                            "Tracking_enabled_RW_default": [
+                                "False"
                             ]
                         }
                     }
@@ -598,7 +608,9 @@
                 "PSOC": {
                     "STAT/PSOC/1": {
                         "properties": {
-                            "SNMP_host": ["10.87.2.145"],
+                            "SNMP_host": [
+                                "10.87.2.145"
+                            ],
                             "PSOC_sockets": [
                                 "ccd_socket",
                                 "sdptr0_socket",
@@ -619,7 +631,9 @@
                 "PCON": {
                     "STAT/PCON/1": {
                         "properties": {
-                            "SNMP_host": ["10.151.225.5"]
+                            "SNMP_host": [
+                                "10.151.225.5"
+                            ]
                         }
                     }
                 }

CDB/test_environment_ConfigDb.json

 {
     "servers": {
+        "TileBeam": {
+            "STAT": {
+                "TileBeam": {
+                    "STAT/TileBeam/1": {
+                        "properties": {
+                            "Tracking_enabled_RW_default": [ "True" ]
+                        }
+                    }
+                }
+            }
+        },
         "Observation": {
             "STAT": {
                 "Observation": {

README.md

@@ -17,7 +17,7 @@ Station Control software related to Tango devices.
   * [Attribute wrapper documentation](tangostationcontrol/tangostationcontrol/clients/README.md)
   * [Archiver documentation](tangostationcontrol/tangostationcontrol/toolkit/README.md)
   * [Adding a new tango device](tangostationcontrol/tangostationcontrol/devices/README.md)
-  * [HDF5 statistics](tangostationcontrol/tangostationcontrol/statistics_writer/README.md)
+  * [HDF5 statistics](tangostationcontrol/tangostationcontrol/statistics/README.md)
 * [Unit tests](tangostationcontrol/tangostationcontrol/test/README.md)
 * [Integration tests](tangostationcontrol/tangostationcontrol/integration_test/README.md)
 

bin/start-ds.sh

 #!/bin/bash
 
+set -e
+
 # Serves as entrypoint script for docker containers
 
 if [[ ! -d "/opt/lofar/tango" ]]; then
@@ -31,8 +33,11 @@ if [[ $TANGOSTATIONCONTROL ]]; then
   exit 2
 else
   # Install the package, exit 1 if it fails
-  cd tangostationcontrol || exit 1
-  pip install --upgrade --force-reinstall ./
+  # pip install ./ will _NOT_ install dependencies in requirements.txt!
+  rm -rf /tmp/tangostationcontrol
+  cp -R /opt/lofar/tango/tangostationcontrol /tmp/
+  cd /tmp/tangostationcontrol || exit 1
+  pip -vvv install --upgrade --force-reinstall ./
 fi
 
 # Return to the stored the directory, this preserves the working_dir argument in

docker-compose/Makefile

@@ -150,20 +150,21 @@ DOCKER_COMPOSE_ARGS := DISPLAY=$(DISPLAY) \
     CONTAINER_NAME_PREFIX=$(CONTAINER_NAME_PREFIX) \
     COMPOSE_IGNORE_ORPHANS=true \
     CONTAINER_EXECUTION_UID=$(shell id -u) \
-    DOCKER_GID=$(DOCKER_GID)
+    DOCKER_GID=$(DOCKER_GID) \
+    TEST_MODULE=$(INTEGRATION_MODULE)
 
 
-.PHONY: up down minimal run integration start stop restart build build-nocache status clean pull help
+.PHONY: up down minimal context run integration start stop restart build build-nocache status clean pull help
 .DEFAULT_GOAL := help
 
 pull: ## pull the images from the Docker hub
 	$(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) pull
 
-build: ## rebuild images
+build: context ## rebuild images
 	# docker-compose does not support build dependencies, so manage those here
 	$(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) build --parallel --progress=plain $(SERVICE)
 
-build-nocache: ## rebuild images from scratch
+build-nocache: context ## rebuild images from scratch
 	# docker-compose does not support build dependencies, so manage those here
 	$(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) build --no-cache --progress=plain $(SERVICE)
 
@@ -174,7 +175,7 @@ run: minimal  ## run a service using arguments and delete it afterwards
 	$(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) run --no-deps --rm $(SERVICE) $(SERVICE_ARGS)
 
 integration: minimal  ## run a service using arguments and delete it afterwards
-	TEST_MODULE=$(INTEGRATION_MODULE) $(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) run --no-deps --rm integration-test $(INTEGRATION_ARGS)
+	$(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) run --no-deps --rm integration-test $(INTEGRATION_ARGS)
 
 down:  ## stop all services and tear down the system
 	$(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) down
@@ -183,13 +184,18 @@ ifneq ($(NETWORK_MODE),host)
 	docker network inspect 9000-$(NETWORK_MODE) &> /dev/null && ([ $$? -eq 0 ] && docker network rm 9000-$(NETWORK_MODE)) || true
 endif
 
-minimal: ## start the base TANGO system
+minimal: context ## start the base TANGO system
 ifneq ($(NETWORK_MODE),host)
 	docker network inspect $(NETWORK_MODE) &> /dev/null || ([ $$? -ne 0 ] && docker network create $(NETWORK_MODE))
 	docker network inspect 9000-$(NETWORK_MODE) &> /dev/null || ([ $$? -ne 0 ] && docker network create 9000-$(NETWORK_MODE) -o com.docker.network.driver.mtu=9000)
 endif
+
 	$(DOCKER_COMPOSE_ARGS) docker-compose -f tango.yml -f networks.yml up --no-recreate -d
 
+context: ## Move the necessary files to create minimal docker context
+	@mkdir -p tmp
+	@cp ../tangostationcontrol/requirements.txt tmp/
+
 bootstrap: pull build # first start, initialise from scratch
 	$(MAKE) start elk-configure-host # configure host kernel for elk container
 	$(MAKE) start dsconfig # boot up containers to load configurations

docker-compose/device-antennafield.yml

@@ -19,8 +19,8 @@ services:
     # build explicitly, as docker-compose does not understand a local image
     # being shared among services.
     build:
-        context: ..
-        dockerfile: docker-compose/lofar-device-base/Dockerfile
+        context: .
+        dockerfile: lofar-device-base/Dockerfile
         args:
             SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}device-antennafield

docker-compose/device-apsct.yml

@@ -18,8 +18,8 @@ services:
     # build explicitly, as docker-compose does not understand a local image
     # being shared among services.
     build:
-        context: ..
-        dockerfile: docker-compose/lofar-device-base/Dockerfile
+        context: .
+        dockerfile: lofar-device-base/Dockerfile
         args:
             SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}device-apsct

docker-compose/device-apspu.yml

@@ -18,8 +18,8 @@ services:
     # build explicitly, as docker-compose does not understand a local image
     # being shared among services.
     build:
-        context: ..
-        dockerfile: docker-compose/lofar-device-base/Dockerfile
+        context: .
+        dockerfile: lofar-device-base/Dockerfile
         args:
             SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}device-apspu

docker-compose/device-beamlet.yml

@@ -18,8 +18,8 @@ services:
     # build explicitly, as docker-compose does not understand a local image
     # being shared among services.
     build:
-        context: ..
-        dockerfile: docker-compose/lofar-device-base/Dockerfile
+        context: .
+        dockerfile: lofar-device-base/Dockerfile
         args:
             SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}device-beamlet

docker-compose/device-boot.yml

@@ -17,8 +17,8 @@ services:
     # build explicitly, as docker-compose does not understand a local image
     # being shared among services.
     build:
-        context: ..
-        dockerfile: docker-compose/lofar-device-base/Dockerfile
+        context: .
+        dockerfile: lofar-device-base/Dockerfile
         args:
             SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}device-boot

docker-compose/device-bst.yml

@@ -18,8 +18,8 @@ services:
     # build explicitly, as docker-compose does not understand a local image
     # being shared among services.
     build:
-        context: ..
-        dockerfile: docker-compose/lofar-device-base/Dockerfile
+        context: .
+        dockerfile: lofar-device-base/Dockerfile
         args:
             SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}device-bst

docker-compose/device-digitalbeam.yml

@@ -18,8 +18,8 @@ services:
     # build explicitly, as docker-compose does not understand a local image
     # being shared among services.
     build:
-        context: ..
-        dockerfile: docker-compose/lofar-device-base/Dockerfile
+        context: .
+        dockerfile: lofar-device-base/Dockerfile
         args:
             SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}device-digitalbeam

docker-compose/device-docker.yml

@@ -18,8 +18,8 @@ services:
     # build explicitly, as docker-compose does not understand a local image
     # being shared among services.
     build:
-        context: ..
-        dockerfile: docker-compose/lofar-device-base/Dockerfile
+        context: .
+        dockerfile: lofar-device-base/Dockerfile
         args:
             SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}device-docker

docker-compose/device-observation-control.yml

@@ -17,8 +17,8 @@ services:
     # build explicitly, as docker-compose does not understand a local image
     # being shared among services.
     build:
-        context: ..
-        dockerfile: docker-compose/lofar-device-base/Dockerfile
+        context: .
+        dockerfile: lofar-device-base/Dockerfile
         args:
             SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}device-observation-control

docker-compose/device-observation.yml

@@ -16,8 +16,8 @@ services:
     # build explicitly, as docker-compose does not understand a local image
     # being shared among services.
     build:
-        context: ..
-        dockerfile: docker-compose/lofar-device-base/Dockerfile
+        context: .
+        dockerfile: lofar-device-base/Dockerfile
         args:
             SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}device-observation

docker-compose/device-pcon.yml

@@ -13,8 +13,8 @@ services:
     # build explicitly, as docker-compose does not understand a local image
     # being shared among services.
     build:
-        context: ..
-        dockerfile: docker-compose/lofar-device-base/Dockerfile
+        context: .
+        dockerfile: lofar-device-base/Dockerfile
         args:
             SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}device-pcon

docker-compose/device-psoc.yml

@@ -13,8 +13,8 @@ services:
     # build explicitly, as docker-compose does not understand a local image
     # being shared among services.
     build:
-        context: ..
-        dockerfile: docker-compose/lofar-device-base/Dockerfile
+        context: .
+        dockerfile: lofar-device-base/Dockerfile
         args:
             SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}device-psoc

docker-compose/device-recv.yml

@@ -18,8 +18,8 @@ services:
     # build explicitly, as docker-compose does not understand a local image
     # being shared among services.
     build:
-        context: ..
-        dockerfile: docker-compose/lofar-device-base/Dockerfile
+        context: .
+        dockerfile: lofar-device-base/Dockerfile
         args:
             SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}device-recv