Merge branch 'master' into L2SS-822-cache-archiver-ts-docker-image

.gitlab-ci.yml

@@ -42,23 +42,34 @@ stages:
     - . bootstrap/etc/lofar20rc.sh || true
 ##    Allow docker image script to execute
 #    - chmod u+x $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh
+
+# Template for docker images NOT on tagged or master builds
 .base_docker_images_except:
   extends: .base_docker_images
   except:
     refs:
       - tags
       - master
+
+# Template to download all remote images and store them on our image registry
+# (call tag_and_push without arguments)
 .base_docker_store_images:
   extends: .base_docker_images
   script:
 #    Do not remove 'bash' or statement will be ignored by primitive docker shell
     - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh
+
+# Download all remote images and store them on our image registry for tagged
+# master builds
 docker_store_images_master_tag:
   extends: .base_docker_store_images
   only:
     refs:
       - tags
       - master
+
+# Download all remote images and store them on our image registry if .env changes
+# on a merge request
 docker_store_images_changes:
   extends: .base_docker_store_images
   # This will spawn as detached pipeline but atleast ensures the changes rule
@@ -70,6 +81,8 @@ docker_store_images_changes:
       - merge_requests
     changes:
       - docker-compose/.env
+
+# Build and push all our custom images on tagged or master builds
 docker_build_image_all:
   extends: .base_docker_images
   only:
@@ -115,30 +128,32 @@ docker_build_image_all:
     - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh hdbpp latest
     - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh hdbppts-cm latest
     - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh hdbppts-es latest
-docker_build_image_elk:
+
+# Build and push custom images on merge request if relevant files changed
+docker_build_image_lofar_device_base:
   extends: .base_docker_images_except
   only:
     refs:
       - merge_requests
     changes:
-      - docker-compose/elk.yml
-      - docker-compose/elk/*
-      - docker-compose/elk-configure-host/*
+      - docker-compose/lofar-device-base.yml
+      - docker-compose/lofar-device-base/*
   script:
 #    Do not remove 'bash' or statement will be ignored by primitive docker shell
-    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh elk $tag
-    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh elk-configure-host $tag
-docker_build_image_lofar_device_base:
+    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh lofar-device-base $tag
+docker_build_image_elk:
   extends: .base_docker_images_except
   only:
     refs:
       - merge_requests
     changes:
-      - docker-compose/lofar-device-base.yml
-      - docker-compose/lofar-device-base/*
+      - docker-compose/elk.yml
+      - docker-compose/elk/*
+      - docker-compose/elk-configure-host/*
   script:
 #    Do not remove 'bash' or statement will be ignored by primitive docker shell
-    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh lofar-device-base $tag
+    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh elk $tag
+    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh elk-configure-host $tag
 docker_build_image_prometheus:
   extends: .base_docker_images_except
   only:
@@ -622,10 +637,6 @@ wheel_packaging:
   artifacts:
     paths:
       - tangostationcontrol/dist/*.whl
-  before_script:
-    - pip3 install build
-    - pip3 install -r tangostationcontrol/test-requirements.txt
-    - pip3 install -r docker-compose/itango/lofar-requirements.txt
   script:
     - cd tangostationcontrol
-    - python -m build
+    - tox -e build

CDB/LOFAR_ConfigDb.json

@@ -93,7 +93,7 @@
                                 "RECV, RECV_TEMP_error_R"
                             ],
                             "Shutdown_Device_List":[
-                                "STAT/SDP/1", "STAT/UNB2/1", "STAT/RECV/1", "STAT/APSCT/1", "STAT/APSPU/1"
+                                "STAT/SDP/1", "STAT/UNB2/1", "STAT/RECV/1", "STAT/APSCT/1", "STAT/CCD/1", "STAT/APSPU/1"
                             ]
                         }
                     }

CDB/stations/simulators_ConfigDb.json

@@ -140,12 +140,6 @@
                 "TemperatureManager": {
                     "STAT/TemperatureManager/1": {
                         "properties": {
-                            "Alarm_Error_List": [
-                                "RECV, HBAT_LED_on_RW"
-                            ],
-                            "Shutdown_Device_List":[
-                                "STAT/SDP/1", "STAT/UNB2/1", "STAT/RECV/1", "STAT/APSCT/1", "STAT/CCD/1","STAT/APSPU/1"
-                            ]
                         }
                     }
                 }

bin/start-ds.sh

@@ -35,10 +35,10 @@ else
   # Install the package, exit 1 if it fails
   # pip install ./ will _NOT_ install dependencies in requirements.txt!
   rm -rf /tmp/tangostationcontrol
-  cp -R /opt/lofar/tango/tangostationcontrol /tmp/
+  # Ideally we would use git copy but it can't copy on subdirectory level
+  # DO NOT PUT SPACES IN THE EXCLUDE LIST!
+  rsync -av --progress --exclude={".tox","*.egg-info","dist","build",".git","*.pyc"} /opt/lofar/tango/tangostationcontrol /tmp/
   cd /tmp/tangostationcontrol || exit 1
-  # Remove the build directory if copied from the source
-  rm -rf build
   pip -vvv install --upgrade --force-reinstall ./
 fi
 

docker-compose/device-pcon.yml

@@ -16,7 +16,7 @@ services:
         context: .
         dockerfile: lofar-device-base/Dockerfile
         args:
-            SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
+            SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}device-pcon
     logging:
       driver: "json-file"

docker-compose/device-psoc.yml

@@ -16,7 +16,7 @@ services:
         context: .
         dockerfile: lofar-device-base/Dockerfile
         args:
-            SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
+            SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}device-psoc
     logging:
       driver: "json-file"

docker-compose/integration-test.yml

@@ -30,7 +30,12 @@ services:
       - --timeout=30
       - --strict
       - --
-      - tox --recreate -e integration
+      - tox -e integration
+# TODO(L2SS-992): Update me to use sitepackages once L2SS-992 is fixed.
+# sitepackages can be enabled for inside docker once pytango is installed from
+# requirements.txt (To ensure reliable builds / all environments same versions)
+# When run outside of docker, environment will be recreated if mismatched
+#      - tox --sitepackages -e integration
     command:
 #     Allow for arguments to be passed that wil be put after the entrypoint
 #     tox is configured to take these arguments as integration test directory

docker-compose/lofar-device-base/Dockerfile

 ARG SOURCE_IMAGE
 FROM ${SOURCE_IMAGE}
 
-RUN sudo apt-get update && sudo apt-get install -y git g++ gcc shellcheck graphviz python3-dev && sudo apt-get clean
-
+RUN sudo apt-get update
+RUN sudo apt-get install -y git && sudo apt-get clean
+RUN sudo apt-get install -y g++ gcc && sudo apt-get clean
+RUN sudo apt-get install -y shellcheck graphviz && sudo apt-get clean
+RUN sudo apt-get install -y python3-dev libboost-python-dev pkg-config && sudo apt-get clean
+RUN sudo apt-get install -y rsync && sudo apt-get clean
 
 COPY lofar-device-base/lofar-requirements.txt /lofar-requirements.txt
 RUN sudo pip3 install -r /lofar-requirements.txt

sbin/run_integration_test.sh

@@ -32,7 +32,8 @@ function integration_test {
     echo "make restart ${restarts[@]} ..."
     make restart "${restarts[@]}"
   fi
-  sleep 5
+  #TODO(L2SS-988): Use healthcheck to wait for containers to become ready
+  sleep 30
   echo "make integration ${1} ..."
   make integration "${1}"
 }
@@ -64,6 +65,8 @@ fi
 
 cd "$LOFAR20_DIR/docker-compose" || exit 1
 
+
+
 # Start the database server first
 make build databaseds dsconfig
 make start databaseds dsconfig
@@ -90,8 +93,22 @@ make build archiver-timescale hdbppts-cm hdbppts-es
 # shellcheck disable=SC2086
 make stop $DEVICES $SIMULATORS hdbppts-es hdbppts-cm archiver-timescale
 make stop device-docker # this one does not test well in docker-in-docker
+make stop elk
+
+# Run dummy integration test to install pytango in tox virtualenv without
+# the memory pressure of the ELK stack.
+# Alternatively this step can be avoided if we use:
+# `tox --sitepackages -e integration` for the integration docker container,
+# however, that does require creating a container specific integration job.
+# TODO(L2SS-992): Remove me and above documentation
+integration_test dummy
+
 make start elk
 
+# Give elk time to start
+# TODO(L2SS-988): Use a nicer more reliable mechanism
+sleep 10
+
 # Update the dsconfig
 # Do not remove `bash`, otherwise statement ignored by gitlab ci shell!
 bash "${LOFAR20_DIR}"/sbin/update_ConfigDb.sh "${LOFAR20_DIR}"/CDB/LOFAR_ConfigDb.json
@@ -113,8 +130,8 @@ make start $DEVICES
 make start archiver-timescale
 
 # Give devices time to restart
-# TODO(Corne Lukken): Use a nicer more reliable mechanism
-sleep 60
+# TODO(L2SS-988): Use a nicer more reliable mechanism
+sleep 70
 
 # Give archiver-timescale time to start
 # shellcheck disable=SC2016

sbin/tag_and_push_docker_image.sh

@@ -166,7 +166,7 @@ if [ ! -z "${1+x}" ] && [ "${1}" != "pull" ]; then
         docker pull "${local_url}:latest" || true
       fi
 
-      make build "${1}"
+      make build "${1}" || exit 1
       docker tag "${2}" "${local_url}:${tag}" || docker tag "${2/_/-}" "${local_url}:${tag}"
       docker push "${local_url}:${tag}"
     fi

tangostationcontrol/docs/docs-requirements.txt

+importlib-metadata<2.0.0,>=0.12;python_version<"3.8"
 sphinx>=4.3.2 # BSD
 sphinx-rtd-theme>=1.0.0 # MIT
\ No newline at end of file

tangostationcontrol/requirements.txt

@@ -2,7 +2,10 @@
 # order of appearance. Changing the order has an impact on the overall
 # integration process, which may cause wedges in the gate later.
 
+importlib-metadata<2.0.0,>=0.12;python_version<"3.8"
 lofar-station-client@git+https://git.astron.nl/lofar2.0/lofar-station-client@0.6.0
+numpy
+mock
 asyncua >= 0.9.90 # LGPLv3
 PyMySQL[rsa] >= 1.0.2 # MIT
 psycopg2-binary >= 2.9.2 # LGPL

tangostationcontrol/tangostationcontrol/integration_test/README.md

@@ -3,6 +3,13 @@
 Integration tests are separated into multi modules. Each module requires a
 different state and configuration. These configurations are managed externally.
 
+To minimize runtime overhead and memory pressure it is best to execute the
+dummy integration test module before any other container is created. This will
+ensure  pytango is installed in the `.tox/integration` virtual environment
+before the system experiences the severe memory pressure of the ELK stack.
+
+Simply run `make integration dummy` from inside the `docker-compose` folder.
+
 In total the orchestration of integration tests is handled through four separate
 layers, each one calling the next:
 
@@ -17,7 +24,7 @@ value can be left empty for the `default` module:
 
 Individual tests can be invoked using arguments:
 
-`TEST_MODULE=default tox -e integration`
+`TEST_MODULE=default tox -e integration import.path.class.functionname`
 
 These arguments and modules can also be passed at the level of the Makefile
 instead of through tox directly:
@@ -48,7 +55,7 @@ cd tangostationcontrol
 # Single test to significantly reduce runtime
 tox -e integration tangostationcontrol.integration_test.default.devices.test_device_digitalbeam.TestDeviceDigitalBeam.test_pointing_to_zenith
 source .tox/integration/bin/activate
-# Add import pdb; pdb.set_trace() somehwere
+# Add import pdb; pdb.set_trace() somewhere
 nano integration tangostationcontrol.integration_test.default.devices.test_device_digitalbeam.py
 python -m testtools.run tangostationcontrol.integration_test.default.devices.test_device_digitalbeam.TestDeviceDigitalBeam.test_pointing_to_zenith
 ```
@@ -71,3 +78,10 @@ are running and are in the required state.
 ```shell
 sbin/run_integration_test.sh
 ```
+
+## Cleanup, recovery or starting over
+
+All docker content including images, containers, networks and volumes can
+be deleted using the following:
+
+`docker stop $(docker ps | tail -n+2 | awk '{NF=1}1' | awk '{printf("%s ",$0)} END { printf "\n" }'); docker system prune --all; docker volume prune`

tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_beamlet.py

@@ -41,7 +41,8 @@ class TestDeviceBeamlet(AbstractTestBases.TestDeviceBase):
         sdp_proxy = self.setup_sdp()
 
         self.proxy.initialise()
-        self.proxy.subband_select = [0] * 488
+        # TODO(Corne): Update test so effects of attribute are asserted L2SS-984
+        # self.proxy.subband_select_RW = [0] * 488
         self.proxy.on()
 
         # The subband frequency of HBA subband 0 is 200 MHz,
@@ -63,7 +64,8 @@ class TestDeviceBeamlet(AbstractTestBases.TestDeviceBase):
         sdp_proxy = self.setup_sdp()
 
         self.proxy.initialise()
-        self.proxy.subband_select = list(range(488))
+        # TODO(Corne): Update test so effects of attribute are asserted L2SS-984
+        # self.proxy.subband_select_RW = list(range(488))
         self.proxy.on()
 
         # any non-zero delay should result in different weights for different clocks

tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_digitalbeam.py

@@ -66,8 +66,9 @@ class TestDeviceDigitalBeam(AbstractTestBases.TestDeviceBase):
         antennafield_proxy.put_property({
             "RECV_devices": [self.recv_iden],
             "Control_to_RECV_mapping": numpy.array(control_mapping).flatten(),
-            'Antenna_Quality': antenna_qualities, 'Antenna_Use': antenna_use}
-        )
+            "Antenna_Quality": antenna_qualities,
+            "Antenna_Use": antenna_use
+        })
         antennafield_proxy.off()
         antennafield_proxy.boot()
         return antennafield_proxy
@@ -81,9 +82,10 @@ class TestDeviceDigitalBeam(AbstractTestBases.TestDeviceBase):
         self.setup_antennafield_proxy(self.antenna_qualities_ok, self.antenna_use_ok)
         self.setup_sdp_proxy()
 
+        # TODO(Corne): Update these and ensure their effects is asserted in tests L2SS-984
         # Setup beamlet configuration
-        self.beamlet_proxy.clock_RW = 200 * 1000000
-        self.beamlet_proxy.subband_select = list(range(488))
+        # self.beamlet_proxy.clock_RW = 200 * 1000000
+        # self.beamlet_proxy.subband_select_RW = list(range(488))
 
         self.proxy.initialise()
         self.proxy.Tracking_enabled_RW = False
@@ -106,10 +108,6 @@ class TestDeviceDigitalBeam(AbstractTestBases.TestDeviceBase):
         self.setup_antennafield_proxy(self.antenna_qualities_ok, self.antenna_use_ok)
         self.setup_sdp_proxy()
 
-        # Setup beamlet configuration
-        self.beamlet_proxy.clock_RW = 200 * 1000000
-        self.beamlet_proxy.subband_select = list(range(488))
-
         self.proxy.initialise()
         self.proxy.Tracking_enabled_RW = False
         self.proxy.on()
@@ -140,10 +138,6 @@ class TestDeviceDigitalBeam(AbstractTestBases.TestDeviceBase):
         self.setup_antennafield_proxy(self.antenna_qualities_ok, self.antenna_use_ok)
         self.setup_sdp_proxy()
 
-        # Setup beamlet configuration
-        self.beamlet_proxy.clock_RW = 200 * 1000000
-        self.beamlet_proxy.subband_select = list(range(488))
-
         self.proxy.initialise()
         self.proxy.Tracking_enabled_RW = False
         self.proxy.on()

tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_temperature_manager.py

@@ -48,6 +48,7 @@ class TestDeviceTemperatureManager(AbstractTestBases.TestDeviceBase):
     def test_alarm(self):
         # Exclude other devices which raise a TimeoutError, since they wait for the attribute *_translator_busy_R to become False 
         # (set instead to True in this test environment)
+        self.proxy.put_property({"Alarm_Error_List": ["RECV, HBAT_LED_on_RW"]})
         self.proxy.put_property({"Shutdown_Device_List": ["STAT/SDP/1"]})
         devices = [DeviceProxy("STAT/SDP/1")]
         

tangostationcontrol/tangostationcontrol/integration_test/dummy/test_dummy.py

+# -*- coding: utf-8 -*-
+#
+# This file is part of the LOFAR 2.0 Station Software
+#
+#
+#
+# Distributed under the terms of the APACHE license.
+# See LICENSE.txt for more info.
+
+from tangostationcontrol.integration_test import base
+
+
+class TestDummy(base.IntegrationTestCase):
+    def test_dummy(self):
+        self.assertTrue(True)

tangostationcontrol/test-requirements.txt

@@ -2,8 +2,11 @@
 # order of appearance. Changing the order has an impact on the overall
 # integration process, which may cause wedges in the gate later.
 
+importlib-metadata<2.0.0,>=0.12;python_version<"3.8"
 asynctest>=0.13.0 # Apache-2.0
 bandit>=1.6.0 # Apache-2.0
+virtualenv>=20.16.0 # MIT
+build>=0.8.0 # MIT
 coverage>=5.2.0 # Apache-2.0
 doc8>=0.8.0 # Apache-2.0
 flake8>=3.8.0 # MIT

tangostationcontrol/tox.ini

 [tox]
 minversion = 3.20
-envlist = py37,py38,py39,py310,pep8
+envlist = py3{7,8,9,10},pep8
 skipsdist = True
 
 [testenv]
 usedevelop = True
-; Module access is a bit of an ugly hack. This is due to testenv inheritance
-; with sitepackages = True`, meaning that global packages can be accessed by the
-; tox environment. Our tango images already have several dependencies system
-; wide installed, however, the system wide installation will never look inside
-; tox its virtualenv for packages. So accessing stestr and others fail.. We
-; can't remove `sitepackages = True` either as we need access to tango and
-; installing this package is non-trivial. The solution is to prevent calling
-; binaries directly and utilizing python and tox variables to resolve the
-; requested module.
-sitepackages = True
+; Python and tox variables are used to access modules and binaries instead of
+; directly. This makes the setup robust for using sitepackages=True.
 install_command = {envbindir}/pip3 install {opts} {packages}
 passenv = HOME
 setenv =
    VIRTUAL_ENV={envdir}
    PYTHONWARNINGS=default::DeprecationWarning
+; Share the same envdir with as many jobs as possible due to extensive time it
+; takes to compile the pytango wheel, in addition to its large install size.
+; should the environment change (such as the Python version) the environment
+; will automatically be recreated.
+envdir = {toxworkdir}/testenv
 deps =
     -r{toxinidir}/requirements.txt
     -r{toxinidir}/../docker-compose/lofar-device-base/lofar-requirements.txt
     -r{toxinidir}/test-requirements.txt
 commands_pre =
     {envpython} --version
+    pip install --no-cache pytango
 commands =
     {envpython} -m stestr --version
     {envpython} -m stestr run {posargs}
 
+; We can't detect the current Python version for an environment dynamically
+; so each Python version specific job needs its own envdir.
+[testenv:py37]
+envdir = {toxworkdir}/testenvpy37
+
+[testenv:py38]
+envdir = {toxworkdir}/testenvpy38
+
+[testenv:py39]
+envdir = {toxworkdir}/testenvpy39
+
+[testenv:py310]
+envdir = {toxworkdir}/testenvpy310
+
 [testenv:integration]
-; Warning running integration tests will make changes to your docker system!
-; These tests should only be run by the integration-test docker container.
 allowlist_externals = echo
 passenv = TANGO_HOST
 setenv =
@@ -55,10 +65,6 @@ commands =
 setenv =
     VIRTUAL_ENV={envdir}
     PYTHON={envpython} -m coverage run --source tangostationcontrol --parallel-mode
-deps =
-    -r{toxinidir}/requirements.txt
-    -r{toxinidir}/../docker-compose/lofar-device-base/lofar-requirements.txt
-    -r{toxinidir}/test-requirements.txt
 commands =
     {envpython} -m stestr --version
     {envpython} -m coverage --version
@@ -91,7 +97,12 @@ commands =
     {envpython} -m xenon --version
     {envpython} -m xenon tangostationcontrol -b B -m A -a A -i libhdbpp-python
 
+[testenv:build]
+usedevelop = False
+commands = {envpython} -m build
+
 [testenv:docs]
+envdir = {toxworkdir}/docs
 deps =
     -r{toxinidir}/../docker-compose/lofar-device-base/lofar-requirements.txt
     -r{toxinidir}/docs/docs-requirements.txt