diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8d2e2c7cdd208fd6f9fcc07228c37190cf3b0a91..5e7b26942b89dbc2a42372599da5e6a31275f517 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -42,23 +42,34 @@ stages: - . bootstrap/etc/lofar20rc.sh || true ## Allow docker image script to execute # - chmod u+x $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh + +# Template for docker images NOT on tagged or master builds .base_docker_images_except: extends: .base_docker_images except: refs: - tags - master + +# Template to download all remote images and store them on our image registry +# (call tag_and_push without arguments) .base_docker_store_images: extends: .base_docker_images script: # Do not remove 'bash' or statement will be ignored by primitive docker shell - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh + +# Download all remote images and store them on our image registry for tagged +# master builds docker_store_images_master_tag: extends: .base_docker_store_images only: refs: - tags - master + +# Download all remote images and store them on our image registry if .env changes +# on a merge request docker_store_images_changes: extends: .base_docker_store_images # This will spawn as detached pipeline but atleast ensures the changes rule @@ -70,6 +81,8 @@ docker_store_images_changes: - merge_requests changes: - docker-compose/.env + +# Build and push all our custom images on tagged or master builds docker_build_image_all: extends: .base_docker_images only: @@ -114,30 +127,32 @@ docker_build_image_all: - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh archiver-timescale latest - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh hdbppts-cm latest - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh hdbppts-es latest -docker_build_image_elk: + +# Build and push custom images on merge request if relevant files changed +docker_build_image_lofar_device_base: extends: .base_docker_images_except only: refs: - merge_requests changes: - - docker-compose/elk.yml - - docker-compose/elk/* - - docker-compose/elk-configure-host/* + - docker-compose/lofar-device-base.yml + - docker-compose/lofar-device-base/* script: # Do not remove 'bash' or statement will be ignored by primitive docker shell - - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh elk $tag - - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh elk-configure-host $tag -docker_build_image_lofar_device_base: + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh lofar-device-base $tag +docker_build_image_elk: extends: .base_docker_images_except only: refs: - merge_requests changes: - - docker-compose/lofar-device-base.yml - - docker-compose/lofar-device-base/* + - docker-compose/elk.yml + - docker-compose/elk/* + - docker-compose/elk-configure-host/* script: # Do not remove 'bash' or statement will be ignored by primitive docker shell - - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh lofar-device-base $tag + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh elk $tag + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh elk-configure-host $tag docker_build_image_prometheus: extends: .base_docker_images_except only: @@ -610,10 +625,6 @@ wheel_packaging: artifacts: paths: - tangostationcontrol/dist/*.whl - before_script: - - pip3 install build - - pip3 install -r tangostationcontrol/test-requirements.txt - - pip3 install -r docker-compose/itango/lofar-requirements.txt script: - cd tangostationcontrol - - python -m build + - tox -e build diff --git a/bin/start-ds.sh b/bin/start-ds.sh index 71cb29a29bfa977fbf6033c8479c9e59435485b5..86ed80dd8b044027b967b46074295b15e9a271a5 100755 --- a/bin/start-ds.sh +++ b/bin/start-ds.sh @@ -35,10 +35,10 @@ else # Install the package, exit 1 if it fails # pip install ./ will _NOT_ install dependencies in requirements.txt! rm -rf /tmp/tangostationcontrol - cp -R /opt/lofar/tango/tangostationcontrol /tmp/ + # Ideally we would use git copy but it can't copy on subdirectory level + # DO NOT PUT SPACES IN THE EXCLUDE LIST! + rsync -av --progress --exclude={".tox","*.egg-info","dist","build",".git","*.pyc"} /opt/lofar/tango/tangostationcontrol /tmp/ cd /tmp/tangostationcontrol || exit 1 - # Remove the build directory if copied from the source - rm -rf build pip -vvv install --upgrade --force-reinstall ./ fi diff --git a/docker-compose/device-pcon.yml b/docker-compose/device-pcon.yml index cb6b2c3cdde164405d8dfb18015fd77cc9a5851b..1c8140aaac4d459c81cd9bcaf5c9e44d03b74334 100644 --- a/docker-compose/device-pcon.yml +++ b/docker-compose/device-pcon.yml @@ -16,7 +16,7 @@ services: context: . dockerfile: lofar-device-base/Dockerfile args: - SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION} + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION} container_name: ${CONTAINER_NAME_PREFIX}device-pcon logging: driver: "json-file" diff --git a/docker-compose/device-psoc.yml b/docker-compose/device-psoc.yml index a32d2374b28ae5f7649cea742ab6c606ce06391f..b493627aaaaf2a6af5c66153b68de814c7bc6b7c 100644 --- a/docker-compose/device-psoc.yml +++ b/docker-compose/device-psoc.yml @@ -16,7 +16,7 @@ services: context: . dockerfile: lofar-device-base/Dockerfile args: - SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION} + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION} container_name: ${CONTAINER_NAME_PREFIX}device-psoc logging: driver: "json-file" diff --git a/docker-compose/integration-test.yml b/docker-compose/integration-test.yml index 6e7e5407d422afd1989ef7a127d1f54217307cc3..5d27f716ab07c69d061cfaa6dd883798e0c2bab5 100644 --- a/docker-compose/integration-test.yml +++ b/docker-compose/integration-test.yml @@ -30,7 +30,12 @@ services: - --timeout=30 - --strict - -- - - tox --recreate -e integration + - tox -e integration +# TODO(L2SS-992): Update me to use sitepackages once L2SS-992 is fixed. +# sitepackages can be enabled for inside docker once pytango is installed from +# requirements.txt (To ensure reliable builds / all environments same versions) +# When run outside of docker, environment will be recreated if mismatched +# - tox --sitepackages -e integration command: # Allow for arguments to be passed that wil be put after the entrypoint # tox is configured to take these arguments as integration test directory diff --git a/docker-compose/lofar-device-base/Dockerfile b/docker-compose/lofar-device-base/Dockerfile index becc95d0833408a8e4a04f074bff9ed0b0b2b6da..d280bf4f3f76f4b71e17cf0fcf6c8fb7ff32e572 100644 --- a/docker-compose/lofar-device-base/Dockerfile +++ b/docker-compose/lofar-device-base/Dockerfile @@ -1,8 +1,12 @@ ARG SOURCE_IMAGE FROM ${SOURCE_IMAGE} -RUN sudo apt-get update && sudo apt-get install -y git g++ gcc shellcheck graphviz python3-dev && sudo apt-get clean - +RUN sudo apt-get update +RUN sudo apt-get install -y git && sudo apt-get clean +RUN sudo apt-get install -y g++ gcc && sudo apt-get clean +RUN sudo apt-get install -y shellcheck graphviz && sudo apt-get clean +RUN sudo apt-get install -y python3-dev libboost-python-dev pkg-config && sudo apt-get clean +RUN sudo apt-get install -y rsync && sudo apt-get clean COPY lofar-device-base/lofar-requirements.txt /lofar-requirements.txt RUN sudo pip3 install -r /lofar-requirements.txt diff --git a/sbin/run_integration_test.sh b/sbin/run_integration_test.sh index 3047557a3678940748746acc6f7e7e3a59ea6090..367d96f7f418755d0f28a645fb4167b233dbb7cb 100755 --- a/sbin/run_integration_test.sh +++ b/sbin/run_integration_test.sh @@ -32,7 +32,7 @@ function integration_test { echo "make restart ${restarts[@]} ..." make restart "${restarts[@]}" fi - sleep 5 + sleep 10 echo "make integration ${1} ..." make integration "${1}" } @@ -64,6 +64,8 @@ fi cd "$LOFAR20_DIR/docker-compose" || exit 1 + + # Start the database server first make build databaseds dsconfig make start databaseds dsconfig @@ -90,8 +92,21 @@ make build archiver-timescale hdbppts-cm hdbppts-es # shellcheck disable=SC2086 make stop $DEVICES $SIMULATORS hdbppts-es hdbppts-cm archiver-timescale make stop device-docker # this one does not test well in docker-in-docker +make stop elk + +# Run dummy integration test to install pytango in tox virtualenv without +# the memory pressure of the ELK stack. +# Alternatively this step can be avoided if we use: +# `tox --sitepackages -e integration` for the integration docker container, +# however, that does require creating a container specific integration job. +# TODO(L2SS-992): Remove me and above documentation +integration_test dummy + make start elk +# Give elk time to start +sleep 10 + # Update the dsconfig # Do not remove `bash`, otherwise statement ignored by gitlab ci shell! bash "${LOFAR20_DIR}"/sbin/update_ConfigDb.sh "${LOFAR20_DIR}"/CDB/LOFAR_ConfigDb.json @@ -114,7 +129,7 @@ make start archiver-timescale # Give devices time to restart # TODO(Corne Lukken): Use a nicer more reliable mechanism -sleep 60 +sleep 70 # Give archiver-timescale time to start # shellcheck disable=SC2016 diff --git a/sbin/tag_and_push_docker_image.sh b/sbin/tag_and_push_docker_image.sh index 44605c52c7eed433cf89d0fab66d51946ce305cd..dfba35a67ca1578918a6a5bb42e78aa9a656b967 100755 --- a/sbin/tag_and_push_docker_image.sh +++ b/sbin/tag_and_push_docker_image.sh @@ -166,7 +166,7 @@ if [ ! -z "${1+x}" ] && [ "${1}" != "pull" ]; then docker pull "${local_url}:latest" || true fi - make build "${1}" + make build "${1}" || exit 1 docker tag "${2}" "${local_url}:${tag}" || docker tag "${2/_/-}" "${local_url}:${tag}" docker push "${local_url}:${tag}" fi diff --git a/tangostationcontrol/requirements.txt b/tangostationcontrol/requirements.txt index b252910091df6eaca22673b2931b1ab73405c2f0..690513a0d46fdd97e9353275918fc20e1d174fd9 100644 --- a/tangostationcontrol/requirements.txt +++ b/tangostationcontrol/requirements.txt @@ -3,6 +3,8 @@ # integration process, which may cause wedges in the gate later. lofar-station-client@git+https://git.astron.nl/lofar2.0/lofar-station-client@0.6.0 +numpy +mock asyncua >= 0.9.90 # LGPLv3 PyMySQL[rsa] >= 1.0.2 # MIT psycopg2-binary >= 2.9.2 # LGPL diff --git a/tangostationcontrol/tangostationcontrol/integration_test/README.md b/tangostationcontrol/tangostationcontrol/integration_test/README.md index d06aa9b504ed46b8bffd711f7ed640729b0d301d..6c2321ad67ffade31894fcb2d71d160fb9e6f849 100644 --- a/tangostationcontrol/tangostationcontrol/integration_test/README.md +++ b/tangostationcontrol/tangostationcontrol/integration_test/README.md @@ -3,6 +3,13 @@ Integration tests are separated into multi modules. Each module requires a different state and configuration. These configurations are managed externally. +To minimize runtime overhead and memory pressure it is best to execute the +dummy integration test module before any other container is created. This will +ensure pytango is installed in the `.tox/integration` virtual environment +before the system experiences the severe memory pressure of the ELK stack. + +Simply run `make integration dummy` from inside the `docker-compose` folder. + In total the orchestration of integration tests is handled through four separate layers, each one calling the next: @@ -17,7 +24,7 @@ value can be left empty for the `default` module: Individual tests can be invoked using arguments: -`TEST_MODULE=default tox -e integration` +`TEST_MODULE=default tox -e integration import.path.class.functionname` These arguments and modules can also be passed at the level of the Makefile instead of through tox directly: @@ -48,7 +55,7 @@ cd tangostationcontrol # Single test to significantly reduce runtime tox -e integration tangostationcontrol.integration_test.default.devices.test_device_digitalbeam.TestDeviceDigitalBeam.test_pointing_to_zenith source .tox/integration/bin/activate -# Add import pdb; pdb.set_trace() somehwere +# Add import pdb; pdb.set_trace() somewhere nano integration tangostationcontrol.integration_test.default.devices.test_device_digitalbeam.py python -m testtools.run tangostationcontrol.integration_test.default.devices.test_device_digitalbeam.TestDeviceDigitalBeam.test_pointing_to_zenith ``` @@ -71,3 +78,10 @@ are running and are in the required state. ```shell sbin/run_integration_test.sh ``` + +## Cleanup, recovery or starting over + +All docker content including images, containers, networks and volumes can +be deleted using the following: + +`docker stop $(docker ps | tail -n+2 | awk '{NF=1}1' | awk '{printf("%s ",$0)} END { printf "\n" }'); docker system prune --all; docker volume prune` diff --git a/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_beamlet.py b/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_beamlet.py index b04808948129dd3f37ab01dd35b149a606483d6d..b5d7c9f85e4051e7bcd1f7bc5482166583fb1939 100644 --- a/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_beamlet.py +++ b/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_beamlet.py @@ -41,7 +41,8 @@ class TestDeviceBeamlet(AbstractTestBases.TestDeviceBase): sdp_proxy = self.setup_sdp() self.proxy.initialise() - self.proxy.subband_select = [0] * 488 + # TODO(Corne): Update test so effects of attribute are asserted L2SS-984 + # self.proxy.subband_select_RW = [0] * 488 self.proxy.on() # The subband frequency of HBA subband 0 is 200 MHz, @@ -63,7 +64,8 @@ class TestDeviceBeamlet(AbstractTestBases.TestDeviceBase): sdp_proxy = self.setup_sdp() self.proxy.initialise() - self.proxy.subband_select = list(range(488)) + # TODO(Corne): Update test so effects of attribute are asserted L2SS-984 + # self.proxy.subband_select_RW = list(range(488)) self.proxy.on() # any non-zero delay should result in different weights for different clocks diff --git a/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_digitalbeam.py b/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_digitalbeam.py index aed6b32314ddc6a2a69fe3f939a0c7ccaa4b6fda..f905e72ec57463ea5c7a41abc5b5a40c31582303 100644 --- a/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_digitalbeam.py +++ b/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_digitalbeam.py @@ -66,8 +66,9 @@ class TestDeviceDigitalBeam(AbstractTestBases.TestDeviceBase): antennafield_proxy.put_property({ "RECV_devices": [self.recv_iden], "Control_to_RECV_mapping": numpy.array(control_mapping).flatten(), - 'Antenna_Quality': antenna_qualities, 'Antenna_Use': antenna_use} - ) + "Antenna_Quality": antenna_qualities, + "Antenna_Use": antenna_use + }) antennafield_proxy.off() antennafield_proxy.boot() return antennafield_proxy @@ -81,9 +82,10 @@ class TestDeviceDigitalBeam(AbstractTestBases.TestDeviceBase): self.setup_antennafield_proxy(self.antenna_qualities_ok, self.antenna_use_ok) self.setup_sdp_proxy() + # TODO(Corne): Update these and ensure their effects is asserted in tests L2SS-984 # Setup beamlet configuration - self.beamlet_proxy.clock_RW = 200 * 1000000 - self.beamlet_proxy.subband_select = list(range(488)) + # self.beamlet_proxy.clock_RW = 200 * 1000000 + # self.beamlet_proxy.subband_select_RW = list(range(488)) self.proxy.initialise() self.proxy.Tracking_enabled_RW = False @@ -106,10 +108,6 @@ class TestDeviceDigitalBeam(AbstractTestBases.TestDeviceBase): self.setup_antennafield_proxy(self.antenna_qualities_ok, self.antenna_use_ok) self.setup_sdp_proxy() - # Setup beamlet configuration - self.beamlet_proxy.clock_RW = 200 * 1000000 - self.beamlet_proxy.subband_select = list(range(488)) - self.proxy.initialise() self.proxy.Tracking_enabled_RW = False self.proxy.on() @@ -140,10 +138,6 @@ class TestDeviceDigitalBeam(AbstractTestBases.TestDeviceBase): self.setup_antennafield_proxy(self.antenna_qualities_ok, self.antenna_use_ok) self.setup_sdp_proxy() - # Setup beamlet configuration - self.beamlet_proxy.clock_RW = 200 * 1000000 - self.beamlet_proxy.subband_select = list(range(488)) - self.proxy.initialise() self.proxy.Tracking_enabled_RW = False self.proxy.on() diff --git a/tangostationcontrol/tangostationcontrol/integration_test/dummy/__init__.py b/tangostationcontrol/tangostationcontrol/integration_test/dummy/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tangostationcontrol/tangostationcontrol/integration_test/dummy/test_dummy.py b/tangostationcontrol/tangostationcontrol/integration_test/dummy/test_dummy.py new file mode 100644 index 0000000000000000000000000000000000000000..61956173c0142948f0b1137e4d5bf28489eee8d1 --- /dev/null +++ b/tangostationcontrol/tangostationcontrol/integration_test/dummy/test_dummy.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +# +# This file is part of the LOFAR 2.0 Station Software +# +# +# +# Distributed under the terms of the APACHE license. +# See LICENSE.txt for more info. + +from tangostationcontrol.integration_test import base + + +class TestDummy(base.IntegrationTestCase): + def test_dummy(self): + self.assertTrue(True) diff --git a/tangostationcontrol/test-requirements.txt b/tangostationcontrol/test-requirements.txt index b8fe7099dac8cd826106e9f74136dd2c78f41bc3..dbd81d74d77b574e349b3bb7203284fc41b401cf 100644 --- a/tangostationcontrol/test-requirements.txt +++ b/tangostationcontrol/test-requirements.txt @@ -4,6 +4,8 @@ asynctest>=0.13.0 # Apache-2.0 bandit>=1.6.0 # Apache-2.0 +virtualenv>=20.16.0 # MIT +build>=0.8.0 # MIT coverage>=5.2.0 # Apache-2.0 doc8>=0.8.0 # Apache-2.0 flake8>=3.8.0 # MIT diff --git a/tangostationcontrol/tox.ini b/tangostationcontrol/tox.ini index 52d27b6eae221651e0c526a3e09552cb6129168e..f4d8ddce0312c73dc9d43a7b439c6a2beae8a535 100644 --- a/tangostationcontrol/tox.ini +++ b/tangostationcontrol/tox.ini @@ -1,38 +1,48 @@ [tox] minversion = 3.20 -envlist = py37,py38,py39,py310,pep8 +envlist = py3{7,8,9,10},pep8 skipsdist = True [testenv] usedevelop = True -; Module access is a bit of an ugly hack. This is due to testenv inheritance -; with sitepackages = True`, meaning that global packages can be accessed by the -; tox environment. Our tango images already have several dependencies system -; wide installed, however, the system wide installation will never look inside -; tox its virtualenv for packages. So accessing stestr and others fail.. We -; can't remove `sitepackages = True` either as we need access to tango and -; installing this package is non-trivial. The solution is to prevent calling -; binaries directly and utilizing python and tox variables to resolve the -; requested module. -sitepackages = True +; Python and tox variables are used to access modules and binaries instead of +; directly. This makes the setup robust for using sitepackages=True. install_command = {envbindir}/pip3 install {opts} {packages} passenv = HOME setenv = VIRTUAL_ENV={envdir} PYTHONWARNINGS=default::DeprecationWarning +; Share the same envdir with as many jobs as possible due to extensive time it +; takes to compile the pytango wheel, in addition to its large install size. +; should the environment change (such as the Python version) the environment +; will automatically be recreated. +envdir = {toxworkdir}/testenv deps = -r{toxinidir}/requirements.txt -r{toxinidir}/../docker-compose/lofar-device-base/lofar-requirements.txt -r{toxinidir}/test-requirements.txt commands_pre = {envpython} --version + pip install --no-cache pytango commands = {envpython} -m stestr --version {envpython} -m stestr run {posargs} +; We can't detect the current Python version for an environment dynamically +; so each Python version specific job needs its own envdir. +[testenv:py37] +envdir = {toxworkdir}/testenvpy37 + +[testenv:py38] +envdir = {toxworkdir}/testenvpy38 + +[testenv:py39] +envdir = {toxworkdir}/testenvpy39 + +[testenv:py310] +envdir = {toxworkdir}/testenvpy310 + [testenv:integration] -; Warning running integration tests will make changes to your docker system! -; These tests should only be run by the integration-test docker container. allowlist_externals = echo passenv = TANGO_HOST setenv = @@ -55,10 +65,6 @@ commands = setenv = VIRTUAL_ENV={envdir} PYTHON={envpython} -m coverage run --source tangostationcontrol --parallel-mode -deps = - -r{toxinidir}/requirements.txt - -r{toxinidir}/../docker-compose/lofar-device-base/lofar-requirements.txt - -r{toxinidir}/test-requirements.txt commands = {envpython} -m stestr --version {envpython} -m coverage --version @@ -91,7 +97,12 @@ commands = {envpython} -m xenon --version {envpython} -m xenon tangostationcontrol -b B -m A -a A -i libhdbpp-python +[testenv:build] +usedevelop = False +commands = {envpython} -m build + [testenv:docs] +envdir = {toxworkdir}/docs deps = -r{toxinidir}/../docker-compose/lofar-device-base/lofar-requirements.txt -r{toxinidir}/docs/docs-requirements.txt