diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0371a7f7f806a9a6e9ac5c22d8c5562c6880a4eb..14ee77667f38df99badcb69c35a823b7ca62a283 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,6 +2,7 @@ # images is in place. image: artefact.skao.int/ska-tango-images-tango-itango:9.3.5 variables: + GIT_SUBMODULE_STRATEGY: recursive PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" # The PBR dependency requires a set version, not actually used # Instead `util/lofar_git.py:get_version()` is used. @@ -16,11 +17,12 @@ stages: - unit-tests - integration-tests newline_at_eof: - stage: linting - before_script: - - pip3 install -r devices/test-requirements.txt - script: - - flake8 --filename *.sh,*.conf,*.md,*.yml --select=W292 --exclude .tox,.egg-info,docker + stage: linting + before_script: + - pip3 install -r devices/test-requirements.txt + script: +# TODO(Corne): Ignore shell files in submodules more cleanly + - flake8 --filename *.sh,*.conf,*.md,*.yml --select=W292 --exclude docker-compose/tango-prometheus-exporter,.tox,.egg-info,docker python_linting: stage: linting script: @@ -37,6 +39,7 @@ shellcheck: - sudo apt-get update - sudo apt-get install -y shellcheck script: +# TODO(Corne): Ignore shell files in submodules - shellcheck **/*.sh unit_test: stage: unit-tests @@ -45,28 +48,33 @@ unit_test: - sudo apt-get install -y git script: - cd devices - - tox --recreate -e py37 -integration_test: + - tox -e py37 +integration_test_docker: stage: integration-tests - allow_failure: true + image: docker:latest tags: - privileged services: - - name: docker:20.10.8-dind + - name: docker:dind variables: DOCKER_TLS_CERTDIR: "/certs" -# Everything below does not work currently, we need a privileged container -# that can run the dind service before_script: - - sudo apt update - - sudo apt install -y docker.io - - export USER=$(id | awk -F'=' '{print $2}' | awk -F'(' '{print $2}' | awk -F')' '{print $1}') - - echo $USER -# - sudo usermod -aG docker $USER - - sudo docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + - apk add --update make bash docker-compose + - apk add --update bind-tools + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY script: - - touch /home/$USER/.Xauthority - - source bootstrap/etc/lofar20rc.sh - - export HOSTNAME=$(cat /run/systemd/netif/leases/2 | grep ^ADDRESS= | awk -F'=' '{print $2}') - - echo $HOSTNAME - - sudo $CI_PROJECT_DIR/sbin/run_integration_test.sh + - touch /root/.Xauthority +# Hack BASH_SOURCE into sourced files, docker its sh shell won't set this + - export BASH_SOURCE=$(pwd)/bootstrap/etc/lofar20rc.sh +# Hack HOSTNAME env variable into host.docker.internal, set in docker-compose + - export HOSTNAME=host.docker.internal +# - export HOSTNAME=$(hostname -i) +# - export HOSTNAME=$(cat /run/systemd/netif/leases/2 | grep ^ADDRESS= | awk -F'=' '{print $2}') +# source the lofarrc file and mask its non zero exit code + - . bootstrap/etc/lofar20rc.sh || true +# TANGO_HOST must be unset our databaseds will be unreachable + - unset TANGO_HOST +# Allow integration test to execute + - chmod u+x $CI_PROJECT_DIR/sbin/run_integration_test.sh +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/run_integration_test.sh diff --git a/bin/dump_ConfigDb.sh b/bin/dump_ConfigDb.sh index 7745c18482000fe2e7a726e27b6fa5eeae57e88e..0dc634c458b76cd5d3c13e2d7dab6e905f66248a 100755 --- a/bin/dump_ConfigDb.sh +++ b/bin/dump_ConfigDb.sh @@ -1,4 +1,4 @@ #!/bin/bash -# writes the JSON dump to stdout -docker exec -it dsconfig python -m dsconfig.dump +# writes the JSON dump to stdout, Do not change -i into -it incompatible with gitlab ci! +docker exec -i "${CONTAINER_NAME_PREFIX}"dsconfig python -m dsconfig. diff --git a/bin/itango_console.sh b/bin/itango_console.sh index fb4c9b8a285adca3211633fb89e6e9e59bd01087..c2474781787257c6ab3ee0656659415f09380b29 100755 --- a/bin/itango_console.sh +++ b/bin/itango_console.sh @@ -1,2 +1,2 @@ #!/bin/bash -exec docker exec -it itango itango3 +exec docker exec -it "${CONTAINER_NAME_PREFIX}"itango itango3 diff --git a/bin/itango_shell.sh b/bin/itango_shell.sh index abab9ef8515fd5ce1a1bf2d6d452a538426a499a..334953de3cca29ff459f6c861637c0859b1da97b 100755 --- a/bin/itango_shell.sh +++ b/bin/itango_shell.sh @@ -1,2 +1,2 @@ #!/bin/bash -exec docker exec -it itango /bin/bash +exec docker exec -it "${CONTAINER_NAME_PREFIX}"itango /bin/bash diff --git a/bin/start-DS.sh b/bin/start-DS.sh deleted file mode 100755 index 83a6eec6dd30f2e496fa03ffc6f7351d8e9a664d..0000000000000000000000000000000000000000 --- a/bin/start-DS.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -function help() -{ - why="${1}" - echo -e "*** Cannot start the Python device server.\\n${why}\\n\\n* The Python file for the device server must be the 1st parameter that is provided.\\n* The instance of this device server must be the 2nd parameter that is provided." - exit -1 -} - -# Check if the mandatory parameters are present: -# ${1}: device server's Python file -# ${2}: instance of the device server's executable in the configDB -case ${#} in - 0) - help "The device server's Python file and the instance are missing." - ;; - 1) - help "The device server's instance is missing." - ;; - *) - deviceServer="${1}" - shift - instance="${1}" - shift - ;; -esac - -# Find the path to the device server's Python file that is -# relative to the /hosthome directory (in Docker the user's -# mounted ${HOME}). -# ATTENTION -# This is assuming that the device server's Python file exists -# on the Docker's host in the user's ${HOME} directory. -runThis=$(basename "${deviceServer}") -runThis=${runThis//.sh/.py} -if [ -f "${runThis}" ]; then - myDir=${PWD} -else - myDir=${PWD}/$(dirname "${deviceServer}") -fi -deviceServerPath=${myDir/${HOME}/\/hosthome} - -# Tango log lines start with a UNIX timestamp. Replace them with the UTC time. -docker exec -it itango python3 "${deviceServerPath}/${runThis}" "${instance}" "${@}" | perl -ne 'use Time::Piece; s/^([0-9]+)/gmtime($1)->strftime("%F %T")/e; print;' diff --git a/bootstrap/etc/lofar20rc.sh b/bootstrap/etc/lofar20rc.sh index 9a9dd658b56f5d30bb1ff8c5de692bd8fe2164de..6e4a5c9bc8d6a78c1b61cca02159ee01291d3805 100755 --- a/bootstrap/etc/lofar20rc.sh +++ b/bootstrap/etc/lofar20rc.sh @@ -16,6 +16,14 @@ if [ ! -f "${LOFAR20_DIR}/.git/hooks/post-checkout" ]; then alias git="cp ${LOFAR20_DIR}/bin/update_submodules.sh ${LOFAR20_DIR}/.git/hooks/post-checkout; cp ${LOFAR20_DIR}/bin/update_submodules.sh ${LOFAR20_DIR}/.git/hooks/post-merge; unalias git; git" fi +if [ ! -z ${CI_BUILD_ID+x} ]; then + export CONTAINER_NAME_PREFIX=${CI_BUILD_ID}- +elif [ ! -z ${CI_JOB_ID+x} ]; then + export CONTAINER_NAME_PREFIX=${CI_JOB_ID}- +else + unset CONTAINER_NAME_PREFIX +fi + # This needs to be modified for a development environment. # In case you run multiple Docker networks on the same host in parallel, you need to specify a unique # network name for each of them. @@ -26,7 +34,6 @@ export NETWORK_MODE=tangonet # Example: export TANGO_HOST=station-xk25.astron.nl:10000 export TANGO_HOST=$(hostname):10000 - # # NO MODIFICATION BEYOND THIS POINT! # diff --git a/devices/integration_test/devices/test_device_recv.py b/devices/integration_test/devices/test_device_recv.py index 3a010a000c03d3c039f8f93a68c0f6437bc30db1..d7ba9e2a00884d36d45bd6036e426d832af23617 100644 --- a/devices/integration_test/devices/test_device_recv.py +++ b/devices/integration_test/devices/test_device_recv.py @@ -42,7 +42,7 @@ class TestDeviceRECV(base.IntegrationTestCase): d = DeviceProxy("LTS/RECV/1") - d.initialise() + d.Initialise() self.assertEqual(DevState.STANDBY, d.state()) @@ -51,7 +51,7 @@ class TestDeviceRECV(base.IntegrationTestCase): d = DeviceProxy("LTS/RECV/1") - d.initialise() + d.Initialise() d.on() diff --git a/devices/integration_test/devices/test_device_sdp.py b/devices/integration_test/devices/test_device_sdp.py index 5f064128f858e0bd2c44768a4f13057e5dc20266..eef62dc907b91b826ab9768c293c2b7f4ef26508 100644 --- a/devices/integration_test/devices/test_device_sdp.py +++ b/devices/integration_test/devices/test_device_sdp.py @@ -38,12 +38,19 @@ class TestDeviceSDP(base.IntegrationTestCase): self.assertEqual(DevState.OFF, d.state()) + def test_device_sdp_ping(self): + """Test if we can successfully ping the device server""" + + d = DeviceProxy("LTS/SDP/1") + + self.assertGreater(d.ping(), 0) + def test_device_sdp_initialize(self): """Test if we can transition to standby""" d = DeviceProxy("LTS/SDP/1") - d.initialise() + d.Initialise() self.assertEqual(DevState.STANDBY, d.state()) @@ -52,7 +59,7 @@ class TestDeviceSDP(base.IntegrationTestCase): d = DeviceProxy("LTS/SDP/1") - d.initialise() + d.Initialise() d.on() diff --git a/devices/integration_test/devices/test_tango_database.py b/devices/integration_test/devices/test_tango_database.py new file mode 100644 index 0000000000000000000000000000000000000000..74404a24dafd195bd25d35ba14d9ff42ca012b9f --- /dev/null +++ b/devices/integration_test/devices/test_tango_database.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# +# This file is part of the LOFAR 2.0 Station Software +# +# +# +# Distributed under the terms of the APACHE license. +# See LICENSE.txt for more info. + +import time + +from tango import Database +from tango._tango import DevState + +from integration_test import base + + +class TestTangoDatabase(base.IntegrationTestCase): + + def setUp(self): + """Intentionally recreate the device object in each test""" + super(TestTangoDatabase, self).setUp() + + def test_database_servers(self): + """Connect to the database and find at least 3 servers + + One for SDP, RECV and the databaseds itself. + """ + + d = Database() + + # Ensure this value is close to actual amount of servers defined by + # integration_ConfigDb.json + self.assertGreater(len(d.get_server_list()), 16) diff --git a/docker-compose/archiver.yml b/docker-compose/archiver.yml index 84dded354d22c97eeccd51ea97d8ff41b909f01e..31ed4177010eef66eed0d80b053f19079620d91f 100644 --- a/docker-compose/archiver.yml +++ b/docker-compose/archiver.yml @@ -3,7 +3,7 @@ version: '2' services: archiver-maria-db: image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/mariadb_hdbpp:2021-05-28 - container_name: archiver-maria-db + container_name: ${CONTAINER_NAME_PREFIX}archiver-maria-db networks: - control ports: @@ -23,7 +23,7 @@ services: image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-archiver:2021-05-28 networks: - control - container_name: hdbpp-es + container_name: ${CONTAINER_NAME_PREFIX}hdbpp-es depends_on: - databaseds - dsconfig @@ -42,7 +42,7 @@ services: image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-archiver:${TANGO_ARCHIVER_VERSION} networks: - control - container_name: hdbpp-cm + container_name: ${CONTAINER_NAME_PREFIX}hdbpp-cm depends_on: - databaseds - dsconfig @@ -58,7 +58,7 @@ services: dsconfig: image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-dsconfig:${TANGO_DSCONFIG_VERSION} - container_name: dsconfig + container_name: ${CONTAINER_NAME_PREFIX}dsconfig networks: - control depends_on: diff --git a/docker-compose/device-apsct.yml b/docker-compose/device-apsct.yml index f17919a227a12b9846ba7a272254faaaee4d496a..36a21fb6c81bce72c0f708e5aca924eb0de9e14f 100644 --- a/docker-compose/device-apsct.yml +++ b/docker-compose/device-apsct.yml @@ -26,6 +26,8 @@ services: - control ports: - "5709:5709" # unique port for this DS + extra_hosts: + - "host.docker.internal:host-gateway" volumes: - ..:/opt/lofar/tango:rw environment: diff --git a/docker-compose/device-apspu.yml b/docker-compose/device-apspu.yml index 30da5a6c2aba5daef2452f906693c41e7fece330..f1ea50c89d5194c6829fcf29446090edae33e686 100644 --- a/docker-compose/device-apspu.yml +++ b/docker-compose/device-apspu.yml @@ -26,6 +26,8 @@ services: - control ports: - "5710:5710" # unique port for this DS + extra_hosts: + - "host.docker.internal:host-gateway" volumes: - ..:/opt/lofar/tango:rw environment: diff --git a/docker-compose/device-boot.yml b/docker-compose/device-boot.yml index 58a9aa7df81eab368464f4ca69ddab54129b7ace..d51f91614a483feb8c5956d3d35942843c4be3ce 100644 --- a/docker-compose/device-boot.yml +++ b/docker-compose/device-boot.yml @@ -25,6 +25,8 @@ services: - control ports: - "5708:5708" # unique port for this DS + extra_hosts: + - "host.docker.internal:host-gateway" volumes: - ..:/opt/lofar/tango:rw environment: diff --git a/docker-compose/device-docker.yml b/docker-compose/device-docker.yml index d9e1e1e35233177ab271db395773538ed8c74ffa..49144b7bb61ed1ca30c83b26e0c5443761411c54 100644 --- a/docker-compose/device-docker.yml +++ b/docker-compose/device-docker.yml @@ -26,6 +26,8 @@ services: - control ports: - "5705:5705" # unique port for this DS + extra_hosts: + - "host.docker.internal:host-gateway" volumes: - ..:/opt/lofar/tango:rw - /var/run/docker.sock:/var/run/docker.sock:rw # we want to control our sibling containers, NOT do docker-in-docker (dind) diff --git a/docker-compose/device-observation_control.yml b/docker-compose/device-observation_control.yml index 827a558a10167d29f3e0bd3402f3f84debcd3c23..b057df808d9f7c0fc3a2e5fb8bb9a3f0504ff388 100644 --- a/docker-compose/device-observation_control.yml +++ b/docker-compose/device-observation_control.yml @@ -25,6 +25,8 @@ services: - control ports: - "5703:5703" # unique port for this DS + extra_hosts: + - "host.docker.internal:host-gateway" volumes: - ..:/opt/lofar/tango:rw environment: diff --git a/docker-compose/device-recv.yml b/docker-compose/device-recv.yml index f3bc3eea12b51b44cacbeb790d0666ced24ae169..0b860949bb44cd8301986f732df99a8a3d58d8aa 100644 --- a/docker-compose/device-recv.yml +++ b/docker-compose/device-recv.yml @@ -26,6 +26,8 @@ services: - control ports: - "5707:5707" # unique port for this DS + extra_hosts: + - "host.docker.internal:host-gateway" volumes: - ..:/opt/lofar/tango:rw environment: diff --git a/docker-compose/device-sdp.yml b/docker-compose/device-sdp.yml index 8fefa3f355eda485ea757f0859924e317b9245ee..c1685235467277ac60f781918aa98fde6394877d 100644 --- a/docker-compose/device-sdp.yml +++ b/docker-compose/device-sdp.yml @@ -26,6 +26,8 @@ services: - control ports: - "5701:5701" # unique port for this DS + extra_hosts: + - "host.docker.internal:host-gateway" volumes: - ..:/opt/lofar/tango:rw environment: diff --git a/docker-compose/device-sst.yml b/docker-compose/device-sst.yml index 7d922a61badf6575d15c6f0a0489a6fac3683367..68164379ed1e7d3fa985cb9f3c5f620c09b71d50 100644 --- a/docker-compose/device-sst.yml +++ b/docker-compose/device-sst.yml @@ -29,6 +29,8 @@ services: - "5001:5001/udp" # port to receive SST UDP packets on - "5101:5101/tcp" # port to emit SST TCP packets on - "5702:5702" # unique port for this DS + extra_hosts: + - "host.docker.internal:host-gateway" volumes: - ..:/opt/lofar/tango:rw environment: diff --git a/docker-compose/device-unb2.yml b/docker-compose/device-unb2.yml index 6844ee2b6994c11fb53469535925284be42410c0..bd9b6496332994171959967ff358115ac91e5ca7 100644 --- a/docker-compose/device-unb2.yml +++ b/docker-compose/device-unb2.yml @@ -26,6 +26,8 @@ services: - control ports: - "5704:5704" # unique port for this DS + extra_hosts: + - "host.docker.internal:host-gateway" volumes: - ..:/opt/lofar/tango:rw environment: diff --git a/docker-compose/device-xst.yml b/docker-compose/device-xst.yml index c634e5d83fc7b28f2b8438ae59dffb7157a03f54..619b532e694f502b93638c2427bb7d9464138e82 100644 --- a/docker-compose/device-xst.yml +++ b/docker-compose/device-xst.yml @@ -29,6 +29,8 @@ services: - "5002:5002/udp" # port to receive XST UDP packets on - "5102:5102/tcp" # port to emit XST TCP packets on - "5706:5706" # unique port for this DS + extra_hosts: + - "host.docker.internal:host-gateway" volumes: - ..:/opt/lofar/tango:rw environment: diff --git a/docker-compose/integration-test.yml b/docker-compose/integration-test.yml index e0d1c6baf58948cdbee5a71ff2f859ab429dcd4b..1811645b06f4d576a71a66d2f36a475c976a2308 100644 --- a/docker-compose/integration-test.yml +++ b/docker-compose/integration-test.yml @@ -15,6 +15,8 @@ services: container_name: ${CONTAINER_NAME_PREFIX}integration-test networks: - control + extra_hosts: + - "host.docker.internal:host-gateway" volumes: - ..:/opt/lofar/tango:rw environment: diff --git a/docker-compose/itango.yml b/docker-compose/itango.yml index 5b0874f2f3c936c1b57915580ac79be81b5edcb9..9b01c4ea25e2abc5849c9a98c29cc7601ba1115f 100644 --- a/docker-compose/itango.yml +++ b/docker-compose/itango.yml @@ -21,6 +21,8 @@ services: container_name: ${CONTAINER_NAME_PREFIX}itango networks: - control + extra_hosts: + - "host.docker.internal:host-gateway" volumes: - ..:/opt/lofar/tango:rw - ${HOME}:/hosthome diff --git a/docker-compose/lofar-device-base.yml b/docker-compose/lofar-device-base.yml index 22ecabca37c526867afe52461d7ce432964f8385..ce110ed85ba0cfb20b607ab7d08e70505d2392e8 100644 --- a/docker-compose/lofar-device-base.yml +++ b/docker-compose/lofar-device-base.yml @@ -20,5 +20,9 @@ services: args: SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION} container_name: ${CONTAINER_NAME_PREFIX}lofar-device-base + # These parameters are just visual queues, you have to define them again + # in derived docker-compose files! networks: - control + extra_hosts: + - "host.docker.internal:host-gateway" diff --git a/sbin/load_ConfigDb.sh b/sbin/load_ConfigDb.sh index 03ab449a026b5de41056f16de0d2e566a00adfbb..0fe57087a89dc08ebef51d4679a687ebbf6a144a 100755 --- a/sbin/load_ConfigDb.sh +++ b/sbin/load_ConfigDb.sh @@ -9,10 +9,10 @@ fi # copy file into container to read it from container, as the file's location # in the container won't be the same as on the host. -docker cp "${file}" dsconfig:/tmp/dsconfig-load-settings.json || exit 1 +docker cp "${file}" "${CONTAINER_NAME_PREFIX}"dsconfig:/tmp/dsconfig-update-settings.json || exit 1 -# write settings -docker exec -it dsconfig json2tango --write /tmp/dsconfig-load-settings.json +# update settings, Do not change -i into -it this will break integration tests in gitlab ci! +docker exec -i "${CONTAINER_NAME_PREFIX}"dsconfig json2tango --write /tmp/dsconfig-update-settings.json # somehow json2tango does not return 0 on success exit 0 diff --git a/sbin/run_integration_test.sh b/sbin/run_integration_test.sh index 2175f2774e771ac31e9dcd1cb1ade68da51f923e..e4e22e0d7e037a0e7c6c117dcff96e24e62d4246 100755 --- a/sbin/run_integration_test.sh +++ b/sbin/run_integration_test.sh @@ -12,13 +12,14 @@ fi # Start and stop sequence cd "$LOFAR20_DIR/docker-compose" || exit 1 make stop device-sdp device-recv device-sst device-unb2 device-xst sdptr-sim recv-sim unb2-sim apsct-sim apspu-sim -make start databaseds dsconfig jupyter elk +make start databaseds dsconfig elk # Give dsconfig and databaseds time to start -sleep 15 +sleep 60 # Update the dsconfig -"${LOFAR20_DIR}"/sbin/update_ConfigDb.sh "${LOFAR20_DIR}"/CDB/integration_ConfigDb.json +# Do not remove `bash`, otherwise statement ignored by gitlab ci shell! +bash "${LOFAR20_DIR}"/sbin/update_ConfigDb.sh "${LOFAR20_DIR}"/CDB/integration_ConfigDb.json cd "$LOFAR20_DIR/docker-compose" || exit 1 make start sdptr-sim recv-sim unb2-sim apsct-sim apspu-sim @@ -28,12 +29,16 @@ sleep 5 make start device-sdp device-recv device-sst device-unb2 device-xst -# Give the devices time to start -sleep 5 +# Give devices time to restart +# TODO(Corne Lukken): Use a nicer more reliable mechanism +sleep 60 # Start the integration test cd "$LOFAR20_DIR/docker-compose" || exit 1 make start integration-test +# Give devices time to restart +sleep 60 + # Run the integration test with the output displayed on stdout -docker start -a integration-test +docker start -a "${CONTAINER_NAME_PREFIX}"integration-test diff --git a/sbin/update_ConfigDb.sh b/sbin/update_ConfigDb.sh index 8d71c312fc94ba4dba45b17c05a966f62fa9ff34..1255f1ea141a75940f2cd858dfc2b40818bd6ec2 100755 --- a/sbin/update_ConfigDb.sh +++ b/sbin/update_ConfigDb.sh @@ -9,10 +9,10 @@ fi # copy file into container to read it from container, as the file's location # in the container won't be the same as on the host. -docker cp "${file}" dsconfig:/tmp/dsconfig-update-settings.json || exit 1 +docker cp "${file}" "${CONTAINER_NAME_PREFIX}"dsconfig:/tmp/dsconfig-update-settings.json || exit 1 -# update settings -docker exec -it dsconfig json2tango --write --update /tmp/dsconfig-update-settings.json +# update settings, Do not change -i into -it this will break integration tests in gitlab ci! +docker exec -i "${CONTAINER_NAME_PREFIX}"dsconfig json2tango --write --update /tmp/dsconfig-update-settings.json # somehow json2tango does not return 0 on success exit 0