diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f458c94bb5c002a69aac660d28736b4b0aa65f8a..24124fc8a01456cb9cd10eddee0070db77b3f82e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,4 @@ -# TODO(Corne): Update this image to use our own registry once building -# images is in place. -image: artefact.skao.int/ska-tango-images-tango-itango:9.3.7 +image: git.astron.nl:5000/lofar2.0/tango/tango-itango:9.3.7 variables: GIT_SUBMODULE_STRATEGY: recursive PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" @@ -8,12 +6,291 @@ cache: paths: - .cache/pip stages: + - images - building - linting - static-analysis - unit-tests - integration-tests - packaging +# See docker-compose/README.md for docker image behavior and explanation +.base_docker_images: + stage: images + image: docker:latest + tags: + - privileged + services: + - name: docker:dind + variables: + DOCKER_TLS_CERTDIR: "/certs" + before_script: + - | + if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" && -z "$CI_COMMIT_TAG" ]]; then + tag="latest" + echo "Running on tagged default branch '$CI_DEFAULT_BRANCH': tag = 'latest'" + else + tag="$CI_COMMIT_REF_SLUG" + echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" + fi + - apk add --update make bash docker-compose + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + - touch /root/.Xauthority +# Hack BASH_SOURCE into sourced files, docker its sh shell won't set this + - export BASH_SOURCE=$(pwd)/bootstrap/etc/lofar20rc.sh +# source the lofarrc file and mask its non zero exit code + - . bootstrap/etc/lofar20rc.sh || true +## Allow docker image script to execute +# - chmod u+x $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh +.base_docker_images_except: + extends: .base_docker_images + except: + refs: + - tags + - master +.base_docker_store_images: + extends: .base_docker_images + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh +docker_store_images_master_tag: + extends: .base_docker_store_images + only: + refs: + - tags + - master +docker_store_images_changes: + extends: .base_docker_store_images + only: + changes: + - docker-compose/.env + except: + refs: + - tags + - master +docker_build_image_all: + extends: .base_docker_images + only: + refs: + - tags + - master + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh elk latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh elk-configure-host latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh lofar-device-base latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh prometheus latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh itango latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh grafana latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh jupyter latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh apsct-sim latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh apspu-sim latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh recv-sim latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh sdptr-sim latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh unb2-sim latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-apsct latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-apspu latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-boot latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-docker latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-observation_control latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-recv latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-sdp latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-sst latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-unb2 latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-xst latest +docker_build_image_elk: + extends: .base_docker_images_except + only: + changes: + - docker-compose/elk.yml + - docker-compose/elk/* + - docker-compose/elk-configure-host/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh elk $tag + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh elk-configure-host $tag +docker_build_image_lofar_device_base: + extends: .base_docker_images_except + only: + changes: + - docker-compose/lofar-device-base.yml + - docker-compose/lofar-device-base/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh lofar-device-base $tag +docker_build_image_prometheus: + extends: .base_docker_images_except + only: + changes: + - docker-compose/prometheus.yml + - docker-compose/prometheus/* + except: + refs: + - tags + - master + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh prometheus $tag +docker_build_image_itango: + extends: .base_docker_images_except + only: + changes: + - docker-compose/itango.yml + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh itango $tag +docker_build_image_grafana: + extends: .base_docker_images_except + only: + changes: + - docker-compose/grafana.yml + - docker-compose/grafana/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh grafana $tag +docker_build_image_jupyter: + extends: .base_docker_images_except + only: + changes: + - docker-compose/jupyter.yml + - docker-compose/jupyter/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh jupyter $tag +docker_build_image_apsct_sim: + extends: .base_docker_images_except + only: + changes: + - docker-compose/aspct-sim.yml + - docker-compose/pypcc-sim-base/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh apsct-sim $tag +docker_build_image_apspu_sim: + extends: .base_docker_images_except + only: + changes: + - docker-compose/apspu-sim.yml + - docker-compose/pypcc-sim-base/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh apspu-sim $tag +docker_build_image_recv_sim: + extends: .base_docker_images_except + only: + changes: + - docker-compose/recv-sim.yml + - docker-compose/pypcc-sim-base/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh recv-sim $tag +docker_build_image_sdptr_sim: + extends: .base_docker_images_except + only: + changes: + - docker-compose/sdptr-sim.yml + - docker-compose/sdptr-sim/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh sdptr-sim $tag +docker_build_image_unb2_sim: + extends: .base_docker_images_except + only: + changes: + - docker-compose/unb2-sim.yml + - docker-compose/pypcc-sim-base/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh unb2-sim $tag +docker_build_image_device_apsct: + extends: .base_docker_images_except + only: + changes: + - docker-compose/device-aspct.yml + - docker-compose/lofar-device-base/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-aspct $tag +docker_build_image_device_apspu: + extends: .base_docker_images_except + only: + changes: + - docker-compose/device-apspu.yml + - docker-compose/lofar-device-base/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-apspu $tag +docker_build_image_device_boot: + extends: .base_docker_images_except + only: + changes: + - docker-compose/device-boot.yml + - docker-compose/lofar-device-base/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-boot $tag +docker_build_image_device_docker: + extends: .base_docker_images_except + only: + changes: + - docker-compose/device-docker.yml + - docker-compose/lofar-device-base/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-docker $tag +docker_build_image_device_ovservation_control: + extends: .base_docker_images_except + only: + changes: + - docker-compose/device-observation_control.yml + - docker-compose/lofar-device-base/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-observation_control $tag +docker_build_image_device_recv: + extends: .base_docker_images_except + only: + changes: + - docker-compose/device-recv.yml + - docker-compose/lofar-device-base/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-recv $tag +docker_build_image_device_sdp: + extends: .base_docker_images_except + only: + changes: + - docker-compose/device-sdp.yml + - docker-compose/lofar-device-base/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-sdp $tag +docker_build_image_device_sst: + extends: .base_docker_images_except + only: + changes: + - docker-compose/device-sst.yml + - docker-compose/lofar-device-base/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-sst $tag +docker_build_image_device_unb2: + extends: .base_docker_images_except + only: + changes: + - docker-compose/device-unb2.yml + - docker-compose/lofar-device-base/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-unb2 $tag +docker_build_image_device_xst: + extends: .base_docker_images_except + only: + changes: + - docker-compose/device-xst.yml + - docker-compose/lofar-device-base/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-xst $tag newline_at_eof: stage: linting before_script: @@ -63,6 +340,14 @@ integration_test_docker: variables: DOCKER_TLS_CERTDIR: "/certs" before_script: + - | + if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" && -z "$CI_COMMIT_TAG" ]]; then + tag="latest" + echo "Running on tagged default branch '$CI_DEFAULT_BRANCH': tag = 'latest'" + else + tag="$CI_COMMIT_REF_SLUG" + echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" + fi - apk add --update make bash docker-compose - apk add --update bind-tools - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY @@ -78,8 +363,12 @@ integration_test_docker: - . bootstrap/etc/lofar20rc.sh || true # TANGO_HOST must be unset our databaseds will be unreachable - unset TANGO_HOST -# Allow integration test to execute - - chmod u+x $CI_PROJECT_DIR/sbin/run_integration_test.sh +## Allow docker image script to execute +# - chmod u+x $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh pull $tag +## Allow integration test to execute +# - chmod u+x $CI_PROJECT_DIR/sbin/run_integration_test.sh # Do not remove 'bash' or statement will be ignored by primitive docker shell - bash $CI_PROJECT_DIR/sbin/run_integration_test.sh wheel_packaging: diff --git a/bootstrap/etc/lofar20rc.sh b/bootstrap/etc/lofar20rc.sh index 6e4a5c9bc8d6a78c1b61cca02159ee01291d3805..4b9d806d819816a86c5fea3ab8eb59135d8edcfc 100755 --- a/bootstrap/etc/lofar20rc.sh +++ b/bootstrap/etc/lofar20rc.sh @@ -16,6 +16,7 @@ if [ ! -f "${LOFAR20_DIR}/.git/hooks/post-checkout" ]; then alias git="cp ${LOFAR20_DIR}/bin/update_submodules.sh ${LOFAR20_DIR}/.git/hooks/post-checkout; cp ${LOFAR20_DIR}/bin/update_submodules.sh ${LOFAR20_DIR}/.git/hooks/post-merge; unalias git; git" fi +# CI_BUILD_ID does not exist see https://docs.gitlab.com/ee/ci/variables/predefined_variables.html if [ ! -z ${CI_BUILD_ID+x} ]; then export CONTAINER_NAME_PREFIX=${CI_BUILD_ID}- elif [ ! -z ${CI_JOB_ID+x} ]; then diff --git a/docker-compose/.env b/docker-compose/.env index c1956e315f8cde0d48b4b5279807025bede69261..53937727c24d398a5d82b24c31f205db50064163 100644 --- a/docker-compose/.env +++ b/docker-compose/.env @@ -1,6 +1,7 @@ DOCKER_REGISTRY_HOST=artefact.skao.int DOCKER_REGISTRY_USER=ska-tango-images LOCAL_DOCKER_REGISTRY_HOST=git.astron.nl:5000 +LOCAL_DOCKER_REGISTRY_LOFAR=lofar2.0 LOCAL_DOCKER_REGISTRY_USER=lofar2.0/tango TANGO_ARCHIVER_VERSION=2021-05-28 diff --git a/docker-compose/Makefile b/docker-compose/Makefile index d85ff1df88d91db097bdd22b060cfc03b681a04f..6c6e3c888eaa77b7c23ad0fca6ec0d94a8099bf7 100644 --- a/docker-compose/Makefile +++ b/docker-compose/Makefile @@ -33,6 +33,8 @@ else ifeq (stop,$(firstword $(MAKECMDGOALS))) SERVICE_TARGET = true else ifeq (restart,$(firstword $(MAKECMDGOALS))) SERVICE_TARGET = true +else ifeq (up,$(firstword $(MAKECMDGOALS))) + SERVICE_TARGET = true else ifeq (build,$(firstword $(MAKECMDGOALS))) SERVICE_TARGET = true else ifeq (build-nocache,$(firstword $(MAKECMDGOALS))) @@ -143,8 +145,8 @@ build-nocache: ## rebuild images from scratch $(DOCKER_COMPOSE_ARGS) docker-compose -f lofar-device-base.yml -f networks.yml build --progress=plain $(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) build --no-cache --progress=plain $(SERVICE) -up: minimal ## start the base TANGO system and prepare all services - $(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) up --no-start --no-recreate +up: minimal ## start the base TANGO system and prepare requested services + $(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) up --no-start --no-recreate $(SERVICE) down: ## stop all services and tear down the system $(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) down diff --git a/docker-compose/README.md b/docker-compose/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d76a75b0c79dad1574d9d24912a671a39998abee --- /dev/null +++ b/docker-compose/README.md @@ -0,0 +1,69 @@ +# Docker Compose + +Documentation on how the LOFAR station control software utilizes docker-compose. +This documentation is intended for developers listing strategies and their +respective advantages and disadvantages. In addition, this documentation +contains developer expectations that they should uphold to. + +## Image tagging and change detection + +Preventing unnecessary builds of docker images reduces build times and increases +iteration speed. In order to achieve this the project requires properly tagged +images and mechanisms for change detection. + +For change detection the system relies on git. Git is used to determine the +directories and files that have changes between the current and previous commit. +All image related change detection mechanisms are based on this difference. + +Using docker cache within the dind service is impractical see: +https://gitlab.com/gitlab-org/gitlab-foss/-/issues/17861 + +### Types of containers and specific strategies. + +- Devices +- Simulators +- Base images +- Services + +Devices, these are detected by changes to the .yml file or directory of the +respective service inside the docker-compose directory. + +Simulators, Since the source code for simulators is maintained by other teams +we can not accurately determine from our repository if the simulator has +changed. Instead, the images is build by the teams their respective CI +pipelines. We simply pull these images as base images. + +Base images, these are detected by changes to the .env file in the +docker-compose directory. When changed they will be downloaded from the remote +registry and uploaded to our own using matching tags. + +Services, same mechanism as devices. + +### Setup and maintenance + +All behavioral logic to orchestrate change detection and image pushing can be +found in the sbin/tag_and_push_docker_images.sh script as well as the +.gitlab-ci.yml. The shell script relies on the fact that each .yml file in the +docker-compose directory corresponds to one image. + +### Gitlab CI phases + +Docker images are managed in three phases. First is remote image storing, second +is image building and change detection with finally image pulling. + +Remote images are downloaded and stored on the local registry when the .env +file for docker-compose has changes. + +Local images are build when either the files in the base context directory +change or if the docker compose file itself has changes. See the gitlab-ci.yml +for how these changes are detected. All local images will be rebuild and tagged +latest when a tagged commit is pushed to master. + +Local images download the latest image from the registry as cache unless it is +a tagged commit on master. + +Finally, the integration test downloads all images from the registry either +tagged with the current pipeline or with latest. Should both tags be unavailable +than the integration test fails. Not all images are needed for the integration +test. See sbin/tag_and_push_docker_image.sh for how these images are +differentiated. diff --git a/docker-compose/apsct-sim.yml b/docker-compose/apsct-sim.yml index d30f5a026f734bb72ee91c7bf533df677f37ca88..b9742fdb97ec3f30026d441c668a13732013201e 100644 --- a/docker-compose/apsct-sim.yml +++ b/docker-compose/apsct-sim.yml @@ -10,6 +10,9 @@ services: apsct-sim: build: context: pypcc-sim-base + args: + - LOCAL_DOCKER_REGISTRY_HOST=${LOCAL_DOCKER_REGISTRY_HOST} + - LOCAL_DOCKER_REGISTRY_LOFAR=${LOCAL_DOCKER_REGISTRY_LOFAR} container_name: ${CONTAINER_NAME_PREFIX}apsct-sim networks: - control diff --git a/docker-compose/apspu-sim.yml b/docker-compose/apspu-sim.yml index d3fc5fa04f6ce0d6ddfe4c8f87887ab7500720e3..f5677048fbe1fe28082b219177bc67a2986c31fe 100644 --- a/docker-compose/apspu-sim.yml +++ b/docker-compose/apspu-sim.yml @@ -10,6 +10,9 @@ services: apspu-sim: build: context: pypcc-sim-base + args: + - LOCAL_DOCKER_REGISTRY_HOST=${LOCAL_DOCKER_REGISTRY_HOST} + - LOCAL_DOCKER_REGISTRY_LOFAR=${LOCAL_DOCKER_REGISTRY_LOFAR} container_name: ${CONTAINER_NAME_PREFIX}apspu-sim networks: - control diff --git a/docker-compose/archiver.yml b/docker-compose/archiver.yml index f1f2a1ec65dd4259b99e675c43cb7500862049f4..12ec2d88959fca75b047cff6004dd6e2b22c294a 100644 --- a/docker-compose/archiver.yml +++ b/docker-compose/archiver.yml @@ -94,7 +94,7 @@ services: tag: "{{.Name}}" dsconfig: - image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-dsconfig:${TANGO_DSCONFIG_VERSION} + image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-dsconfig:${TANGO_DSCONFIG_VERSION} container_name: ${CONTAINER_NAME_PREFIX}dsconfig networks: - control diff --git a/docker-compose/astor.yml b/docker-compose/astor.yml index 7010a82afa2fbcf5cb3dd797bda384bb516354f8..502472fc4eedd022388cd13d76e74135e00ff3db 100644 --- a/docker-compose/astor.yml +++ b/docker-compose/astor.yml @@ -13,7 +13,7 @@ version: '2' services: astor: - image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-java:${TANGO_JAVA_VERSION} + image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-java:${TANGO_JAVA_VERSION} container_name: ${CONTAINER_NAME_PREFIX}astor networks: - control diff --git a/docker-compose/device-apsct.yml b/docker-compose/device-apsct.yml index 60f65fc47ed81822242282fc743846221acec2d9..0e258fecdb3a96c3a73714ae2c28cf2e847457a1 100644 --- a/docker-compose/device-apsct.yml +++ b/docker-compose/device-apsct.yml @@ -20,7 +20,7 @@ services: build: context: lofar-device-base args: - SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION} + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION} container_name: ${CONTAINER_NAME_PREFIX}device-apsct networks: - control diff --git a/docker-compose/device-apspu.yml b/docker-compose/device-apspu.yml index b694b09518215e293d19e1ff551f4f608e6f818d..5f325b19fb357e83ab3d35e3acfa1a5cbbb2896a 100644 --- a/docker-compose/device-apspu.yml +++ b/docker-compose/device-apspu.yml @@ -20,7 +20,7 @@ services: build: context: lofar-device-base args: - SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION} + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION} container_name: ${CONTAINER_NAME_PREFIX}device-apspu networks: - control diff --git a/docker-compose/device-boot.yml b/docker-compose/device-boot.yml index 3db111410fafde9901fd8f91cb40a1c3560e4242..330cb723ed3bb5ee8ccd50bf4cb933da4e1fe09c 100644 --- a/docker-compose/device-boot.yml +++ b/docker-compose/device-boot.yml @@ -19,7 +19,7 @@ services: build: context: lofar-device-base args: - SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION} + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION} container_name: ${CONTAINER_NAME_PREFIX}device-boot networks: - control diff --git a/docker-compose/device-docker.yml b/docker-compose/device-docker.yml index 5a2641e9871f163f27ed7a60d872d30d4fe855e1..a9e4ccfdd6f66eda66f05ea5244fcf0fd732a382 100644 --- a/docker-compose/device-docker.yml +++ b/docker-compose/device-docker.yml @@ -20,7 +20,7 @@ services: build: context: lofar-device-base args: - SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION} + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION} container_name: ${CONTAINER_NAME_PREFIX}device-docker networks: - control diff --git a/docker-compose/device-observation_control.yml b/docker-compose/device-observation_control.yml index 33fb0d066fd76b8eb4a9c7753266f16d04157726..d4f6f15d1f4eb80d02cd0c5738dc0a011b9dfc72 100644 --- a/docker-compose/device-observation_control.yml +++ b/docker-compose/device-observation_control.yml @@ -19,7 +19,7 @@ services: build: context: lofar-device-base args: - SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION} + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION} container_name: ${CONTAINER_NAME_PREFIX}device-observation_control networks: - control diff --git a/docker-compose/device-recv.yml b/docker-compose/device-recv.yml index a08f566e7b39e095403f00cb5b086420b689d66b..25e767726f139ff532dbe649ccb230fabbec0602 100644 --- a/docker-compose/device-recv.yml +++ b/docker-compose/device-recv.yml @@ -20,7 +20,7 @@ services: build: context: lofar-device-base args: - SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION} + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION} container_name: ${CONTAINER_NAME_PREFIX}device-recv networks: - control diff --git a/docker-compose/device-sdp.yml b/docker-compose/device-sdp.yml index f32c34394475c6a7483cb98cd03def1f62cf9ff0..06a523f606d67811986bd7a13b9a3202cb74e91d 100644 --- a/docker-compose/device-sdp.yml +++ b/docker-compose/device-sdp.yml @@ -20,7 +20,7 @@ services: build: context: lofar-device-base args: - SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION} + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION} container_name: ${CONTAINER_NAME_PREFIX}device-sdp networks: - control diff --git a/docker-compose/device-sst.yml b/docker-compose/device-sst.yml index 7464cb01f45e584ab705fe9098e0229a1b762295..86651c7878d844646528b41fb0969dfd19af6eea 100644 --- a/docker-compose/device-sst.yml +++ b/docker-compose/device-sst.yml @@ -20,7 +20,7 @@ services: build: context: lofar-device-base args: - SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION} + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION} container_name: ${CONTAINER_NAME_PREFIX}device-sst networks: - control diff --git a/docker-compose/device-unb2.yml b/docker-compose/device-unb2.yml index af1329d21a905f3c150c092529978e17f0c0ee37..2b9b47146a405440ebd36fd84162935fb6b8a56d 100644 --- a/docker-compose/device-unb2.yml +++ b/docker-compose/device-unb2.yml @@ -20,7 +20,7 @@ services: build: context: lofar-device-base args: - SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION} + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION} container_name: ${CONTAINER_NAME_PREFIX}device-unb2 networks: - control diff --git a/docker-compose/device-xst.yml b/docker-compose/device-xst.yml index c4ea684fd94e34fcaaa857a5717ca47745eccc72..54ca5a21f911084160d2cec772df06da55ef5cf1 100644 --- a/docker-compose/device-xst.yml +++ b/docker-compose/device-xst.yml @@ -20,7 +20,7 @@ services: build: context: lofar-device-base args: - SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION} + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION} container_name: ${CONTAINER_NAME_PREFIX}device-xst networks: - control diff --git a/docker-compose/elk.yml b/docker-compose/elk.yml index 67f13baee061a74ebd08320f1e9f2f9f3e72f646..25bb1b218669baebff50ddc830b049b691349f71 100644 --- a/docker-compose/elk.yml +++ b/docker-compose/elk.yml @@ -6,6 +6,7 @@ # - elk-configure-host: Configures the hosts's kernel to be able to use the ELK stack # - elk: ELK stack # + version: '2' volumes: diff --git a/docker-compose/grafana.yml b/docker-compose/grafana.yml index 29c93c52c4dc05849aad10fabac12712c12dd4d7..f298db2746961b7d30d2e147192d0dfc58530725 100644 --- a/docker-compose/grafana.yml +++ b/docker-compose/grafana.yml @@ -4,6 +4,7 @@ # Defines: # - grafana: Grafana # + version: '2' #volumes: diff --git a/docker-compose/integration-test.yml b/docker-compose/integration-test.yml index defb45e3c3183516131795b283372ca784635d8c..e2be9144ef7d73b7108609a917529c019e109c62 100644 --- a/docker-compose/integration-test.yml +++ b/docker-compose/integration-test.yml @@ -11,7 +11,7 @@ services: build: context: itango args: - SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION} + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION} container_name: ${CONTAINER_NAME_PREFIX}integration-test networks: - control diff --git a/docker-compose/itango.yml b/docker-compose/itango.yml index 9b01c4ea25e2abc5849c9a98c29cc7601ba1115f..02d6801bd8a2f748a4b3d3336352891c78d4882b 100644 --- a/docker-compose/itango.yml +++ b/docker-compose/itango.yml @@ -17,7 +17,7 @@ services: build: context: itango args: - SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION} + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION} container_name: ${CONTAINER_NAME_PREFIX}itango networks: - control diff --git a/docker-compose/jive.yml b/docker-compose/jive.yml index 456ae1fc96771bad1ab6b99e52e3b0c9c046c20c..5a2caea9a1d9d6fb19d235781abc33a3230412e8 100644 --- a/docker-compose/jive.yml +++ b/docker-compose/jive.yml @@ -18,7 +18,7 @@ version: '2' services: jive: - image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-java:${TANGO_JAVA_VERSION} + image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-java:${TANGO_JAVA_VERSION} container_name: ${CONTAINER_NAME_PREFIX}jive network_mode: host volumes: diff --git a/docker-compose/jupyter.yml b/docker-compose/jupyter.yml index 1e1deea6f0e22299544f988602efc676bbe6200c..f32c6c1395fe6e41764e4569d472178620f731dd 100644 --- a/docker-compose/jupyter.yml +++ b/docker-compose/jupyter.yml @@ -7,6 +7,7 @@ # Defines: # - jupyter: Jupyter Notebook with iTango support # + version: '2' services: @@ -15,7 +16,7 @@ services: context: jupyter args: CONTAINER_EXECUTION_UID: ${CONTAINER_EXECUTION_UID} - SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION} + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION} container_name: ${CONTAINER_NAME_PREFIX}jupyter networks: - control diff --git a/docker-compose/lofar-device-base.yml b/docker-compose/lofar-device-base.yml index ce110ed85ba0cfb20b607ab7d08e70505d2392e8..f01faac2d2f41647708229106a895d3dad23c3e4 100644 --- a/docker-compose/lofar-device-base.yml +++ b/docker-compose/lofar-device-base.yml @@ -10,6 +10,7 @@ # Requires: # - tango.yml # + version: '2' services: @@ -18,7 +19,7 @@ services: build: context: lofar-device-base args: - SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION} + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION} container_name: ${CONTAINER_NAME_PREFIX}lofar-device-base # These parameters are just visual queues, you have to define them again # in derived docker-compose files! diff --git a/docker-compose/lofar-device-base/lofar-requirements.txt b/docker-compose/lofar-device-base/lofar-requirements.txt index 10ad55d977c97793a352c13da323d84d3c826c0e..95ed439cd121c0dc72b0c9a1c69d409e0bacc57e 100644 --- a/docker-compose/lofar-device-base/lofar-requirements.txt +++ b/docker-compose/lofar-device-base/lofar-requirements.txt @@ -1,5 +1,2 @@ -# Do not put tangostationcontrol dependencies here -astropy - -# requirements to build tangocontrol +# Do not put tangostationcontrol dependencies here, only setup.py / __init__.py GitPython >= 3.1.24 # BSD diff --git a/docker-compose/logviewer.yml b/docker-compose/logviewer.yml index bf0c9b2d51cbdb7334a579184114de6925fd37a1..08da4000b23925980e1683465fa4fdd4c05f04ae 100644 --- a/docker-compose/logviewer.yml +++ b/docker-compose/logviewer.yml @@ -12,7 +12,7 @@ version: '2' services: logviewer: - image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-java:${TANGO_JAVA_VERSION} + image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-java:${TANGO_JAVA_VERSION} container_name: ${CONTAINER_NAME_PREFIX}logviewer networks: - control diff --git a/docker-compose/pogo.yml b/docker-compose/pogo.yml index 826daac9fbd6ef3226a690832eedab505bbeaba3..954841746b9f0338d4a84fdae7e043fde04be460 100644 --- a/docker-compose/pogo.yml +++ b/docker-compose/pogo.yml @@ -20,7 +20,7 @@ volumes: services: pogo: - image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-pogo:${TANGO_POGO_VERSION} + image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-pogo:${TANGO_POGO_VERSION} container_name: ${CONTAINER_NAME_PREFIX}pogo networks: - control diff --git a/docker-compose/prometheus.yml b/docker-compose/prometheus.yml index 604f4bf4bde93dd6d68aaf7f3b1da2fd3f884e83..1e9ce6f1aa2cd050565f48a4b991865641fd1566 100644 --- a/docker-compose/prometheus.yml +++ b/docker-compose/prometheus.yml @@ -4,6 +4,7 @@ # Defines: # - prometheus: Prometheus # + version: '2' services: diff --git a/docker-compose/pypcc-sim-base/Dockerfile b/docker-compose/pypcc-sim-base/Dockerfile index c65c5b6f836e889f9b3c364ceace5f7b9b821628..f0f37dec5613b988ba3c471428aa426606cf9d5a 100644 --- a/docker-compose/pypcc-sim-base/Dockerfile +++ b/docker-compose/pypcc-sim-base/Dockerfile @@ -1,10 +1,6 @@ -FROM ubuntu:20.04 +ARG LOCAL_DOCKER_REGISTRY_HOST +ARG LOCAL_DOCKER_REGISTRY_LOFAR -COPY requirements.txt /requirements.txt +FROM ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_LOFAR}/pypcc:latest -RUN apt-get update && apt-get install -y python3 python3-pip python3-yaml git && \ - pip3 install -r requirements.txt && \ - git clone --depth 1 --branch master https://git.astron.nl/lofar2.0/pypcc - -WORKDIR /pypcc -CMD ["python3","pypcc2.py","--simulator","--port","4843"] +CMD ["python3", "pypcc2.py", "--simulator", "--port","4843"] diff --git a/docker-compose/pypcc-sim-base/requirements.txt b/docker-compose/pypcc-sim-base/requirements.txt deleted file mode 100644 index 2cd015945c044fcd1e39a823f49a807fc519ac67..0000000000000000000000000000000000000000 --- a/docker-compose/pypcc-sim-base/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -opcua -numpy -recordclass>=0.16,<0.16.1 \ No newline at end of file diff --git a/docker-compose/recv-sim.yml b/docker-compose/recv-sim.yml index effee8b298b855e2007e50c379fa3df45010bd05..8fd795be60ef89b23491895dd9809ff67b1c67ae 100644 --- a/docker-compose/recv-sim.yml +++ b/docker-compose/recv-sim.yml @@ -10,6 +10,9 @@ services: recv-sim: build: context: pypcc-sim-base + args: + - LOCAL_DOCKER_REGISTRY_HOST=${LOCAL_DOCKER_REGISTRY_HOST} + - LOCAL_DOCKER_REGISTRY_LOFAR=${LOCAL_DOCKER_REGISTRY_LOFAR} container_name: ${CONTAINER_NAME_PREFIX}recv-sim networks: - control diff --git a/docker-compose/rest.yml b/docker-compose/rest.yml index 467319399d6f3fec12a74068fea182195014b59e..94e1168455ddfefa20796c352e92d27e07f9a115 100644 --- a/docker-compose/rest.yml +++ b/docker-compose/rest.yml @@ -13,7 +13,7 @@ version: '2' services: rest: - image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-rest:${TANGO_REST_VERSION} + image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-rest:${TANGO_REST_VERSION} container_name: ${CONTAINER_NAME_PREFIX}tango-rest networks: - control diff --git a/docker-compose/sdptr-sim.yml b/docker-compose/sdptr-sim.yml index c81c3db9ae4744e013b5a92f1ebb5e9bdaa6e92c..badf707e37621c8b3030121424bacd1393910b87 100644 --- a/docker-compose/sdptr-sim.yml +++ b/docker-compose/sdptr-sim.yml @@ -10,6 +10,9 @@ services: sdptr-sim: build: context: sdptr-sim + args: + - LOCAL_DOCKER_REGISTRY_HOST=${LOCAL_DOCKER_REGISTRY_HOST} + - LOCAL_DOCKER_REGISTRY_LOFAR=${LOCAL_DOCKER_REGISTRY_LOFAR} container_name: ${CONTAINER_NAME_PREFIX}sdptr-sim networks: - control diff --git a/docker-compose/sdptr-sim/Dockerfile b/docker-compose/sdptr-sim/Dockerfile index 57fe98141f180a4d15a1e2d87c2c67be8f5894ff..4e64ca2a67229e602a705c9e61b0de999e64fad4 100644 --- a/docker-compose/sdptr-sim/Dockerfile +++ b/docker-compose/sdptr-sim/Dockerfile @@ -1,20 +1,7 @@ -FROM ubuntu:20.04 +ARG LOCAL_DOCKER_REGISTRY_HOST +ARG LOCAL_DOCKER_REGISTRY_LOFAR -# Install build tools for sdptr and the C language OPC-UA lib -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common && \ - DEBIAN_FRONTEND=noninteractive add-apt-repository ppa:open62541-team/ppa && \ - apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y autoconf automake git make g++ build-essential pkg-config libboost-dev libboost-regex-dev libboost-system-dev libboost-program-options-dev libopen62541-1-dev libopen62541-1-tools && \ - apt-get clean - -# Install SDPTR -RUN cd / && git clone --depth 1 --branch master https://git.astron.nl/lofar2.0/sdptr - -RUN cd /sdptr && \ - autoreconf -v -f -i && \ - ./configure && \ - bash -c "make -j `nproc` install" +FROM ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_LOFAR}/sdptr:latest COPY simulator.conf /sdptr/src/simulator.conf diff --git a/docker-compose/tango.yml b/docker-compose/tango.yml index 937cc5c8ecbe00b553d4692988e6cc2e5d7c51ef..19500fca1eeba859f74e7ba54fc3cbb021ea0ce6 100644 --- a/docker-compose/tango.yml +++ b/docker-compose/tango.yml @@ -15,7 +15,7 @@ volumes: services: tangodb: - image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-db:${TANGO_DB_VERSION} + image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-db:${TANGO_DB_VERSION} container_name: ${CONTAINER_NAME_PREFIX}tangodb networks: - control @@ -37,7 +37,7 @@ services: restart: unless-stopped databaseds: - image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-cpp:${TANGO_CPP_VERSION} + image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-cpp:${TANGO_CPP_VERSION} container_name: ${CONTAINER_NAME_PREFIX}databaseds networks: - control diff --git a/docker-compose/tangotest.yml b/docker-compose/tangotest.yml index 357c91df487b51379db221f7cb984bc05018f5e3..a97290d48f437b1c65b0bef01f6788fb525b2275 100644 --- a/docker-compose/tangotest.yml +++ b/docker-compose/tangotest.yml @@ -11,7 +11,7 @@ version: '2' services: tangotest: - image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-java:${TANGO_JAVA_VERSION} + image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-java:${TANGO_JAVA_VERSION} container_name: ${CONTAINER_NAME_PREFIX}tangotest networks: - control diff --git a/docker-compose/unb2-sim.yml b/docker-compose/unb2-sim.yml index d1ecaaa70a3c1e52f39ab1453d2ec8eb191f8831..b01802cd0526abe325c710f08fe965d6244cb2ba 100644 --- a/docker-compose/unb2-sim.yml +++ b/docker-compose/unb2-sim.yml @@ -10,6 +10,9 @@ services: unb2-sim: build: context: pypcc-sim-base + args: + - LOCAL_DOCKER_REGISTRY_HOST=${LOCAL_DOCKER_REGISTRY_HOST} + - LOCAL_DOCKER_REGISTRY_LOFAR=${LOCAL_DOCKER_REGISTRY_LOFAR} container_name: ${CONTAINER_NAME_PREFIX}unb2-sim networks: - control diff --git a/sbin/run_integration_test.sh b/sbin/run_integration_test.sh index 5ec1d299b13199f299b0a9ec73bf4bdeafa0350c..05e076ed00f79858fa44be6e23f3b94370672cb9 100755 --- a/sbin/run_integration_test.sh +++ b/sbin/run_integration_test.sh @@ -11,8 +11,12 @@ fi cd "$LOFAR20_DIR/docker-compose" || exit 1 -# Make sure builds are recent, and use our building parameters. -make build +# Build only the required images, please do not build everything that makes CI +# take really long to finish, especially grafana / jupyter / prometheus. +# jupyter is physically large > 2.5gb and overlayfs is really slow. +make build device-sdp device-recv device-sst device-unb2 device-xst +make build sdptr-sim recv-sim unb2-sim apsct-sim apspu-sim +make build databaseds dsconfig elk integration-test # Start and stop sequence make stop device-boot device-docker device-apsct device-apspu device-sdp device-recv device-sst device-unb2 device-xst device-beam sdptr-sim recv-sim unb2-sim apsct-sim apspu-sim diff --git a/sbin/tag_and_push_docker_image.sh b/sbin/tag_and_push_docker_image.sh index 799ab1cd779bb5caf840685f339080b57916063b..631413235d4ba9b2f39b9b0e216d31cd21913bdf 100755 --- a/sbin/tag_and_push_docker_image.sh +++ b/sbin/tag_and_push_docker_image.sh @@ -1,40 +1,177 @@ #!/bin/bash -e -# Tag and push which image version? -DOCKER_TAG=latest +function usage { + echo "./$(basename "$0") + no arguments, downloads remote images and pushes these to ASTRON registry. + The versions downloaded are controlled by the docker-compose/.env file" + echo "" + echo "./$(basename "$0") -h + displays this help message" + echo "" + echo "./$(basename "$0") <docker service name> <tag> + downloads latest version of image from the ASTRON registry, builds the + specified service and pushes the image with the specified tag to the + ASTRON registry" + echo "" + echo "./$(basename "$0") pull <tag> + downloads all images for the integration test with the specified tag + falling back to 'latest' if unavailable. Should neither exist on the + ASTRON registry the script will exit 1. The images are retagged to match + the output of docker-compose." +} -# Change to git tag or git hash if no tag -VERSION=$(date +"%Y-%M-%d") +# list of arguments expected in the input +optstring=":h" -SKA_REPO="nexus.engageska-portugal.pt/ska-docker" -LOFAR_REPO="git.astron.nl:5000/lofar2.0/tango" +while getopts ${optstring} arg; do + case ${arg} in + h) + usage + exit 0 + ;; + :) + echo "$0: Must supply an argument to -$OPTARG." >&2 + exit 1 + ;; + ?) + echo "Invalid option: -${OPTARG}." + exit 2 + ;; + esac +done -# Compile a list of the SKA images -SKA_IMAGES=$(for i in $(docker images | grep -E ${DOCKER_TAG} | grep -E ${SKA_REPO} | cut -d' ' -f1); do printf "%s " "${i}"; done) +if [ -z "${LOFAR20_DIR+x}" ]; then + echo "LOFAR20_DIR not set, did you forget to source lofar20rc.sh?" + exit 1 +fi -# Compile a list of LOFAR2.0 images -LOFAR_IMAGES=$(for i in $(docker images | grep -E ${DOCKER_TAG} | grep -E -v "${SKA_REPO}|${LOFAR_REPO}" | cut -d' ' -f1); do printf "%s " "${i}"; done) +# shellcheck disable=SC1090 +. "${LOFAR20_DIR}/docker-compose/.env" || exit 1 -function tag_and_push() -{ - ( - docker tag "${1}" "${2}" - docker push "${2}" - ) & -} +# List of images and their tag +REMOTE_IMAGES=( + "tango-dsconfig:${TANGO_DSCONFIG_VERSION}" "tango-java:${TANGO_JAVA_VERSION}" + "tango-itango:${TANGO_ITANGO_VERSION}" "tango-pogo:${TANGO_POGO_VERSION}" + "tango-cpp:${TANGO_CPP_VERSION}" "tango-db:${TANGO_DB_VERSION}" + "tango-dsconfig:${TANGO_DSCONFIG_VERSION}" "tango-rest:${TANGO_REST_VERSION}" +) -# Rename the SKA images for the LOFAR2.0 repo -# and push them to the LOFAR2.0 repo -for IMAGE in ${SKA_IMAGES}; do - PUSH_IMAGE=${IMAGE//${SKA_REPO}/${LOFAR_REPO}}:${VERSION} - tag_and_push "${IMAGE}" "${PUSH_IMAGE}" -done +# If first argument of bash script not set run first stage +if [ -z "${1+x}" ]; then + echo "Pulling and retagging remote images" -# Rename the LOFAR2.0 images for the LOFAR2.0 repo -# and push them to the LOFAR2.0 repo -for IMAGE in ${LOFAR_IMAGES}; do - PUSH_IMAGE=${LOFAR_REPO}/${IMAGE}:${VERSION} - tag_and_push "${IMAGE}" "${PUSH_IMAGE}" -done + # Iterate over al the REMOTE_IMAGES and pull them from remote and push local + for image in "${REMOTE_IMAGES[@]}"; do + remote_url="${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-${image}" + local_url="${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/${image}" + docker pull "${remote_url}" + docker tag "${remote_url}" "${local_url}" + docker push "${local_url}" + done + + exit 0 +fi + +# Triple tuple of docker-compose names, image names and if necessary for +# integration tests. +# TODO(Corne): Have this list generated from the .yml files +LOCAL_IMAGES=( + "elk elk y" "elk-configure-host elk-configure-host y" + "lofar-device-base lofar-device-base y" + + "apsct-sim docker-compose_apsct-sim y" "apspu-sim docker-compose_apspu-sim y" + "recv-sim docker-compose_recv-sim y" "sdptr-sim docker-compose_sdptr-sim y" + "unb2-sim docker-compose_unb2-sim y" + + "device-apsct device-apsct y" "device-apspu device-apspu y" + "device-boot device-boot y" "device-docker device-docker y" + "device-observation_control device-observation_control y" + "device-recv device-recv y" "device-sdp device-sdp y" + "device-sst device-sst y" "device-unb2 device-unb2 y" + "device-xst device-xst y" + + "itango docker-compose_itango y" + + "grafana grafana n" "prometheus prometheus n" + "jupyter docker-compose_jupyter n" + "integration-test docker-compose_integration-test n" + "tango-prometheus-exporter docker-compose_tango-prometheus-exporter n" +) + + + +# If first argument set run second stage, determine LOCAL_IMAGE to build and +# push from the argument +if [ ! -z "${1+x}" ] && [ "${1}" != "pull" ]; then + + # The second argument must pass the tag variable must be set + if [ -z "${2+x}" ]; then + echo "Error, second argument must pass tag variable" + exit 1 + fi + + # Set the tag and image variable, variables $1 and $2 are shadowed later + local_image="${1}" + tag="${2}" + + cd "${LOFAR20_DIR}/docker-compose" || exit 1 + + # Loop through images and find the specified one + for image in "${LOCAL_IMAGES[@]}"; do + # Set, splits tuple into $1 and $2. this shadows previous variables + # shellcheck disable=SC2086 + set -- $image + if [ "${local_image}" == "${1}" ]; then + echo "Building image for ${1} container" + local_url="${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/${2}" + + # If tag is not latest, than it is not a tagged master build and we can + # pull the latest image as cache. + if [ "${tag}" != "latest" ]; then + docker pull "${local_url}:latest" + fi + + make build "${1}" + docker tag "${2}" "${local_url}:${tag}" + docker push "${local_url}:${tag}" + fi + done + + exit 0 +fi + +# Final stage, pull images for integration cache try special tag image first +# if it fails download latest instead +if [ ! -z "${1+x}" ] && [ "${1}" == "pull" ]; then + echo "Pulling images for integration test cache" + + # The second argument must pass the tag variable must be set + if [ -z "${2+x}" ]; then + echo "Error, second argument must pass tag variable" + exit 1 + fi + + # Set the tag variable + tag="${2}" + + for image in "${LOCAL_IMAGES[@]}"; do + # Set, splits tuple into $1 and $2. this shadows previous variables + # shellcheck disable=SC2086 + set -- $image + + # Only download images which are needed for integration test + if [ "${3}" == "y" ]; then + local_url="${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/${2}" + # Pull images, at least one of the two images must succeed + echo "docker pull ${local_url}:${tag}" + docker pull "${local_url}:${tag}" || docker pull "${local_url}:latest" || exit 1 + # Ensure the images will have the same tags as generated by docker-compose + docker tag "${local_url}:${tag}" "${2}" || docker tag "${local_url}:latest" "${2}" || exit 1 + fi + done + + exit 0 +fi -wait +# Somehow nothing ran, that is an error do not fail silently +exit 1 diff --git a/tangostationcontrol/requirements.txt b/tangostationcontrol/requirements.txt index b1620255b5a45c9e3f653661e65de6732fd93a07..087bb148d057ef2e6d10f1727999932e4828c33a 100644 --- a/tangostationcontrol/requirements.txt +++ b/tangostationcontrol/requirements.txt @@ -5,7 +5,6 @@ asyncua >= 0.9.90 # LGPLv3 PyMySQL[rsa] >= 1.0.2 # MIT sqlalchemy >= 1.4.26 #MIT -GitPython >= 3.1.24 # BSD snmp >= 0.1.7 # GPL3 h5py >= 3.5.0 # BSD psutil >= 5.8.0 # BSD diff --git a/tangostationcontrol/tangostationcontrol/integration_test/device_proxy.py b/tangostationcontrol/tangostationcontrol/integration_test/device_proxy.py index 00ba0904c7bd01fa2ce1453a4c6701d6a4246e14..25c92411ecaababad20007868cfd19bdc3e9e18a 100644 --- a/tangostationcontrol/tangostationcontrol/integration_test/device_proxy.py +++ b/tangostationcontrol/tangostationcontrol/integration_test/device_proxy.py @@ -1,8 +1,25 @@ +import logging +import time + from tango import DeviceProxy +logger = logging.getLogger() + class TestDeviceProxy(DeviceProxy): def __init__(self, *args, **kwargs): super(TestDeviceProxy, self).__init__(*args, **kwargs) self.set_timeout_millis(10000) + + @staticmethod + def test_device_turn_off(endpoint): + d = TestDeviceProxy(endpoint) + try: + d.Off() + except Exception as e: + """Failing to turn Off devices should not raise errors here""" + logger.error(f"Failed to turn device off in teardown {e}") + + """Wait for 1 second to prevent propagating reconnection errors""" + time.sleep(1) diff --git a/tangostationcontrol/tangostationcontrol/integration_test/devices/base.py b/tangostationcontrol/tangostationcontrol/integration_test/devices/base.py index 555f7256ea49d68465b1c45bf038d47b39beeb25..ef4854a8241aaee7e6099ae7d417d7b94acfa21e 100644 --- a/tangostationcontrol/tangostationcontrol/integration_test/devices/base.py +++ b/tangostationcontrol/tangostationcontrol/integration_test/devices/base.py @@ -34,11 +34,19 @@ class AbstractTestBases: # make sure the device starts in Off self.proxy.Off() + self.addCleanup(TestDeviceProxy.test_device_turn_off, self.name) + super().setUp() - def tearDown(self): - """Turn device Off in teardown to prevent blocking tests""" - self.proxy.Off() + def test_device_fetch_state(self): + """Test if we can successfully fetch state""" + + self.assertEqual(DevState.OFF, self.proxy.state()) + + def test_device_ping(self): + """Test if we can successfully ping the device server""" + + self.assertGreater(self.proxy.ping(), 0) def test_device_initialize(self): """Test if we can transition to standby""" diff --git a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_apsct.py b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_apsct.py index ca73fc236a7486b858298e863663cf997c70ccc8..d973581e88cceb7510d2d257b006aa81ebbfbdfb 100644 --- a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_apsct.py +++ b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_apsct.py @@ -9,6 +9,7 @@ from .base import AbstractTestBases + class TestDeviceAPSCT(AbstractTestBases.TestDeviceBase): def setUp(self): diff --git a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_apspu.py b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_apspu.py index b9d2bc3d44acf0d4c2d3e1dd9b0adc095c86037b..5ebadc24029ce4a8e4a8b56805685d69aec73f2b 100644 --- a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_apspu.py +++ b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_apspu.py @@ -9,6 +9,7 @@ from .base import AbstractTestBases + class TestDeviceAPSPU(AbstractTestBases.TestDeviceBase): def setUp(self): diff --git a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_boot.py b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_boot.py index cc39c45d0a35da050aa041f0e5f2063df6312169..6d93080cc59ca176aea226893927f3321f503d89 100644 --- a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_boot.py +++ b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_boot.py @@ -11,6 +11,7 @@ import time from .base import AbstractTestBases + class TestDeviceBoot(AbstractTestBases.TestDeviceBase): def setUp(self): diff --git a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_recv.py b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_recv.py index 26e02ef312214df4a9304eafd050ea1438002da2..e96c385a7f976bc3ecb76d48b509b61d80454819 100644 --- a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_recv.py +++ b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_recv.py @@ -9,6 +9,7 @@ from .base import AbstractTestBases + class TestDeviceRECV(AbstractTestBases.TestDeviceBase): def setUp(self): diff --git a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_sdp.py b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_sdp.py index 2df399ed4607ea802abcf647c02078193f1b7f03..7de27c34b9746c1541c2b7091c977ed32ce9e535 100644 --- a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_sdp.py +++ b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_sdp.py @@ -7,11 +7,14 @@ # Distributed under the terms of the APACHE license. # See LICENSE.txt for more info. +from tango._tango import DevState from .base import AbstractTestBases + class TestDeviceSDP(AbstractTestBases.TestDeviceBase): def setUp(self): + """Intentionally recreate the device object in each test""" super().setUp("STAT/SDP/1") def test_device_sdp_read_attribute(self): diff --git a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_sst.py b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_sst.py index 38f3528f531660704baa00f70a7073974256a19f..60675e121364b52fb692b2f8461001bfdc78b50a 100644 --- a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_sst.py +++ b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_sst.py @@ -15,24 +15,13 @@ from tango._tango import DevState from .base import AbstractTestBases + class TestDeviceSST(AbstractTestBases.TestDeviceBase): def setUp(self): + """Intentionally recreate the device object in each test""" super().setUp("STAT/SST/1") - def test_device_on(self): - """Test if we can transition to on""" - - port_property = {"Statistics_Client_TCP_Port": "4999"} - self.proxy.put_property(port_property) - self.proxy.initialise() - - self.assertEqual(DevState.STANDBY, self.proxy.state()) - - self.proxy.on() - - self.assertEqual(DevState.ON, self.proxy.state()) - def test_device_sst_send_udp(self): port_property = {"Statistics_Client_TCP_Port": "4998"} self.proxy.put_property(port_property) diff --git a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_unb2.py b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_unb2.py index a35f70adde7af58408882e3b5ee3256a4838db84..d5731630188879e5f79f94f98951f8d6c1637ace 100644 --- a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_unb2.py +++ b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_unb2.py @@ -9,7 +9,9 @@ from .base import AbstractTestBases + class TestDeviceUNB2(AbstractTestBases.TestDeviceBase): def setUp(self): + """Intentionally recreate the device object in each test""" super().setUp("STAT/UNB2/1")