diff --git a/.gitignore b/.gitignore index 60c6519f7724a7ca08cac3263b595400dba9fdd2..6841e3ee2682fb0d5660e2491c5b03ac34f6957e 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ **/.project **/.pydevproject **/.settings/org.eclipse.core.resources.prefs +docs/build tangostationcontrol/dist tangostationcontrol/build **/.ipynb_checkpoints diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f458c94bb5c002a69aac660d28736b4b0aa65f8a..24124fc8a01456cb9cd10eddee0070db77b3f82e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,4 @@ -# TODO(Corne): Update this image to use our own registry once building -# images is in place. -image: artefact.skao.int/ska-tango-images-tango-itango:9.3.7 +image: git.astron.nl:5000/lofar2.0/tango/tango-itango:9.3.7 variables: GIT_SUBMODULE_STRATEGY: recursive PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" @@ -8,12 +6,291 @@ cache: paths: - .cache/pip stages: + - images - building - linting - static-analysis - unit-tests - integration-tests - packaging +# See docker-compose/README.md for docker image behavior and explanation +.base_docker_images: + stage: images + image: docker:latest + tags: + - privileged + services: + - name: docker:dind + variables: + DOCKER_TLS_CERTDIR: "/certs" + before_script: + - | + if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" && -z "$CI_COMMIT_TAG" ]]; then + tag="latest" + echo "Running on tagged default branch '$CI_DEFAULT_BRANCH': tag = 'latest'" + else + tag="$CI_COMMIT_REF_SLUG" + echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" + fi + - apk add --update make bash docker-compose + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + - touch /root/.Xauthority +# Hack BASH_SOURCE into sourced files, docker its sh shell won't set this + - export BASH_SOURCE=$(pwd)/bootstrap/etc/lofar20rc.sh +# source the lofarrc file and mask its non zero exit code + - . bootstrap/etc/lofar20rc.sh || true +## Allow docker image script to execute +# - chmod u+x $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh +.base_docker_images_except: + extends: .base_docker_images + except: + refs: + - tags + - master +.base_docker_store_images: + extends: .base_docker_images + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh +docker_store_images_master_tag: + extends: .base_docker_store_images + only: + refs: + - tags + - master +docker_store_images_changes: + extends: .base_docker_store_images + only: + changes: + - docker-compose/.env + except: + refs: + - tags + - master +docker_build_image_all: + extends: .base_docker_images + only: + refs: + - tags + - master + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh elk latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh elk-configure-host latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh lofar-device-base latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh prometheus latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh itango latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh grafana latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh jupyter latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh apsct-sim latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh apspu-sim latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh recv-sim latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh sdptr-sim latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh unb2-sim latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-apsct latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-apspu latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-boot latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-docker latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-observation_control latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-recv latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-sdp latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-sst latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-unb2 latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-xst latest +docker_build_image_elk: + extends: .base_docker_images_except + only: + changes: + - docker-compose/elk.yml + - docker-compose/elk/* + - docker-compose/elk-configure-host/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh elk $tag + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh elk-configure-host $tag +docker_build_image_lofar_device_base: + extends: .base_docker_images_except + only: + changes: + - docker-compose/lofar-device-base.yml + - docker-compose/lofar-device-base/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh lofar-device-base $tag +docker_build_image_prometheus: + extends: .base_docker_images_except + only: + changes: + - docker-compose/prometheus.yml + - docker-compose/prometheus/* + except: + refs: + - tags + - master + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh prometheus $tag +docker_build_image_itango: + extends: .base_docker_images_except + only: + changes: + - docker-compose/itango.yml + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh itango $tag +docker_build_image_grafana: + extends: .base_docker_images_except + only: + changes: + - docker-compose/grafana.yml + - docker-compose/grafana/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh grafana $tag +docker_build_image_jupyter: + extends: .base_docker_images_except + only: + changes: + - docker-compose/jupyter.yml + - docker-compose/jupyter/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh jupyter $tag +docker_build_image_apsct_sim: + extends: .base_docker_images_except + only: + changes: + - docker-compose/aspct-sim.yml + - docker-compose/pypcc-sim-base/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh apsct-sim $tag +docker_build_image_apspu_sim: + extends: .base_docker_images_except + only: + changes: + - docker-compose/apspu-sim.yml + - docker-compose/pypcc-sim-base/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh apspu-sim $tag +docker_build_image_recv_sim: + extends: .base_docker_images_except + only: + changes: + - docker-compose/recv-sim.yml + - docker-compose/pypcc-sim-base/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh recv-sim $tag +docker_build_image_sdptr_sim: + extends: .base_docker_images_except + only: + changes: + - docker-compose/sdptr-sim.yml + - docker-compose/sdptr-sim/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh sdptr-sim $tag +docker_build_image_unb2_sim: + extends: .base_docker_images_except + only: + changes: + - docker-compose/unb2-sim.yml + - docker-compose/pypcc-sim-base/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh unb2-sim $tag +docker_build_image_device_apsct: + extends: .base_docker_images_except + only: + changes: + - docker-compose/device-aspct.yml + - docker-compose/lofar-device-base/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-aspct $tag +docker_build_image_device_apspu: + extends: .base_docker_images_except + only: + changes: + - docker-compose/device-apspu.yml + - docker-compose/lofar-device-base/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-apspu $tag +docker_build_image_device_boot: + extends: .base_docker_images_except + only: + changes: + - docker-compose/device-boot.yml + - docker-compose/lofar-device-base/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-boot $tag +docker_build_image_device_docker: + extends: .base_docker_images_except + only: + changes: + - docker-compose/device-docker.yml + - docker-compose/lofar-device-base/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-docker $tag +docker_build_image_device_ovservation_control: + extends: .base_docker_images_except + only: + changes: + - docker-compose/device-observation_control.yml + - docker-compose/lofar-device-base/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-observation_control $tag +docker_build_image_device_recv: + extends: .base_docker_images_except + only: + changes: + - docker-compose/device-recv.yml + - docker-compose/lofar-device-base/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-recv $tag +docker_build_image_device_sdp: + extends: .base_docker_images_except + only: + changes: + - docker-compose/device-sdp.yml + - docker-compose/lofar-device-base/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-sdp $tag +docker_build_image_device_sst: + extends: .base_docker_images_except + only: + changes: + - docker-compose/device-sst.yml + - docker-compose/lofar-device-base/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-sst $tag +docker_build_image_device_unb2: + extends: .base_docker_images_except + only: + changes: + - docker-compose/device-unb2.yml + - docker-compose/lofar-device-base/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-unb2 $tag +docker_build_image_device_xst: + extends: .base_docker_images_except + only: + changes: + - docker-compose/device-xst.yml + - docker-compose/lofar-device-base/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-xst $tag newline_at_eof: stage: linting before_script: @@ -63,6 +340,14 @@ integration_test_docker: variables: DOCKER_TLS_CERTDIR: "/certs" before_script: + - | + if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" && -z "$CI_COMMIT_TAG" ]]; then + tag="latest" + echo "Running on tagged default branch '$CI_DEFAULT_BRANCH': tag = 'latest'" + else + tag="$CI_COMMIT_REF_SLUG" + echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" + fi - apk add --update make bash docker-compose - apk add --update bind-tools - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY @@ -78,8 +363,12 @@ integration_test_docker: - . bootstrap/etc/lofar20rc.sh || true # TANGO_HOST must be unset our databaseds will be unreachable - unset TANGO_HOST -# Allow integration test to execute - - chmod u+x $CI_PROJECT_DIR/sbin/run_integration_test.sh +## Allow docker image script to execute +# - chmod u+x $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh pull $tag +## Allow integration test to execute +# - chmod u+x $CI_PROJECT_DIR/sbin/run_integration_test.sh # Do not remove 'bash' or statement will be ignored by primitive docker shell - bash $CI_PROJECT_DIR/sbin/run_integration_test.sh wheel_packaging: diff --git a/CDB/LOFAR_ConfigDb.json b/CDB/LOFAR_ConfigDb.json index 6886437ce7cf02ee741b5b70f07ebfbf71b08b4d..60b0115925b2ccf01bd27b3e0d072a17c78d580f 100644 --- a/CDB/LOFAR_ConfigDb.json +++ b/CDB/LOFAR_ConfigDb.json @@ -21,6 +21,13 @@ } } }, + "Beam": { + "STAT": { + "Beam": { + "STAT/Beam/1": {} + } + } + }, "boot": { "STAT": { "Boot": { diff --git a/bin/start-ds.sh b/bin/start-ds.sh index 7b601c4c8f5e24ac56755ad08a1203a6cbba62d2..a48b0b4554cd2ef9380cbc482b07edfc38203043 100755 --- a/bin/start-ds.sh +++ b/bin/start-ds.sh @@ -2,6 +2,11 @@ # Serves as entrypoint script for docker containers +if [[ ! -d "/opt/lofar/tango" ]]; then + >&2 echo "/opt/lofar/tango volume does not exist!" + exit 1 +fi + # Check required support file exists if [[ ! -f "/usr/local/bin/wait-for-it.sh" ]]; then >&2 echo "/usr/local/bin/wait-for-it.sh file does not exist!" @@ -14,6 +19,11 @@ if [[ ! $TANGO_HOST ]]; then exit 1 fi +# Store directory so we can return to it after installation +CWD=$(pwd) + +cd /opt/lofar/tango || exit 1 + # Check if configured for specific version if [[ $TANGOSTATIONCONTROL ]]; then # TODO (Corne): Download version from artifacts or pypi. @@ -28,4 +38,8 @@ else sudo pip install --force-reinstall "$(ls -Art /tmp/tangostationcontrol/*.whl | tail -n 1)" fi +# Return to the stored the directory, this preserves the working_dir argument in +# docker-compose files. +cd "$CWD" || exit 1 + /usr/local/bin/wait-for-it.sh "$TANGO_HOST" --timeout=30 --strict -- "$@" diff --git a/bootstrap/etc/lofar20rc.sh b/bootstrap/etc/lofar20rc.sh index 6e4a5c9bc8d6a78c1b61cca02159ee01291d3805..4b9d806d819816a86c5fea3ab8eb59135d8edcfc 100755 --- a/bootstrap/etc/lofar20rc.sh +++ b/bootstrap/etc/lofar20rc.sh @@ -16,6 +16,7 @@ if [ ! -f "${LOFAR20_DIR}/.git/hooks/post-checkout" ]; then alias git="cp ${LOFAR20_DIR}/bin/update_submodules.sh ${LOFAR20_DIR}/.git/hooks/post-checkout; cp ${LOFAR20_DIR}/bin/update_submodules.sh ${LOFAR20_DIR}/.git/hooks/post-merge; unalias git; git" fi +# CI_BUILD_ID does not exist see https://docs.gitlab.com/ee/ci/variables/predefined_variables.html if [ ! -z ${CI_BUILD_ID+x} ]; then export CONTAINER_NAME_PREFIX=${CI_BUILD_ID}- elif [ ! -z ${CI_JOB_ID+x} ]; then diff --git a/docker-compose/.env b/docker-compose/.env index c1956e315f8cde0d48b4b5279807025bede69261..53937727c24d398a5d82b24c31f205db50064163 100644 --- a/docker-compose/.env +++ b/docker-compose/.env @@ -1,6 +1,7 @@ DOCKER_REGISTRY_HOST=artefact.skao.int DOCKER_REGISTRY_USER=ska-tango-images LOCAL_DOCKER_REGISTRY_HOST=git.astron.nl:5000 +LOCAL_DOCKER_REGISTRY_LOFAR=lofar2.0 LOCAL_DOCKER_REGISTRY_USER=lofar2.0/tango TANGO_ARCHIVER_VERSION=2021-05-28 diff --git a/docker-compose/Makefile b/docker-compose/Makefile index d85ff1df88d91db097bdd22b060cfc03b681a04f..6c6e3c888eaa77b7c23ad0fca6ec0d94a8099bf7 100644 --- a/docker-compose/Makefile +++ b/docker-compose/Makefile @@ -33,6 +33,8 @@ else ifeq (stop,$(firstword $(MAKECMDGOALS))) SERVICE_TARGET = true else ifeq (restart,$(firstword $(MAKECMDGOALS))) SERVICE_TARGET = true +else ifeq (up,$(firstword $(MAKECMDGOALS))) + SERVICE_TARGET = true else ifeq (build,$(firstword $(MAKECMDGOALS))) SERVICE_TARGET = true else ifeq (build-nocache,$(firstword $(MAKECMDGOALS))) @@ -143,8 +145,8 @@ build-nocache: ## rebuild images from scratch $(DOCKER_COMPOSE_ARGS) docker-compose -f lofar-device-base.yml -f networks.yml build --progress=plain $(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) build --no-cache --progress=plain $(SERVICE) -up: minimal ## start the base TANGO system and prepare all services - $(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) up --no-start --no-recreate +up: minimal ## start the base TANGO system and prepare requested services + $(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) up --no-start --no-recreate $(SERVICE) down: ## stop all services and tear down the system $(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) down diff --git a/docker-compose/README.md b/docker-compose/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d76a75b0c79dad1574d9d24912a671a39998abee --- /dev/null +++ b/docker-compose/README.md @@ -0,0 +1,69 @@ +# Docker Compose + +Documentation on how the LOFAR station control software utilizes docker-compose. +This documentation is intended for developers listing strategies and their +respective advantages and disadvantages. In addition, this documentation +contains developer expectations that they should uphold to. + +## Image tagging and change detection + +Preventing unnecessary builds of docker images reduces build times and increases +iteration speed. In order to achieve this the project requires properly tagged +images and mechanisms for change detection. + +For change detection the system relies on git. Git is used to determine the +directories and files that have changes between the current and previous commit. +All image related change detection mechanisms are based on this difference. + +Using docker cache within the dind service is impractical see: +https://gitlab.com/gitlab-org/gitlab-foss/-/issues/17861 + +### Types of containers and specific strategies. + +- Devices +- Simulators +- Base images +- Services + +Devices, these are detected by changes to the .yml file or directory of the +respective service inside the docker-compose directory. + +Simulators, Since the source code for simulators is maintained by other teams +we can not accurately determine from our repository if the simulator has +changed. Instead, the images is build by the teams their respective CI +pipelines. We simply pull these images as base images. + +Base images, these are detected by changes to the .env file in the +docker-compose directory. When changed they will be downloaded from the remote +registry and uploaded to our own using matching tags. + +Services, same mechanism as devices. + +### Setup and maintenance + +All behavioral logic to orchestrate change detection and image pushing can be +found in the sbin/tag_and_push_docker_images.sh script as well as the +.gitlab-ci.yml. The shell script relies on the fact that each .yml file in the +docker-compose directory corresponds to one image. + +### Gitlab CI phases + +Docker images are managed in three phases. First is remote image storing, second +is image building and change detection with finally image pulling. + +Remote images are downloaded and stored on the local registry when the .env +file for docker-compose has changes. + +Local images are build when either the files in the base context directory +change or if the docker compose file itself has changes. See the gitlab-ci.yml +for how these changes are detected. All local images will be rebuild and tagged +latest when a tagged commit is pushed to master. + +Local images download the latest image from the registry as cache unless it is +a tagged commit on master. + +Finally, the integration test downloads all images from the registry either +tagged with the current pipeline or with latest. Should both tags be unavailable +than the integration test fails. Not all images are needed for the integration +test. See sbin/tag_and_push_docker_image.sh for how these images are +differentiated. diff --git a/docker-compose/apsct-sim.yml b/docker-compose/apsct-sim.yml index d30f5a026f734bb72ee91c7bf533df677f37ca88..b9742fdb97ec3f30026d441c668a13732013201e 100644 --- a/docker-compose/apsct-sim.yml +++ b/docker-compose/apsct-sim.yml @@ -10,6 +10,9 @@ services: apsct-sim: build: context: pypcc-sim-base + args: + - LOCAL_DOCKER_REGISTRY_HOST=${LOCAL_DOCKER_REGISTRY_HOST} + - LOCAL_DOCKER_REGISTRY_LOFAR=${LOCAL_DOCKER_REGISTRY_LOFAR} container_name: ${CONTAINER_NAME_PREFIX}apsct-sim networks: - control diff --git a/docker-compose/apspu-sim.yml b/docker-compose/apspu-sim.yml index d3fc5fa04f6ce0d6ddfe4c8f87887ab7500720e3..f5677048fbe1fe28082b219177bc67a2986c31fe 100644 --- a/docker-compose/apspu-sim.yml +++ b/docker-compose/apspu-sim.yml @@ -10,6 +10,9 @@ services: apspu-sim: build: context: pypcc-sim-base + args: + - LOCAL_DOCKER_REGISTRY_HOST=${LOCAL_DOCKER_REGISTRY_HOST} + - LOCAL_DOCKER_REGISTRY_LOFAR=${LOCAL_DOCKER_REGISTRY_LOFAR} container_name: ${CONTAINER_NAME_PREFIX}apspu-sim networks: - control diff --git a/docker-compose/archiver.yml b/docker-compose/archiver.yml index f1f2a1ec65dd4259b99e675c43cb7500862049f4..12ec2d88959fca75b047cff6004dd6e2b22c294a 100644 --- a/docker-compose/archiver.yml +++ b/docker-compose/archiver.yml @@ -94,7 +94,7 @@ services: tag: "{{.Name}}" dsconfig: - image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-dsconfig:${TANGO_DSCONFIG_VERSION} + image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-dsconfig:${TANGO_DSCONFIG_VERSION} container_name: ${CONTAINER_NAME_PREFIX}dsconfig networks: - control diff --git a/docker-compose/astor.yml b/docker-compose/astor.yml index 7010a82afa2fbcf5cb3dd797bda384bb516354f8..502472fc4eedd022388cd13d76e74135e00ff3db 100644 --- a/docker-compose/astor.yml +++ b/docker-compose/astor.yml @@ -13,7 +13,7 @@ version: '2' services: astor: - image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-java:${TANGO_JAVA_VERSION} + image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-java:${TANGO_JAVA_VERSION} container_name: ${CONTAINER_NAME_PREFIX}astor networks: - control diff --git a/docker-compose/device-apsct.yml b/docker-compose/device-apsct.yml index 60f65fc47ed81822242282fc743846221acec2d9..0e258fecdb3a96c3a73714ae2c28cf2e847457a1 100644 --- a/docker-compose/device-apsct.yml +++ b/docker-compose/device-apsct.yml @@ -20,7 +20,7 @@ services: build: context: lofar-device-base args: - SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION} + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION} container_name: ${CONTAINER_NAME_PREFIX}device-apsct networks: - control diff --git a/docker-compose/device-apspu.yml b/docker-compose/device-apspu.yml index b694b09518215e293d19e1ff551f4f608e6f818d..5f325b19fb357e83ab3d35e3acfa1a5cbbb2896a 100644 --- a/docker-compose/device-apspu.yml +++ b/docker-compose/device-apspu.yml @@ -20,7 +20,7 @@ services: build: context: lofar-device-base args: - SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION} + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION} container_name: ${CONTAINER_NAME_PREFIX}device-apspu networks: - control diff --git a/docker-compose/device-beam.yml b/docker-compose/device-beam.yml new file mode 100644 index 0000000000000000000000000000000000000000..97385f16492ec123044033713d0c7b835d2062fd --- /dev/null +++ b/docker-compose/device-beam.yml @@ -0,0 +1,33 @@ +# +# Requires: +# - lofar-device-base.yml +# +version: '2' + +services: + device-beam: + image: device-beam + # build explicitly, as docker-compose does not understand a local image + # being shared among services. + build: + context: lofar-device-base + args: + SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION} + container_name: ${CONTAINER_NAME_PREFIX}device-beam + networks: + - control + ports: + - "5711:5711" # unique port for this DS + extra_hosts: + - "host.docker.internal:host-gateway" + volumes: + - ..:/opt/lofar/tango:rw + environment: + - TANGO_HOST=${TANGO_HOST} + working_dir: /opt/lofar/tango + entrypoint: + - bin/start-ds.sh + # configure CORBA to _listen_ on 0:port, but tell others we're _reachable_ through ${HOSTNAME}:port, since CORBA + # can't know about our Docker port forwarding + - l2ss-beam Beam STAT -v -ORBendPoint giop:tcp:0:5711 -ORBendPointPublish giop:tcp:${HOSTNAME}:5711 + restart: unless-stopped diff --git a/docker-compose/device-boot.yml b/docker-compose/device-boot.yml index 3db111410fafde9901fd8f91cb40a1c3560e4242..330cb723ed3bb5ee8ccd50bf4cb933da4e1fe09c 100644 --- a/docker-compose/device-boot.yml +++ b/docker-compose/device-boot.yml @@ -19,7 +19,7 @@ services: build: context: lofar-device-base args: - SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION} + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION} container_name: ${CONTAINER_NAME_PREFIX}device-boot networks: - control diff --git a/docker-compose/device-docker.yml b/docker-compose/device-docker.yml index 5a2641e9871f163f27ed7a60d872d30d4fe855e1..a9e4ccfdd6f66eda66f05ea5244fcf0fd732a382 100644 --- a/docker-compose/device-docker.yml +++ b/docker-compose/device-docker.yml @@ -20,7 +20,7 @@ services: build: context: lofar-device-base args: - SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION} + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION} container_name: ${CONTAINER_NAME_PREFIX}device-docker networks: - control diff --git a/docker-compose/device-observation_control.yml b/docker-compose/device-observation_control.yml index 33fb0d066fd76b8eb4a9c7753266f16d04157726..d4f6f15d1f4eb80d02cd0c5738dc0a011b9dfc72 100644 --- a/docker-compose/device-observation_control.yml +++ b/docker-compose/device-observation_control.yml @@ -19,7 +19,7 @@ services: build: context: lofar-device-base args: - SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION} + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION} container_name: ${CONTAINER_NAME_PREFIX}device-observation_control networks: - control diff --git a/docker-compose/device-recv.yml b/docker-compose/device-recv.yml index a08f566e7b39e095403f00cb5b086420b689d66b..25e767726f139ff532dbe649ccb230fabbec0602 100644 --- a/docker-compose/device-recv.yml +++ b/docker-compose/device-recv.yml @@ -20,7 +20,7 @@ services: build: context: lofar-device-base args: - SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION} + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION} container_name: ${CONTAINER_NAME_PREFIX}device-recv networks: - control diff --git a/docker-compose/device-sdp.yml b/docker-compose/device-sdp.yml index f32c34394475c6a7483cb98cd03def1f62cf9ff0..06a523f606d67811986bd7a13b9a3202cb74e91d 100644 --- a/docker-compose/device-sdp.yml +++ b/docker-compose/device-sdp.yml @@ -20,7 +20,7 @@ services: build: context: lofar-device-base args: - SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION} + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION} container_name: ${CONTAINER_NAME_PREFIX}device-sdp networks: - control diff --git a/docker-compose/device-sst.yml b/docker-compose/device-sst.yml index 7464cb01f45e584ab705fe9098e0229a1b762295..86651c7878d844646528b41fb0969dfd19af6eea 100644 --- a/docker-compose/device-sst.yml +++ b/docker-compose/device-sst.yml @@ -20,7 +20,7 @@ services: build: context: lofar-device-base args: - SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION} + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION} container_name: ${CONTAINER_NAME_PREFIX}device-sst networks: - control diff --git a/docker-compose/device-unb2.yml b/docker-compose/device-unb2.yml index af1329d21a905f3c150c092529978e17f0c0ee37..2b9b47146a405440ebd36fd84162935fb6b8a56d 100644 --- a/docker-compose/device-unb2.yml +++ b/docker-compose/device-unb2.yml @@ -20,7 +20,7 @@ services: build: context: lofar-device-base args: - SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION} + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION} container_name: ${CONTAINER_NAME_PREFIX}device-unb2 networks: - control diff --git a/docker-compose/device-xst.yml b/docker-compose/device-xst.yml index c4ea684fd94e34fcaaa857a5717ca47745eccc72..54ca5a21f911084160d2cec772df06da55ef5cf1 100644 --- a/docker-compose/device-xst.yml +++ b/docker-compose/device-xst.yml @@ -20,7 +20,7 @@ services: build: context: lofar-device-base args: - SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION} + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION} container_name: ${CONTAINER_NAME_PREFIX}device-xst networks: - control diff --git a/docker-compose/elk.yml b/docker-compose/elk.yml index 67f13baee061a74ebd08320f1e9f2f9f3e72f646..25bb1b218669baebff50ddc830b049b691349f71 100644 --- a/docker-compose/elk.yml +++ b/docker-compose/elk.yml @@ -6,6 +6,7 @@ # - elk-configure-host: Configures the hosts's kernel to be able to use the ELK stack # - elk: ELK stack # + version: '2' volumes: diff --git a/docker-compose/grafana.yml b/docker-compose/grafana.yml index 29c93c52c4dc05849aad10fabac12712c12dd4d7..f298db2746961b7d30d2e147192d0dfc58530725 100644 --- a/docker-compose/grafana.yml +++ b/docker-compose/grafana.yml @@ -4,6 +4,7 @@ # Defines: # - grafana: Grafana # + version: '2' #volumes: diff --git a/docker-compose/integration-test.yml b/docker-compose/integration-test.yml index defb45e3c3183516131795b283372ca784635d8c..e2be9144ef7d73b7108609a917529c019e109c62 100644 --- a/docker-compose/integration-test.yml +++ b/docker-compose/integration-test.yml @@ -11,7 +11,7 @@ services: build: context: itango args: - SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION} + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION} container_name: ${CONTAINER_NAME_PREFIX}integration-test networks: - control diff --git a/docker-compose/itango.yml b/docker-compose/itango.yml index 9b01c4ea25e2abc5849c9a98c29cc7601ba1115f..02d6801bd8a2f748a4b3d3336352891c78d4882b 100644 --- a/docker-compose/itango.yml +++ b/docker-compose/itango.yml @@ -17,7 +17,7 @@ services: build: context: itango args: - SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION} + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION} container_name: ${CONTAINER_NAME_PREFIX}itango networks: - control diff --git a/docker-compose/jive.yml b/docker-compose/jive.yml index 456ae1fc96771bad1ab6b99e52e3b0c9c046c20c..5a2caea9a1d9d6fb19d235781abc33a3230412e8 100644 --- a/docker-compose/jive.yml +++ b/docker-compose/jive.yml @@ -18,7 +18,7 @@ version: '2' services: jive: - image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-java:${TANGO_JAVA_VERSION} + image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-java:${TANGO_JAVA_VERSION} container_name: ${CONTAINER_NAME_PREFIX}jive network_mode: host volumes: diff --git a/docker-compose/jupyter.yml b/docker-compose/jupyter.yml index 1e1deea6f0e22299544f988602efc676bbe6200c..bbc20f269f8a44acff3ce9f36bf11eeef17cea8f 100644 --- a/docker-compose/jupyter.yml +++ b/docker-compose/jupyter.yml @@ -7,6 +7,7 @@ # Defines: # - jupyter: Jupyter Notebook with iTango support # + version: '2' services: @@ -15,7 +16,7 @@ services: context: jupyter args: CONTAINER_EXECUTION_UID: ${CONTAINER_EXECUTION_UID} - SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION} + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION} container_name: ${CONTAINER_NAME_PREFIX}jupyter networks: - control @@ -30,10 +31,6 @@ services: user: ${CONTAINER_EXECUTION_UID} working_dir: /jupyter-notebooks entrypoint: - - /usr/local/bin/wait-for-it.sh - - ${TANGO_HOST} - - --timeout=30 - - --strict - - -- + - /opt/lofar/tango/bin/start-ds.sh - /usr/bin/tini -- /usr/local/bin/jupyter-notebook --port=8888 --no-browser --ip=0.0.0.0 --allow-root --NotebookApp.token= --NotebookApp.password= restart: unless-stopped diff --git a/docker-compose/jupyter/Dockerfile b/docker-compose/jupyter/Dockerfile index 122a2d83f29591e6dea88f0ec53e8f8b1d96ced8..cc1652e4a45bc14805632ec1d4056beaab1fd34c 100644 --- a/docker-compose/jupyter/Dockerfile +++ b/docker-compose/jupyter/Dockerfile @@ -10,23 +10,13 @@ ENV HOME=/home/user RUN sudo mkdir -p ${HOME} RUN sudo chown ${CONTAINER_EXECUTION_UID} -R ${HOME} -# ipython 7.28 is broken in combination with Jupyter, it causes connection errors with notebooks -RUN sudo pip3 install ipython==7.27.0 - -RUN sudo pip3 install jupyter -RUN sudo pip3 install ipykernel -RUN sudo pip3 install jupyter_bokeh -# Install matplotlib, jupyterplot -RUN sudo pip3 install matplotlib jupyterplot - -# Allow Download as -> PDF via html -RUN sudo pip3 install nbconvert -RUN sudo pip3 install notebook-as-pdf +COPY requirements.txt ./ +RUN sudo pip3 install -r requirements.txt # see https://github.com/jupyter/nbconvert/issues/1434 RUN sudo bash -c "echo DEFAULT_ARGS += [\\\"--no-sandbox\\\"] >> /usr/local/lib/python3.7/dist-packages/pyppeteer/launcher.py" RUN sudo apt-get update -y -RUN sudo apt-get install -y gconf-service libasound2 libatk1.0-0 libatk-bridge2.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget libcairo-gobject2 libxinerama1 libgtk2.0-0 libpangoft2-1.0-0 libthai0 libpixman-1-0 libxcb-render0 libharfbuzz0b libdatrie1 libgraphite2-3 libgbm1 +RUN sudo apt-get install -y git gconf-service libasound2 libatk1.0-0 libatk-bridge2.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget libcairo-gobject2 libxinerama1 libgtk2.0-0 libpangoft2-1.0-0 libthai0 libpixman-1-0 libxcb-render0 libharfbuzz0b libdatrie1 libgraphite2-3 libgbm1 # Allow Download as -> PDF via LaTeX RUN sudo apt-get install -y texlive-xetex texlive-fonts-recommended texlive-latex-recommended @@ -43,15 +33,8 @@ RUN sudo chown ${CONTAINER_EXECUTION_UID} -R /opt/ipython-profiles COPY jupyter-kernels /usr/local/share/jupyter/kernels/ # Install patched jupyter executable -RUN sudo pip3 install python-logstash-async COPY jupyter-notebook /usr/local/bin/jupyter-notebook -#Install further python modules -RUN sudo pip3 install PyMySQL[rsa] psycopg2-binary sqlalchemy - -# Packages to interface with testing hardware directly -RUN sudo pip3 install pyvisa pyvisa-py opcua - # Add Tini. Tini operates as a process subreaper for jupyter. This prevents kernel crashes. ENV TINI_VERSION v0.6.0 ENV JUPYTER_RUNTIME_DIR=/tmp diff --git a/docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py b/docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py index 74692f10fff62684c21047eb373aebd70a876155..03e7b75b4649c79647c5bb573b22e41eb330c159 100644 --- a/docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py +++ b/docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py @@ -7,7 +7,8 @@ sst = DeviceProxy("STAT/SST/1") xst = DeviceProxy("STAT/XST/1") unb2 = DeviceProxy("STAT/UNB2/1") boot = DeviceProxy("STAT/Boot/1") +beam = DeviceProxy("STAT/Beam/1") docker = DeviceProxy("STAT/Docker/1") # Put them in a list in case one wants to iterate -devices = [apsct, apspu, recv, sdp, sst, xst, unb2, boot, docker] +devices = [apsct, apspu, recv, sdp, sst, xst, unb2, boot, beam, docker] diff --git a/docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/02-stationcontrol.py b/docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/02-stationcontrol.py new file mode 100644 index 0000000000000000000000000000000000000000..d2d810b1ad5915504e4a5b508e9a128bd2981628 --- /dev/null +++ b/docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/02-stationcontrol.py @@ -0,0 +1 @@ +from tangostationcontrol import * \ No newline at end of file diff --git a/docker-compose/jupyter/requirements.txt b/docker-compose/jupyter/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..3be9c3d4c8f1052920b577d77442a348c09aa7ca --- /dev/null +++ b/docker-compose/jupyter/requirements.txt @@ -0,0 +1,15 @@ +GitPython >= 3.1.24 # BSD +ipython >=7.27.0,!=7.28.0 # BSD +jupyter +ipykernel +jupyter_bokeh +matplotlib +jupyterplot +nbconvert +notebook-as-pdf +python-logstash-async +PyMySQL[rsa] +sqlalchemy +pyvisa +pyvisa-py +opcua \ No newline at end of file diff --git a/docker-compose/lofar-device-base.yml b/docker-compose/lofar-device-base.yml index ce110ed85ba0cfb20b607ab7d08e70505d2392e8..f01faac2d2f41647708229106a895d3dad23c3e4 100644 --- a/docker-compose/lofar-device-base.yml +++ b/docker-compose/lofar-device-base.yml @@ -10,6 +10,7 @@ # Requires: # - tango.yml # + version: '2' services: @@ -18,7 +19,7 @@ services: build: context: lofar-device-base args: - SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION} + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION} container_name: ${CONTAINER_NAME_PREFIX}lofar-device-base # These parameters are just visual queues, you have to define them again # in derived docker-compose files! diff --git a/docker-compose/lofar-device-base/lofar-requirements.txt b/docker-compose/lofar-device-base/lofar-requirements.txt index 10ad55d977c97793a352c13da323d84d3c826c0e..95ed439cd121c0dc72b0c9a1c69d409e0bacc57e 100644 --- a/docker-compose/lofar-device-base/lofar-requirements.txt +++ b/docker-compose/lofar-device-base/lofar-requirements.txt @@ -1,5 +1,2 @@ -# Do not put tangostationcontrol dependencies here -astropy - -# requirements to build tangocontrol +# Do not put tangostationcontrol dependencies here, only setup.py / __init__.py GitPython >= 3.1.24 # BSD diff --git a/docker-compose/logviewer.yml b/docker-compose/logviewer.yml index bf0c9b2d51cbdb7334a579184114de6925fd37a1..08da4000b23925980e1683465fa4fdd4c05f04ae 100644 --- a/docker-compose/logviewer.yml +++ b/docker-compose/logviewer.yml @@ -12,7 +12,7 @@ version: '2' services: logviewer: - image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-java:${TANGO_JAVA_VERSION} + image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-java:${TANGO_JAVA_VERSION} container_name: ${CONTAINER_NAME_PREFIX}logviewer networks: - control diff --git a/docker-compose/pogo.yml b/docker-compose/pogo.yml index 826daac9fbd6ef3226a690832eedab505bbeaba3..954841746b9f0338d4a84fdae7e043fde04be460 100644 --- a/docker-compose/pogo.yml +++ b/docker-compose/pogo.yml @@ -20,7 +20,7 @@ volumes: services: pogo: - image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-pogo:${TANGO_POGO_VERSION} + image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-pogo:${TANGO_POGO_VERSION} container_name: ${CONTAINER_NAME_PREFIX}pogo networks: - control diff --git a/docker-compose/prometheus.yml b/docker-compose/prometheus.yml index 604f4bf4bde93dd6d68aaf7f3b1da2fd3f884e83..1e9ce6f1aa2cd050565f48a4b991865641fd1566 100644 --- a/docker-compose/prometheus.yml +++ b/docker-compose/prometheus.yml @@ -4,6 +4,7 @@ # Defines: # - prometheus: Prometheus # + version: '2' services: diff --git a/docker-compose/pypcc-sim-base/Dockerfile b/docker-compose/pypcc-sim-base/Dockerfile index c65c5b6f836e889f9b3c364ceace5f7b9b821628..f0f37dec5613b988ba3c471428aa426606cf9d5a 100644 --- a/docker-compose/pypcc-sim-base/Dockerfile +++ b/docker-compose/pypcc-sim-base/Dockerfile @@ -1,10 +1,6 @@ -FROM ubuntu:20.04 +ARG LOCAL_DOCKER_REGISTRY_HOST +ARG LOCAL_DOCKER_REGISTRY_LOFAR -COPY requirements.txt /requirements.txt +FROM ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_LOFAR}/pypcc:latest -RUN apt-get update && apt-get install -y python3 python3-pip python3-yaml git && \ - pip3 install -r requirements.txt && \ - git clone --depth 1 --branch master https://git.astron.nl/lofar2.0/pypcc - -WORKDIR /pypcc -CMD ["python3","pypcc2.py","--simulator","--port","4843"] +CMD ["python3", "pypcc2.py", "--simulator", "--port","4843"] diff --git a/docker-compose/pypcc-sim-base/requirements.txt b/docker-compose/pypcc-sim-base/requirements.txt deleted file mode 100644 index 2cd015945c044fcd1e39a823f49a807fc519ac67..0000000000000000000000000000000000000000 --- a/docker-compose/pypcc-sim-base/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -opcua -numpy -recordclass>=0.16,<0.16.1 \ No newline at end of file diff --git a/docker-compose/recv-sim.yml b/docker-compose/recv-sim.yml index effee8b298b855e2007e50c379fa3df45010bd05..8fd795be60ef89b23491895dd9809ff67b1c67ae 100644 --- a/docker-compose/recv-sim.yml +++ b/docker-compose/recv-sim.yml @@ -10,6 +10,9 @@ services: recv-sim: build: context: pypcc-sim-base + args: + - LOCAL_DOCKER_REGISTRY_HOST=${LOCAL_DOCKER_REGISTRY_HOST} + - LOCAL_DOCKER_REGISTRY_LOFAR=${LOCAL_DOCKER_REGISTRY_LOFAR} container_name: ${CONTAINER_NAME_PREFIX}recv-sim networks: - control diff --git a/docker-compose/rest.yml b/docker-compose/rest.yml index 467319399d6f3fec12a74068fea182195014b59e..94e1168455ddfefa20796c352e92d27e07f9a115 100644 --- a/docker-compose/rest.yml +++ b/docker-compose/rest.yml @@ -13,7 +13,7 @@ version: '2' services: rest: - image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-rest:${TANGO_REST_VERSION} + image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-rest:${TANGO_REST_VERSION} container_name: ${CONTAINER_NAME_PREFIX}tango-rest networks: - control diff --git a/docker-compose/sdptr-sim.yml b/docker-compose/sdptr-sim.yml index c81c3db9ae4744e013b5a92f1ebb5e9bdaa6e92c..badf707e37621c8b3030121424bacd1393910b87 100644 --- a/docker-compose/sdptr-sim.yml +++ b/docker-compose/sdptr-sim.yml @@ -10,6 +10,9 @@ services: sdptr-sim: build: context: sdptr-sim + args: + - LOCAL_DOCKER_REGISTRY_HOST=${LOCAL_DOCKER_REGISTRY_HOST} + - LOCAL_DOCKER_REGISTRY_LOFAR=${LOCAL_DOCKER_REGISTRY_LOFAR} container_name: ${CONTAINER_NAME_PREFIX}sdptr-sim networks: - control diff --git a/docker-compose/sdptr-sim/Dockerfile b/docker-compose/sdptr-sim/Dockerfile index 57fe98141f180a4d15a1e2d87c2c67be8f5894ff..4e64ca2a67229e602a705c9e61b0de999e64fad4 100644 --- a/docker-compose/sdptr-sim/Dockerfile +++ b/docker-compose/sdptr-sim/Dockerfile @@ -1,20 +1,7 @@ -FROM ubuntu:20.04 +ARG LOCAL_DOCKER_REGISTRY_HOST +ARG LOCAL_DOCKER_REGISTRY_LOFAR -# Install build tools for sdptr and the C language OPC-UA lib -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common && \ - DEBIAN_FRONTEND=noninteractive add-apt-repository ppa:open62541-team/ppa && \ - apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y autoconf automake git make g++ build-essential pkg-config libboost-dev libboost-regex-dev libboost-system-dev libboost-program-options-dev libopen62541-1-dev libopen62541-1-tools && \ - apt-get clean - -# Install SDPTR -RUN cd / && git clone --depth 1 --branch master https://git.astron.nl/lofar2.0/sdptr - -RUN cd /sdptr && \ - autoreconf -v -f -i && \ - ./configure && \ - bash -c "make -j `nproc` install" +FROM ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_LOFAR}/sdptr:latest COPY simulator.conf /sdptr/src/simulator.conf diff --git a/docker-compose/tango.yml b/docker-compose/tango.yml index 937cc5c8ecbe00b553d4692988e6cc2e5d7c51ef..19500fca1eeba859f74e7ba54fc3cbb021ea0ce6 100644 --- a/docker-compose/tango.yml +++ b/docker-compose/tango.yml @@ -15,7 +15,7 @@ volumes: services: tangodb: - image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-db:${TANGO_DB_VERSION} + image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-db:${TANGO_DB_VERSION} container_name: ${CONTAINER_NAME_PREFIX}tangodb networks: - control @@ -37,7 +37,7 @@ services: restart: unless-stopped databaseds: - image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-cpp:${TANGO_CPP_VERSION} + image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-cpp:${TANGO_CPP_VERSION} container_name: ${CONTAINER_NAME_PREFIX}databaseds networks: - control diff --git a/docker-compose/tangotest.yml b/docker-compose/tangotest.yml index 357c91df487b51379db221f7cb984bc05018f5e3..a97290d48f437b1c65b0bef01f6788fb525b2275 100644 --- a/docker-compose/tangotest.yml +++ b/docker-compose/tangotest.yml @@ -11,7 +11,7 @@ version: '2' services: tangotest: - image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-java:${TANGO_JAVA_VERSION} + image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-java:${TANGO_JAVA_VERSION} container_name: ${CONTAINER_NAME_PREFIX}tangotest networks: - control diff --git a/docker-compose/unb2-sim.yml b/docker-compose/unb2-sim.yml index d1ecaaa70a3c1e52f39ab1453d2ec8eb191f8831..b01802cd0526abe325c710f08fe965d6244cb2ba 100644 --- a/docker-compose/unb2-sim.yml +++ b/docker-compose/unb2-sim.yml @@ -10,6 +10,9 @@ services: unb2-sim: build: context: pypcc-sim-base + args: + - LOCAL_DOCKER_REGISTRY_HOST=${LOCAL_DOCKER_REGISTRY_HOST} + - LOCAL_DOCKER_REGISTRY_LOFAR=${LOCAL_DOCKER_REGISTRY_LOFAR} container_name: ${CONTAINER_NAME_PREFIX}unb2-sim networks: - control diff --git a/docs/source/devices/beam.rst b/docs/source/devices/beam.rst new file mode 100644 index 0000000000000000000000000000000000000000..c469162dfe25fafc1c79cecae15c7e1b36cf47e4 --- /dev/null +++ b/docs/source/devices/beam.rst @@ -0,0 +1,8 @@ +Beam +==================== + +The ``beam == DeviceProxy("STAT/Beam/1")`` device sets up the beamforming on the station: + +- The HBA tiles in RECV need *analog beamforming* to combine their 16 antennas into a single input for the RCU, +- THe LBAs or HBA tiles need *digital beamforming* in SDP to combine their signals into beamlets. + diff --git a/docs/source/devices/sst-xst.rst b/docs/source/devices/sst-xst.rst index cdb689e457dc2d6abebcfc1391f057135f30b722..092bdda832eb0493f93956f4aec8025410d0bf9b 100644 --- a/docs/source/devices/sst-xst.rst +++ b/docs/source/devices/sst-xst.rst @@ -63,7 +63,7 @@ Typically, ``N_ant == 192``, and ``N_blocks == 136``. The metadata refers to the *blocks*, which are emitted by the FPGAs to represent the XSTs between 12 x 12 consecutive antennas. The following code converts block numbers to the indices of the first antenna pair in a block:: - from common.baselines import baseline_from_index + from tangostationcontrol.common.baselines import baseline_from_index def first_antenna_pair(block_nr: int) -> int: coarse_a, coarse_b = baseline_from_index(block_nr) @@ -71,7 +71,7 @@ The metadata refers to the *blocks*, which are emitted by the FPGAs to represent Conversely, to calculate the block index for an antenna pair ``(a,b)``, use:: - from common.baselines import baseline_index + from tangostationcontrol.common.baselines import baseline_index def block_nr(a: int, b: int) -> int: return baseline_index(a // 12, b // 12) diff --git a/docs/source/index.rst b/docs/source/index.rst index 119d95164313538606e9f310f2e65ca6aca233d8..7baab29343d3f7c4015e6ed60e6ea4692e6073e3 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -19,6 +19,7 @@ Even without having access to any LOFAR2.0 hardware, you can install the full st installation interfaces/overview devices/using + devices/beam devices/boot devices/docker devices/recv diff --git a/sbin/run_integration_test.sh b/sbin/run_integration_test.sh index c3c37983ae63688658211337ebc17e83fa7ff546..05e076ed00f79858fa44be6e23f3b94370672cb9 100755 --- a/sbin/run_integration_test.sh +++ b/sbin/run_integration_test.sh @@ -11,11 +11,15 @@ fi cd "$LOFAR20_DIR/docker-compose" || exit 1 -# Make sure builds are recent, and use our building parameters. -make build +# Build only the required images, please do not build everything that makes CI +# take really long to finish, especially grafana / jupyter / prometheus. +# jupyter is physically large > 2.5gb and overlayfs is really slow. +make build device-sdp device-recv device-sst device-unb2 device-xst +make build sdptr-sim recv-sim unb2-sim apsct-sim apspu-sim +make build databaseds dsconfig elk integration-test # Start and stop sequence -make stop device-boot device-docker device-apsct device-apspu device-sdp device-recv device-sst device-unb2 device-xst sdptr-sim recv-sim unb2-sim apsct-sim apspu-sim +make stop device-boot device-docker device-apsct device-apspu device-sdp device-recv device-sst device-unb2 device-xst device-beam sdptr-sim recv-sim unb2-sim apsct-sim apspu-sim make start databaseds dsconfig elk # Give dsconfig and databaseds time to start @@ -32,7 +36,7 @@ make start sdptr-sim recv-sim unb2-sim apsct-sim apspu-sim # Give the simulators time to start sleep 5 -make start device-boot device-apsct device-apspu device-sdp device-recv device-sst device-unb2 device-xst +make start device-boot device-apsct device-apspu device-sdp device-recv device-sst device-unb2 device-xst device-beam # Give devices time to restart # TODO(Corne Lukken): Use a nicer more reliable mechanism diff --git a/sbin/tag_and_push_docker_image.sh b/sbin/tag_and_push_docker_image.sh index 799ab1cd779bb5caf840685f339080b57916063b..631413235d4ba9b2f39b9b0e216d31cd21913bdf 100755 --- a/sbin/tag_and_push_docker_image.sh +++ b/sbin/tag_and_push_docker_image.sh @@ -1,40 +1,177 @@ #!/bin/bash -e -# Tag and push which image version? -DOCKER_TAG=latest +function usage { + echo "./$(basename "$0") + no arguments, downloads remote images and pushes these to ASTRON registry. + The versions downloaded are controlled by the docker-compose/.env file" + echo "" + echo "./$(basename "$0") -h + displays this help message" + echo "" + echo "./$(basename "$0") <docker service name> <tag> + downloads latest version of image from the ASTRON registry, builds the + specified service and pushes the image with the specified tag to the + ASTRON registry" + echo "" + echo "./$(basename "$0") pull <tag> + downloads all images for the integration test with the specified tag + falling back to 'latest' if unavailable. Should neither exist on the + ASTRON registry the script will exit 1. The images are retagged to match + the output of docker-compose." +} -# Change to git tag or git hash if no tag -VERSION=$(date +"%Y-%M-%d") +# list of arguments expected in the input +optstring=":h" -SKA_REPO="nexus.engageska-portugal.pt/ska-docker" -LOFAR_REPO="git.astron.nl:5000/lofar2.0/tango" +while getopts ${optstring} arg; do + case ${arg} in + h) + usage + exit 0 + ;; + :) + echo "$0: Must supply an argument to -$OPTARG." >&2 + exit 1 + ;; + ?) + echo "Invalid option: -${OPTARG}." + exit 2 + ;; + esac +done -# Compile a list of the SKA images -SKA_IMAGES=$(for i in $(docker images | grep -E ${DOCKER_TAG} | grep -E ${SKA_REPO} | cut -d' ' -f1); do printf "%s " "${i}"; done) +if [ -z "${LOFAR20_DIR+x}" ]; then + echo "LOFAR20_DIR not set, did you forget to source lofar20rc.sh?" + exit 1 +fi -# Compile a list of LOFAR2.0 images -LOFAR_IMAGES=$(for i in $(docker images | grep -E ${DOCKER_TAG} | grep -E -v "${SKA_REPO}|${LOFAR_REPO}" | cut -d' ' -f1); do printf "%s " "${i}"; done) +# shellcheck disable=SC1090 +. "${LOFAR20_DIR}/docker-compose/.env" || exit 1 -function tag_and_push() -{ - ( - docker tag "${1}" "${2}" - docker push "${2}" - ) & -} +# List of images and their tag +REMOTE_IMAGES=( + "tango-dsconfig:${TANGO_DSCONFIG_VERSION}" "tango-java:${TANGO_JAVA_VERSION}" + "tango-itango:${TANGO_ITANGO_VERSION}" "tango-pogo:${TANGO_POGO_VERSION}" + "tango-cpp:${TANGO_CPP_VERSION}" "tango-db:${TANGO_DB_VERSION}" + "tango-dsconfig:${TANGO_DSCONFIG_VERSION}" "tango-rest:${TANGO_REST_VERSION}" +) -# Rename the SKA images for the LOFAR2.0 repo -# and push them to the LOFAR2.0 repo -for IMAGE in ${SKA_IMAGES}; do - PUSH_IMAGE=${IMAGE//${SKA_REPO}/${LOFAR_REPO}}:${VERSION} - tag_and_push "${IMAGE}" "${PUSH_IMAGE}" -done +# If first argument of bash script not set run first stage +if [ -z "${1+x}" ]; then + echo "Pulling and retagging remote images" -# Rename the LOFAR2.0 images for the LOFAR2.0 repo -# and push them to the LOFAR2.0 repo -for IMAGE in ${LOFAR_IMAGES}; do - PUSH_IMAGE=${LOFAR_REPO}/${IMAGE}:${VERSION} - tag_and_push "${IMAGE}" "${PUSH_IMAGE}" -done + # Iterate over al the REMOTE_IMAGES and pull them from remote and push local + for image in "${REMOTE_IMAGES[@]}"; do + remote_url="${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-${image}" + local_url="${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/${image}" + docker pull "${remote_url}" + docker tag "${remote_url}" "${local_url}" + docker push "${local_url}" + done + + exit 0 +fi + +# Triple tuple of docker-compose names, image names and if necessary for +# integration tests. +# TODO(Corne): Have this list generated from the .yml files +LOCAL_IMAGES=( + "elk elk y" "elk-configure-host elk-configure-host y" + "lofar-device-base lofar-device-base y" + + "apsct-sim docker-compose_apsct-sim y" "apspu-sim docker-compose_apspu-sim y" + "recv-sim docker-compose_recv-sim y" "sdptr-sim docker-compose_sdptr-sim y" + "unb2-sim docker-compose_unb2-sim y" + + "device-apsct device-apsct y" "device-apspu device-apspu y" + "device-boot device-boot y" "device-docker device-docker y" + "device-observation_control device-observation_control y" + "device-recv device-recv y" "device-sdp device-sdp y" + "device-sst device-sst y" "device-unb2 device-unb2 y" + "device-xst device-xst y" + + "itango docker-compose_itango y" + + "grafana grafana n" "prometheus prometheus n" + "jupyter docker-compose_jupyter n" + "integration-test docker-compose_integration-test n" + "tango-prometheus-exporter docker-compose_tango-prometheus-exporter n" +) + + + +# If first argument set run second stage, determine LOCAL_IMAGE to build and +# push from the argument +if [ ! -z "${1+x}" ] && [ "${1}" != "pull" ]; then + + # The second argument must pass the tag variable must be set + if [ -z "${2+x}" ]; then + echo "Error, second argument must pass tag variable" + exit 1 + fi + + # Set the tag and image variable, variables $1 and $2 are shadowed later + local_image="${1}" + tag="${2}" + + cd "${LOFAR20_DIR}/docker-compose" || exit 1 + + # Loop through images and find the specified one + for image in "${LOCAL_IMAGES[@]}"; do + # Set, splits tuple into $1 and $2. this shadows previous variables + # shellcheck disable=SC2086 + set -- $image + if [ "${local_image}" == "${1}" ]; then + echo "Building image for ${1} container" + local_url="${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/${2}" + + # If tag is not latest, than it is not a tagged master build and we can + # pull the latest image as cache. + if [ "${tag}" != "latest" ]; then + docker pull "${local_url}:latest" + fi + + make build "${1}" + docker tag "${2}" "${local_url}:${tag}" + docker push "${local_url}:${tag}" + fi + done + + exit 0 +fi + +# Final stage, pull images for integration cache try special tag image first +# if it fails download latest instead +if [ ! -z "${1+x}" ] && [ "${1}" == "pull" ]; then + echo "Pulling images for integration test cache" + + # The second argument must pass the tag variable must be set + if [ -z "${2+x}" ]; then + echo "Error, second argument must pass tag variable" + exit 1 + fi + + # Set the tag variable + tag="${2}" + + for image in "${LOCAL_IMAGES[@]}"; do + # Set, splits tuple into $1 and $2. this shadows previous variables + # shellcheck disable=SC2086 + set -- $image + + # Only download images which are needed for integration test + if [ "${3}" == "y" ]; then + local_url="${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/${2}" + # Pull images, at least one of the two images must succeed + echo "docker pull ${local_url}:${tag}" + docker pull "${local_url}:${tag}" || docker pull "${local_url}:latest" || exit 1 + # Ensure the images will have the same tags as generated by docker-compose + docker tag "${local_url}:${tag}" "${2}" || docker tag "${local_url}:latest" "${2}" || exit 1 + fi + done + + exit 0 +fi -wait +# Somehow nothing ran, that is an error do not fail silently +exit 1 diff --git a/tangostationcontrol/requirements.txt b/tangostationcontrol/requirements.txt index 359d803b6d959d5029b51580bcb36411fc420c48..f2d3743cd07dcea9c3d88255f9a583dfe67f714b 100644 --- a/tangostationcontrol/requirements.txt +++ b/tangostationcontrol/requirements.txt @@ -6,7 +6,6 @@ asyncua >= 0.9.90 # LGPLv3 PyMySQL[rsa] >= 1.0.2 # MIT psycopg2-binary >= 2.9.2 #LGPL sqlalchemy >= 1.4.26 #MIT -GitPython >= 3.1.24 # BSD snmp >= 0.1.7 # GPL3 h5py >= 3.5.0 # BSD psutil >= 5.8.0 # BSD diff --git a/tangostationcontrol/setup.cfg b/tangostationcontrol/setup.cfg index e6101c890893c961e1eff27ce3d3ab4e019ea312..31662e40d639c2af2de417e734e794dc5679d4a4 100644 --- a/tangostationcontrol/setup.cfg +++ b/tangostationcontrol/setup.cfg @@ -36,6 +36,7 @@ where=./ console_scripts = l2ss-apsct = tangostationcontrol.devices.apsct:main l2ss-apspu = tangostationcontrol.devices.apspu:main + l2ss-beam = tangostationcontrol.devices.beam:main l2ss-boot = tangostationcontrol.devices.boot:main l2ss-docker-device = tangostationcontrol.devices.docker_device:main l2ss-observation = tangostationcontrol.devices.observation:main diff --git a/tangostationcontrol/tangostationcontrol/devices/README.md b/tangostationcontrol/tangostationcontrol/devices/README.md index 0c87cc56c040ce971ac012fcee534a61bb711ecf..f546fb339077c328e55dfd359c8cb23ccfbd64d1 100644 --- a/tangostationcontrol/tangostationcontrol/devices/README.md +++ b/tangostationcontrol/tangostationcontrol/devices/README.md @@ -10,6 +10,9 @@ If a new device is added, it will (likely) need to be referenced in several plac - Adjust `docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py` to make an alias for it available in Jupyter, - Adjust `tangostationcontrol/tangostationcontrol/devices/boot.py` to add the device to the station initialisation sequence, - Add to `docker-compose/` to create a YaML file to start the device in a docker container. NOTE: it needs a unique 57xx port assigned, +- Adjust `tangostationcontrol/setup.cfg` to add an entry point for the device in the package installation, - Add to `tangostationcontrol/tangostationcontrol/integration_test/devices/` to add an integration test, - Adjust `sbin/run_integration_test.sh` to have the device started when running the integration tests, - Add to `docs/source/devices/` to mention the device in the end-user documentation. +- Adjust `docs/source/index.rst` to include the newly created file in `docs/source/devices/`. + diff --git a/tangostationcontrol/tangostationcontrol/devices/beam.py b/tangostationcontrol/tangostationcontrol/devices/beam.py new file mode 100644 index 0000000000000000000000000000000000000000..1711a8a89f9db0e73cc4bf2fd38e81a7a95ded01 --- /dev/null +++ b/tangostationcontrol/tangostationcontrol/devices/beam.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# +# Distributed under the terms of the APACHE license. +# See LICENSE.txt for more info. + +""" Beam Device Server for LOFAR2.0 + +""" + +# PyTango imports +from tango import AttrWriteType +import numpy +# Additional import + +from tangostationcontrol.devices.device_decorators import * +from tangostationcontrol.common.entrypoint import entry +from tangostationcontrol.clients.attribute_wrapper import attribute_wrapper +from tangostationcontrol.devices.lofar_device import lofar_device +from tangostationcontrol.common.lofar_logging import device_logging_to_python, log_exceptions + +__all__ = ["Beam", "main"] + + +@device_logging_to_python() +class Beam(lofar_device): + # ----------------- + # Device Properties + # ----------------- + + # ---------- + # Attributes + # ---------- + + pass + + # -------- + # overloaded functions + # -------- + + + # -------- + # Commands + # -------- + + +# ---------- +# Run server +# ---------- +def main(**kwargs): + """Main function of the ObservationControl module.""" + return entry(Beam, **kwargs) diff --git a/tangostationcontrol/tangostationcontrol/devices/boot.py b/tangostationcontrol/tangostationcontrol/devices/boot.py index c4df70bf5027a8068a1e58326c5ad442e63ad4fc..5f2e9b76e47a9ed61207a07fbc0584ec355b6620 100644 --- a/tangostationcontrol/tangostationcontrol/devices/boot.py +++ b/tangostationcontrol/tangostationcontrol/devices/boot.py @@ -251,6 +251,7 @@ class Boot(lofar_device): "STAT/SDP/1", # SDP controls the mask for SST/XST/BST, so initialise it first "STAT/SST/1", "STAT/XST/1", + "STAT/Beam/1", # Accesses RECV and SDP ], ) diff --git a/tangostationcontrol/tangostationcontrol/integration_test/device_proxy.py b/tangostationcontrol/tangostationcontrol/integration_test/device_proxy.py index 00ba0904c7bd01fa2ce1453a4c6701d6a4246e14..25c92411ecaababad20007868cfd19bdc3e9e18a 100644 --- a/tangostationcontrol/tangostationcontrol/integration_test/device_proxy.py +++ b/tangostationcontrol/tangostationcontrol/integration_test/device_proxy.py @@ -1,8 +1,25 @@ +import logging +import time + from tango import DeviceProxy +logger = logging.getLogger() + class TestDeviceProxy(DeviceProxy): def __init__(self, *args, **kwargs): super(TestDeviceProxy, self).__init__(*args, **kwargs) self.set_timeout_millis(10000) + + @staticmethod + def test_device_turn_off(endpoint): + d = TestDeviceProxy(endpoint) + try: + d.Off() + except Exception as e: + """Failing to turn Off devices should not raise errors here""" + logger.error(f"Failed to turn device off in teardown {e}") + + """Wait for 1 second to prevent propagating reconnection errors""" + time.sleep(1) diff --git a/tangostationcontrol/tangostationcontrol/integration_test/devices/base.py b/tangostationcontrol/tangostationcontrol/integration_test/devices/base.py index 555f7256ea49d68465b1c45bf038d47b39beeb25..ef4854a8241aaee7e6099ae7d417d7b94acfa21e 100644 --- a/tangostationcontrol/tangostationcontrol/integration_test/devices/base.py +++ b/tangostationcontrol/tangostationcontrol/integration_test/devices/base.py @@ -34,11 +34,19 @@ class AbstractTestBases: # make sure the device starts in Off self.proxy.Off() + self.addCleanup(TestDeviceProxy.test_device_turn_off, self.name) + super().setUp() - def tearDown(self): - """Turn device Off in teardown to prevent blocking tests""" - self.proxy.Off() + def test_device_fetch_state(self): + """Test if we can successfully fetch state""" + + self.assertEqual(DevState.OFF, self.proxy.state()) + + def test_device_ping(self): + """Test if we can successfully ping the device server""" + + self.assertGreater(self.proxy.ping(), 0) def test_device_initialize(self): """Test if we can transition to standby""" diff --git a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_apsct.py b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_apsct.py index ca73fc236a7486b858298e863663cf997c70ccc8..d973581e88cceb7510d2d257b006aa81ebbfbdfb 100644 --- a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_apsct.py +++ b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_apsct.py @@ -9,6 +9,7 @@ from .base import AbstractTestBases + class TestDeviceAPSCT(AbstractTestBases.TestDeviceBase): def setUp(self): diff --git a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_apspu.py b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_apspu.py index b9d2bc3d44acf0d4c2d3e1dd9b0adc095c86037b..5ebadc24029ce4a8e4a8b56805685d69aec73f2b 100644 --- a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_apspu.py +++ b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_apspu.py @@ -9,6 +9,7 @@ from .base import AbstractTestBases + class TestDeviceAPSPU(AbstractTestBases.TestDeviceBase): def setUp(self): diff --git a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_beam.py b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_beam.py new file mode 100644 index 0000000000000000000000000000000000000000..b6be958cfde3b9f236947e9b1b76195c844bad40 --- /dev/null +++ b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_beam.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +# +# This file is part of the LOFAR 2.0 Station Software +# +# +# +# Distributed under the terms of the APACHE license. +# See LICENSE.txt for more info. + +from .base import AbstractTestBases + +class TestDeviceBeam(AbstractTestBases.TestDeviceBase): + + def setUp(self): + super().setUp("STAT/Beam/1") diff --git a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_boot.py b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_boot.py index cc39c45d0a35da050aa041f0e5f2063df6312169..6d93080cc59ca176aea226893927f3321f503d89 100644 --- a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_boot.py +++ b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_boot.py @@ -11,6 +11,7 @@ import time from .base import AbstractTestBases + class TestDeviceBoot(AbstractTestBases.TestDeviceBase): def setUp(self): diff --git a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_recv.py b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_recv.py index 26e02ef312214df4a9304eafd050ea1438002da2..e96c385a7f976bc3ecb76d48b509b61d80454819 100644 --- a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_recv.py +++ b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_recv.py @@ -9,6 +9,7 @@ from .base import AbstractTestBases + class TestDeviceRECV(AbstractTestBases.TestDeviceBase): def setUp(self): diff --git a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_sdp.py b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_sdp.py index 2df399ed4607ea802abcf647c02078193f1b7f03..7de27c34b9746c1541c2b7091c977ed32ce9e535 100644 --- a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_sdp.py +++ b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_sdp.py @@ -7,11 +7,14 @@ # Distributed under the terms of the APACHE license. # See LICENSE.txt for more info. +from tango._tango import DevState from .base import AbstractTestBases + class TestDeviceSDP(AbstractTestBases.TestDeviceBase): def setUp(self): + """Intentionally recreate the device object in each test""" super().setUp("STAT/SDP/1") def test_device_sdp_read_attribute(self): diff --git a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_sst.py b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_sst.py index 38f3528f531660704baa00f70a7073974256a19f..60675e121364b52fb692b2f8461001bfdc78b50a 100644 --- a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_sst.py +++ b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_sst.py @@ -15,24 +15,13 @@ from tango._tango import DevState from .base import AbstractTestBases + class TestDeviceSST(AbstractTestBases.TestDeviceBase): def setUp(self): + """Intentionally recreate the device object in each test""" super().setUp("STAT/SST/1") - def test_device_on(self): - """Test if we can transition to on""" - - port_property = {"Statistics_Client_TCP_Port": "4999"} - self.proxy.put_property(port_property) - self.proxy.initialise() - - self.assertEqual(DevState.STANDBY, self.proxy.state()) - - self.proxy.on() - - self.assertEqual(DevState.ON, self.proxy.state()) - def test_device_sst_send_udp(self): port_property = {"Statistics_Client_TCP_Port": "4998"} self.proxy.put_property(port_property) diff --git a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_unb2.py b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_unb2.py index a35f70adde7af58408882e3b5ee3256a4838db84..d5731630188879e5f79f94f98951f8d6c1637ace 100644 --- a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_unb2.py +++ b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_unb2.py @@ -9,7 +9,9 @@ from .base import AbstractTestBases + class TestDeviceUNB2(AbstractTestBases.TestDeviceBase): def setUp(self): + """Intentionally recreate the device object in each test""" super().setUp("STAT/UNB2/1")