diff --git a/.gitignore b/.gitignore
index 60c6519f7724a7ca08cac3263b595400dba9fdd2..6841e3ee2682fb0d5660e2491c5b03ac34f6957e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,6 +12,7 @@
**/.project
**/.pydevproject
**/.settings/org.eclipse.core.resources.prefs
+docs/build
tangostationcontrol/dist
tangostationcontrol/build
**/.ipynb_checkpoints
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f458c94bb5c002a69aac660d28736b4b0aa65f8a..24124fc8a01456cb9cd10eddee0070db77b3f82e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,6 +1,4 @@
-# TODO(Corne): Update this image to use our own registry once building
-# images is in place.
-image: artefact.skao.int/ska-tango-images-tango-itango:9.3.7
+image: git.astron.nl:5000/lofar2.0/tango/tango-itango:9.3.7
variables:
GIT_SUBMODULE_STRATEGY: recursive
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
@@ -8,12 +6,291 @@ cache:
paths:
- .cache/pip
stages:
+ - images
- building
- linting
- static-analysis
- unit-tests
- integration-tests
- packaging
+# See docker-compose/README.md for docker image behavior and explanation
+.base_docker_images:
+ stage: images
+ image: docker:latest
+ tags:
+ - privileged
+ services:
+ - name: docker:dind
+ variables:
+ DOCKER_TLS_CERTDIR: "/certs"
+ before_script:
+ - |
+ if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" && -z "$CI_COMMIT_TAG" ]]; then
+ tag="latest"
+ echo "Running on tagged default branch '$CI_DEFAULT_BRANCH': tag = 'latest'"
+ else
+ tag="$CI_COMMIT_REF_SLUG"
+ echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag"
+ fi
+ - apk add --update make bash docker-compose
+ - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+ - touch /root/.Xauthority
+# Hack BASH_SOURCE into sourced files, docker its sh shell won't set this
+ - export BASH_SOURCE=$(pwd)/bootstrap/etc/lofar20rc.sh
+# source the lofarrc file and mask its non zero exit code
+ - . bootstrap/etc/lofar20rc.sh || true
+## Allow docker image script to execute
+# - chmod u+x $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh
+.base_docker_images_except:
+ extends: .base_docker_images
+ except:
+ refs:
+ - tags
+ - master
+.base_docker_store_images:
+ extends: .base_docker_images
+ script:
+# Do not remove 'bash' or statement will be ignored by primitive docker shell
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh
+docker_store_images_master_tag:
+ extends: .base_docker_store_images
+ only:
+ refs:
+ - tags
+ - master
+docker_store_images_changes:
+ extends: .base_docker_store_images
+ only:
+ changes:
+ - docker-compose/.env
+ except:
+ refs:
+ - tags
+ - master
+docker_build_image_all:
+ extends: .base_docker_images
+ only:
+ refs:
+ - tags
+ - master
+ script:
+# Do not remove 'bash' or statement will be ignored by primitive docker shell
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh elk latest
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh elk-configure-host latest
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh lofar-device-base latest
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh prometheus latest
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh itango latest
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh grafana latest
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh jupyter latest
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh apsct-sim latest
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh apspu-sim latest
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh recv-sim latest
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh sdptr-sim latest
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh unb2-sim latest
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-apsct latest
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-apspu latest
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-boot latest
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-docker latest
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-observation_control latest
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-recv latest
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-sdp latest
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-sst latest
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-unb2 latest
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-xst latest
+docker_build_image_elk:
+ extends: .base_docker_images_except
+ only:
+ changes:
+ - docker-compose/elk.yml
+ - docker-compose/elk/*
+ - docker-compose/elk-configure-host/*
+ script:
+# Do not remove 'bash' or statement will be ignored by primitive docker shell
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh elk $tag
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh elk-configure-host $tag
+docker_build_image_lofar_device_base:
+ extends: .base_docker_images_except
+ only:
+ changes:
+ - docker-compose/lofar-device-base.yml
+ - docker-compose/lofar-device-base/*
+ script:
+# Do not remove 'bash' or statement will be ignored by primitive docker shell
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh lofar-device-base $tag
+docker_build_image_prometheus:
+ extends: .base_docker_images_except
+ only:
+ changes:
+ - docker-compose/prometheus.yml
+ - docker-compose/prometheus/*
+ except:
+ refs:
+ - tags
+ - master
+ script:
+# Do not remove 'bash' or statement will be ignored by primitive docker shell
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh prometheus $tag
+docker_build_image_itango:
+ extends: .base_docker_images_except
+ only:
+ changes:
+ - docker-compose/itango.yml
+ script:
+# Do not remove 'bash' or statement will be ignored by primitive docker shell
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh itango $tag
+docker_build_image_grafana:
+ extends: .base_docker_images_except
+ only:
+ changes:
+ - docker-compose/grafana.yml
+ - docker-compose/grafana/*
+ script:
+# Do not remove 'bash' or statement will be ignored by primitive docker shell
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh grafana $tag
+docker_build_image_jupyter:
+ extends: .base_docker_images_except
+ only:
+ changes:
+ - docker-compose/jupyter.yml
+ - docker-compose/jupyter/*
+ script:
+# Do not remove 'bash' or statement will be ignored by primitive docker shell
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh jupyter $tag
+docker_build_image_apsct_sim:
+ extends: .base_docker_images_except
+ only:
+ changes:
+ - docker-compose/aspct-sim.yml
+ - docker-compose/pypcc-sim-base/*
+ script:
+# Do not remove 'bash' or statement will be ignored by primitive docker shell
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh apsct-sim $tag
+docker_build_image_apspu_sim:
+ extends: .base_docker_images_except
+ only:
+ changes:
+ - docker-compose/apspu-sim.yml
+ - docker-compose/pypcc-sim-base/*
+ script:
+# Do not remove 'bash' or statement will be ignored by primitive docker shell
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh apspu-sim $tag
+docker_build_image_recv_sim:
+ extends: .base_docker_images_except
+ only:
+ changes:
+ - docker-compose/recv-sim.yml
+ - docker-compose/pypcc-sim-base/*
+ script:
+# Do not remove 'bash' or statement will be ignored by primitive docker shell
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh recv-sim $tag
+docker_build_image_sdptr_sim:
+ extends: .base_docker_images_except
+ only:
+ changes:
+ - docker-compose/sdptr-sim.yml
+ - docker-compose/sdptr-sim/*
+ script:
+# Do not remove 'bash' or statement will be ignored by primitive docker shell
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh sdptr-sim $tag
+docker_build_image_unb2_sim:
+ extends: .base_docker_images_except
+ only:
+ changes:
+ - docker-compose/unb2-sim.yml
+ - docker-compose/pypcc-sim-base/*
+ script:
+# Do not remove 'bash' or statement will be ignored by primitive docker shell
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh unb2-sim $tag
+docker_build_image_device_apsct:
+ extends: .base_docker_images_except
+ only:
+ changes:
+ - docker-compose/device-aspct.yml
+ - docker-compose/lofar-device-base/*
+ script:
+# Do not remove 'bash' or statement will be ignored by primitive docker shell
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-aspct $tag
+docker_build_image_device_apspu:
+ extends: .base_docker_images_except
+ only:
+ changes:
+ - docker-compose/device-apspu.yml
+ - docker-compose/lofar-device-base/*
+ script:
+# Do not remove 'bash' or statement will be ignored by primitive docker shell
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-apspu $tag
+docker_build_image_device_boot:
+ extends: .base_docker_images_except
+ only:
+ changes:
+ - docker-compose/device-boot.yml
+ - docker-compose/lofar-device-base/*
+ script:
+# Do not remove 'bash' or statement will be ignored by primitive docker shell
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-boot $tag
+docker_build_image_device_docker:
+ extends: .base_docker_images_except
+ only:
+ changes:
+ - docker-compose/device-docker.yml
+ - docker-compose/lofar-device-base/*
+ script:
+# Do not remove 'bash' or statement will be ignored by primitive docker shell
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-docker $tag
+docker_build_image_device_ovservation_control:
+ extends: .base_docker_images_except
+ only:
+ changes:
+ - docker-compose/device-observation_control.yml
+ - docker-compose/lofar-device-base/*
+ script:
+# Do not remove 'bash' or statement will be ignored by primitive docker shell
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-observation_control $tag
+docker_build_image_device_recv:
+ extends: .base_docker_images_except
+ only:
+ changes:
+ - docker-compose/device-recv.yml
+ - docker-compose/lofar-device-base/*
+ script:
+# Do not remove 'bash' or statement will be ignored by primitive docker shell
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-recv $tag
+docker_build_image_device_sdp:
+ extends: .base_docker_images_except
+ only:
+ changes:
+ - docker-compose/device-sdp.yml
+ - docker-compose/lofar-device-base/*
+ script:
+# Do not remove 'bash' or statement will be ignored by primitive docker shell
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-sdp $tag
+docker_build_image_device_sst:
+ extends: .base_docker_images_except
+ only:
+ changes:
+ - docker-compose/device-sst.yml
+ - docker-compose/lofar-device-base/*
+ script:
+# Do not remove 'bash' or statement will be ignored by primitive docker shell
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-sst $tag
+docker_build_image_device_unb2:
+ extends: .base_docker_images_except
+ only:
+ changes:
+ - docker-compose/device-unb2.yml
+ - docker-compose/lofar-device-base/*
+ script:
+# Do not remove 'bash' or statement will be ignored by primitive docker shell
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-unb2 $tag
+docker_build_image_device_xst:
+ extends: .base_docker_images_except
+ only:
+ changes:
+ - docker-compose/device-xst.yml
+ - docker-compose/lofar-device-base/*
+ script:
+# Do not remove 'bash' or statement will be ignored by primitive docker shell
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-xst $tag
newline_at_eof:
stage: linting
before_script:
@@ -63,6 +340,14 @@ integration_test_docker:
variables:
DOCKER_TLS_CERTDIR: "/certs"
before_script:
+ - |
+ if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" && -z "$CI_COMMIT_TAG" ]]; then
+ tag="latest"
+ echo "Running on tagged default branch '$CI_DEFAULT_BRANCH': tag = 'latest'"
+ else
+ tag="$CI_COMMIT_REF_SLUG"
+ echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag"
+ fi
- apk add --update make bash docker-compose
- apk add --update bind-tools
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
@@ -78,8 +363,12 @@ integration_test_docker:
- . bootstrap/etc/lofar20rc.sh || true
# TANGO_HOST must be unset our databaseds will be unreachable
- unset TANGO_HOST
-# Allow integration test to execute
- - chmod u+x $CI_PROJECT_DIR/sbin/run_integration_test.sh
+## Allow docker image script to execute
+# - chmod u+x $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh
+# Do not remove 'bash' or statement will be ignored by primitive docker shell
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh pull $tag
+## Allow integration test to execute
+# - chmod u+x $CI_PROJECT_DIR/sbin/run_integration_test.sh
# Do not remove 'bash' or statement will be ignored by primitive docker shell
- bash $CI_PROJECT_DIR/sbin/run_integration_test.sh
wheel_packaging:
diff --git a/CDB/LOFAR_ConfigDb.json b/CDB/LOFAR_ConfigDb.json
index 6886437ce7cf02ee741b5b70f07ebfbf71b08b4d..60b0115925b2ccf01bd27b3e0d072a17c78d580f 100644
--- a/CDB/LOFAR_ConfigDb.json
+++ b/CDB/LOFAR_ConfigDb.json
@@ -21,6 +21,13 @@
}
}
},
+ "Beam": {
+ "STAT": {
+ "Beam": {
+ "STAT/Beam/1": {}
+ }
+ }
+ },
"boot": {
"STAT": {
"Boot": {
diff --git a/bin/start-ds.sh b/bin/start-ds.sh
index 7b601c4c8f5e24ac56755ad08a1203a6cbba62d2..a48b0b4554cd2ef9380cbc482b07edfc38203043 100755
--- a/bin/start-ds.sh
+++ b/bin/start-ds.sh
@@ -2,6 +2,11 @@
# Serves as entrypoint script for docker containers
+if [[ ! -d "/opt/lofar/tango" ]]; then
+ >&2 echo "/opt/lofar/tango volume does not exist!"
+ exit 1
+fi
+
# Check required support file exists
if [[ ! -f "/usr/local/bin/wait-for-it.sh" ]]; then
>&2 echo "/usr/local/bin/wait-for-it.sh file does not exist!"
@@ -14,6 +19,11 @@ if [[ ! $TANGO_HOST ]]; then
exit 1
fi
+# Store directory so we can return to it after installation
+CWD=$(pwd)
+
+cd /opt/lofar/tango || exit 1
+
# Check if configured for specific version
if [[ $TANGOSTATIONCONTROL ]]; then
# TODO (Corne): Download version from artifacts or pypi.
@@ -28,4 +38,8 @@ else
sudo pip install --force-reinstall "$(ls -Art /tmp/tangostationcontrol/*.whl | tail -n 1)"
fi
+# Return to the stored the directory, this preserves the working_dir argument in
+# docker-compose files.
+cd "$CWD" || exit 1
+
/usr/local/bin/wait-for-it.sh "$TANGO_HOST" --timeout=30 --strict -- "$@"
diff --git a/bootstrap/etc/lofar20rc.sh b/bootstrap/etc/lofar20rc.sh
index 6e4a5c9bc8d6a78c1b61cca02159ee01291d3805..4b9d806d819816a86c5fea3ab8eb59135d8edcfc 100755
--- a/bootstrap/etc/lofar20rc.sh
+++ b/bootstrap/etc/lofar20rc.sh
@@ -16,6 +16,7 @@ if [ ! -f "${LOFAR20_DIR}/.git/hooks/post-checkout" ]; then
alias git="cp ${LOFAR20_DIR}/bin/update_submodules.sh ${LOFAR20_DIR}/.git/hooks/post-checkout; cp ${LOFAR20_DIR}/bin/update_submodules.sh ${LOFAR20_DIR}/.git/hooks/post-merge; unalias git; git"
fi
+# CI_BUILD_ID does not exist see https://docs.gitlab.com/ee/ci/variables/predefined_variables.html
if [ ! -z ${CI_BUILD_ID+x} ]; then
export CONTAINER_NAME_PREFIX=${CI_BUILD_ID}-
elif [ ! -z ${CI_JOB_ID+x} ]; then
diff --git a/docker-compose/.env b/docker-compose/.env
index c1956e315f8cde0d48b4b5279807025bede69261..53937727c24d398a5d82b24c31f205db50064163 100644
--- a/docker-compose/.env
+++ b/docker-compose/.env
@@ -1,6 +1,7 @@
DOCKER_REGISTRY_HOST=artefact.skao.int
DOCKER_REGISTRY_USER=ska-tango-images
LOCAL_DOCKER_REGISTRY_HOST=git.astron.nl:5000
+LOCAL_DOCKER_REGISTRY_LOFAR=lofar2.0
LOCAL_DOCKER_REGISTRY_USER=lofar2.0/tango
TANGO_ARCHIVER_VERSION=2021-05-28
diff --git a/docker-compose/Makefile b/docker-compose/Makefile
index d85ff1df88d91db097bdd22b060cfc03b681a04f..6c6e3c888eaa77b7c23ad0fca6ec0d94a8099bf7 100644
--- a/docker-compose/Makefile
+++ b/docker-compose/Makefile
@@ -33,6 +33,8 @@ else ifeq (stop,$(firstword $(MAKECMDGOALS)))
SERVICE_TARGET = true
else ifeq (restart,$(firstword $(MAKECMDGOALS)))
SERVICE_TARGET = true
+else ifeq (up,$(firstword $(MAKECMDGOALS)))
+ SERVICE_TARGET = true
else ifeq (build,$(firstword $(MAKECMDGOALS)))
SERVICE_TARGET = true
else ifeq (build-nocache,$(firstword $(MAKECMDGOALS)))
@@ -143,8 +145,8 @@ build-nocache: ## rebuild images from scratch
$(DOCKER_COMPOSE_ARGS) docker-compose -f lofar-device-base.yml -f networks.yml build --progress=plain
$(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) build --no-cache --progress=plain $(SERVICE)
-up: minimal ## start the base TANGO system and prepare all services
- $(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) up --no-start --no-recreate
+up: minimal ## start the base TANGO system and prepare requested services
+ $(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) up --no-start --no-recreate $(SERVICE)
down: ## stop all services and tear down the system
$(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) down
diff --git a/docker-compose/README.md b/docker-compose/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..d76a75b0c79dad1574d9d24912a671a39998abee
--- /dev/null
+++ b/docker-compose/README.md
@@ -0,0 +1,69 @@
+# Docker Compose
+
+Documentation on how the LOFAR station control software utilizes docker-compose.
+This documentation is intended for developers listing strategies and their
+respective advantages and disadvantages. In addition, this documentation
+contains developer expectations that they should uphold to.
+
+## Image tagging and change detection
+
+Preventing unnecessary builds of docker images reduces build times and increases
+iteration speed. In order to achieve this the project requires properly tagged
+images and mechanisms for change detection.
+
+For change detection the system relies on git. Git is used to determine the
+directories and files that have changes between the current and previous commit.
+All image related change detection mechanisms are based on this difference.
+
+Using docker cache within the dind service is impractical see:
+https://gitlab.com/gitlab-org/gitlab-foss/-/issues/17861
+
+### Types of containers and specific strategies.
+
+- Devices
+- Simulators
+- Base images
+- Services
+
+Devices, these are detected by changes to the .yml file or directory of the
+respective service inside the docker-compose directory.
+
+Simulators, Since the source code for simulators is maintained by other teams
+we can not accurately determine from our repository if the simulator has
+changed. Instead, the images is build by the teams their respective CI
+pipelines. We simply pull these images as base images.
+
+Base images, these are detected by changes to the .env file in the
+docker-compose directory. When changed they will be downloaded from the remote
+registry and uploaded to our own using matching tags.
+
+Services, same mechanism as devices.
+
+### Setup and maintenance
+
+All behavioral logic to orchestrate change detection and image pushing can be
+found in the sbin/tag_and_push_docker_images.sh script as well as the
+.gitlab-ci.yml. The shell script relies on the fact that each .yml file in the
+docker-compose directory corresponds to one image.
+
+### Gitlab CI phases
+
+Docker images are managed in three phases. First is remote image storing, second
+is image building and change detection with finally image pulling.
+
+Remote images are downloaded and stored on the local registry when the .env
+file for docker-compose has changes.
+
+Local images are build when either the files in the base context directory
+change or if the docker compose file itself has changes. See the gitlab-ci.yml
+for how these changes are detected. All local images will be rebuild and tagged
+latest when a tagged commit is pushed to master.
+
+Local images download the latest image from the registry as cache unless it is
+a tagged commit on master.
+
+Finally, the integration test downloads all images from the registry either
+tagged with the current pipeline or with latest. Should both tags be unavailable
+than the integration test fails. Not all images are needed for the integration
+test. See sbin/tag_and_push_docker_image.sh for how these images are
+differentiated.
diff --git a/docker-compose/apsct-sim.yml b/docker-compose/apsct-sim.yml
index d30f5a026f734bb72ee91c7bf533df677f37ca88..b9742fdb97ec3f30026d441c668a13732013201e 100644
--- a/docker-compose/apsct-sim.yml
+++ b/docker-compose/apsct-sim.yml
@@ -10,6 +10,9 @@ services:
apsct-sim:
build:
context: pypcc-sim-base
+ args:
+ - LOCAL_DOCKER_REGISTRY_HOST=${LOCAL_DOCKER_REGISTRY_HOST}
+ - LOCAL_DOCKER_REGISTRY_LOFAR=${LOCAL_DOCKER_REGISTRY_LOFAR}
container_name: ${CONTAINER_NAME_PREFIX}apsct-sim
networks:
- control
diff --git a/docker-compose/apspu-sim.yml b/docker-compose/apspu-sim.yml
index d3fc5fa04f6ce0d6ddfe4c8f87887ab7500720e3..f5677048fbe1fe28082b219177bc67a2986c31fe 100644
--- a/docker-compose/apspu-sim.yml
+++ b/docker-compose/apspu-sim.yml
@@ -10,6 +10,9 @@ services:
apspu-sim:
build:
context: pypcc-sim-base
+ args:
+ - LOCAL_DOCKER_REGISTRY_HOST=${LOCAL_DOCKER_REGISTRY_HOST}
+ - LOCAL_DOCKER_REGISTRY_LOFAR=${LOCAL_DOCKER_REGISTRY_LOFAR}
container_name: ${CONTAINER_NAME_PREFIX}apspu-sim
networks:
- control
diff --git a/docker-compose/archiver.yml b/docker-compose/archiver.yml
index f1f2a1ec65dd4259b99e675c43cb7500862049f4..12ec2d88959fca75b047cff6004dd6e2b22c294a 100644
--- a/docker-compose/archiver.yml
+++ b/docker-compose/archiver.yml
@@ -94,7 +94,7 @@ services:
tag: "{{.Name}}"
dsconfig:
- image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-dsconfig:${TANGO_DSCONFIG_VERSION}
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-dsconfig:${TANGO_DSCONFIG_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}dsconfig
networks:
- control
diff --git a/docker-compose/astor.yml b/docker-compose/astor.yml
index 7010a82afa2fbcf5cb3dd797bda384bb516354f8..502472fc4eedd022388cd13d76e74135e00ff3db 100644
--- a/docker-compose/astor.yml
+++ b/docker-compose/astor.yml
@@ -13,7 +13,7 @@ version: '2'
services:
astor:
- image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-java:${TANGO_JAVA_VERSION}
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-java:${TANGO_JAVA_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}astor
networks:
- control
diff --git a/docker-compose/device-apsct.yml b/docker-compose/device-apsct.yml
index 60f65fc47ed81822242282fc743846221acec2d9..0e258fecdb3a96c3a73714ae2c28cf2e847457a1 100644
--- a/docker-compose/device-apsct.yml
+++ b/docker-compose/device-apsct.yml
@@ -20,7 +20,7 @@ services:
build:
context: lofar-device-base
args:
- SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
+ SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-apsct
networks:
- control
diff --git a/docker-compose/device-apspu.yml b/docker-compose/device-apspu.yml
index b694b09518215e293d19e1ff551f4f608e6f818d..5f325b19fb357e83ab3d35e3acfa1a5cbbb2896a 100644
--- a/docker-compose/device-apspu.yml
+++ b/docker-compose/device-apspu.yml
@@ -20,7 +20,7 @@ services:
build:
context: lofar-device-base
args:
- SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
+ SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-apspu
networks:
- control
diff --git a/docker-compose/device-beam.yml b/docker-compose/device-beam.yml
new file mode 100644
index 0000000000000000000000000000000000000000..97385f16492ec123044033713d0c7b835d2062fd
--- /dev/null
+++ b/docker-compose/device-beam.yml
@@ -0,0 +1,33 @@
+#
+# Requires:
+# - lofar-device-base.yml
+#
+version: '2'
+
+services:
+ device-beam:
+ image: device-beam
+ # build explicitly, as docker-compose does not understand a local image
+ # being shared among services.
+ build:
+ context: lofar-device-base
+ args:
+ SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
+ container_name: ${CONTAINER_NAME_PREFIX}device-beam
+ networks:
+ - control
+ ports:
+ - "5711:5711" # unique port for this DS
+ extra_hosts:
+ - "host.docker.internal:host-gateway"
+ volumes:
+ - ..:/opt/lofar/tango:rw
+ environment:
+ - TANGO_HOST=${TANGO_HOST}
+ working_dir: /opt/lofar/tango
+ entrypoint:
+ - bin/start-ds.sh
+ # configure CORBA to _listen_ on 0:port, but tell others we're _reachable_ through ${HOSTNAME}:port, since CORBA
+ # can't know about our Docker port forwarding
+ - l2ss-beam Beam STAT -v -ORBendPoint giop:tcp:0:5711 -ORBendPointPublish giop:tcp:${HOSTNAME}:5711
+ restart: unless-stopped
diff --git a/docker-compose/device-boot.yml b/docker-compose/device-boot.yml
index 3db111410fafde9901fd8f91cb40a1c3560e4242..330cb723ed3bb5ee8ccd50bf4cb933da4e1fe09c 100644
--- a/docker-compose/device-boot.yml
+++ b/docker-compose/device-boot.yml
@@ -19,7 +19,7 @@ services:
build:
context: lofar-device-base
args:
- SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
+ SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-boot
networks:
- control
diff --git a/docker-compose/device-docker.yml b/docker-compose/device-docker.yml
index 5a2641e9871f163f27ed7a60d872d30d4fe855e1..a9e4ccfdd6f66eda66f05ea5244fcf0fd732a382 100644
--- a/docker-compose/device-docker.yml
+++ b/docker-compose/device-docker.yml
@@ -20,7 +20,7 @@ services:
build:
context: lofar-device-base
args:
- SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
+ SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-docker
networks:
- control
diff --git a/docker-compose/device-observation_control.yml b/docker-compose/device-observation_control.yml
index 33fb0d066fd76b8eb4a9c7753266f16d04157726..d4f6f15d1f4eb80d02cd0c5738dc0a011b9dfc72 100644
--- a/docker-compose/device-observation_control.yml
+++ b/docker-compose/device-observation_control.yml
@@ -19,7 +19,7 @@ services:
build:
context: lofar-device-base
args:
- SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
+ SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-observation_control
networks:
- control
diff --git a/docker-compose/device-recv.yml b/docker-compose/device-recv.yml
index a08f566e7b39e095403f00cb5b086420b689d66b..25e767726f139ff532dbe649ccb230fabbec0602 100644
--- a/docker-compose/device-recv.yml
+++ b/docker-compose/device-recv.yml
@@ -20,7 +20,7 @@ services:
build:
context: lofar-device-base
args:
- SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
+ SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-recv
networks:
- control
diff --git a/docker-compose/device-sdp.yml b/docker-compose/device-sdp.yml
index f32c34394475c6a7483cb98cd03def1f62cf9ff0..06a523f606d67811986bd7a13b9a3202cb74e91d 100644
--- a/docker-compose/device-sdp.yml
+++ b/docker-compose/device-sdp.yml
@@ -20,7 +20,7 @@ services:
build:
context: lofar-device-base
args:
- SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
+ SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-sdp
networks:
- control
diff --git a/docker-compose/device-sst.yml b/docker-compose/device-sst.yml
index 7464cb01f45e584ab705fe9098e0229a1b762295..86651c7878d844646528b41fb0969dfd19af6eea 100644
--- a/docker-compose/device-sst.yml
+++ b/docker-compose/device-sst.yml
@@ -20,7 +20,7 @@ services:
build:
context: lofar-device-base
args:
- SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
+ SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-sst
networks:
- control
diff --git a/docker-compose/device-unb2.yml b/docker-compose/device-unb2.yml
index af1329d21a905f3c150c092529978e17f0c0ee37..2b9b47146a405440ebd36fd84162935fb6b8a56d 100644
--- a/docker-compose/device-unb2.yml
+++ b/docker-compose/device-unb2.yml
@@ -20,7 +20,7 @@ services:
build:
context: lofar-device-base
args:
- SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
+ SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-unb2
networks:
- control
diff --git a/docker-compose/device-xst.yml b/docker-compose/device-xst.yml
index c4ea684fd94e34fcaaa857a5717ca47745eccc72..54ca5a21f911084160d2cec772df06da55ef5cf1 100644
--- a/docker-compose/device-xst.yml
+++ b/docker-compose/device-xst.yml
@@ -20,7 +20,7 @@ services:
build:
context: lofar-device-base
args:
- SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
+ SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-xst
networks:
- control
diff --git a/docker-compose/elk.yml b/docker-compose/elk.yml
index 67f13baee061a74ebd08320f1e9f2f9f3e72f646..25bb1b218669baebff50ddc830b049b691349f71 100644
--- a/docker-compose/elk.yml
+++ b/docker-compose/elk.yml
@@ -6,6 +6,7 @@
# - elk-configure-host: Configures the hosts's kernel to be able to use the ELK stack
# - elk: ELK stack
#
+
version: '2'
volumes:
diff --git a/docker-compose/grafana.yml b/docker-compose/grafana.yml
index 29c93c52c4dc05849aad10fabac12712c12dd4d7..f298db2746961b7d30d2e147192d0dfc58530725 100644
--- a/docker-compose/grafana.yml
+++ b/docker-compose/grafana.yml
@@ -4,6 +4,7 @@
# Defines:
# - grafana: Grafana
#
+
version: '2'
#volumes:
diff --git a/docker-compose/grafana/dashboards/archiver_mariadb.json b/docker-compose/grafana/dashboards/archiver_mariadb.json
new file mode 100644
index 0000000000000000000000000000000000000000..73b4aa07650e9c8cac3a4bcf12537cd2fe381c6a
--- /dev/null
+++ b/docker-compose/grafana/dashboards/archiver_mariadb.json
@@ -0,0 +1,799 @@
+{
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": "-- Grafana --",
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "target": {
+ "limit": 100,
+ "matchAny": false,
+ "tags": [],
+ "type": "dashboard"
+ },
+ "type": "dashboard"
+ }
+ ]
+ },
+ "editable": true,
+ "fiscalYearStartMonth": 0,
+ "gnetId": null,
+ "graphTooltip": 0,
+ "links": [],
+ "liveNow": false,
+ "panels": [
+ {
+ "datasource": null,
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "align": "auto",
+ "displayMode": "auto"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 0
+ },
+ "id": 4,
+ "options": {
+ "showHeader": true
+ },
+ "pluginVersion": "8.2.3",
+ "targets": [
+ {
+ "format": "table",
+ "group": [],
+ "hide": false,
+ "metricColumn": "none",
+ "rawQuery": true,
+ "rawSql": "select name as \"Attribute\"\r\nfrom att_conf ac \r\nwhere ac.domain ='stat' and ac.family = 'sdp' and ac.member ='1'\r\norder by ac.name",
+ "refId": "A",
+ "select": [
+ [
+ {
+ "params": [
+ "name"
+ ],
+ "type": "column"
+ }
+ ]
+ ],
+ "table": "att_conf",
+ "timeColumn": "data_time",
+ "timeColumnType": "timestamp",
+ "where": []
+ }
+ ],
+ "title": "STAT/SDP/1 archived attributes",
+ "type": "table"
+ },
+ {
+ "datasource": null,
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "align": "auto",
+ "displayMode": "auto"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 0
+ },
+ "id": 6,
+ "options": {
+ "showHeader": true
+ },
+ "pluginVersion": "8.2.3",
+ "targets": [
+ {
+ "format": "table",
+ "group": [],
+ "metricColumn": "none",
+ "rawQuery": true,
+ "rawSql": "select name as \"Attribute\" \r\nfrom att_conf \r\nwhere domain ='stat' and family = 'recv' and member ='1'\r\norder by name",
+ "refId": "A",
+ "select": [
+ [
+ {
+ "params": [
+ "att_conf_id"
+ ],
+ "type": "column"
+ }
+ ]
+ ],
+ "table": "att_array_devfloat_rw",
+ "timeColumn": "data_time",
+ "timeColumnType": "timestamp",
+ "where": [
+ {
+ "name": "$__timeFilter",
+ "params": [],
+ "type": "macro"
+ }
+ ]
+ }
+ ],
+ "title": "STAT/RECV/1 archived attributes",
+ "type": "table"
+ },
+ {
+ "datasource": null,
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 4,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "__systemRef": "hideSeriesFrom",
+ "matcher": {
+ "id": "byNames",
+ "options": {
+ "mode": "exclude",
+ "names": [
+ "temperature [1]"
+ ],
+ "prefix": "All except:",
+ "readOnly": true
+ }
+ },
+ "properties": [
+ {
+ "id": "custom.hideFrom",
+ "value": {
+ "legend": false,
+ "tooltip": false,
+ "viz": true
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 8
+ },
+ "id": 2,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
+ },
+ "pluginVersion": "8.2.1",
+ "targets": [
+ {
+ "format": "time_series",
+ "group": [],
+ "hide": true,
+ "metricColumn": "none",
+ "rawQuery": true,
+ "rawSql": "SELECT\ndata_time as \"time\",\nvalue_r as \"value\"\nFROM att_array_devdouble_ro\nJOIN att_conf\nON att_array_devdouble_ro.att_conf_id = att_conf.att_conf_id \nWHERE\natt_conf.domain = 'lts'\nand att_conf.family = 'sdp'\nand att_conf.member = 1\nand att_conf.name = 'fpga_temp_r'\nORDER BY data_time,idx\n",
+ "refId": "A",
+ "select": [
+ [
+ {
+ "params": [
+ "value"
+ ],
+ "type": "column"
+ }
+ ]
+ ],
+ "timeColumn": "time",
+ "where": [
+ {
+ "name": "$__timeFilter",
+ "params": [],
+ "type": "macro"
+ }
+ ]
+ },
+ {
+ "format": "time_series",
+ "group": [],
+ "hide": true,
+ "metricColumn": "none",
+ "rawQuery": true,
+ "rawSql": "SELECT\ndata_time as \"time\",\nvalue_r as \"mask\"\nFROM att_array_devboolean_ro\nJOIN att_conf\nON att_array_devboolean_ro.att_conf_id = att_conf.att_conf_id \nWHERE\natt_conf.domain = 'lts'\nand att_conf.family = 'sdp'\nand att_conf.member = 1\nand att_conf.name = 'tr_fpga_mask_r'\nORDER BY data_time,idx\n",
+ "refId": "B",
+ "select": [
+ [
+ {
+ "params": [
+ "value"
+ ],
+ "type": "column"
+ }
+ ]
+ ],
+ "timeColumn": "time",
+ "where": [
+ {
+ "name": "$__timeFilter",
+ "params": [],
+ "type": "macro"
+ }
+ ]
+ },
+ {
+ "format": "time_series",
+ "group": [],
+ "hide": false,
+ "metricColumn": "none",
+ "rawQuery": true,
+ "rawSql": "SELECT t.`time` as \"time\",\r\nt.value as \"temperature [0]\"\r\nFROM SDP_FPGA_temp as t INNER JOIN SDP_TR_FPGA_mask as m\r\nON t.`time`=m.`time` and t.`index`=m.`index`\r\nWHERE m.value = 1\r\nAND t.`time` > $__timeFrom()\r\nAND t.`time` < $__timeTo()\r\nAND t.`index` = 0\r\nORDER BY t.`time`, t.`index`",
+ "refId": "Masked Values",
+ "select": [
+ [
+ {
+ "params": [
+ "att_conf_id"
+ ],
+ "type": "column"
+ }
+ ]
+ ],
+ "table": "att_array_devfloat_rw",
+ "timeColumn": "data_time",
+ "timeColumnType": "timestamp",
+ "where": [
+ {
+ "name": "$__timeFilter",
+ "params": [],
+ "type": "macro"
+ }
+ ]
+ },
+ {
+ "format": "time_series",
+ "group": [],
+ "hide": false,
+ "metricColumn": "none",
+ "rawQuery": true,
+ "rawSql": "SELECT t.`time` as \"time\",\r\nt.value as \"temperature [1]\"\r\nFROM SDP_FPGA_temp as t INNER JOIN SDP_TR_FPGA_mask as m\r\nON t.`time`=m.`time` and t.`index`=m.`index`\r\nWHERE m.value = 1\r\nAND t.`time` > $__timeFrom()\r\nAND t.`time` < $__timeTo()\r\nAND t.`index` = 1\r\nORDER BY t.`time`, t.`index`",
+ "refId": "C",
+ "select": [
+ [
+ {
+ "params": [
+ "att_conf_id"
+ ],
+ "type": "column"
+ }
+ ]
+ ],
+ "table": "att_array_devfloat_rw",
+ "timeColumn": "data_time",
+ "timeColumnType": "timestamp",
+ "where": [
+ {
+ "name": "$__timeFilter",
+ "params": [],
+ "type": "macro"
+ }
+ ]
+ },
+ {
+ "format": "time_series",
+ "group": [],
+ "hide": false,
+ "metricColumn": "none",
+ "rawQuery": true,
+ "rawSql": "SELECT t.`time` as \"time\",\r\nt.value as \"temperature [2]\"\r\nFROM SDP_FPGA_temp as t INNER JOIN SDP_TR_FPGA_mask as m\r\nON t.`time`=m.`time` and t.`index`=m.`index`\r\nWHERE m.value = 1\r\nAND t.`time` > $__timeFrom()\r\nAND t.`time` < $__timeTo()\r\nAND t.`index` = 2\r\nORDER BY t.`time`, t.`index`",
+ "refId": "D",
+ "select": [
+ [
+ {
+ "params": [
+ "att_conf_id"
+ ],
+ "type": "column"
+ }
+ ]
+ ],
+ "table": "att_array_devfloat_rw",
+ "timeColumn": "data_time",
+ "timeColumnType": "timestamp",
+ "where": [
+ {
+ "name": "$__timeFilter",
+ "params": [],
+ "type": "macro"
+ }
+ ]
+ },
+ {
+ "format": "time_series",
+ "group": [],
+ "hide": false,
+ "metricColumn": "none",
+ "rawQuery": true,
+ "rawSql": "SELECT t.`time` as \"time\",\r\nt.value as \"temperature [3]\"\r\nFROM SDP_FPGA_temp as t INNER JOIN SDP_TR_FPGA_mask as m\r\nON t.`time`=m.`time` and t.`index`=m.`index`\r\nWHERE m.value = 1\r\nAND t.`time` > $__timeFrom()\r\nAND t.`time` < $__timeTo()\r\nAND t.`index` = 3\r\nORDER BY t.`time`, t.`index`",
+ "refId": "E",
+ "select": [
+ [
+ {
+ "params": [
+ "att_conf_id"
+ ],
+ "type": "column"
+ }
+ ]
+ ],
+ "table": "att_array_devfloat_rw",
+ "timeColumn": "data_time",
+ "timeColumnType": "timestamp",
+ "where": [
+ {
+ "name": "$__timeFilter",
+ "params": [],
+ "type": "macro"
+ }
+ ]
+ },
+ {
+ "format": "time_series",
+ "group": [],
+ "hide": false,
+ "metricColumn": "none",
+ "rawQuery": true,
+ "rawSql": "SELECT t.`time` as \"time\",\r\nt.value as \"temperature [4]\"\r\nFROM SDP_FPGA_temp as t INNER JOIN SDP_TR_FPGA_mask as m\r\nON t.`time`=m.`time` and t.`index`=m.`index`\r\nWHERE m.value = 1\r\nAND t.`time` > $__timeFrom()\r\nAND t.`time` < $__timeTo()\r\nAND t.`index` = 4\r\nORDER BY t.`time`, t.`index`",
+ "refId": "F",
+ "select": [
+ [
+ {
+ "params": [
+ "att_conf_id"
+ ],
+ "type": "column"
+ }
+ ]
+ ],
+ "table": "att_array_devfloat_rw",
+ "timeColumn": "data_time",
+ "timeColumnType": "timestamp",
+ "where": [
+ {
+ "name": "$__timeFilter",
+ "params": [],
+ "type": "macro"
+ }
+ ]
+ },
+ {
+ "format": "time_series",
+ "group": [],
+ "hide": false,
+ "metricColumn": "none",
+ "rawQuery": true,
+ "rawSql": "SELECT t.`time` as \"time\",\r\nt.value as \"temperature [5]\"\r\nFROM SDP_FPGA_temp as t INNER JOIN SDP_TR_FPGA_mask as m\r\nON t.`time`=m.`time` and t.`index`=m.`index`\r\nWHERE m.value = 1\r\nAND t.`time` > $__timeFrom()\r\nAND t.`time` < $__timeTo()\r\nAND t.`index` = 5\r\nORDER BY t.`time`, t.`index`",
+ "refId": "G",
+ "select": [
+ [
+ {
+ "params": [
+ "att_conf_id"
+ ],
+ "type": "column"
+ }
+ ]
+ ],
+ "table": "att_array_devfloat_rw",
+ "timeColumn": "data_time",
+ "timeColumnType": "timestamp",
+ "where": [
+ {
+ "name": "$__timeFilter",
+ "params": [],
+ "type": "macro"
+ }
+ ]
+ },
+ {
+ "format": "time_series",
+ "group": [],
+ "hide": false,
+ "metricColumn": "none",
+ "rawQuery": true,
+ "rawSql": "SELECT t.`time` as \"time\",\r\nt.value as \"temperature [6]\"\r\nFROM SDP_FPGA_temp as t INNER JOIN SDP_TR_FPGA_mask as m\r\nON t.`time`=m.`time` and t.`index`=m.`index`\r\nWHERE m.value = 1\r\nAND t.`time` > $__timeFrom()\r\nAND t.`time` < $__timeTo()\r\nAND t.`index` = 6\r\nORDER BY t.`time`, t.`index`",
+ "refId": "H",
+ "select": [
+ [
+ {
+ "params": [
+ "att_conf_id"
+ ],
+ "type": "column"
+ }
+ ]
+ ],
+ "table": "att_array_devfloat_rw",
+ "timeColumn": "data_time",
+ "timeColumnType": "timestamp",
+ "where": [
+ {
+ "name": "$__timeFilter",
+ "params": [],
+ "type": "macro"
+ }
+ ]
+ },
+ {
+ "format": "time_series",
+ "group": [],
+ "hide": false,
+ "metricColumn": "none",
+ "rawQuery": true,
+ "rawSql": "SELECT t.`time` as \"time\",\r\nt.value as \"temperature [7]\"\r\nFROM SDP_FPGA_temp as t INNER JOIN SDP_TR_FPGA_mask as m\r\nON t.`time`=m.`time` and t.`index`=m.`index`\r\nWHERE m.value = 1\r\nAND t.`time` > $__timeFrom()\r\nAND t.`time` < $__timeTo()\r\nAND t.`index` = 7\r\nORDER BY t.`time`, t.`index`",
+ "refId": "I",
+ "select": [
+ [
+ {
+ "params": [
+ "att_conf_id"
+ ],
+ "type": "column"
+ }
+ ]
+ ],
+ "table": "att_array_devfloat_rw",
+ "timeColumn": "data_time",
+ "timeColumnType": "timestamp",
+ "where": [
+ {
+ "name": "$__timeFilter",
+ "params": [],
+ "type": "macro"
+ }
+ ]
+ },
+ {
+ "format": "time_series",
+ "group": [],
+ "hide": false,
+ "metricColumn": "none",
+ "rawQuery": true,
+ "rawSql": "SELECT t.`time` as \"time\",\r\nt.value as \"temperature [8]\"\r\nFROM SDP_FPGA_temp as t INNER JOIN SDP_TR_FPGA_mask as m\r\nON t.`time`=m.`time` and t.`index`=m.`index`\r\nWHERE m.value = 1\r\nAND t.`time` > $__timeFrom()\r\nAND t.`time` < $__timeTo()\r\nAND t.`index` = 8\r\nORDER BY t.`time`, t.`index`",
+ "refId": "J",
+ "select": [
+ [
+ {
+ "params": [
+ "att_conf_id"
+ ],
+ "type": "column"
+ }
+ ]
+ ],
+ "table": "att_array_devfloat_rw",
+ "timeColumn": "data_time",
+ "timeColumnType": "timestamp",
+ "where": [
+ {
+ "name": "$__timeFilter",
+ "params": [],
+ "type": "macro"
+ }
+ ]
+ },
+ {
+ "format": "time_series",
+ "group": [],
+ "hide": false,
+ "metricColumn": "none",
+ "rawQuery": true,
+ "rawSql": "SELECT t.`time` as \"time\",\r\nt.value as \"temperature [9]\"\r\nFROM SDP_FPGA_temp as t INNER JOIN SDP_TR_FPGA_mask as m\r\nON t.`time`=m.`time` and t.`index`=m.`index`\r\nWHERE m.value = 1\r\nAND t.`time` > $__timeFrom()\r\nAND t.`time` < $__timeTo()\r\nAND t.`index` = 9\r\nORDER BY t.`time`, t.`index`",
+ "refId": "K",
+ "select": [
+ [
+ {
+ "params": [
+ "att_conf_id"
+ ],
+ "type": "column"
+ }
+ ]
+ ],
+ "table": "att_array_devfloat_rw",
+ "timeColumn": "data_time",
+ "timeColumnType": "timestamp",
+ "where": [
+ {
+ "name": "$__timeFilter",
+ "params": [],
+ "type": "macro"
+ }
+ ]
+ },
+ {
+ "format": "time_series",
+ "group": [],
+ "hide": false,
+ "metricColumn": "none",
+ "rawQuery": true,
+ "rawSql": "SELECT t.`time` as \"time\",\r\nt.value as \"temperature [10]\"\r\nFROM SDP_FPGA_temp as t INNER JOIN SDP_TR_FPGA_mask as m\r\nON t.`time`=m.`time` and t.`index`=m.`index`\r\nWHERE m.value = 1\r\nAND t.`time` > $__timeFrom()\r\nAND t.`time` < $__timeTo()\r\nAND t.`index` = 10\r\nORDER BY t.`time`, t.`index`",
+ "refId": "L",
+ "select": [
+ [
+ {
+ "params": [
+ "att_conf_id"
+ ],
+ "type": "column"
+ }
+ ]
+ ],
+ "table": "att_array_devfloat_rw",
+ "timeColumn": "data_time",
+ "timeColumnType": "timestamp",
+ "where": [
+ {
+ "name": "$__timeFilter",
+ "params": [],
+ "type": "macro"
+ }
+ ]
+ },
+ {
+ "format": "time_series",
+ "group": [],
+ "hide": false,
+ "metricColumn": "none",
+ "rawQuery": true,
+ "rawSql": "SELECT t.`time` as \"time\",\r\nt.value as \"temperature [11]\"\r\nFROM SDP_FPGA_temp as t INNER JOIN SDP_TR_FPGA_mask as m\r\nON t.`time`=m.`time` and t.`index`=m.`index`\r\nWHERE m.value = 1\r\nAND t.`time` > $__timeFrom()\r\nAND t.`time` < $__timeTo()\r\nAND t.`index` = 11\r\nORDER BY t.`time`, t.`index`",
+ "refId": "M",
+ "select": [
+ [
+ {
+ "params": [
+ "att_conf_id"
+ ],
+ "type": "column"
+ }
+ ]
+ ],
+ "table": "att_array_devfloat_rw",
+ "timeColumn": "data_time",
+ "timeColumnType": "timestamp",
+ "where": [
+ {
+ "name": "$__timeFilter",
+ "params": [],
+ "type": "macro"
+ }
+ ]
+ },
+ {
+ "format": "time_series",
+ "group": [],
+ "hide": false,
+ "metricColumn": "none",
+ "rawQuery": true,
+ "rawSql": "SELECT t.`time` as \"time\",\r\nt.value as \"temperature [12]\"\r\nFROM SDP_FPGA_temp as t INNER JOIN SDP_TR_FPGA_mask as m\r\nON t.`time`=m.`time` and t.`index`=m.`index`\r\nWHERE m.value = 1\r\nAND t.`time` > $__timeFrom()\r\nAND t.`time` < $__timeTo()\r\nAND t.`index` = 12\r\nORDER BY t.`time`, t.`index`",
+ "refId": "N",
+ "select": [
+ [
+ {
+ "params": [
+ "att_conf_id"
+ ],
+ "type": "column"
+ }
+ ]
+ ],
+ "table": "att_array_devfloat_rw",
+ "timeColumn": "data_time",
+ "timeColumnType": "timestamp",
+ "where": [
+ {
+ "name": "$__timeFilter",
+ "params": [],
+ "type": "macro"
+ }
+ ]
+ },
+ {
+ "format": "time_series",
+ "group": [],
+ "hide": false,
+ "metricColumn": "none",
+ "rawQuery": true,
+ "rawSql": "SELECT t.`time` as \"time\",\r\nt.value as \"temperature [13]\"\r\nFROM SDP_FPGA_temp as t INNER JOIN SDP_TR_FPGA_mask as m\r\nON t.`time`=m.`time` and t.`index`=m.`index`\r\nWHERE m.value = 1\r\nAND t.`time` > $__timeFrom()\r\nAND t.`time` < $__timeTo()\r\nAND t.`index` = 13\r\nORDER BY t.`time`, t.`index`",
+ "refId": "O",
+ "select": [
+ [
+ {
+ "params": [
+ "att_conf_id"
+ ],
+ "type": "column"
+ }
+ ]
+ ],
+ "table": "att_array_devfloat_rw",
+ "timeColumn": "data_time",
+ "timeColumnType": "timestamp",
+ "where": [
+ {
+ "name": "$__timeFilter",
+ "params": [],
+ "type": "macro"
+ }
+ ]
+ },
+ {
+ "format": "time_series",
+ "group": [],
+ "hide": false,
+ "metricColumn": "none",
+ "rawQuery": true,
+ "rawSql": "SELECT t.`time` as \"time\",\r\nt.value as \"temperature [14]\"\r\nFROM SDP_FPGA_temp as t INNER JOIN SDP_TR_FPGA_mask as m\r\nON t.`time`=m.`time` and t.`index`=m.`index`\r\nWHERE m.value = 1\r\nAND t.`time` > $__timeFrom()\r\nAND t.`time` < $__timeTo()\r\nAND t.`index` = 14\r\nORDER BY t.`time`, t.`index`",
+ "refId": "P",
+ "select": [
+ [
+ {
+ "params": [
+ "att_conf_id"
+ ],
+ "type": "column"
+ }
+ ]
+ ],
+ "table": "att_array_devfloat_rw",
+ "timeColumn": "data_time",
+ "timeColumnType": "timestamp",
+ "where": [
+ {
+ "name": "$__timeFilter",
+ "params": [],
+ "type": "macro"
+ }
+ ]
+ },
+ {
+ "format": "time_series",
+ "group": [],
+ "hide": false,
+ "metricColumn": "none",
+ "rawQuery": true,
+ "rawSql": "SELECT t.`time` as \"time\",\r\nt.value as \"temperature [15]\"\r\nFROM SDP_FPGA_temp as t INNER JOIN SDP_TR_FPGA_mask as m\r\nON t.`time`=m.`time` and t.`index`=m.`index`\r\nWHERE m.value = 1\r\nAND t.`time` > $__timeFrom()\r\nAND t.`time` < $__timeTo()\r\nAND t.`index` = 15\r\nORDER BY t.`time`, t.`index`",
+ "refId": "Q",
+ "select": [
+ [
+ {
+ "params": [
+ "att_conf_id"
+ ],
+ "type": "column"
+ }
+ ]
+ ],
+ "table": "att_array_devfloat_rw",
+ "timeColumn": "data_time",
+ "timeColumnType": "timestamp",
+ "where": [
+ {
+ "name": "$__timeFilter",
+ "params": [],
+ "type": "macro"
+ }
+ ]
+ }
+ ],
+ "title": "SDP - FPGA Temperature",
+ "type": "timeseries"
+ }
+ ],
+ "refresh": "",
+ "schemaVersion": 31,
+ "style": "dark",
+ "tags": [],
+ "templating": {
+ "list": []
+ },
+ "time": {
+ "from": "now-24h",
+ "to": "now"
+ },
+ "timepicker": {},
+ "timezone": "",
+ "title": "Archiver-MariaDB",
+ "uid": "2zuNvev7z",
+ "version": 1
+}
diff --git a/docker-compose/grafana/dashboards/archiver_timescale.json b/docker-compose/grafana/dashboards/archiver_timescale.json
new file mode 100644
index 0000000000000000000000000000000000000000..bec06ef691e2a89e8e75e264a7f3d9f120a82ea4
--- /dev/null
+++ b/docker-compose/grafana/dashboards/archiver_timescale.json
@@ -0,0 +1,273 @@
+{
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": "-- Grafana --",
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "target": {
+ "limit": 100,
+ "matchAny": false,
+ "tags": [],
+ "type": "dashboard"
+ },
+ "type": "dashboard"
+ }
+ ]
+ },
+ "editable": true,
+ "fiscalYearStartMonth": 0,
+ "gnetId": null,
+ "graphTooltip": 0,
+ "links": [],
+ "liveNow": false,
+ "panels": [
+ {
+ "datasource": "TimescaleDB",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "align": "auto",
+ "displayMode": "auto"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 0
+ },
+ "id": 6,
+ "options": {
+ "showHeader": true
+ },
+ "pluginVersion": "8.2.5",
+ "targets": [
+ {
+ "format": "table",
+ "group": [],
+ "metricColumn": "none",
+ "rawQuery": true,
+ "rawSql": "select name as \"Attribute\",\r\ntable_name as \"Type\"\r\nfrom att_conf ac \r\nwhere \"domain\" ='stat' and \"family\" = 'sdp' and \"member\" ='1'\r\norder by name",
+ "refId": "A",
+ "select": [
+ [
+ {
+ "params": [
+ "value_r"
+ ],
+ "type": "column"
+ }
+ ]
+ ],
+ "table": "att_scalar_devdouble",
+ "timeColumn": "data_time",
+ "timeColumnType": "timestamp",
+ "where": [
+ {
+ "name": "$__timeFilter",
+ "params": [],
+ "type": "macro"
+ }
+ ]
+ }
+ ],
+ "title": "STAT/SDP/1 archived attributes",
+ "type": "table"
+ },
+ {
+ "datasource": "TimescaleDB",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "align": "auto",
+ "displayMode": "auto"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 0
+ },
+ "id": 8,
+ "options": {
+ "showHeader": true
+ },
+ "pluginVersion": "8.2.5",
+ "targets": [
+ {
+ "format": "table",
+ "group": [],
+ "metricColumn": "none",
+ "rawQuery": true,
+ "rawSql": "select name as \"Attribute\",\r\ntable_name as \"Type\" \r\nfrom att_conf ac \r\nwhere \"domain\" ='stat' and \"family\" = 'recv' and \"member\" ='1'\r\norder by name",
+ "refId": "A",
+ "select": [
+ [
+ {
+ "params": [
+ "value_r"
+ ],
+ "type": "column"
+ }
+ ]
+ ],
+ "table": "att_scalar_devdouble",
+ "timeColumn": "data_time",
+ "timeColumnType": "timestamp",
+ "where": [
+ {
+ "name": "$__timeFilter",
+ "params": [],
+ "type": "macro"
+ }
+ ]
+ }
+ ],
+ "title": "LTS/RECV/1 archived attributes",
+ "type": "table"
+ },
+ {
+ "datasource": "TimescaleDB",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "continuous-BlYlRd"
+ },
+ "custom": {
+ "fillOpacity": 70,
+ "lineWidth": 1
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 8
+ },
+ "id": 9,
+ "interval": "10s",
+ "maxDataPoints": 1000,
+ "options": {
+ "colWidth": 0.3,
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom"
+ },
+ "rowHeight": 0.78,
+ "showValue": "auto",
+ "tooltip": {
+ "mode": "single"
+ }
+ },
+ "pluginVersion": "8.2.4",
+ "targets": [
+ {
+ "format": "time_series",
+ "group": [],
+ "hide": false,
+ "metricColumn": "none",
+ "rawQuery": true,
+ "rawSql": "SELECT time_bucket('00:00:01'::interval, q1.\"time\") AS time,\r\n mask(array[q1.value[1]], array[q2.\"value\"[1]]) AS \"temperature[1]\",\r\n mask(array[q1.value[2]], array[q2.\"value\"[2]]) AS \"temperature[2]\",\r\n mask(array[q1.value[3]], array[q2.\"value\"[3]]) AS \"temperature[3]\",\r\n mask(array[q1.value[4]], array[q2.\"value\"[4]]) AS \"temperature[4]\",\r\n mask(array[q1.value[5]], array[q2.\"value\"[5]]) AS \"temperature[5]\",\r\n mask(array[q1.value[6]], array[q2.\"value\"[6]]) AS \"temperature[6]\",\r\n mask(array[q1.value[7]], array[q2.\"value\"[7]]) AS \"temperature[7]\",\r\n mask(array[q1.value[8]], array[q2.\"value\"[8]]) AS \"temperature[8]\",\r\n mask(array[q1.value[9]], array[q2.\"value\"[9]]) AS \"temperature[9]\",\r\n mask(array[q1.value[10]], array[q2.\"value\"[10]]) AS \"temperature[10]\",\r\n mask(array[q1.value[11]], array[q2.\"value\"[11]]) AS \"temperature[11]\",\r\n mask(array[q1.value[12]], array[q2.\"value\"[12]]) AS \"temperature[12]\",\r\n mask(array[q1.value[13]], array[q2.\"value\"[13]]) AS \"temperature[13]\",\r\n mask(array[q1.value[14]], array[q2.\"value\"[14]]) AS \"temperature[14]\",\r\n mask(array[q1.value[15]], array[q2.\"value\"[15]]) AS \"temperature[15]\"\r\n FROM ( SELECT aad.data_time AS \"time\",\r\n aad.value_r AS value\r\n FROM att_array_devdouble aad\r\n JOIN att_conf ac ON aad.att_conf_id = ac.att_conf_id\r\n WHERE aad.value_r IS NOT NULL AND ac.domain = 'stat'::text AND ac.family = 'sdp'::text AND ac.member = '1'::text AND ac.name = 'fpga_temp_r'::text\r\n ORDER BY aad.data_time) q1\r\n JOIN ( SELECT aab.data_time AS \"time\",\r\n aab.value_r AS value\r\n FROM att_array_devboolean aab\r\n JOIN att_conf ac ON aab.att_conf_id = ac.att_conf_id\r\n WHERE aab.value_r IS NOT NULL AND ac.domain = 'stat'::text AND ac.family = 'sdp'::text AND ac.member = '1'::text AND ac.name = 'tr_fpga_mask_r'::text\r\n ORDER BY aab.data_time) q2 ON time_bucket('00:00:01'::interval, q1.\"time\") = time_bucket('00:00:01'::interval, q2.\"time\")\r\n WHERE $__timeFilter(q1.\"time\")\r\n ORDER BY q1.\"time\";",
+ "refId": "B",
+ "select": [
+ [
+ {
+ "params": [
+ "temperature[1]"
+ ],
+ "type": "column"
+ }
+ ]
+ ],
+ "table": "sdp_masked_temp_values",
+ "timeColumn": "temp_time",
+ "timeColumnType": "timestamptz",
+ "where": [
+ {
+ "name": "$__timeFilter",
+ "params": [],
+ "type": "macro"
+ }
+ ]
+ }
+ ],
+ "title": "SDP - FPGA Temperature",
+ "type": "status-history"
+ }
+ ],
+ "refresh": "",
+ "schemaVersion": 32,
+ "style": "dark",
+ "tags": [],
+ "templating": {
+ "list": []
+ },
+ "time": {
+ "from": "now-5m",
+ "to": "now"
+ },
+ "timepicker": {},
+ "timezone": "",
+ "title": "Archiver-Timescale",
+ "uid": "M7zA7Hc7k",
+ "version": 1
+}
diff --git a/docker-compose/grafana/datasources/archiver-timescale.yaml b/docker-compose/grafana/datasources/archiver-timescale.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..69a896dae776283d41283c33688088700f85cadf
--- /dev/null
+++ b/docker-compose/grafana/datasources/archiver-timescale.yaml
@@ -0,0 +1,45 @@
+apiVersion: 1
+
+datasources:
+ # <string, required> name of the datasource. Required
+ - name: TimescaleDB
+ # <string, required> datasource type. Required
+ type: postgres
+ # <string, required> access mode. proxy or direct (Server or Browser in the UI). Required
+ access: proxy
+ # <int> org id. will default to orgId 1 if not specified
+ orgId: 1
+ # <string> custom UID which can be used to reference this datasource in other parts of the configuration, if not specified will be generated automatically
+ uid: ZqArtG97z
+ # <string> url
+ url: archiver-timescale
+ # <string> Deprecated, use secureJsonData.password
+ password:
+ # <string> database user, if used
+ user: postgres
+ # <string> database name, if used
+ database: hdb
+ # <bool> enable/disable basic auth
+ basicAuth: false
+ # <string> basic auth username
+ basicAuthUser:
+ # <string> Deprecated, use secureJsonData.basicAuthPassword
+ basicAuthPassword:
+ # <bool> enable/disable with credentials headers
+ withCredentials:
+ # <bool> mark as default datasource. Max one per org
+ isDefault: false
+ # <map> fields that will be converted to json and stored in jsonData
+ jsonData:
+ # <string> determines whether or with what priority a secure TLS/SSL TCP/IP connection will be negotiated with the server.
+ sslmode: "disable"
+ # <bool> enable TimescaleDB
+ timescaledb: true
+ # <string> json object of data that will be encrypted.
+ secureJsonData:
+ # <string> database password, if used
+ password: password
+ version: 1
+ # <bool> allow users to edit datasources from the UI.
+ editable: false
+
diff --git a/docker-compose/integration-test.yml b/docker-compose/integration-test.yml
index defb45e3c3183516131795b283372ca784635d8c..e2be9144ef7d73b7108609a917529c019e109c62 100644
--- a/docker-compose/integration-test.yml
+++ b/docker-compose/integration-test.yml
@@ -11,7 +11,7 @@ services:
build:
context: itango
args:
- SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
+ SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}integration-test
networks:
- control
diff --git a/docker-compose/itango.yml b/docker-compose/itango.yml
index 9b01c4ea25e2abc5849c9a98c29cc7601ba1115f..02d6801bd8a2f748a4b3d3336352891c78d4882b 100644
--- a/docker-compose/itango.yml
+++ b/docker-compose/itango.yml
@@ -17,7 +17,7 @@ services:
build:
context: itango
args:
- SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
+ SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}itango
networks:
- control
diff --git a/docker-compose/jive.yml b/docker-compose/jive.yml
index 456ae1fc96771bad1ab6b99e52e3b0c9c046c20c..5a2caea9a1d9d6fb19d235781abc33a3230412e8 100644
--- a/docker-compose/jive.yml
+++ b/docker-compose/jive.yml
@@ -18,7 +18,7 @@ version: '2'
services:
jive:
- image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-java:${TANGO_JAVA_VERSION}
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-java:${TANGO_JAVA_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}jive
network_mode: host
volumes:
diff --git a/docker-compose/jupyter.yml b/docker-compose/jupyter.yml
index 1e1deea6f0e22299544f988602efc676bbe6200c..bbc20f269f8a44acff3ce9f36bf11eeef17cea8f 100644
--- a/docker-compose/jupyter.yml
+++ b/docker-compose/jupyter.yml
@@ -7,6 +7,7 @@
# Defines:
# - jupyter: Jupyter Notebook with iTango support
#
+
version: '2'
services:
@@ -15,7 +16,7 @@ services:
context: jupyter
args:
CONTAINER_EXECUTION_UID: ${CONTAINER_EXECUTION_UID}
- SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
+ SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}jupyter
networks:
- control
@@ -30,10 +31,6 @@ services:
user: ${CONTAINER_EXECUTION_UID}
working_dir: /jupyter-notebooks
entrypoint:
- - /usr/local/bin/wait-for-it.sh
- - ${TANGO_HOST}
- - --timeout=30
- - --strict
- - --
+ - /opt/lofar/tango/bin/start-ds.sh
- /usr/bin/tini -- /usr/local/bin/jupyter-notebook --port=8888 --no-browser --ip=0.0.0.0 --allow-root --NotebookApp.token= --NotebookApp.password=
restart: unless-stopped
diff --git a/docker-compose/jupyter/Dockerfile b/docker-compose/jupyter/Dockerfile
index 5393cece6a74ff1de85e9c37ce6a8307e3a66cf5..cc1652e4a45bc14805632ec1d4056beaab1fd34c 100644
--- a/docker-compose/jupyter/Dockerfile
+++ b/docker-compose/jupyter/Dockerfile
@@ -10,23 +10,13 @@ ENV HOME=/home/user
RUN sudo mkdir -p ${HOME}
RUN sudo chown ${CONTAINER_EXECUTION_UID} -R ${HOME}
-# ipython 7.28 is broken in combination with Jupyter, it causes connection errors with notebooks
-RUN sudo pip3 install ipython==7.27.0
-
-RUN sudo pip3 install jupyter
-RUN sudo pip3 install ipykernel
-RUN sudo pip3 install jupyter_bokeh
-# Install matplotlib, jupyterplot
-RUN sudo pip3 install matplotlib jupyterplot
-
-# Allow Download as -> PDF via html
-RUN sudo pip3 install nbconvert
-RUN sudo pip3 install notebook-as-pdf
+COPY requirements.txt ./
+RUN sudo pip3 install -r requirements.txt
# see https://github.com/jupyter/nbconvert/issues/1434
RUN sudo bash -c "echo DEFAULT_ARGS += [\\\"--no-sandbox\\\"] >> /usr/local/lib/python3.7/dist-packages/pyppeteer/launcher.py"
RUN sudo apt-get update -y
-RUN sudo apt-get install -y gconf-service libasound2 libatk1.0-0 libatk-bridge2.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget libcairo-gobject2 libxinerama1 libgtk2.0-0 libpangoft2-1.0-0 libthai0 libpixman-1-0 libxcb-render0 libharfbuzz0b libdatrie1 libgraphite2-3 libgbm1
+RUN sudo apt-get install -y git gconf-service libasound2 libatk1.0-0 libatk-bridge2.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget libcairo-gobject2 libxinerama1 libgtk2.0-0 libpangoft2-1.0-0 libthai0 libpixman-1-0 libxcb-render0 libharfbuzz0b libdatrie1 libgraphite2-3 libgbm1
# Allow Download as -> PDF via LaTeX
RUN sudo apt-get install -y texlive-xetex texlive-fonts-recommended texlive-latex-recommended
@@ -43,15 +33,8 @@ RUN sudo chown ${CONTAINER_EXECUTION_UID} -R /opt/ipython-profiles
COPY jupyter-kernels /usr/local/share/jupyter/kernels/
# Install patched jupyter executable
-RUN sudo pip3 install python-logstash-async
COPY jupyter-notebook /usr/local/bin/jupyter-notebook
-#Install further python modules
-RUN sudo pip3 install PyMySQL[rsa] sqlalchemy
-
-# Packages to interface with testing hardware directly
-RUN sudo pip3 install pyvisa pyvisa-py opcua
-
# Add Tini. Tini operates as a process subreaper for jupyter. This prevents kernel crashes.
ENV TINI_VERSION v0.6.0
ENV JUPYTER_RUNTIME_DIR=/tmp
diff --git a/docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py b/docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py
index 74692f10fff62684c21047eb373aebd70a876155..03e7b75b4649c79647c5bb573b22e41eb330c159 100644
--- a/docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py
+++ b/docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py
@@ -7,7 +7,8 @@ sst = DeviceProxy("STAT/SST/1")
xst = DeviceProxy("STAT/XST/1")
unb2 = DeviceProxy("STAT/UNB2/1")
boot = DeviceProxy("STAT/Boot/1")
+beam = DeviceProxy("STAT/Beam/1")
docker = DeviceProxy("STAT/Docker/1")
# Put them in a list in case one wants to iterate
-devices = [apsct, apspu, recv, sdp, sst, xst, unb2, boot, docker]
+devices = [apsct, apspu, recv, sdp, sst, xst, unb2, boot, beam, docker]
diff --git a/docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/02-stationcontrol.py b/docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/02-stationcontrol.py
new file mode 100644
index 0000000000000000000000000000000000000000..d2d810b1ad5915504e4a5b508e9a128bd2981628
--- /dev/null
+++ b/docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/02-stationcontrol.py
@@ -0,0 +1 @@
+from tangostationcontrol import *
\ No newline at end of file
diff --git a/docker-compose/jupyter/requirements.txt b/docker-compose/jupyter/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..3be9c3d4c8f1052920b577d77442a348c09aa7ca
--- /dev/null
+++ b/docker-compose/jupyter/requirements.txt
@@ -0,0 +1,15 @@
+GitPython >= 3.1.24 # BSD
+ipython >=7.27.0,!=7.28.0 # BSD
+jupyter
+ipykernel
+jupyter_bokeh
+matplotlib
+jupyterplot
+nbconvert
+notebook-as-pdf
+python-logstash-async
+PyMySQL[rsa]
+sqlalchemy
+pyvisa
+pyvisa-py
+opcua
\ No newline at end of file
diff --git a/docker-compose/lofar-device-base.yml b/docker-compose/lofar-device-base.yml
index ce110ed85ba0cfb20b607ab7d08e70505d2392e8..f01faac2d2f41647708229106a895d3dad23c3e4 100644
--- a/docker-compose/lofar-device-base.yml
+++ b/docker-compose/lofar-device-base.yml
@@ -10,6 +10,7 @@
# Requires:
# - tango.yml
#
+
version: '2'
services:
@@ -18,7 +19,7 @@ services:
build:
context: lofar-device-base
args:
- SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
+ SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}lofar-device-base
# These parameters are just visual queues, you have to define them again
# in derived docker-compose files!
diff --git a/docker-compose/lofar-device-base/lofar-requirements.txt b/docker-compose/lofar-device-base/lofar-requirements.txt
index 10ad55d977c97793a352c13da323d84d3c826c0e..95ed439cd121c0dc72b0c9a1c69d409e0bacc57e 100644
--- a/docker-compose/lofar-device-base/lofar-requirements.txt
+++ b/docker-compose/lofar-device-base/lofar-requirements.txt
@@ -1,5 +1,2 @@
-# Do not put tangostationcontrol dependencies here
-astropy
-
-# requirements to build tangocontrol
+# Do not put tangostationcontrol dependencies here, only setup.py / __init__.py
GitPython >= 3.1.24 # BSD
diff --git a/docker-compose/logviewer.yml b/docker-compose/logviewer.yml
index bf0c9b2d51cbdb7334a579184114de6925fd37a1..08da4000b23925980e1683465fa4fdd4c05f04ae 100644
--- a/docker-compose/logviewer.yml
+++ b/docker-compose/logviewer.yml
@@ -12,7 +12,7 @@ version: '2'
services:
logviewer:
- image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-java:${TANGO_JAVA_VERSION}
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-java:${TANGO_JAVA_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}logviewer
networks:
- control
diff --git a/docker-compose/pogo.yml b/docker-compose/pogo.yml
index 826daac9fbd6ef3226a690832eedab505bbeaba3..954841746b9f0338d4a84fdae7e043fde04be460 100644
--- a/docker-compose/pogo.yml
+++ b/docker-compose/pogo.yml
@@ -20,7 +20,7 @@ volumes:
services:
pogo:
- image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-pogo:${TANGO_POGO_VERSION}
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-pogo:${TANGO_POGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}pogo
networks:
- control
diff --git a/docker-compose/prometheus.yml b/docker-compose/prometheus.yml
index 604f4bf4bde93dd6d68aaf7f3b1da2fd3f884e83..1e9ce6f1aa2cd050565f48a4b991865641fd1566 100644
--- a/docker-compose/prometheus.yml
+++ b/docker-compose/prometheus.yml
@@ -4,6 +4,7 @@
# Defines:
# - prometheus: Prometheus
#
+
version: '2'
services:
diff --git a/docker-compose/pypcc-sim-base/Dockerfile b/docker-compose/pypcc-sim-base/Dockerfile
index c65c5b6f836e889f9b3c364ceace5f7b9b821628..f0f37dec5613b988ba3c471428aa426606cf9d5a 100644
--- a/docker-compose/pypcc-sim-base/Dockerfile
+++ b/docker-compose/pypcc-sim-base/Dockerfile
@@ -1,10 +1,6 @@
-FROM ubuntu:20.04
+ARG LOCAL_DOCKER_REGISTRY_HOST
+ARG LOCAL_DOCKER_REGISTRY_LOFAR
-COPY requirements.txt /requirements.txt
+FROM ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_LOFAR}/pypcc:latest
-RUN apt-get update && apt-get install -y python3 python3-pip python3-yaml git && \
- pip3 install -r requirements.txt && \
- git clone --depth 1 --branch master https://git.astron.nl/lofar2.0/pypcc
-
-WORKDIR /pypcc
-CMD ["python3","pypcc2.py","--simulator","--port","4843"]
+CMD ["python3", "pypcc2.py", "--simulator", "--port","4843"]
diff --git a/docker-compose/pypcc-sim-base/requirements.txt b/docker-compose/pypcc-sim-base/requirements.txt
deleted file mode 100644
index 2cd015945c044fcd1e39a823f49a807fc519ac67..0000000000000000000000000000000000000000
--- a/docker-compose/pypcc-sim-base/requirements.txt
+++ /dev/null
@@ -1,3 +0,0 @@
-opcua
-numpy
-recordclass>=0.16,<0.16.1
\ No newline at end of file
diff --git a/docker-compose/recv-sim.yml b/docker-compose/recv-sim.yml
index effee8b298b855e2007e50c379fa3df45010bd05..8fd795be60ef89b23491895dd9809ff67b1c67ae 100644
--- a/docker-compose/recv-sim.yml
+++ b/docker-compose/recv-sim.yml
@@ -10,6 +10,9 @@ services:
recv-sim:
build:
context: pypcc-sim-base
+ args:
+ - LOCAL_DOCKER_REGISTRY_HOST=${LOCAL_DOCKER_REGISTRY_HOST}
+ - LOCAL_DOCKER_REGISTRY_LOFAR=${LOCAL_DOCKER_REGISTRY_LOFAR}
container_name: ${CONTAINER_NAME_PREFIX}recv-sim
networks:
- control
diff --git a/docker-compose/rest.yml b/docker-compose/rest.yml
index 467319399d6f3fec12a74068fea182195014b59e..94e1168455ddfefa20796c352e92d27e07f9a115 100644
--- a/docker-compose/rest.yml
+++ b/docker-compose/rest.yml
@@ -13,7 +13,7 @@ version: '2'
services:
rest:
- image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-rest:${TANGO_REST_VERSION}
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-rest:${TANGO_REST_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}tango-rest
networks:
- control
diff --git a/docker-compose/sdptr-sim.yml b/docker-compose/sdptr-sim.yml
index c81c3db9ae4744e013b5a92f1ebb5e9bdaa6e92c..badf707e37621c8b3030121424bacd1393910b87 100644
--- a/docker-compose/sdptr-sim.yml
+++ b/docker-compose/sdptr-sim.yml
@@ -10,6 +10,9 @@ services:
sdptr-sim:
build:
context: sdptr-sim
+ args:
+ - LOCAL_DOCKER_REGISTRY_HOST=${LOCAL_DOCKER_REGISTRY_HOST}
+ - LOCAL_DOCKER_REGISTRY_LOFAR=${LOCAL_DOCKER_REGISTRY_LOFAR}
container_name: ${CONTAINER_NAME_PREFIX}sdptr-sim
networks:
- control
diff --git a/docker-compose/sdptr-sim/Dockerfile b/docker-compose/sdptr-sim/Dockerfile
index 57fe98141f180a4d15a1e2d87c2c67be8f5894ff..4e64ca2a67229e602a705c9e61b0de999e64fad4 100644
--- a/docker-compose/sdptr-sim/Dockerfile
+++ b/docker-compose/sdptr-sim/Dockerfile
@@ -1,20 +1,7 @@
-FROM ubuntu:20.04
+ARG LOCAL_DOCKER_REGISTRY_HOST
+ARG LOCAL_DOCKER_REGISTRY_LOFAR
-# Install build tools for sdptr and the C language OPC-UA lib
-RUN apt-get update && \
- DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common && \
- DEBIAN_FRONTEND=noninteractive add-apt-repository ppa:open62541-team/ppa && \
- apt-get update && \
- DEBIAN_FRONTEND=noninteractive apt-get install -y autoconf automake git make g++ build-essential pkg-config libboost-dev libboost-regex-dev libboost-system-dev libboost-program-options-dev libopen62541-1-dev libopen62541-1-tools && \
- apt-get clean
-
-# Install SDPTR
-RUN cd / && git clone --depth 1 --branch master https://git.astron.nl/lofar2.0/sdptr
-
-RUN cd /sdptr && \
- autoreconf -v -f -i && \
- ./configure && \
- bash -c "make -j `nproc` install"
+FROM ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_LOFAR}/sdptr:latest
COPY simulator.conf /sdptr/src/simulator.conf
diff --git a/docker-compose/tango-prometheus-exporter/ska-tango-grafana-exporter b/docker-compose/tango-prometheus-exporter/ska-tango-grafana-exporter
index 774d39a40ca19c9d979ad22565e57b4af3e9a831..dddb23ff587f6e9c837cdb77e7955e94272eca6f 160000
--- a/docker-compose/tango-prometheus-exporter/ska-tango-grafana-exporter
+++ b/docker-compose/tango-prometheus-exporter/ska-tango-grafana-exporter
@@ -1 +1 @@
-Subproject commit 774d39a40ca19c9d979ad22565e57b4af3e9a831
+Subproject commit dddb23ff587f6e9c837cdb77e7955e94272eca6f
diff --git a/docker-compose/tango.yml b/docker-compose/tango.yml
index 937cc5c8ecbe00b553d4692988e6cc2e5d7c51ef..19500fca1eeba859f74e7ba54fc3cbb021ea0ce6 100644
--- a/docker-compose/tango.yml
+++ b/docker-compose/tango.yml
@@ -15,7 +15,7 @@ volumes:
services:
tangodb:
- image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-db:${TANGO_DB_VERSION}
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-db:${TANGO_DB_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}tangodb
networks:
- control
@@ -37,7 +37,7 @@ services:
restart: unless-stopped
databaseds:
- image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-cpp:${TANGO_CPP_VERSION}
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-cpp:${TANGO_CPP_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}databaseds
networks:
- control
diff --git a/docker-compose/tangotest.yml b/docker-compose/tangotest.yml
index 357c91df487b51379db221f7cb984bc05018f5e3..a97290d48f437b1c65b0bef01f6788fb525b2275 100644
--- a/docker-compose/tangotest.yml
+++ b/docker-compose/tangotest.yml
@@ -11,7 +11,7 @@ version: '2'
services:
tangotest:
- image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-java:${TANGO_JAVA_VERSION}
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-java:${TANGO_JAVA_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}tangotest
networks:
- control
diff --git a/docker-compose/timescaledb/Dockerfile b/docker-compose/timescaledb/Dockerfile
index 86e7a820c3fc4e4ba0eaa8edaa9c7e421c87231f..4ca5dd07826a14bc17d7d982cdd1198fbe657980 100644
--- a/docker-compose/timescaledb/Dockerfile
+++ b/docker-compose/timescaledb/Dockerfile
@@ -5,5 +5,5 @@ COPY resources/01_admin.sh docker-entrypoint-initdb.d/002_admin.sh
COPY resources/02_hdb_schema.sql docker-entrypoint-initdb.d/003_hdb_schema.sql
COPY resources/03_hdb_roles.sql docker-entrypoint-initdb.d/004_hdb_roles.sql
COPY resources/04_hdb_ext_aggregates.sql docker-entrypoint-initdb.d/005_hdb_ext_aggregates.sql
-COPY resources/05_lofar_views.sql docker-entrypoint-initdb.d/006_lofar_views.sql
+COPY resources/05_lofar_func.sh docker-entrypoint-initdb.d/006_lofar_func.sh
COPY resources/06_cleanup.sql docker-entrypoint-initdb.d/007_cleanup.sql
diff --git a/docker-compose/timescaledb/resources/05_lofar_func.sh b/docker-compose/timescaledb/resources/05_lofar_func.sh
new file mode 100644
index 0000000000000000000000000000000000000000..4797c7a6df2acf11e3709221a9bd4fd335128264
--- /dev/null
+++ b/docker-compose/timescaledb/resources/05_lofar_func.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+psql << EOF
+\c hdb
+CREATE OR REPLACE FUNCTION mask(double precision[], boolean[])
+RETURNS double precision[] LANGUAGE sql
+AS \$function\$ SELECT ARRAY(SELECT
+case when \$2[i] then \$1[i]
+else '0'::double precision end
+FROM generate_subscripts(\$1,1) g(i)) \$function\$;
+EOF
diff --git a/docker-compose/timescaledb/resources/05_lofar_views.sql b/docker-compose/timescaledb/resources/05_lofar_views.sql
deleted file mode 100644
index 82aa8e5de360b434ecf081d1b02c000dff5d50f7..0000000000000000000000000000000000000000
--- a/docker-compose/timescaledb/resources/05_lofar_views.sql
+++ /dev/null
@@ -1,54 +0,0 @@
--- LOFAR 2.0 CUSTOMIZED VIEWS
-
-\c hdb
--- SDP FPGA Temperature
-create or replace view "sdp_fpga_temp" as
-select
- ac.att_name as "attribute",
- aad.data_time AS "time",
- aad.value_r
-FROM att_array_devdouble aad join att_conf ac
-on aad.att_conf_id = ac.att_conf_id
-where aad.value_R is not null
-and ac."domain" ='stat' and ac."family" ='sdp' and ac."member" ='1'
-ORDER BY aad.data_time;
-
--- SDP FPGA Mask
-create or replace view "sdp_tr_fpga_mask" as
-select
- ac.att_name as "attribute",
- aab.data_time AS "time",
- aab.value_r
-FROM att_array_devboolean aab join att_conf ac
-on aab.att_conf_id = ac.att_conf_id
-where aab.value_R is not null
-and ac."domain" ='stat' and ac."family" ='sdp' and ac."member" ='1'
-ORDER BY aab.data_time;
-
--- SDP Masked values (rounded to 1 second)
-create or replace view "sdp_masked_temp_values" as
-select time_bucket('1 second',t.time) as "temp_time",
-time_bucket('1 second',m.time) as "mask_time",
-t.value_r as "temperature",
-m.value_r as "mask"
-from sdp_fpga_temp as t
-inner join sdp_tr_fpga_mask as m
-on time_bucket('1 second',t.time) = time_bucket('1 second',m.time)
-/* Replace if possible with SQL loop */
-where m.value_r[1]=true and
-m.value_r[2]=true and
-m.value_r[3]=true and
-m.value_r[4]=true and
-m.value_r[5]=true and
-m.value_r[6]=true and
-m.value_r[7]=true and
-m.value_r[8]=true and
-m.value_r[9]=true and
-m.value_r[10]=true and
-m.value_r[11]=true and
-m.value_r[12]=true and
-m.value_r[13]=true and
-m.value_r[14]=true and
-m.value_r[15]=true and
-m.value_r[16]=true
-order by t."time" ;
diff --git a/docker-compose/unb2-sim.yml b/docker-compose/unb2-sim.yml
index d1ecaaa70a3c1e52f39ab1453d2ec8eb191f8831..b01802cd0526abe325c710f08fe965d6244cb2ba 100644
--- a/docker-compose/unb2-sim.yml
+++ b/docker-compose/unb2-sim.yml
@@ -10,6 +10,9 @@ services:
unb2-sim:
build:
context: pypcc-sim-base
+ args:
+ - LOCAL_DOCKER_REGISTRY_HOST=${LOCAL_DOCKER_REGISTRY_HOST}
+ - LOCAL_DOCKER_REGISTRY_LOFAR=${LOCAL_DOCKER_REGISTRY_LOFAR}
container_name: ${CONTAINER_NAME_PREFIX}unb2-sim
networks:
- control
diff --git a/docs/source/devices/beam.rst b/docs/source/devices/beam.rst
new file mode 100644
index 0000000000000000000000000000000000000000..c469162dfe25fafc1c79cecae15c7e1b36cf47e4
--- /dev/null
+++ b/docs/source/devices/beam.rst
@@ -0,0 +1,8 @@
+Beam
+====================
+
+The ``beam == DeviceProxy("STAT/Beam/1")`` device sets up the beamforming on the station:
+
+- The HBA tiles in RECV need *analog beamforming* to combine their 16 antennas into a single input for the RCU,
+- THe LBAs or HBA tiles need *digital beamforming* in SDP to combine their signals into beamlets.
+
diff --git a/docs/source/devices/sst-xst.rst b/docs/source/devices/sst-xst.rst
index cdb689e457dc2d6abebcfc1391f057135f30b722..092bdda832eb0493f93956f4aec8025410d0bf9b 100644
--- a/docs/source/devices/sst-xst.rst
+++ b/docs/source/devices/sst-xst.rst
@@ -63,7 +63,7 @@ Typically, ``N_ant == 192``, and ``N_blocks == 136``.
The metadata refers to the *blocks*, which are emitted by the FPGAs to represent the XSTs between 12 x 12 consecutive antennas. The following code converts block numbers to the indices of the first antenna pair in a block::
- from common.baselines import baseline_from_index
+ from tangostationcontrol.common.baselines import baseline_from_index
def first_antenna_pair(block_nr: int) -> int:
coarse_a, coarse_b = baseline_from_index(block_nr)
@@ -71,7 +71,7 @@ The metadata refers to the *blocks*, which are emitted by the FPGAs to represent
Conversely, to calculate the block index for an antenna pair ``(a,b)``, use::
- from common.baselines import baseline_index
+ from tangostationcontrol.common.baselines import baseline_index
def block_nr(a: int, b: int) -> int:
return baseline_index(a // 12, b // 12)
diff --git a/docs/source/index.rst b/docs/source/index.rst
index 119d95164313538606e9f310f2e65ca6aca233d8..7baab29343d3f7c4015e6ed60e6ea4692e6073e3 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -19,6 +19,7 @@ Even without having access to any LOFAR2.0 hardware, you can install the full st
installation
interfaces/overview
devices/using
+ devices/beam
devices/boot
devices/docker
devices/recv
diff --git a/sbin/run_integration_test.sh b/sbin/run_integration_test.sh
index c3c37983ae63688658211337ebc17e83fa7ff546..05e076ed00f79858fa44be6e23f3b94370672cb9 100755
--- a/sbin/run_integration_test.sh
+++ b/sbin/run_integration_test.sh
@@ -11,11 +11,15 @@ fi
cd "$LOFAR20_DIR/docker-compose" || exit 1
-# Make sure builds are recent, and use our building parameters.
-make build
+# Build only the required images, please do not build everything that makes CI
+# take really long to finish, especially grafana / jupyter / prometheus.
+# jupyter is physically large > 2.5gb and overlayfs is really slow.
+make build device-sdp device-recv device-sst device-unb2 device-xst
+make build sdptr-sim recv-sim unb2-sim apsct-sim apspu-sim
+make build databaseds dsconfig elk integration-test
# Start and stop sequence
-make stop device-boot device-docker device-apsct device-apspu device-sdp device-recv device-sst device-unb2 device-xst sdptr-sim recv-sim unb2-sim apsct-sim apspu-sim
+make stop device-boot device-docker device-apsct device-apspu device-sdp device-recv device-sst device-unb2 device-xst device-beam sdptr-sim recv-sim unb2-sim apsct-sim apspu-sim
make start databaseds dsconfig elk
# Give dsconfig and databaseds time to start
@@ -32,7 +36,7 @@ make start sdptr-sim recv-sim unb2-sim apsct-sim apspu-sim
# Give the simulators time to start
sleep 5
-make start device-boot device-apsct device-apspu device-sdp device-recv device-sst device-unb2 device-xst
+make start device-boot device-apsct device-apspu device-sdp device-recv device-sst device-unb2 device-xst device-beam
# Give devices time to restart
# TODO(Corne Lukken): Use a nicer more reliable mechanism
diff --git a/sbin/tag_and_push_docker_image.sh b/sbin/tag_and_push_docker_image.sh
index 799ab1cd779bb5caf840685f339080b57916063b..631413235d4ba9b2f39b9b0e216d31cd21913bdf 100755
--- a/sbin/tag_and_push_docker_image.sh
+++ b/sbin/tag_and_push_docker_image.sh
@@ -1,40 +1,177 @@
#!/bin/bash -e
-# Tag and push which image version?
-DOCKER_TAG=latest
+function usage {
+ echo "./$(basename "$0")
+ no arguments, downloads remote images and pushes these to ASTRON registry.
+ The versions downloaded are controlled by the docker-compose/.env file"
+ echo ""
+ echo "./$(basename "$0") -h
+ displays this help message"
+ echo ""
+ echo "./$(basename "$0") <docker service name> <tag>
+ downloads latest version of image from the ASTRON registry, builds the
+ specified service and pushes the image with the specified tag to the
+ ASTRON registry"
+ echo ""
+ echo "./$(basename "$0") pull <tag>
+ downloads all images for the integration test with the specified tag
+ falling back to 'latest' if unavailable. Should neither exist on the
+ ASTRON registry the script will exit 1. The images are retagged to match
+ the output of docker-compose."
+}
-# Change to git tag or git hash if no tag
-VERSION=$(date +"%Y-%M-%d")
+# list of arguments expected in the input
+optstring=":h"
-SKA_REPO="nexus.engageska-portugal.pt/ska-docker"
-LOFAR_REPO="git.astron.nl:5000/lofar2.0/tango"
+while getopts ${optstring} arg; do
+ case ${arg} in
+ h)
+ usage
+ exit 0
+ ;;
+ :)
+ echo "$0: Must supply an argument to -$OPTARG." >&2
+ exit 1
+ ;;
+ ?)
+ echo "Invalid option: -${OPTARG}."
+ exit 2
+ ;;
+ esac
+done
-# Compile a list of the SKA images
-SKA_IMAGES=$(for i in $(docker images | grep -E ${DOCKER_TAG} | grep -E ${SKA_REPO} | cut -d' ' -f1); do printf "%s " "${i}"; done)
+if [ -z "${LOFAR20_DIR+x}" ]; then
+ echo "LOFAR20_DIR not set, did you forget to source lofar20rc.sh?"
+ exit 1
+fi
-# Compile a list of LOFAR2.0 images
-LOFAR_IMAGES=$(for i in $(docker images | grep -E ${DOCKER_TAG} | grep -E -v "${SKA_REPO}|${LOFAR_REPO}" | cut -d' ' -f1); do printf "%s " "${i}"; done)
+# shellcheck disable=SC1090
+. "${LOFAR20_DIR}/docker-compose/.env" || exit 1
-function tag_and_push()
-{
- (
- docker tag "${1}" "${2}"
- docker push "${2}"
- ) &
-}
+# List of images and their tag
+REMOTE_IMAGES=(
+ "tango-dsconfig:${TANGO_DSCONFIG_VERSION}" "tango-java:${TANGO_JAVA_VERSION}"
+ "tango-itango:${TANGO_ITANGO_VERSION}" "tango-pogo:${TANGO_POGO_VERSION}"
+ "tango-cpp:${TANGO_CPP_VERSION}" "tango-db:${TANGO_DB_VERSION}"
+ "tango-dsconfig:${TANGO_DSCONFIG_VERSION}" "tango-rest:${TANGO_REST_VERSION}"
+)
-# Rename the SKA images for the LOFAR2.0 repo
-# and push them to the LOFAR2.0 repo
-for IMAGE in ${SKA_IMAGES}; do
- PUSH_IMAGE=${IMAGE//${SKA_REPO}/${LOFAR_REPO}}:${VERSION}
- tag_and_push "${IMAGE}" "${PUSH_IMAGE}"
-done
+# If first argument of bash script not set run first stage
+if [ -z "${1+x}" ]; then
+ echo "Pulling and retagging remote images"
-# Rename the LOFAR2.0 images for the LOFAR2.0 repo
-# and push them to the LOFAR2.0 repo
-for IMAGE in ${LOFAR_IMAGES}; do
- PUSH_IMAGE=${LOFAR_REPO}/${IMAGE}:${VERSION}
- tag_and_push "${IMAGE}" "${PUSH_IMAGE}"
-done
+ # Iterate over al the REMOTE_IMAGES and pull them from remote and push local
+ for image in "${REMOTE_IMAGES[@]}"; do
+ remote_url="${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-${image}"
+ local_url="${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/${image}"
+ docker pull "${remote_url}"
+ docker tag "${remote_url}" "${local_url}"
+ docker push "${local_url}"
+ done
+
+ exit 0
+fi
+
+# Triple tuple of docker-compose names, image names and if necessary for
+# integration tests.
+# TODO(Corne): Have this list generated from the .yml files
+LOCAL_IMAGES=(
+ "elk elk y" "elk-configure-host elk-configure-host y"
+ "lofar-device-base lofar-device-base y"
+
+ "apsct-sim docker-compose_apsct-sim y" "apspu-sim docker-compose_apspu-sim y"
+ "recv-sim docker-compose_recv-sim y" "sdptr-sim docker-compose_sdptr-sim y"
+ "unb2-sim docker-compose_unb2-sim y"
+
+ "device-apsct device-apsct y" "device-apspu device-apspu y"
+ "device-boot device-boot y" "device-docker device-docker y"
+ "device-observation_control device-observation_control y"
+ "device-recv device-recv y" "device-sdp device-sdp y"
+ "device-sst device-sst y" "device-unb2 device-unb2 y"
+ "device-xst device-xst y"
+
+ "itango docker-compose_itango y"
+
+ "grafana grafana n" "prometheus prometheus n"
+ "jupyter docker-compose_jupyter n"
+ "integration-test docker-compose_integration-test n"
+ "tango-prometheus-exporter docker-compose_tango-prometheus-exporter n"
+)
+
+
+
+# If first argument set run second stage, determine LOCAL_IMAGE to build and
+# push from the argument
+if [ ! -z "${1+x}" ] && [ "${1}" != "pull" ]; then
+
+ # The second argument must pass the tag variable must be set
+ if [ -z "${2+x}" ]; then
+ echo "Error, second argument must pass tag variable"
+ exit 1
+ fi
+
+ # Set the tag and image variable, variables $1 and $2 are shadowed later
+ local_image="${1}"
+ tag="${2}"
+
+ cd "${LOFAR20_DIR}/docker-compose" || exit 1
+
+ # Loop through images and find the specified one
+ for image in "${LOCAL_IMAGES[@]}"; do
+ # Set, splits tuple into $1 and $2. this shadows previous variables
+ # shellcheck disable=SC2086
+ set -- $image
+ if [ "${local_image}" == "${1}" ]; then
+ echo "Building image for ${1} container"
+ local_url="${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/${2}"
+
+ # If tag is not latest, than it is not a tagged master build and we can
+ # pull the latest image as cache.
+ if [ "${tag}" != "latest" ]; then
+ docker pull "${local_url}:latest"
+ fi
+
+ make build "${1}"
+ docker tag "${2}" "${local_url}:${tag}"
+ docker push "${local_url}:${tag}"
+ fi
+ done
+
+ exit 0
+fi
+
+# Final stage, pull images for integration cache try special tag image first
+# if it fails download latest instead
+if [ ! -z "${1+x}" ] && [ "${1}" == "pull" ]; then
+ echo "Pulling images for integration test cache"
+
+ # The second argument must pass the tag variable must be set
+ if [ -z "${2+x}" ]; then
+ echo "Error, second argument must pass tag variable"
+ exit 1
+ fi
+
+ # Set the tag variable
+ tag="${2}"
+
+ for image in "${LOCAL_IMAGES[@]}"; do
+ # Set, splits tuple into $1 and $2. this shadows previous variables
+ # shellcheck disable=SC2086
+ set -- $image
+
+ # Only download images which are needed for integration test
+ if [ "${3}" == "y" ]; then
+ local_url="${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/${2}"
+ # Pull images, at least one of the two images must succeed
+ echo "docker pull ${local_url}:${tag}"
+ docker pull "${local_url}:${tag}" || docker pull "${local_url}:latest" || exit 1
+ # Ensure the images will have the same tags as generated by docker-compose
+ docker tag "${local_url}:${tag}" "${2}" || docker tag "${local_url}:latest" "${2}" || exit 1
+ fi
+ done
+
+ exit 0
+fi
-wait
+# Somehow nothing ran, that is an error do not fail silently
+exit 1
diff --git a/tangostationcontrol/requirements.txt b/tangostationcontrol/requirements.txt
index b1620255b5a45c9e3f653661e65de6732fd93a07..087bb148d057ef2e6d10f1727999932e4828c33a 100644
--- a/tangostationcontrol/requirements.txt
+++ b/tangostationcontrol/requirements.txt
@@ -5,7 +5,6 @@
asyncua >= 0.9.90 # LGPLv3
PyMySQL[rsa] >= 1.0.2 # MIT
sqlalchemy >= 1.4.26 #MIT
-GitPython >= 3.1.24 # BSD
snmp >= 0.1.7 # GPL3
h5py >= 3.5.0 # BSD
psutil >= 5.8.0 # BSD
diff --git a/tangostationcontrol/setup.cfg b/tangostationcontrol/setup.cfg
index e6101c890893c961e1eff27ce3d3ab4e019ea312..31662e40d639c2af2de417e734e794dc5679d4a4 100644
--- a/tangostationcontrol/setup.cfg
+++ b/tangostationcontrol/setup.cfg
@@ -36,6 +36,7 @@ where=./
console_scripts =
l2ss-apsct = tangostationcontrol.devices.apsct:main
l2ss-apspu = tangostationcontrol.devices.apspu:main
+ l2ss-beam = tangostationcontrol.devices.beam:main
l2ss-boot = tangostationcontrol.devices.boot:main
l2ss-docker-device = tangostationcontrol.devices.docker_device:main
l2ss-observation = tangostationcontrol.devices.observation:main
diff --git a/tangostationcontrol/tangostationcontrol/devices/README.md b/tangostationcontrol/tangostationcontrol/devices/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..f546fb339077c328e55dfd359c8cb23ccfbd64d1
--- /dev/null
+++ b/tangostationcontrol/tangostationcontrol/devices/README.md
@@ -0,0 +1,18 @@
+# Devices
+
+This directory contains the sources for our custom Tango devices.
+
+## Adding a new device
+
+If a new device is added, it will (likely) need to be referenced in several places. Adjust or add the following files (referenced from the repository root), following the pattern shown by the devices already there:
+
+- Adjust `CDB/LOFAR_ConfigDb.json` to create the device in the Tango device database,
+- Adjust `docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py` to make an alias for it available in Jupyter,
+- Adjust `tangostationcontrol/tangostationcontrol/devices/boot.py` to add the device to the station initialisation sequence,
+- Add to `docker-compose/` to create a YaML file to start the device in a docker container. NOTE: it needs a unique 57xx port assigned,
+- Adjust `tangostationcontrol/setup.cfg` to add an entry point for the device in the package installation,
+- Add to `tangostationcontrol/tangostationcontrol/integration_test/devices/` to add an integration test,
+- Adjust `sbin/run_integration_test.sh` to have the device started when running the integration tests,
+- Add to `docs/source/devices/` to mention the device in the end-user documentation.
+- Adjust `docs/source/index.rst` to include the newly created file in `docs/source/devices/`.
+
diff --git a/tangostationcontrol/tangostationcontrol/devices/beam.py b/tangostationcontrol/tangostationcontrol/devices/beam.py
new file mode 100644
index 0000000000000000000000000000000000000000..1711a8a89f9db0e73cc4bf2fd38e81a7a95ded01
--- /dev/null
+++ b/tangostationcontrol/tangostationcontrol/devices/beam.py
@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+#
+# Distributed under the terms of the APACHE license.
+# See LICENSE.txt for more info.
+
+""" Beam Device Server for LOFAR2.0
+
+"""
+
+# PyTango imports
+from tango import AttrWriteType
+import numpy
+# Additional import
+
+from tangostationcontrol.devices.device_decorators import *
+from tangostationcontrol.common.entrypoint import entry
+from tangostationcontrol.clients.attribute_wrapper import attribute_wrapper
+from tangostationcontrol.devices.lofar_device import lofar_device
+from tangostationcontrol.common.lofar_logging import device_logging_to_python, log_exceptions
+
+__all__ = ["Beam", "main"]
+
+
+@device_logging_to_python()
+class Beam(lofar_device):
+ # -----------------
+ # Device Properties
+ # -----------------
+
+ # ----------
+ # Attributes
+ # ----------
+
+ pass
+
+ # --------
+ # overloaded functions
+ # --------
+
+
+ # --------
+ # Commands
+ # --------
+
+
+# ----------
+# Run server
+# ----------
+def main(**kwargs):
+ """Main function of the ObservationControl module."""
+ return entry(Beam, **kwargs)
diff --git a/tangostationcontrol/tangostationcontrol/devices/boot.py b/tangostationcontrol/tangostationcontrol/devices/boot.py
index c4df70bf5027a8068a1e58326c5ad442e63ad4fc..5f2e9b76e47a9ed61207a07fbc0584ec355b6620 100644
--- a/tangostationcontrol/tangostationcontrol/devices/boot.py
+++ b/tangostationcontrol/tangostationcontrol/devices/boot.py
@@ -251,6 +251,7 @@ class Boot(lofar_device):
"STAT/SDP/1", # SDP controls the mask for SST/XST/BST, so initialise it first
"STAT/SST/1",
"STAT/XST/1",
+ "STAT/Beam/1", # Accesses RECV and SDP
],
)
diff --git a/tangostationcontrol/tangostationcontrol/devices/recv.py b/tangostationcontrol/tangostationcontrol/devices/recv.py
index 5070b8cef0f54e5d4165701ccaefe0a637faf2d7..e6f0d1f36ed16790f5532b7f9cafce3419ff8579 100644
--- a/tangostationcontrol/tangostationcontrol/devices/recv.py
+++ b/tangostationcontrol/tangostationcontrol/devices/recv.py
@@ -61,6 +61,9 @@ class RECV(opcua_device):
RCU_LED_colour_R = attribute(dtype=(numpy.uint32,), max_dim_x=32, fget=lambda self: (2 * self.proxy.RCU_LED_green_on_R + 4 * self.proxy.RCU_LED_red_on_R).astype(numpy.uint32))
ANT_mask_RW = attribute_wrapper(comms_annotation=["ANT_mask_RW" ],datatype=numpy.bool_ , dims=(3,32), access=AttrWriteType.READ_WRITE)
+
+ # The HBAT beamformer delays represent 32 delays for each of the 96 inputs.
+ # The 32 delays deconstruct as delays[polarisation][dipole], and each delay is the number of 'delay steps' to apply (0.5ns for HBAT1).
HBAT_BF_delays_R = attribute_wrapper(comms_annotation=["HBAT_BF_delays_R" ],datatype=numpy.int64 , dims=(32,96))
HBAT_BF_delays_RW = attribute_wrapper(comms_annotation=["HBAT_BF_delays_RW" ],datatype=numpy.int64 , dims=(32,96), access=AttrWriteType.READ_WRITE)
HBAT_LED_on_R = attribute_wrapper(comms_annotation=["HBAT_LED_on_R" ],datatype=numpy.bool_ , dims=(32,96))
diff --git a/tangostationcontrol/tangostationcontrol/devices/sdp/statistics_packet.py b/tangostationcontrol/tangostationcontrol/devices/sdp/statistics_packet.py
index c98ae9b5bdc604e8a55480cc5473e658b10cefa1..59c74e296c2eebdd677d448bdae523be8d149934 100644
--- a/tangostationcontrol/tangostationcontrol/devices/sdp/statistics_packet.py
+++ b/tangostationcontrol/tangostationcontrol/devices/sdp/statistics_packet.py
@@ -75,13 +75,13 @@ class StatisticsPacket(object):
"Invalid SDP statistics packet: packet marker (first byte) is {}, not one of 'SBX'.".format(
self.marker))
+ # format string for the header, see unpack below
+ header_format = ">cBL HHB BHL BBH HQ"
+ header_size = struct.calcsize(header_format)
+
def unpack(self):
""" Unpack the packet into properties of this object. """
- # format string for the header, see unpack below
- self.header_format = ">cBL HHB BHL BBH HQ"
- self.header_size = struct.calcsize(self.header_format)
-
# unpack fields
try:
(self.marker_raw,
@@ -335,11 +335,35 @@ def main(args=None, **kwargs):
import sys
import pprint
- # read all of stdin, even though we only parse the first packet. we're too lazy to intelligently decide when
- # the packet is complete and can stop reading.
- data = sys.stdin.buffer.read()
- packet = SSTPacket(data)
-
- # print header & payload
- pprint.pprint(packet.header())
- pprint.pprint(packet.payload)
+ # packet counter
+ nr = 0
+
+ # byte offset in the stream
+ offset = 0
+
+ while True:
+ # read just the header
+ header = sys.stdin.buffer.read(StatisticsPacket.header_size)
+ if not header:
+ break
+
+ # read the payload
+ packet = StatisticsPacket(header)
+ payload_size = packet.expected_size() - len(header)
+ payload = sys.stdin.buffer.read(payload_size)
+
+ # construct the packet based on type
+ if packet.marker == 'S':
+ packet = SSTPacket(header + payload)
+ elif packet.marker == 'X':
+ packet = XSTPacket(header + payload)
+ elif packet.marker == 'B':
+ packet = BSTPacket(header + payload)
+
+ # print header
+ print(f"# Packet {nr} starting at offset {offset}")
+ pprint.pprint(packet.header())
+
+ # increment counters
+ nr += 1
+ offset += len(header) + len(payload)
diff --git a/tangostationcontrol/tangostationcontrol/integration_test/device_proxy.py b/tangostationcontrol/tangostationcontrol/integration_test/device_proxy.py
index 00ba0904c7bd01fa2ce1453a4c6701d6a4246e14..25c92411ecaababad20007868cfd19bdc3e9e18a 100644
--- a/tangostationcontrol/tangostationcontrol/integration_test/device_proxy.py
+++ b/tangostationcontrol/tangostationcontrol/integration_test/device_proxy.py
@@ -1,8 +1,25 @@
+import logging
+import time
+
from tango import DeviceProxy
+logger = logging.getLogger()
+
class TestDeviceProxy(DeviceProxy):
def __init__(self, *args, **kwargs):
super(TestDeviceProxy, self).__init__(*args, **kwargs)
self.set_timeout_millis(10000)
+
+ @staticmethod
+ def test_device_turn_off(endpoint):
+ d = TestDeviceProxy(endpoint)
+ try:
+ d.Off()
+ except Exception as e:
+ """Failing to turn Off devices should not raise errors here"""
+ logger.error(f"Failed to turn device off in teardown {e}")
+
+ """Wait for 1 second to prevent propagating reconnection errors"""
+ time.sleep(1)
diff --git a/tangostationcontrol/tangostationcontrol/integration_test/devices/base.py b/tangostationcontrol/tangostationcontrol/integration_test/devices/base.py
index 555f7256ea49d68465b1c45bf038d47b39beeb25..ef4854a8241aaee7e6099ae7d417d7b94acfa21e 100644
--- a/tangostationcontrol/tangostationcontrol/integration_test/devices/base.py
+++ b/tangostationcontrol/tangostationcontrol/integration_test/devices/base.py
@@ -34,11 +34,19 @@ class AbstractTestBases:
# make sure the device starts in Off
self.proxy.Off()
+ self.addCleanup(TestDeviceProxy.test_device_turn_off, self.name)
+
super().setUp()
- def tearDown(self):
- """Turn device Off in teardown to prevent blocking tests"""
- self.proxy.Off()
+ def test_device_fetch_state(self):
+ """Test if we can successfully fetch state"""
+
+ self.assertEqual(DevState.OFF, self.proxy.state())
+
+ def test_device_ping(self):
+ """Test if we can successfully ping the device server"""
+
+ self.assertGreater(self.proxy.ping(), 0)
def test_device_initialize(self):
"""Test if we can transition to standby"""
diff --git a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_apsct.py b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_apsct.py
index ca73fc236a7486b858298e863663cf997c70ccc8..d973581e88cceb7510d2d257b006aa81ebbfbdfb 100644
--- a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_apsct.py
+++ b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_apsct.py
@@ -9,6 +9,7 @@
from .base import AbstractTestBases
+
class TestDeviceAPSCT(AbstractTestBases.TestDeviceBase):
def setUp(self):
diff --git a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_apspu.py b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_apspu.py
index b9d2bc3d44acf0d4c2d3e1dd9b0adc095c86037b..5ebadc24029ce4a8e4a8b56805685d69aec73f2b 100644
--- a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_apspu.py
+++ b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_apspu.py
@@ -9,6 +9,7 @@
from .base import AbstractTestBases
+
class TestDeviceAPSPU(AbstractTestBases.TestDeviceBase):
def setUp(self):
diff --git a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_beam.py b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_beam.py
new file mode 100644
index 0000000000000000000000000000000000000000..b6be958cfde3b9f236947e9b1b76195c844bad40
--- /dev/null
+++ b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_beam.py
@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of the LOFAR 2.0 Station Software
+#
+#
+#
+# Distributed under the terms of the APACHE license.
+# See LICENSE.txt for more info.
+
+from .base import AbstractTestBases
+
+class TestDeviceBeam(AbstractTestBases.TestDeviceBase):
+
+ def setUp(self):
+ super().setUp("STAT/Beam/1")
diff --git a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_boot.py b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_boot.py
index cc39c45d0a35da050aa041f0e5f2063df6312169..6d93080cc59ca176aea226893927f3321f503d89 100644
--- a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_boot.py
+++ b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_boot.py
@@ -11,6 +11,7 @@ import time
from .base import AbstractTestBases
+
class TestDeviceBoot(AbstractTestBases.TestDeviceBase):
def setUp(self):
diff --git a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_recv.py b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_recv.py
index 26e02ef312214df4a9304eafd050ea1438002da2..e96c385a7f976bc3ecb76d48b509b61d80454819 100644
--- a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_recv.py
+++ b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_recv.py
@@ -9,6 +9,7 @@
from .base import AbstractTestBases
+
class TestDeviceRECV(AbstractTestBases.TestDeviceBase):
def setUp(self):
diff --git a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_sdp.py b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_sdp.py
index 2df399ed4607ea802abcf647c02078193f1b7f03..7de27c34b9746c1541c2b7091c977ed32ce9e535 100644
--- a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_sdp.py
+++ b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_sdp.py
@@ -7,11 +7,14 @@
# Distributed under the terms of the APACHE license.
# See LICENSE.txt for more info.
+from tango._tango import DevState
from .base import AbstractTestBases
+
class TestDeviceSDP(AbstractTestBases.TestDeviceBase):
def setUp(self):
+ """Intentionally recreate the device object in each test"""
super().setUp("STAT/SDP/1")
def test_device_sdp_read_attribute(self):
diff --git a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_sst.py b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_sst.py
index 38f3528f531660704baa00f70a7073974256a19f..60675e121364b52fb692b2f8461001bfdc78b50a 100644
--- a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_sst.py
+++ b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_sst.py
@@ -15,24 +15,13 @@ from tango._tango import DevState
from .base import AbstractTestBases
+
class TestDeviceSST(AbstractTestBases.TestDeviceBase):
def setUp(self):
+ """Intentionally recreate the device object in each test"""
super().setUp("STAT/SST/1")
- def test_device_on(self):
- """Test if we can transition to on"""
-
- port_property = {"Statistics_Client_TCP_Port": "4999"}
- self.proxy.put_property(port_property)
- self.proxy.initialise()
-
- self.assertEqual(DevState.STANDBY, self.proxy.state())
-
- self.proxy.on()
-
- self.assertEqual(DevState.ON, self.proxy.state())
-
def test_device_sst_send_udp(self):
port_property = {"Statistics_Client_TCP_Port": "4998"}
self.proxy.put_property(port_property)
diff --git a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_unb2.py b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_unb2.py
index a35f70adde7af58408882e3b5ee3256a4838db84..d5731630188879e5f79f94f98951f8d6c1637ace 100644
--- a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_unb2.py
+++ b/tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_unb2.py
@@ -9,7 +9,9 @@
from .base import AbstractTestBases
+
class TestDeviceUNB2(AbstractTestBases.TestDeviceBase):
def setUp(self):
+ """Intentionally recreate the device object in each test"""
super().setUp("STAT/UNB2/1")
diff --git a/tangostationcontrol/tangostationcontrol/test/devices/automatic_polling_performance_test/Tango_Controls-Automatic_polling_performance_test.md b/tangostationcontrol/tangostationcontrol/test/devices/automatic_polling_performance_test/Tango_Controls-Automatic_polling_performance_test.md
new file mode 100644
index 0000000000000000000000000000000000000000..3078d28472daf767475b070ad85492e785167b1e
--- /dev/null
+++ b/tangostationcontrol/tangostationcontrol/test/devices/automatic_polling_performance_test/Tango_Controls-Automatic_polling_performance_test.md
@@ -0,0 +1,236 @@
+# Test purpose
+
+Tango Controls Device Servers (DS) can automatically poll device attribute values. The default for the polling is a single thread that runs in the DS process. It has the purpose to call the `read` function of every attribute in all devices that run in the DS.
+The highest polling rate among all the attributes in all devices determines how often the polling thread runs. This can lead to a situation where the single polling thread is unable to finish executing all attribute `read` functions. When that happens, the attributes that did not have their `read` function executed will not have updated values. Since the polling thread always follows the same order of attributes, this will lead to some attribute values never be updated.
+
+We investigate whether using more polling threads can alleviate the situation for a single or multiple devices that run in the same DS.
+
+# References
+
+Please refer to the reference documents for in-depth information about the automatic polling and how to configure a DS to use a dedicated thread per device for polling.
+
+[Attribute polling in Tango Controls](https://tango-controls.readthedocs.io/en/latest/development/device-api/ds-guideline/device-server-guidelines.html#tango-polling-mechanism)
+
+[Device polling in Tango Controls](https://tango-controls.readthedocs.io/en/latest/development/device-api/device-polling.html)
+
+[Configuring a DS to use per Device threads in polling](https://tango-controls.readthedocs.io/en/latest/development/advanced/reference.html#dserver-class-device-properties)
+
+# Test set-up
+
+- Two devices run in the same DS.
+ - Tango DB is modified to have a DS named `monitoring_performance_test/1`.
+
+ - Tango DB is modified to have the DS named `monitoring_performance_test/1` run two devices named `test/monitoring_performance/1` and `test/monitoring_performance/2`. Both devices instantiate the same class `Monitoring_Performance_Device`.
+
+ - Execute the DS like this: `bin/start-DS.sh devices/test/devices/automatic_polling_performance_test/monitoring_performance_test.py 1`
+
+ - Get a DeviceProxy object to both devices like this:
+
+ ```python
+ d1 = DeviceProxy('test/monitoring_performance/1')
+ d2 = DeviceProxy('test/monitoring_performance/2')
+ ```
+
+ This will execute the device code and perform the automatic polling.
+
+ - Devices in the appended data (section Data) are labelled d1 and d2.
+- Each device has 4 read-only attributes (MPs) that are arrays with 2e6 doubles.
+ - In Tango DB automatic polling every 1s is enabled for each array.
+- Two scenarios:
+ 1. On read a random number gets generated and the entire array is populated with it. Populating the array with a new random number on every read access prevents caching.
+ 2. A 0-filled array is created in init_device and copied to the attribute when read is called.
+- The number of polling threads is adjusted in init_device. The number of polling threads is a DS setting. Due to inconsistencies how Tango Controls handles input parameters, it is nor possible to pass parameters to devices.
+- Number of polling threads: 1, 10, 100
+
+# Test execution
+
+- The DS source code is modified for the number of polling threads according to the test set-up outlined above.
+- The DS is started manually.
+- The attribute polling resumes automatically as soon as the device state is set to ON in init_device.
+- The test script creates two Python processes that are assigned to one of the two devices each.
+ - The process creates a `DeviceProxy` object for its device and runs
+ - The process executes `attribute_polling_stats` and the results are printed out.
+ - The process exits.
+- The DS is allowed to run for approximately 10 polling iterations.
+ - The DS processes will print statistics about the polling.
+- The test is manually stopped and the output copied.
+
+Test results are in the attached file.
+
+# Findings
+
+The tests have shown that polling gets significantly delayed under certain circumstances:
+
+- When the read function takes a longer time to return an attribute's value.
+ Examples that have been tried out in this test but that are not excluding other reasons:
+ - Creating a numpy array of size 2e6 on the fly with the same random number in every value.
+ - Reading of a 2e5 values array from an OPC-UA server and converting it to a numpy array.
+
+From this finding other reasons that will have a negative impact on polling become immediately obvious and need to be avoided no matter what:
+
+- Fetching data on-the-fly over a slow communication link in an attribute's ` read` function.
+- Fetching a big amount of data on-the-fly over a fast communication link in an attribute's ` read` function.
+- Computing data on-the-fly in an attribute's `read`function.
+
+Adding more polling threads to a DS does not alleviate this situation. The reason lies in the way how Tango Controls polling works. As a default the polling is performed by one thread per DS or, if the polling thread pool size is increased, by at most one thread per device. Adding twice as many polling threads as devices that are running in a DS does not change the situation as the data suggests.
+
+# Recommendation
+
+For devices that contain attributes with values that are big in byte size, i.e. arrays of significant size, it is strongly recommended to assess the situation, i.e. measure how long reading the data over a communication link takes. If it is essential to poll a high volume attribute at a polling rate that exceeds the performance capabilities of the DS's polling thread, several options are viable:
+
+- Distribute high volume attributes among separate devices which run in their own DS.
+ If necessary create more and more devices with less and less attributes until the desired polling rate can be accomplished. Even if this means that each high volume attribute exists in its own device. To Tango Controls or to device clients it does not matter if a device contains one or many attributes.
+- Distribute high volume attributes among separate devices but continue running them in the same DS. It is necessary to increase the number of polling threads to at least the number of devices in the DS.
+
+The two solutions above are mutually exclusive. Other solutions that alleviate the load on single polling threads can be added to either of the above as well:
+
+- Lower the polling rate so that the polling thread has more time to perform calling the attribute `read` functions. This means that an attribute's read function is allowed to take longer to perform its tasks. Note that this does not solve the original problem that the `read` function is just too slow when it is called.
+- Move the updating of attribute values to their own process. Then the read function can return the current value immediately because the value gets independently updated.
+
+There is an entirely different way to lower the pressure on the automatic polling: manual polling, i.e. performing everything that the polling thread does for a selected set of attributes. Tango Controls allows to manually send events. This opens the possibility to perform the reading of values over communication links, checking for value changes and sending out of archive or on-change events from separate threads or processes when it is convenient.
+
+# Executive summary
+
+Even Tango Controls cannot perform magic. High volume attributes cannot be polled and infinite velocity. One has to distribute the polling load either over more DS, more threads within a DS or more processes within a DS.
+
+# Data
+
+Filling the array with the same newly created random number on ever read.
+
+threads = 1
+d1
+ iterations = 10
+
+ Polling duration
+ min = 0.7053400000000001[s]
+ max = 0.7174940000000001[s]
+ median = 0.7123280000000001[s]
+ mean = 0.7121181[s]
+ stddev = 0.004004320403014733[s]
+
+ Polling delay
+ min = 0.792[s]
+ max = 2.207[s]
+ median = 0.8115[s]
+ mean = 1.2221[s]
+ stddev = 0.6406804897919087[s]
+d2
+ iterations = 10
+
+ Polling duration
+ min = 0.689903[s]
+ max = 0.715033[s]
+ median = 0.7069909999999999[s]
+ mean = 0.7061663000000001[s]
+ stddev = 0.00792590103458277[s]
+
+ Polling delay
+ min = 0.744[s]
+ max = 2.245[s]
+ median = 0.758[s]
+ mean = 1.2010999999999998[s]
+ stddev = 0.681659805181441[s]
+
+threads = 10
+d1
+ iterations = 10
+
+ Polling duration
+ min = 0.700119[s]
+ max = 0.7102459999999999[s]
+ median = 0.710067[s]
+ mean = 0.7068808[s]
+ stddev = 0.004127314376201529[s]
+
+ Polling delay
+ min = 0.802[s]
+ max = 2.196[s]
+ median = 0.806[s]
+ mean = 1.2213[s]
+ stddev = 0.6370044034384692[s]
+d2
+ iterations = 10
+
+ Polling duration
+ min = 0.6984130000000001[s]
+ max = 0.706296[s]
+ median = 0.7044239999999999[s]
+ mean = 0.7036658000000001[s]
+ stddev = 0.0025871636902213896[s]
+
+ Polling delay
+ min = 0.758[s]
+ max = 2.24[s]
+ median = 0.759[s]
+ mean = 1.3504[s]
+ stddev = 0.7247257688256988[s]
+
+threads = 100
+d1
+ iterations = 10
+
+ Polling duration
+ min = 0.690158[s]
+ max = 0.720522[s]
+ median = 0.7119365[s]
+ mean = 0.7107762[s]
+ stddev = 0.008783150821886167[s]
+
+ Polling delay
+ min = 0.79[s]
+ max = 2.209[s]
+ median = 0.8[s]
+ mean = 1.2176000000000002[s]
+ stddev = 0.6462041782594724[s]
+d2
+ iterations = 10
+
+ Polling duration
+ min = 0.702939[s]
+ max = 0.724869[s]
+ median = 0.7119840000000001[s]
+ mean = 0.7122735[s]
+ stddev = 0.006137572716473502[s]
+
+ Polling delay
+ min = 0.749[s]
+ max = 2.25[s]
+ median = 0.755[s]
+ mean = 1.2005[s]
+ stddev = 0.6824934065615579[s]
+
+
+Returning a 0-filled array that was created in init_device
+threads = 100
+d1
+ iterations = 10
+
+ Polling duration
+ min = 0.005712[s]
+ max = 0.008997999999999999[s]
+ median = 0.0065065[s]
+ mean = 0.006732[s]
+ stddev = 0.0009050982267135427[s]
+
+ Polling delay
+ min = 0.998[s]
+ max = 1.001[s]
+ median = 1.0[s]
+ mean = 0.9997[s]
+ stddev = 0.0007810249675906477[s]
+d2
+ iterations = 10
+
+ Polling duration
+ min = 0.0062759999999999995[s]
+ max = 0.008672000000000001[s]
+ median = 0.0069180000000000005[s]
+ mean = 0.0070902[s]
+ stddev = 0.0007260824746542229[s]
+
+ Polling delay
+ min = 0.996[s]
+ max = 1.003[s]
+ median = 0.999[s]
+ mean = 0.9997[s]
+ stddev = 0.002491987158875375[s]
\ No newline at end of file
diff --git a/tangostationcontrol/tangostationcontrol/test/devices/automatic_polling_performance_test/automatic_polling_performance_test.json b/tangostationcontrol/tangostationcontrol/test/devices/automatic_polling_performance_test/automatic_polling_performance_test.json
new file mode 100644
index 0000000000000000000000000000000000000000..0fa271289f3254df9c61158dd1a94b9884945b09
--- /dev/null
+++ b/tangostationcontrol/tangostationcontrol/test/devices/automatic_polling_performance_test/automatic_polling_performance_test.json
@@ -0,0 +1,20 @@
+{
+ "servers":
+ {
+ "monitoring_performance_test":
+ {
+ "1":
+ {
+ "Monitoring_Performance_Device":
+ {
+ "test/monitoring_performance/1":
+ {
+ },
+ "test/monitoring_performance/2":
+ {
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/tangostationcontrol/tangostationcontrol/test/devices/automatic_polling_performance_test/monitoring_performance_test.py b/tangostationcontrol/tangostationcontrol/test/devices/automatic_polling_performance_test/monitoring_performance_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..e5360f26d97ab84564116b1eb98a6f5df58a972e
--- /dev/null
+++ b/tangostationcontrol/tangostationcontrol/test/devices/automatic_polling_performance_test/monitoring_performance_test.py
@@ -0,0 +1,120 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of the LOFAR2.0 project
+#
+#
+#
+# Distributed under the terms of the APACHE license.
+# See LICENSE.txt for more info.
+
+# TODO(Corne): Remove sys.path.append hack once packaging is in place!
+import os, sys
+currentdir = os.path.dirname(os.path.realpath(__file__))
+parentdir = os.path.dirname(currentdir)
+parentdir = os.path.dirname(parentdir)
+sys.path.append(parentdir)
+
+import time
+import numpy
+from tango import DevState, Util
+from tango.server import run, Device, attribute
+from numpy import random
+
+__all__ = ["Monitoring_Performance_Device", "main"]
+
+POLLING_THREADS = 100
+ARRAY_SIZE = 2000000
+
+class Monitoring_Performance_Device(Device):
+ global ARRAY_SIZE
+ def read_array(self):
+ print("{} {}".format(time.time(), self.get_name()))
+ return self._array
+
+ array1_r = attribute(
+ dtype = (numpy.double,),
+ max_dim_x = ARRAY_SIZE,
+ period = 1000,
+ rel_change = 0.1,
+ archive_period = 1000,
+ archive_rel_change = 0.1,
+ max_value = 1.0,
+ min_value = 0.0,
+ fget = read_array,
+ )
+
+ array2_r = attribute(
+ dtype = (numpy.double,),
+ max_dim_x = ARRAY_SIZE,
+ period = 1000,
+ rel_change = 0.1,
+ archive_period = 1000,
+ archive_rel_change = 0.1,
+ max_value = 1.0,
+ min_value = 0.0,
+ fget = read_array,
+ )
+
+ array3_r = attribute(
+ dtype = (numpy.double,),
+ max_dim_x = ARRAY_SIZE,
+ period = 1000,
+ rel_change = 0.1,
+ archive_period = 1000,
+ archive_rel_change = 0.1,
+ max_value = 1.0,
+ min_value = 0.0,
+ fget = read_array,
+ )
+
+ array4_r = attribute(
+ dtype = (numpy.double,),
+ max_dim_x = ARRAY_SIZE,
+ period = 1000,
+ rel_change = 0.1,
+ archive_period = 1000,
+ archive_rel_change = 0.1,
+ max_value = 1.0,
+ min_value = 0.0,
+ fget = read_array,
+ )
+
+ def init_device(self):
+ Device.init_device(self)
+
+ util = Util.instance()
+ print("Current polling thread pool size = {}".format(util.get_polling_threads_pool_size()))
+ util.set_polling_threads_pool_size(POLLING_THREADS)
+ print("New polling thread pool size = {}".format(util.get_polling_threads_pool_size()))
+ print("Array size = {}".format(ARRAY_SIZE))
+
+ self.set_state(DevState.OFF)
+
+ self._array = numpy.zeros(ARRAY_SIZE)
+
+ self.array1_r.set_data_ready_event(True)
+ self.set_change_event("array1_r", True, True)
+ self.set_archive_event("array1_r", True, True)
+
+ self.array2_r.set_data_ready_event(True)
+ self.set_change_event("array2_r", True, True)
+ self.set_archive_event("array2_r", True, True)
+
+ self.array3_r.set_data_ready_event(True)
+ self.set_change_event("array3_r", True, True)
+ self.set_archive_event("array3_r", True, True)
+
+ self.array4_r.set_data_ready_event(True)
+ self.set_change_event("array4_r", True, True)
+ self.set_archive_event("array4_r", True, True)
+
+ self.set_state(DevState.ON)
+
+ def delete_device(self):
+ self.set_state(DevState.OFF)
+
+def main(args = None, **kwargs):
+ return run((Monitoring_Performance_Device, ), args = args, **kwargs)
+
+if __name__ == '__main__':
+ main()
diff --git a/tangostationcontrol/tangostationcontrol/toolkit/archiver.py b/tangostationcontrol/tangostationcontrol/toolkit/archiver.py
index 771fab87a0431defec19ce755b07a5eaa22ca2d6..6c0cd3baa8e2c708d39858191bc56e73e84a9eff 100644
--- a/tangostationcontrol/tangostationcontrol/toolkit/archiver.py
+++ b/tangostationcontrol/tangostationcontrol/toolkit/archiver.py
@@ -578,11 +578,11 @@ class Retriever():
return result
def get_attribute_value_by_interval(self,attribute_fqname: str, start_time: datetime, stop_time: datetime):
- '''
+ """
Takes as input the attribute name and a certain starting and ending point-time.
The datetime format is pretty flexible (e.g. "YYYY-MM-dd hh:mm:ss").
Returns a list of timestamps and a list of values
- '''
+ """
attr_id = self.get_attribute_id(attribute_fqname)
attr_datatype = self.get_attribute_datatype(attribute_fqname)
attr_table_name = 'att_'+str(attr_datatype)
@@ -596,4 +596,33 @@ class Retriever():
except AttributeError as e:
raise Exception(f"Empty result! Attribute {attribute_fqname} not found") from e
return result
-
+
+ def get_masked_fpga_temp(self,start_time: datetime, stop_time: datetime,temp_attr_name:str='LTS/SDP/1/fpga_temp_r',
+ mask_attr_name:str='LTS/SDP/1/tr_fpga_mask_r'):
+ """
+ Returns a list of SDP/fpga_temp_r values, but only if SDP/tr_fpga_mask_r values are TRUE
+ """
+ mask_values = self.get_attribute_value_by_interval(mask_attr_name,start_time,stop_time)
+ temp_values = self.get_attribute_value_by_interval(temp_attr_name,start_time,stop_time)
+ # Since timestamps can be not syncrhonized, remove first or last element from arrays
+ if len(mask_values)==len(temp_values):
+ first_mask_datatime = mask_values[0].data_time
+ first_temp_datatime = temp_values[0].data_time
+ if (first_mask_datatime>first_temp_datatime):
+ mask_values = mask_values[:-int(mask_values[0].dim_x_r)]
+ temp_values = temp_values[int(temp_values[0].dim_x_r):]
+ elif (first_mask_datatime<first_temp_datatime):
+ mask_values = mask_values[int(mask_values[0].dim_x_r)]
+ temp_values = temp_values[:-int(temp_values[0].dim_x_r):]
+ else:
+ raise Exception
+ # Convert DB Array records into Python lists
+ mask_data = build_array_from_record(mask_values,mask_values[0].dim_x_r)
+ temp_data = build_array_from_record(temp_values,temp_values[0].dim_x_r)
+ # Extract only the value from the array
+ mask_array_values = get_values_from_record(mask_data)
+ temp_array_values = get_values_from_record(temp_data)
+ # Multiply the matrix
+ #masked_values = np.multiply(temp_array_values,mask_array_values)
+ masked_values = np.ma.masked_array(temp_array_values,mask=np.invert(mask_array_values.astype(bool)))
+ return masked_values, mask_values, temp_values
diff --git a/tangostationcontrol/tangostationcontrol/toolkit/archiver_config/lofar2.json b/tangostationcontrol/tangostationcontrol/toolkit/archiver_config/lofar2.json
index 90533e18a0fd28b26286cff70be8d5f190b14a6e..6c30be0698175ef914841fa165a4f2a97543d12a 100644
--- a/tangostationcontrol/tangostationcontrol/toolkit/archiver_config/lofar2.json
+++ b/tangostationcontrol/tangostationcontrol/toolkit/archiver_config/lofar2.json
@@ -11,7 +11,7 @@
},
"STAT/SDP/1": {
"environment": "development",
- "include": [],
+ "include": ["FPGA_temp_R","TR_fpga_mask_R"],
"exclude": ["FPGA_scrap_R","FPGA_scrap_RW"]
},
"STAT/SST/1": {
diff --git a/tangostationcontrol/tangostationcontrol/toolkit/attribute_polling_stats.py b/tangostationcontrol/tangostationcontrol/toolkit/attribute_polling_stats.py
new file mode 100644
index 0000000000000000000000000000000000000000..fcfb502320d4de084f219244289f6ddfa064e996
--- /dev/null
+++ b/tangostationcontrol/tangostationcontrol/toolkit/attribute_polling_stats.py
@@ -0,0 +1,39 @@
+import numpy
+import tango
+
+def attribute_polling_stats(dp: tango._tango.DeviceProxy = None, iterations: int = 10, polling_time: float = 1.0, quiet = False):
+ if dp is not None:
+ print('Will sample the device server\'s polling time {} times with a pause of {}s between each sampling.'.format(iterations, polling_time))
+ else:
+ print('A DeviceProxy object is needed!')
+ return
+ import numpy
+ from time import sleep
+ polling_durations = []
+ polling_delays = []
+ value = numpy.double(0)
+ iterations_left = iterations
+ while iterations_left > 0:
+ iterations_left -= 1
+ string = dp.polling_status()[0].split('\n')
+ polling_duration = numpy.double(string[3].split('=')[-1].strip()) / 1e3
+ polling_delay = numpy.double(string[5].split('=')[-1].split(',')[0].strip()) / 1e3
+ polling_durations.append(polling_duration)
+ polling_delays.append(polling_delay)
+ if not quiet:
+ print('Iteration #{}, {} iterations left, polling duration = {}s, polling delay = {}s.'.format(iterations - iterations_left, iterations_left, polling_duration, polling_delay))
+ sleep(polling_time)
+ durations = numpy.array(polling_durations)
+ delays = numpy.array(polling_delays)
+ def compute_and_print(result):
+ min = numpy.min(result)
+ max = numpy.max(result)
+ median = numpy.median(result)
+ mean = numpy.mean(result)
+ std = numpy.std(result)
+ print("\tmin = {}[s]\n\tmax = {}[s]\n\tmedian = {}[s]\n\tmean = {}[s]\n\tstddev = {}[s]".format(min, max, median, mean, std))
+ print("\n\titerations = {}\n\n\tPolling duration".format(iterations))
+ compute_and_print(durations)
+ print("\n\tPolling delay")
+ compute_and_print(delays)
+ return (durations, delays)