Merge branch 'master' into L2SS-528-timescaledb-defaults

docker-compose/recv-sim.yml

@@ -10,6 +10,9 @@ services:
   recv-sim:
     build:
         context: pypcc-sim-base
+        args:
+         - LOCAL_DOCKER_REGISTRY_HOST=${LOCAL_DOCKER_REGISTRY_HOST}
+         - LOCAL_DOCKER_REGISTRY_LOFAR=${LOCAL_DOCKER_REGISTRY_LOFAR}
     container_name: ${CONTAINER_NAME_PREFIX}recv-sim
     networks:
       - control

docker-compose/rest.yml

@@ -13,7 +13,7 @@ version: '2'
 
 services:
   rest:
-    image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-rest:${TANGO_REST_VERSION}
+    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-rest:${TANGO_REST_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}tango-rest
     networks:
       - control

docker-compose/sdptr-sim.yml

@@ -10,6 +10,9 @@ services:
   sdptr-sim:
     build:
         context: sdptr-sim
+        args:
+         - LOCAL_DOCKER_REGISTRY_HOST=${LOCAL_DOCKER_REGISTRY_HOST}
+         - LOCAL_DOCKER_REGISTRY_LOFAR=${LOCAL_DOCKER_REGISTRY_LOFAR}
     container_name: ${CONTAINER_NAME_PREFIX}sdptr-sim
     networks:
       - control

docker-compose/sdptr-sim/Dockerfile

-FROM ubuntu:20.04
+ARG LOCAL_DOCKER_REGISTRY_HOST
+ARG LOCAL_DOCKER_REGISTRY_LOFAR
 
-# Install build tools for sdptr and the C language OPC-UA lib
-RUN apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common && \
-    DEBIAN_FRONTEND=noninteractive add-apt-repository ppa:open62541-team/ppa && \
-    apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y autoconf automake git make g++ build-essential pkg-config libboost-dev libboost-regex-dev libboost-system-dev libboost-program-options-dev libopen62541-1-dev libopen62541-1-tools && \
-    apt-get clean
-
-# Install SDPTR
-RUN cd / && git clone --depth 1 --branch master https://git.astron.nl/lofar2.0/sdptr
-
-RUN cd /sdptr && \
-    autoreconf -v -f -i && \
-    ./configure && \
-    bash -c "make -j `nproc` install"
+FROM ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_LOFAR}/sdptr:latest
 
 COPY simulator.conf /sdptr/src/simulator.conf
 

docker-compose/tango.yml

@@ -15,7 +15,7 @@ volumes:
 
 services:
   tangodb:
-    image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-db:${TANGO_DB_VERSION}
+    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-db:${TANGO_DB_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}tangodb
     networks:
       - control
@@ -37,7 +37,7 @@ services:
     restart: unless-stopped
 
   databaseds:
-    image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-cpp:${TANGO_CPP_VERSION}
+    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-cpp:${TANGO_CPP_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}databaseds
     networks:
       - control

docker-compose/tangotest.yml

@@ -11,7 +11,7 @@ version: '2'
 
 services:
   tangotest:
-    image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-java:${TANGO_JAVA_VERSION}
+    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-java:${TANGO_JAVA_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}tangotest
     networks:
       - control

docker-compose/unb2-sim.yml

@@ -10,6 +10,9 @@ services:
   unb2-sim:
     build:
         context: pypcc-sim-base
+        args:
+         - LOCAL_DOCKER_REGISTRY_HOST=${LOCAL_DOCKER_REGISTRY_HOST}
+         - LOCAL_DOCKER_REGISTRY_LOFAR=${LOCAL_DOCKER_REGISTRY_LOFAR}
     container_name: ${CONTAINER_NAME_PREFIX}unb2-sim
     networks:
       - control

docs/source/devices/beam.rst

+Beam
+====================
+
+The ``beam == DeviceProxy("STAT/Beam/1")`` device sets up the beamforming on the station:
+
+- The HBA tiles in RECV need *analog beamforming* to combine their 16 antennas into a single input for the RCU,
+- THe LBAs or HBA tiles need *digital beamforming* in SDP to combine their signals into beamlets.
+

docs/source/devices/sst-xst.rst

@@ -63,7 +63,7 @@ Typically, ``N_ant == 192``, and ``N_blocks == 136``.
 
 The metadata refers to the *blocks*, which are emitted by the FPGAs to represent the XSTs between 12 x 12 consecutive antennas. The following code converts block numbers to the indices of the first antenna pair in a block::
 
-  from common.baselines import baseline_from_index
+  from tangostationcontrol.common.baselines import baseline_from_index
 
   def first_antenna_pair(block_nr: int) -> int:
       coarse_a, coarse_b = baseline_from_index(block_nr)
@@ -71,7 +71,7 @@ The metadata refers to the *blocks*, which are emitted by the FPGAs to represent
 
 Conversely, to calculate the block index for an antenna pair ``(a,b)``, use::
 
-  from common.baselines import baseline_index
+  from tangostationcontrol.common.baselines import baseline_index
 
   def block_nr(a: int, b: int) -> int:
       return baseline_index(a // 12, b // 12)

docs/source/index.rst

@@ -19,6 +19,7 @@ Even without having access to any LOFAR2.0 hardware, you can install the full st
    installation
    interfaces/overview
    devices/using
+   devices/beam
    devices/boot
    devices/docker
    devices/recv

sbin/run_integration_test.sh

@@ -11,11 +11,15 @@ fi
 
 cd "$LOFAR20_DIR/docker-compose" || exit 1
 
-# Make sure builds are recent, and use our building parameters.
-make build
+# Build only the required images, please do not build everything that makes CI
+# take really long to finish, especially grafana / jupyter / prometheus.
+# jupyter is physically large > 2.5gb and overlayfs is really slow.
+make build device-sdp device-recv device-sst device-unb2 device-xst
+make build sdptr-sim recv-sim unb2-sim apsct-sim apspu-sim
+make build databaseds dsconfig elk integration-test
 
 # Start and stop sequence
-make stop device-boot device-docker device-apsct device-apspu device-sdp device-recv device-sst device-unb2 device-xst sdptr-sim recv-sim unb2-sim apsct-sim apspu-sim
+make stop device-boot device-docker device-apsct device-apspu device-sdp device-recv device-sst device-unb2 device-xst device-beam sdptr-sim recv-sim unb2-sim apsct-sim apspu-sim
 make start databaseds dsconfig elk
 
 # Give dsconfig and databaseds time to start
@@ -32,7 +36,7 @@ make start sdptr-sim recv-sim unb2-sim apsct-sim apspu-sim
 # Give the simulators time to start
 sleep 5
 
-make start device-boot device-apsct device-apspu device-sdp device-recv device-sst device-unb2 device-xst
+make start device-boot device-apsct device-apspu device-sdp device-recv device-sst device-unb2 device-xst device-beam
 
 # Give devices time to restart
 # TODO(Corne Lukken): Use a nicer more reliable mechanism

sbin/tag_and_push_docker_image.sh

 #!/bin/bash -e
 
-# Tag and push which image version?
-DOCKER_TAG=latest
+function usage {
+    echo "./$(basename "$0")
+      no arguments, downloads remote images and pushes these to ASTRON registry.
+      The versions downloaded are controlled by the docker-compose/.env file"
+    echo ""
+    echo "./$(basename "$0") -h
+      displays this help message"
+    echo ""
+    echo "./$(basename "$0") <docker service name> <tag>
+      downloads latest version of image from the ASTRON registry, builds the
+      specified service and pushes the image with the specified tag to the
+      ASTRON registry"
+    echo ""
+    echo "./$(basename "$0") pull <tag>
+      downloads all images for the integration test with the specified tag
+      falling back to 'latest' if unavailable. Should neither exist on the
+      ASTRON registry the script will exit 1. The images are retagged to match
+      the output of docker-compose."
+}
+
+# list of arguments expected in the input
+optstring=":h"
 
-# Change to git tag or git hash if no tag
-VERSION=$(date +"%Y-%M-%d")
+while getopts ${optstring} arg; do
+  case ${arg} in
+    h)
+      usage
+      exit 0
+      ;;
+    :)
+      echo "$0: Must supply an argument to -$OPTARG." >&2
+      exit 1
+      ;;
+    ?)
+      echo "Invalid option: -${OPTARG}."
+      exit 2
+      ;;
+  esac
+done
 
-SKA_REPO="nexus.engageska-portugal.pt/ska-docker"
-LOFAR_REPO="git.astron.nl:5000/lofar2.0/tango"
+if [ -z "${LOFAR20_DIR+x}" ]; then
+  echo "LOFAR20_DIR not set, did you forget to source lofar20rc.sh?"
+  exit 1
+fi
 
-# Compile a list of the SKA images
-SKA_IMAGES=$(for i in $(docker images | grep -E ${DOCKER_TAG} | grep -E ${SKA_REPO} | cut -d' ' -f1); do printf "%s " "${i}"; done)
+# shellcheck disable=SC1090
+. "${LOFAR20_DIR}/docker-compose/.env" || exit 1
 
-# Compile a list of LOFAR2.0 images
-LOFAR_IMAGES=$(for i in $(docker images | grep -E ${DOCKER_TAG} | grep -E -v "${SKA_REPO}|${LOFAR_REPO}" | cut -d' ' -f1); do printf "%s " "${i}"; done)
+# List of images and their tag
+REMOTE_IMAGES=(
+  "tango-dsconfig:${TANGO_DSCONFIG_VERSION}" "tango-java:${TANGO_JAVA_VERSION}"
+  "tango-itango:${TANGO_ITANGO_VERSION}" "tango-pogo:${TANGO_POGO_VERSION}"
+  "tango-cpp:${TANGO_CPP_VERSION}" "tango-db:${TANGO_DB_VERSION}"
+  "tango-dsconfig:${TANGO_DSCONFIG_VERSION}" "tango-rest:${TANGO_REST_VERSION}"
+)
+
+# If first argument of bash script not set run first stage
+if [ -z "${1+x}" ]; then
+  echo "Pulling and retagging remote images"
+
+  # Iterate over al the REMOTE_IMAGES and pull them from remote and push local
+  for image in "${REMOTE_IMAGES[@]}"; do
+    remote_url="${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-${image}"
+    local_url="${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/${image}"
+    docker pull "${remote_url}"
+    docker tag "${remote_url}" "${local_url}"
+    docker push "${local_url}"
+  done
+
+  exit 0
+fi
+
+# Triple tuple of docker-compose names, image names and if necessary for
+# integration tests.
+# TODO(Corne): Have this list generated from the .yml files
+LOCAL_IMAGES=(
+  "elk elk y" "elk-configure-host elk-configure-host y"
+  "lofar-device-base lofar-device-base y"
+
+  "apsct-sim docker-compose_apsct-sim y" "apspu-sim docker-compose_apspu-sim y"
+  "recv-sim docker-compose_recv-sim y" "sdptr-sim docker-compose_sdptr-sim y"
+  "unb2-sim docker-compose_unb2-sim y"
+
+  "device-apsct device-apsct y" "device-apspu device-apspu y"
+  "device-boot device-boot y" "device-docker device-docker y"
+  "device-observation_control device-observation_control y"
+  "device-recv device-recv y" "device-sdp device-sdp y"
+  "device-sst device-sst y" "device-unb2 device-unb2 y"
+  "device-xst device-xst y"
+
+  "itango docker-compose_itango y"
+
+  "grafana grafana n" "prometheus prometheus n"
+  "jupyter docker-compose_jupyter n"
+  "integration-test docker-compose_integration-test n"
+  "tango-prometheus-exporter docker-compose_tango-prometheus-exporter n"
+)
 
-function tag_and_push()
-{
-    (
-        docker tag "${1}" "${2}"
-        docker push "${2}"
-    ) &
-}
 
-# Rename the SKA images for the LOFAR2.0 repo
-# and push them to the LOFAR2.0 repo
-for IMAGE in ${SKA_IMAGES}; do
-    PUSH_IMAGE=${IMAGE//${SKA_REPO}/${LOFAR_REPO}}:${VERSION}
-    tag_and_push "${IMAGE}" "${PUSH_IMAGE}"
+
+# If first argument set run second stage, determine LOCAL_IMAGE to build and
+# push from the argument
+if [ ! -z "${1+x}" ] && [ "${1}" != "pull" ]; then
+
+  # The second argument must pass the tag variable must be set
+  if [ -z "${2+x}" ]; then
+    echo "Error, second argument must pass tag variable"
+    exit 1
+  fi
+
+  # Set the tag and image variable, variables $1 and $2 are shadowed later
+  local_image="${1}"
+  tag="${2}"
+
+  cd "${LOFAR20_DIR}/docker-compose" || exit 1
+
+  # Loop through images and find the specified one
+  for image in "${LOCAL_IMAGES[@]}"; do
+    # Set, splits tuple into $1 and $2. this shadows previous variables
+    # shellcheck disable=SC2086
+    set -- $image
+    if [ "${local_image}" == "${1}" ]; then
+      echo "Building image for ${1} container"
+      local_url="${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/${2}"
+
+      # If tag is not latest, than it is not a tagged master build and we can
+      # pull the latest image as cache.
+      if [ "${tag}" != "latest" ]; then
+        docker pull "${local_url}:latest"
+      fi
+
+      make build "${1}"
+      docker tag "${2}" "${local_url}:${tag}"
+      docker push "${local_url}:${tag}"
+    fi
   done
 
-# Rename the LOFAR2.0 images for the LOFAR2.0 repo
-# and push them to the LOFAR2.0 repo
-for IMAGE in ${LOFAR_IMAGES}; do
-    PUSH_IMAGE=${LOFAR_REPO}/${IMAGE}:${VERSION}
-    tag_and_push "${IMAGE}" "${PUSH_IMAGE}"
+  exit 0
+fi
+
+# Final stage, pull images for integration cache try special tag image first
+# if it fails download latest instead
+if [ ! -z "${1+x}" ] && [ "${1}" == "pull" ]; then
+  echo "Pulling images for integration test cache"
+
+  # The second argument must pass the tag variable must be set
+  if [ -z "${2+x}" ]; then
+    echo "Error, second argument must pass tag variable"
+    exit 1
+  fi
+
+  # Set the tag variable
+  tag="${2}"
+
+  for image in "${LOCAL_IMAGES[@]}"; do
+      # Set, splits tuple into $1 and $2. this shadows previous variables
+    # shellcheck disable=SC2086
+    set -- $image
+
+    # Only download images which are needed for integration test
+    if [ "${3}" == "y" ]; then
+      local_url="${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/${2}"
+      # Pull images, at least one of the two images must succeed
+      echo "docker pull ${local_url}:${tag}"
+      docker pull "${local_url}:${tag}" || docker pull "${local_url}:latest" || exit 1
+      # Ensure the images will have the same tags as generated by docker-compose
+      docker tag "${local_url}:${tag}" "${2}" || docker tag "${local_url}:latest" "${2}" || exit 1
+    fi
   done
 
-wait
+  exit 0
+fi
+
+# Somehow nothing ran, that is an error do not fail silently
+exit 1

tangostationcontrol/requirements.txt

@@ -6,7 +6,6 @@ asyncua >= 0.9.90 # LGPLv3
 PyMySQL[rsa] >= 1.0.2 # MIT
 psycopg2-binary >= 2.9.2 #LGPL
 sqlalchemy >= 1.4.26 #MIT
-GitPython >= 3.1.24 # BSD
 snmp >= 0.1.7 # GPL3
 h5py >= 3.5.0 # BSD
 psutil >= 5.8.0 # BSD

tangostationcontrol/setup.cfg

@@ -36,6 +36,7 @@ where=./
 console_scripts =
     l2ss-apsct = tangostationcontrol.devices.apsct:main
     l2ss-apspu = tangostationcontrol.devices.apspu:main
+    l2ss-beam = tangostationcontrol.devices.beam:main
     l2ss-boot = tangostationcontrol.devices.boot:main
     l2ss-docker-device = tangostationcontrol.devices.docker_device:main
     l2ss-observation = tangostationcontrol.devices.observation:main

tangostationcontrol/tangostationcontrol/devices/README.md

@@ -10,6 +10,9 @@ If a new device is added, it will (likely) need to be referenced in several plac
 - Adjust `docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py` to make an alias for it available in Jupyter,
 - Adjust `tangostationcontrol/tangostationcontrol/devices/boot.py` to add the device to the station initialisation sequence,
 - Add to `docker-compose/` to create a YaML file to start the device in a docker container. NOTE: it needs a unique 57xx port assigned,
+- Adjust `tangostationcontrol/setup.cfg` to add an entry point for the device in the package installation,
 - Add to `tangostationcontrol/tangostationcontrol/integration_test/devices/` to add an integration test,
 - Adjust `sbin/run_integration_test.sh` to have the device started when running the integration tests,
 - Add to `docs/source/devices/` to mention the device in the end-user documentation.
+- Adjust `docs/source/index.rst` to include the newly created file in `docs/source/devices/`.
+

tangostationcontrol/tangostationcontrol/devices/beam.py

+# -*- coding: utf-8 -*-
+#
+# Distributed under the terms of the APACHE license.
+# See LICENSE.txt for more info.
+
+""" Beam Device Server for LOFAR2.0
+
+"""
+
+# PyTango imports
+from tango import AttrWriteType
+import numpy
+# Additional import
+
+from tangostationcontrol.devices.device_decorators import *
+from tangostationcontrol.common.entrypoint import entry
+from tangostationcontrol.clients.attribute_wrapper import attribute_wrapper
+from tangostationcontrol.devices.lofar_device import lofar_device
+from tangostationcontrol.common.lofar_logging import device_logging_to_python, log_exceptions
+
+__all__ = ["Beam", "main"]
+
+
+@device_logging_to_python()
+class Beam(lofar_device):
+    # -----------------
+    # Device Properties
+    # -----------------
+
+    # ----------
+    # Attributes
+    # ----------
+
+    pass
+
+    # --------
+    # overloaded functions
+    # --------
+
+
+    # --------
+    # Commands
+    # --------
+
+
+# ----------
+# Run server
+# ----------
+def main(**kwargs):
+    """Main function of the ObservationControl module."""
+    return entry(Beam, **kwargs)

tangostationcontrol/tangostationcontrol/devices/boot.py

@@ -251,6 +251,7 @@ class Boot(lofar_device):
                        "STAT/SDP/1",    # SDP controls the mask for SST/XST/BST, so initialise it first
                        "STAT/SST/1",
                        "STAT/XST/1",
+                       "STAT/Beam/1",   # Accesses RECV and SDP
                       ],
     )
 

tangostationcontrol/tangostationcontrol/integration_test/device_proxy.py

+import logging
+import time
+
 from tango import DeviceProxy
 
+logger = logging.getLogger()
+
 
 class TestDeviceProxy(DeviceProxy):
 
     def __init__(self, *args, **kwargs):
         super(TestDeviceProxy, self).__init__(*args, **kwargs)
         self.set_timeout_millis(10000)
+
+    @staticmethod
+    def test_device_turn_off(endpoint):
+        d = TestDeviceProxy(endpoint)
+        try:
+            d.Off()
+        except Exception as e:
+            """Failing to turn Off devices should not raise errors here"""
+            logger.error(f"Failed to turn device off in teardown {e}")
+
+            """Wait for 1 second to prevent propagating reconnection errors"""
+            time.sleep(1)

tangostationcontrol/tangostationcontrol/integration_test/devices/base.py

@@ -34,11 +34,19 @@ class AbstractTestBases:
             # make sure the device starts in Off
             self.proxy.Off()
 
+            self.addCleanup(TestDeviceProxy.test_device_turn_off, self.name)
+
             super().setUp()
 
-        def tearDown(self):
-            """Turn device Off in teardown to prevent blocking tests"""
-            self.proxy.Off()
+        def test_device_fetch_state(self):
+            """Test if we can successfully fetch state"""
+
+            self.assertEqual(DevState.OFF, self.proxy.state())
+
+        def test_device_ping(self):
+            """Test if we can successfully ping the device server"""
+
+            self.assertGreater(self.proxy.ping(), 0)
 
         def test_device_initialize(self):
             """Test if we can transition to standby"""

tangostationcontrol/tangostationcontrol/integration_test/devices/test_device_apsct.py

@@ -9,6 +9,7 @@
 
 from .base import AbstractTestBases
 
+
 class TestDeviceAPSCT(AbstractTestBases.TestDeviceBase):
 
     def setUp(self):