Migrate from station client to lotus

.gitlab-ci.yml

@@ -321,7 +321,7 @@ service_test_docker:
   extends: .test_docker
   script:
     # Do not remove 'bash' or statement will be ignored by primitive docker shell
-    - bash -e $CI_PROJECT_DIR/sbin/run_service_test.sh
+    - bash -e $CI_PROJECT_DIR/sbin/run_integration_test.sh --no-build --skip-tests --module="services" --station=cs
 
 multi_project_integration_test:
   stage: integration-tests

README.md

@@ -150,6 +150,7 @@ Next change the version in the following places:
    through [https://git.astron.nl/lofar2.0/tango/-/tags](Deploy Tags)
 
 # Release Notes
+* 0.47.0 Migrate from lofar-station-client to lofar-lotus package. Update various package dependencies.
 * 0.46.1 Include clock_RW in metadata JSON
 * 0.46.0 Expose latest BST/SST/XST from ZMQ over gRPC
 * 0.45.12 Improve syslog parsing for PyPCC logs

docker/jupyter-lab/Dockerfile

@@ -17,7 +17,7 @@ RUN mamba install --yes 'jupyterlab-git>=0.50.0' && \
 RUN jupyter server extension enable --py jupyterlab_git
 
 COPY requirements.txt ./
-RUN pip install --break-system-packages -r requirements.txt --extra-index-url=https://git.astron.nl/api/v4/projects/395/packages/pypi/simple
+RUN pip install --break-system-packages -r requirements.txt --extra-index-url=https://git.astron.nl/api/v4/groups/36/-/packages/pypi/simple
 RUN rm requirements.txt
 
 USER root

docker/jupyter-lab/ipython-profiles/stationcontrol-jupyter/startup/01-stationcontrol.py

@@ -2,7 +2,8 @@
 #  SPDX-License-Identifier: Apache-2.0
 
 # noinspection PyUnresolvedReferences
-import lofar_station_client
+import lofar_lotus as lofar_station_client
+import lofar_lotus
 
 # noinspection PyUnresolvedReferences
 import tangostationcontrol

docker/jupyter-lab/requirements.txt

@@ -11,7 +11,7 @@ jupyter_server_nbmodel[lab,rtc] # BSD-3
 # NB: tangostationcontrol will also install lofar-station-client. The latter
 #     is omitted here to avoid pip getting confused.
 tangostationcontrol >= 0.39
---extra-index-url https://git.astron.nl/api/v4/projects/71/packages/pypi/simple
+--extra-index-url https://git.astron.nl/api/v4/groups/36/-/packages/pypi/simple
 PyTango
 
 # low-level access to station components

docker/lofar-device-base/Dockerfile

@@ -65,12 +65,12 @@ COPY tmp/debug-requirements.txt /tangostationcontrol-debug-requirements.txt
 RUN echo "DEBUG_BUILD: ${DEBUG_BUILD}"
 RUN if [ $DEBUG_BUILD ]; then \
     echo "Installing debug requirements"; \
-    sudo pip3 install -r /tangostationcontrol-debug-requirements.txt --extra-index-url=https://git.astron.nl/api/v4/projects/395/packages/pypi/simple; \
+    sudo pip3 install -r /tangostationcontrol-debug-requirements.txt --extra-index-url=https://git.astron.nl/api/v4/groups/36/-/packages/pypi/simple; \
   fi
 
 COPY tmp/*.whl /
 RUN echo "Installing prebuild Station Control wheel"; \
-    sudo pip3 install /*.whl --extra-index-url=https://git.astron.nl/api/v4/projects/395/packages/pypi/simple;
+    sudo pip3 install /*.whl --extra-index-url=https://git.astron.nl/api/v4/groups/36/-/packages/pypi/simple;
 
 # install and use ephimerides and geodetic ("measures") tables for casacore.
 # we install a _stub_ since the tables need to be deployed explicitly from within the software.

infra/dev/services/services.hcl

@@ -9,17 +9,17 @@ resource "nomad_job" "monitoring" {
   }
 }
 
-resource "nomad_job" "logging" {
-  cluster = variable.nomad_cluster
-
-  paths = ["../jobs/station/logging.nomad"]
-  depends_on = ["resource.nomad_job.monitoring"]
-
-  health_check {
-    timeout = "300s"
-    jobs    = ["log-scraping"]
-  }
-}
+# resource "nomad_job" "logging" {
+#   cluster = variable.nomad_cluster
+#
+#   paths = ["../jobs/station/logging.nomad"]
+#   depends_on = ["resource.nomad_job.monitoring"]
+#
+#   health_check {
+#     timeout = "300s"
+#     jobs    = ["log-scraping"]
+#   }
+# }
 
 resource "ingress" "grafana" {
   port = 3000

infra/dev/tango/tango.hcl

@@ -140,6 +140,19 @@ resource "nomad_job" "device-servers" {
     jobs    = ["device-servers"]
   }
 }
+resource "nomad_job" "rpc-server" {
+  depends_on = [
+    "resource.nomad_job.device-servers"
+  ]
+  cluster = variable.nomad_cluster
+
+  paths = ["../jobs/station/rpc-server.nomad"]
+
+  health_check {
+    timeout = "3000s"
+    jobs    = ["rpc-server"]
+  }
+}
 
 resource "ingress" "minio_s3" {
   port = 9000

infra/jobs/station/rpc-server.levant.nomad

@@ -45,8 +45,9 @@ job "rpc-server" {
         entrypoint = [""]
         command    = "l2ss-rpc-server"
         args       = [
-          "--port", "${NOMAD_PORT_rpc}",
-          "--metrics-port", "${NOMAD_PORT_metrics}"
+          "--port", "${NOMAD_PORT_grpc}",
+          "--metrics-port", "${NOMAD_PORT_metrics}",
+          "--station", "[[.station]]"
     [[ range $name, $tr := $.sdptr.instances ]]
           , "--antenna-field", "[[ $name ]]"
           , "--statistics-zmq", "tcp://stingray-[[ $name ]]-bst-zmq.service.consul:6001"

sbin/run_integration_test.sh

@@ -23,7 +23,7 @@ function usage {
       disables building of docker images"
     echo ""
     echo "./$(basename "$0") --skip-tests
-      Only setup environment, implies --preserve"
+      Only setup environment"
     echo ""
     echo "./$(basename "$0") --preserve
       Prevents tearing down the dev environment afterwards"
@@ -67,9 +67,7 @@ while true; do
       ;;
     --skip-tests)
       echo "Only setup and configure environment don't run any tests"
-      echo "Implies --preserve"
       export no_tests=1
-      export preserve=1
       ;;
     --preserve)
       echo "Preserve test environment"

sbin/run_service_test.sh

-#!/bin/bash -e
-#
-# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy)
-# SPDX-License-Identifier: Apache-2.0
-#
-
-if [ -z "$LOFAR20_DIR" ]; then
-    # We assume we aren't in the PATH, so we can derive our path.
-    # We need our parent directory.
-    LOFAR20_DIR_RELATIVE=$(dirname "$0")/..
-
-    # As an absolute path
-    LOFAR20_DIR=$(readlink -f "${LOFAR20_DIR_RELATIVE}")
-fi
-
-if [ -z "$TAG" ]; then
-  export TAG="latest"
-fi
-
-docker network rm station || true
-
-# prepare a docker volume for nomad
-tmp_volume="test_$(hexdump -n 16 -v -e '/1 "%02X"' /dev/urandom)"
-
-function cleanup {
-  cd "$LOFAR20_DIR"
-  if [ -n "${save_logs}" ]; then
-    mkdir -p log
-    for container in $(docker ps -a --format "{{.Names}}")
-    do
-      echo "Saving log for container $container"
-      docker logs "${container}" >& "log/${container}.log"
-    done
-    bash "${LOFAR20_DIR}"/sbin/dsconfig.sh --dump >& log/dump_ConfigDb.log
-  fi
-
-  if [ -z "${preserve}" ]; then
-    HOME="$JUMPPAD_HOME" jumppad down
-    docker volume rm "$tmp_volume" || true
-  fi
-}
-
-trap cleanup INT EXIT TERM ERR SIGTERM SIGINT
-
-cd "$LOFAR20_DIR" || exit 1
-
-source "${LOFAR20_DIR}"/sbin/prepare_dev_env.sh --volume="$tmp_volume"
-
-if [ -z "$JUMPPAD_HOME" ]; then
-  JUMPPAD_HOME="$HOME"
-fi
-
-rm -rf "$JUMPPAD_HOME/.jumppad/"
-
-jumppad_options=(
-  --var="host_volume=$tmp_volume"
-  --var="image_tag=$TAG"
-)
-
-HOME="$JUMPPAD_HOME" jumppad up "${jumppad_options[@]}" infra/dev/services.hcl --no-browser
-
-echo "success"

tangostationcontrol/VERSION

-0.46.1
+0.47.0

tangostationcontrol/integration_test/common/base_device_classes/power_hierarchy_tests.py

@@ -10,7 +10,7 @@ from tango import DevState, DeviceProxy
 
 from integration_test import base
 from integration_test.device_proxy import TestDeviceProxy
-from lofar_station_client.common import CaseInsensitiveString
+from lofar_lotus.dict import CaseInsensitiveString
 from tangostationcontrol.common.constants import N_rcu, N_rcu_inp
 from tangostationcontrol.devices.base_device_classes.hierarchy_device import (
     NotFoundException,

tangostationcontrol/integration_test/common/device_digitalbeam_tests.py

@@ -5,7 +5,7 @@ import logging
 import time
 
 import numpy
-import timeout_decorator
+import pytest
 
 from integration_test.device_proxy import TestDeviceProxy
 from integration_test.default.devices.base import TestDeviceBase
@@ -223,7 +223,7 @@ class DigitalBeamDeviceTests(TestDeviceBase):
             f"{FPGA_bf_weights_pp_RW}",
         )
 
-    @timeout_decorator.timeout(15)
+    @pytest.mark.timeout(15)
     def test_beam_tracking_90_percent_interval(self):
         """Verify that the beam tracking operates within 95% of interval"""
 

tangostationcontrol/integration_test/common/device_observation_control_tests.py

@@ -7,12 +7,14 @@ import time
 from datetime import datetime
 from datetime import timedelta
 
+
 import numpy
+import pytest
+
 from integration_test.default.devices.base import TestDeviceBase
 from tango import DevFailed, DevState
 
 from tangostationcontrol.common.constants import CS001_TILES
-from timeout_decorator import timeout_decorator
 
 logger = logging.getLogger()
 
@@ -271,7 +273,7 @@ class DeviceObservationControlTests(TestDeviceBase):
         self.on_device_assert(self.proxy)
         self.assertRaises(DevFailed, self.proxy.add_observation, json.dumps(parameters))
 
-    @timeout_decorator.timeout(60)
+    @pytest.mark.timeout(60)
     def test_add_observation_callbacks(self):
         """Test adding an observation and checking start / stop callbacks work"""
 

tangostationcontrol/integration_test/common/device_tilebeam_tests.py

@@ -7,7 +7,7 @@ import json
 import time
 
 import numpy
-import timeout_decorator
+import pytest
 
 from integration_test.device_proxy import TestDeviceProxy
 from integration_test.default.devices.base import TestDeviceBase
@@ -251,7 +251,7 @@ class TileBeamDeviceTests(TestDeviceBase):
             new_pointings[0:2], list(self.proxy.Pointing_direction_R[0:2])
         )
 
-    @timeout_decorator.timeout(120)
+    @pytest.mark.timeout(120)
     def test_beam_tracking_95_percent_interval(self):
         """Verify that the beam tracking operates within 95% of interval"""
 

tangostationcontrol/integration_test/configuration/test_device_configuration.py

@@ -11,8 +11,8 @@ except ImportError:
     from importlib_resources import files  # type: ignore
 
 from integration_test.default.devices.base import TestDeviceBase
-from lofar_station_client.common import CaseInsensitiveDict
-from lofar_station_client.common import CaseInsensitiveString
+from lofar_lotus.dict import CaseInsensitiveDict
+from lofar_lotus.dict import CaseInsensitiveString
 
 
 from tango import DevState

tangostationcontrol/integration_test/default/devices/test_device_protection_control.py

@@ -5,7 +5,7 @@ import logging
 import time
 from typing import List
 
-import timeout_decorator
+import pytest
 from tango import Database, DevState
 
 from tangostationcontrol.devices import UNB2, SDPFirmware, RECVL, RECVH, APSCT, APSPU
@@ -84,7 +84,7 @@ class TestDeviceProtectionControl(TestDeviceBase):
             self.proxy.number_of_connected_devices_R,
         )
 
-    @timeout_decorator.timeout(10)
+    @pytest.mark.timeout(10)
     def test_change_events_received(self):
 
         self.proxy.boot()

tangostationcontrol/integration_test/default/devices/test_observation_client.py

@@ -4,15 +4,17 @@
 from json import loads
 from os import environ
 
-from lofar_station_client.observation.station_observation import (
-    StationObservation,
-)
 from integration_test import base
 from integration_test.device_proxy import TestDeviceProxy
 from tangostationcontrol.test.dummy_observation_settings import (
     get_observation_settings_hba_core_immediate,
 )
-
+import grpc
+from lofar_sid.interface.stationcontrol.observation_pb2 import (
+    StartObservationRequest,
+    StopObservationRequest,
+)
+from lofar_sid.interface.stationcontrol.observation_pb2_grpc import ObservationStub
 from tango import DevState
 
 
@@ -57,25 +59,27 @@ class TestObservation(base.IntegrationTestCase):
         """Test of the observation_wrapper class basic functionality"""
 
         # convert the JSON specification to a dict for this class
-        specification_dict = loads(
-            get_observation_settings_hba_core_immediate().to_json()
-        )
+        specification = get_observation_settings_hba_core_immediate().to_json()
+        specification_dict = loads(specification)
 
         # create an observation class using the dict and as host just get it using a
         # util function
-        observation = StationObservation(
-            specification=specification_dict, host=environ["TANGO_HOST"]
-        )
 
-        # Assert the observation is running after starting it
-        observation.start()
-        self.assertTrue(observation.is_running)
+        tango_host = environ["TANGO_HOST"]
+        obs = TestDeviceProxy(f"tango://{tango_host}/STAT/ObservationControl/1")
+
+        obs_id = specification_dict["antenna_fields"][0]["observation_id"]
 
-        # Assert the proxy is on
-        station = observation.observation
-        proxy = station.observation_field_proxies[0]
-        self.assertTrue(proxy.state() == DevState.ON)
+        with grpc.insecure_channel("rpc.service.consul:50051") as channel:
+            stub = ObservationStub(channel)
+            stub.StartObservation(StartObservationRequest(configuration=specification))
+
+        # Assert the observation is running after starting it
+        self.assertTrue(obs.is_observation_running(obs_id))
 
         # Assert the observation has stopped after aborting
-        observation.stop()
-        self.assertFalse(observation.is_running)
+
+        with grpc.insecure_channel("rpc.service.consul:50051") as channel:
+            stub = ObservationStub(channel)
+            stub.StopObservation(StopObservationRequest(observation_id=obs_id))
+        self.assertFalse(obs.is_observation_running(obs_id))

tangostationcontrol/pyproject.toml

 [build-system]
 requires = ['setuptools>=62.6', 'wheel']
 build-backend = 'setuptools.build_meta'
+[tool.pytest.ini_options]
+markers = [
+    "timeout",
+]