Merge branch 'reshape-highdim-arrays' into 'main'

Added custom LofarDeviceProxy to enhance support for high-dimensional arrays in attribtues

See merge request !4

.gitlab-ci.yml

 default:
-  image: python:3.7 # minimum supported version
+  image: $CI_REGISTRY/lofar2.0/lofar-station-client/ci_python37:$CI_COMMIT_SHORT_SHA # minimum supported version
   # Make sure each step is executed in a virtual environment with some basic dependencies present
   before_script:
     - python --version # For debugging
-    - python -m pip install --upgrade pip
-    - pip install --upgrade tox
   cache:
     paths:
       - .cache/pip
 
 stages:
+  - image
   - lint
   - test
   - package
@@ -20,6 +19,19 @@ stages:
 variables:
   PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
 
+build_test_image_python37:
+  stage: image
+  image: docker
+  services:
+    - name: docker:dind
+  variables:
+    DOCKER_TLS_CERTDIR: "/certs"
+  before_script:
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+  script:
+    - docker build -t $CI_REGISTRY/lofar2.0/lofar-station-client/ci_python37:$CI_COMMIT_SHORT_SHA -f docker/Dockerfile.ci_python37 docker
+    - docker push $CI_REGISTRY/lofar2.0/lofar-station-client/ci_python37:$CI_COMMIT_SHORT_SHA
+
 run_black:
   stage: lint
   script:
@@ -46,24 +58,37 @@ run_unit_tests_py37:
     - echo "run python3.7 unit tests /w coverage"
     - tox -e py37
 
-run_unit_tests_py38:
-  image: python:3.8
+.run_unit_tests_pyXX:
+  # installs the prerequisites explicitly, instead of piggy backing
+  # on the ci_python37 image. This allows us to use different base
+  # images with different python versions.
   stage: test
+  before_script:
+    - apt-get update -y && DEBIAN_FRONTEND=noninteractive apt-get install -y libboost-python-dev libtango-dev # Needed to install pytango
+    - python -m pip install --upgrade pip
+    - pip install --upgrade tox
+
+run_unit_tests_py38:
+  extends: .run_unit_tests_pyXX
+  image: python:3.8-buster
   script:
     - echo "run python3.8 unit tests /w coverage"
     - tox -e py38
 
 run_unit_tests_py39:
-  image: python:3.9
-  stage: test
+  extends: .run_unit_tests_pyXX
+  image: python:3.9-bullseye
   script:
     - echo "run python3.9 unit tests /w coverage"
     - tox -e py39
 
 run_unit_tests_py310:
-  image: python:3.10
-  stage: test
+  extends: .run_unit_tests_pyXX
+  image: python:3.10-bullseye
   script:
+    # Debian Bullseye ships with libboost-python linked to Python 3.9. Use the one from Debian Sid instead.
+    - echo 'deb http://deb.debian.org/debian sid main' >> /etc/apt/sources.list
+    - apt-get update -y && DEBIAN_FRONTEND=noninteractive apt-get install -y libboost-python1.74-dev
     - echo "run python3.10 unit tests /w coverage"
     - tox -e py310
 

README.md

@@ -13,12 +13,32 @@ Client library for using Tango Station Control.
 
 Table of Contents:
 
+- [Prerequisites](#prerequisites)
 - [Installation](#installation)
 - [Usage](#usage)
 - [Development](#development)
 - [Debug](#debug)
 - [Releasenotes](#releasenotes)
 
+##  Prerequisites
+
+This package uses [PyTango](https://pytango.readthedocs.io), which wraps the Tango C++ library. You will need to install the Tango C++ headers & library to allow `pip` to install PyTango as part of this package's requirements:
+
+Debian and Ubuntu provide these natively:
+
+```shell
+apt-get install libtango-dev
+```
+
+Under Arch, install the [tango-cpp AUR package](https://aur.archlinux.org/packages/tango-cpp). For other distros and installation methods, see [Tango's Linux installation manual](https://tango-controls.readthedocs.io/en/latest/installation/tango-on-linux.html).
+
+You will also need the Boost Python C++ headers & libraries to compile PyTango. Under Debian/Ubuntu, obtain these through:
+
+```shell
+apt-get install libboost-python-dev
+```
+
+
 ##  Installation
 
 Wheel distributions are available from the [gitlab package registry](https://git.astron.nl/lofar2.0/lofar-station-client/-/packages/),
@@ -86,6 +106,7 @@ tox -e debug tests.requests.test_prometheus
 
 ## Releasenotes
 
+- 0.9  - Added `devices.LofarDeviceProxy` that transparently exposes >2D attributes
 - 0.8  - Fixed XST packet parsing.
 - 0.7. - Partial rework of DTS outside code, still many possible improvements.
 - 0.6. - Correctly transpose XST blocks in `XSTCollector`.

VERSION

-0.8.0
+0.9.0

docker/Dockerfile.ci_python37

+FROM python:3.7-buster
+
+# Install PyTango dependencies
+RUN apt-get update -y
+RUN DEBIAN_FRONTEND=noninteractive apt-get install -y libboost-python-dev libtango-dev
+
+# Make sure we have the latest tooling for our tests
+RUN python -m pip install --upgrade pip
+RUN pip install --upgrade tox

lofar_station_client/devices.py

+""" Enhanced interfaces towards the station's Tango devices. """
+
+import ast
+from functools import lru_cache
+
+import numpy
+
+from tango import DeviceProxy
+from tango import ExtractAs
+
+
+class LofarDeviceProxy(DeviceProxy):
+    """A LOFAR-specific tango.DeviceProxy that provides
+    a richer experience."""
+
+    @lru_cache()
+    def get_attribute_config(self, name):
+        """Get cached attribute configurations, as they are not expected to change."""
+
+        return super().get_attribute_config(name)
+
+    @lru_cache()
+    def get_attribute_shape(self, name):
+        """Get the shape of the requested attribute, as a tuple."""
+
+        config = self.get_attribute_config(name)
+
+        if config.format and config.format[0] == "(":
+            # For >2D arrays, the "format" property describes actual
+            # the dimensions as a tuple (x, y, z, ...),
+            # so reshape the value accordingly.
+            shape = ast.literal_eval(config.format)
+        elif config.max_dim_y > 0:
+            # 2D array
+            shape = (config.max_dim_y, config.max_dim_x)
+        elif config.max_dim_x > 1:
+            # 1D array
+            shape = (config.max_dim_x,)
+        else:
+            # scalar
+            shape = ()
+
+        return shape
+
+    def read_attribute(self, name, extract_as=ExtractAs.Numpy):
+        """Read an attribute from the server."""
+
+        attr = super().read_attribute(name, extract_as)
+
+        # convert non-scalar values into their actual shape
+        shape = self.get_attribute_shape(name)
+        if shape != ():
+            attr.value = attr.value.reshape(shape)
+
+        return attr
+
+    def write_attribute(self, name, value):
+        """Write an attribute to the server."""
+
+        config = self.get_attribute_config(name)
+        shape = self.get_attribute_shape(name)
+
+        # 2D arrays also represent arrays of higher dimensionality. reshape them
+        # to fit their original Tango shape before writing.
+        if shape != ():
+            value = numpy.array(value)
+
+            if value.shape != shape:
+                raise ValueError(
+                    f"Invalid shape. Given: {value.shape} Expected: {shape}"
+                )
+
+            if len(shape) > 2:
+                # >2D arrays collapse into 2D
+                value = value.reshape((config.max_dim_y, config.max_dim_x))
+
+        return super().write_attribute(name, value)

requirements.txt

@@ -4,3 +4,5 @@ numpy>=1.21.0 # BSD
 nptyping>=2.3.0 # MIT
 matplotlib>=3.5.0 # PSF
 pyDeprecate>=0.3.0 # MIT
+# commented out until https://gitlab.com/tango-controls/pytango/-/issues/468 is resolved
+# PyTango>=9.3.5 # GNU LGPL v3

tests/test_devices.py

+import numpy
+
+from tests import base
+
+from lofar_station_client.devices import LofarDeviceProxy
+
+from tango.test_context import MultiDeviceTestContext
+from tango.server import Device, attribute, AttrWriteType
+
+
+class MyDevice(Device):
+    A = attribute(
+        dtype=((bool,),),
+        max_dim_x=4,
+        max_dim_y=6,
+        format="(2,3,4)",
+        access=AttrWriteType.READ_WRITE,
+    )
+
+    scalar = attribute(dtype=bool, access=AttrWriteType.READ_WRITE)
+
+    spectrum = attribute(dtype=(bool,), max_dim_x=2, access=AttrWriteType.READ_WRITE)
+
+    image = attribute(
+        dtype=((bool,),), max_dim_x=2, max_dim_y=3, access=AttrWriteType.READ_WRITE
+    )
+
+    def init_device(self):
+        self.value_A = numpy.zeros((6, 4), dtype=bool)
+        self.value_scalar = False
+        self.value_spectrum = [False, False]
+        self.value_image = [[False, False]] * 3
+
+    def read_scalar(self):
+        return self.value_scalar
+
+    def write_scalar(self, value):
+        self.value_scalar = value
+
+    def read_spectrum(self):
+        return self.value_spectrum
+
+    def write_spectrum(self, value):
+        self.value_spectrum = value
+
+    def read_image(self):
+        return self.value_image
+
+    def write_image(self, value):
+        self.value_image = value
+
+    def read_A(self):
+        return self.value_A
+
+    def write_A(self, value):
+        self.value_A = value
+
+
+class LofarDeviceProxyTest(base.TestCase):
+    TEST_DEVICE_INFO = [
+        {
+            "class": MyDevice,
+            "devices": [{"name": "STAT/MyDevice/1", "properties": {}, "memorized": {}}],
+        }
+    ]
+
+    @classmethod
+    def setUpClass(cls):
+        # setting up the TestContext takes ~1 second, so do it only once
+        cls.context = MultiDeviceTestContext(
+            cls.TEST_DEVICE_INFO,
+            process=True,
+        )
+
+        cls.context.start()
+        cls.proxy = LofarDeviceProxy(cls.context.get_device_access("STAT/MyDevice/1"))
+
+    @classmethod
+    def tearDownClass(cls):
+        # In Python3.8+, we can use addClassCleanup instead
+        cls.context.stop()
+
+    def test_read_scalar(self):
+        value = self.proxy.scalar
+
+        self.assertEqual(bool, type(value))
+
+    def test_write_scalar(self):
+        self.proxy.scalar = True
+
+    def test_read_spectrum(self):
+        value = self.proxy.spectrum
+
+        self.assertEqual((2,), value.shape)
+        self.assertEqual(numpy.bool_, type(value[0]))
+
+    def test_write_spectrum(self):
+        self.proxy.spectrum = [True, False]
+
+    def test_read_image(self):
+        value = self.proxy.image
+
+        self.assertEqual((3, 2), value.shape)
+        self.assertEqual(numpy.bool_, type(value[0, 0]))
+
+    def test_write_image(self):
+        self.proxy.image = [[True, False]] * 3
+
+    def test_write_3D_attribute_lists(self):
+        self.proxy.A = [
+            [True, True, True, True],
+            [True, True, True, True],
+            [True, True, True, True],
+        ], [
+            [False, False, False, False],
+            [False, False, False, False],
+            [False, False, False, False],
+        ]
+
+    def test_write_3D_attribute_numpy(self):
+        self.proxy.A = numpy.zeros((2, 3, 4), dtype=bool)
+
+    def test_write_3D_attribute_numpy_with_too_few_dimensions(self):
+        # write a 2D shape
+        with self.assertRaises(ValueError):
+            self.proxy.A = numpy.zeros((2, 12), dtype=bool)
+
+    def test_write_3D_attribute_numpy_with_too_many_dimension(self):
+        # write a 4D shape
+        with self.assertRaises(ValueError):
+            self.proxy.A = numpy.zeros((2, 3, 2, 2), dtype=bool)
+
+    def test_read_3D_attribute(self):
+        value = self.proxy.A
+
+        self.assertEqual((2, 3, 4), value.shape)

tox.ini

@@ -14,6 +14,10 @@ setenv =
 deps =
     -r{toxinidir}/requirements.txt
     -r{toxinidir}/test-requirements.txt
+commands_pre =
+    # required until https://gitlab.com/tango-controls/pytango/-/issues/468 is resolved
+    pip install 'numpy>=1.21.0'
+    pip install --no-cache 'PyTango>=9.3.5'
 commands =
     {envpython} --version
     stestr run {posargs}