diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 16240653ae827112f531a2df361948472a9ffc46..e88c625fd80ec7de203f736942946347bcd450f6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,15 +1,14 @@ default: - image: python:3.7 # minimum supported version + image: $CI_REGISTRY/lofar2.0/lofar-station-client/ci_python37:$CI_COMMIT_SHORT_SHA # minimum supported version # Make sure each step is executed in a virtual environment with some basic dependencies present before_script: - python --version # For debugging - - python -m pip install --upgrade pip - - pip install --upgrade tox cache: paths: - .cache/pip stages: + - image - lint - test - package @@ -20,6 +19,19 @@ stages: variables: PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" +build_test_image_python37: + stage: image + image: docker + services: + - name: docker:dind + variables: + DOCKER_TLS_CERTDIR: "/certs" + before_script: + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + script: + - docker build -t $CI_REGISTRY/lofar2.0/lofar-station-client/ci_python37:$CI_COMMIT_SHORT_SHA -f docker/Dockerfile.ci_python37 docker + - docker push $CI_REGISTRY/lofar2.0/lofar-station-client/ci_python37:$CI_COMMIT_SHORT_SHA + run_black: stage: lint script: @@ -46,24 +58,37 @@ run_unit_tests_py37: - echo "run python3.7 unit tests /w coverage" - tox -e py37 -run_unit_tests_py38: - image: python:3.8 +.run_unit_tests_pyXX: + # installs the prerequisites explicitly, instead of piggy backing + # on the ci_python37 image. This allows us to use different base + # images with different python versions. stage: test + before_script: + - apt-get update -y && DEBIAN_FRONTEND=noninteractive apt-get install -y libboost-python-dev libtango-dev # Needed to install pytango + - python -m pip install --upgrade pip + - pip install --upgrade tox + +run_unit_tests_py38: + extends: .run_unit_tests_pyXX + image: python:3.8-buster script: - echo "run python3.8 unit tests /w coverage" - tox -e py38 run_unit_tests_py39: - image: python:3.9 - stage: test + extends: .run_unit_tests_pyXX + image: python:3.9-bullseye script: - echo "run python3.9 unit tests /w coverage" - tox -e py39 run_unit_tests_py310: - image: python:3.10 - stage: test + extends: .run_unit_tests_pyXX + image: python:3.10-bullseye script: + # Debian Bullseye ships with libboost-python linked to Python 3.9. Use the one from Debian Sid instead. + - echo 'deb http://deb.debian.org/debian sid main' >> /etc/apt/sources.list + - apt-get update -y && DEBIAN_FRONTEND=noninteractive apt-get install -y libboost-python1.74-dev - echo "run python3.10 unit tests /w coverage" - tox -e py310 diff --git a/README.md b/README.md index 2af55ed518a276a9a991085269138e876c0e5399..32deb0e52f8e4f1770501e3bb6465af6cd68da12 100644 --- a/README.md +++ b/README.md @@ -13,12 +13,32 @@ Client library for using Tango Station Control. Table of Contents: +- [Prerequisites](#prerequisites) - [Installation](#installation) - [Usage](#usage) - [Development](#development) - [Debug](#debug) - [Releasenotes](#releasenotes) +## Prerequisites + +This package uses [PyTango](https://pytango.readthedocs.io), which wraps the Tango C++ library. You will need to install the Tango C++ headers & library to allow `pip` to install PyTango as part of this package's requirements: + +Debian and Ubuntu provide these natively: + +```shell +apt-get install libtango-dev +``` + +Under Arch, install the [tango-cpp AUR package](https://aur.archlinux.org/packages/tango-cpp). For other distros and installation methods, see [Tango's Linux installation manual](https://tango-controls.readthedocs.io/en/latest/installation/tango-on-linux.html). + +You will also need the Boost Python C++ headers & libraries to compile PyTango. Under Debian/Ubuntu, obtain these through: + +```shell +apt-get install libboost-python-dev +``` + + ## Installation Wheel distributions are available from the [gitlab package registry](https://git.astron.nl/lofar2.0/lofar-station-client/-/packages/), @@ -86,6 +106,7 @@ tox -e debug tests.requests.test_prometheus ## Releasenotes +- 0.9 - Added `devices.LofarDeviceProxy` that transparently exposes >2D attributes - 0.8 - Fixed XST packet parsing. - 0.7. - Partial rework of DTS outside code, still many possible improvements. - 0.6. - Correctly transpose XST blocks in `XSTCollector`. diff --git a/VERSION b/VERSION index a3df0a6959e154733da89a5d6063742ce6d5b851..ac39a106c48515b621e90c028ed94c6f71bc03fa 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.8.0 +0.9.0 diff --git a/docker/Dockerfile.ci_python37 b/docker/Dockerfile.ci_python37 new file mode 100644 index 0000000000000000000000000000000000000000..1f4362fcff091ce1ba35661777eeb3e78c2a8b98 --- /dev/null +++ b/docker/Dockerfile.ci_python37 @@ -0,0 +1,9 @@ +FROM python:3.7-buster + +# Install PyTango dependencies +RUN apt-get update -y +RUN DEBIAN_FRONTEND=noninteractive apt-get install -y libboost-python-dev libtango-dev + +# Make sure we have the latest tooling for our tests +RUN python -m pip install --upgrade pip +RUN pip install --upgrade tox diff --git a/lofar_station_client/devices.py b/lofar_station_client/devices.py new file mode 100644 index 0000000000000000000000000000000000000000..aac73f1b8f89f2c859c5f5fd42f9450b8116ad1f --- /dev/null +++ b/lofar_station_client/devices.py @@ -0,0 +1,77 @@ +""" Enhanced interfaces towards the station's Tango devices. """ + +import ast +from functools import lru_cache + +import numpy + +from tango import DeviceProxy +from tango import ExtractAs + + +class LofarDeviceProxy(DeviceProxy): + """A LOFAR-specific tango.DeviceProxy that provides + a richer experience.""" + + @lru_cache() + def get_attribute_config(self, name): + """Get cached attribute configurations, as they are not expected to change.""" + + return super().get_attribute_config(name) + + @lru_cache() + def get_attribute_shape(self, name): + """Get the shape of the requested attribute, as a tuple.""" + + config = self.get_attribute_config(name) + + if config.format and config.format[0] == "(": + # For >2D arrays, the "format" property describes actual + # the dimensions as a tuple (x, y, z, ...), + # so reshape the value accordingly. + shape = ast.literal_eval(config.format) + elif config.max_dim_y > 0: + # 2D array + shape = (config.max_dim_y, config.max_dim_x) + elif config.max_dim_x > 1: + # 1D array + shape = (config.max_dim_x,) + else: + # scalar + shape = () + + return shape + + def read_attribute(self, name, extract_as=ExtractAs.Numpy): + """Read an attribute from the server.""" + + attr = super().read_attribute(name, extract_as) + + # convert non-scalar values into their actual shape + shape = self.get_attribute_shape(name) + if shape != (): + attr.value = attr.value.reshape(shape) + + return attr + + def write_attribute(self, name, value): + """Write an attribute to the server.""" + + config = self.get_attribute_config(name) + shape = self.get_attribute_shape(name) + + # 2D arrays also represent arrays of higher dimensionality. reshape them + # to fit their original Tango shape before writing. + if shape != (): + value = numpy.array(value) + + if value.shape != shape: + raise ValueError( + f"Invalid shape. Given: {value.shape} Expected: {shape}" + ) + + if len(shape) > 2: + # >2D arrays collapse into 2D + value = value.reshape((config.max_dim_y, config.max_dim_x)) + + return super().write_attribute(name, value) diff --git a/requirements.txt b/requirements.txt index ef6adefd36e51eee51da83af1c324d3780fa9732..5e2e3478a9c1d974fc92a40993834cc03b7fd5c4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,5 @@ numpy>=1.21.0 # BSD nptyping>=2.3.0 # MIT matplotlib>=3.5.0 # PSF pyDeprecate>=0.3.0 # MIT +# commented out until https://gitlab.com/tango-controls/pytango/-/issues/468 is resolved +# PyTango>=9.3.5 # GNU LGPL v3 diff --git a/tests/test_devices.py b/tests/test_devices.py new file mode 100644 index 0000000000000000000000000000000000000000..26701397a49ef7c5d38f07142a6366fe37c6f707 --- /dev/null +++ b/tests/test_devices.py @@ -0,0 +1,136 @@ +import numpy + +from tests import base + +from lofar_station_client.devices import LofarDeviceProxy + +from tango.test_context import MultiDeviceTestContext +from tango.server import Device, attribute, AttrWriteType + + +class MyDevice(Device): + A = attribute( + dtype=((bool,),), + max_dim_x=4, + max_dim_y=6, + format="(2,3,4)", + access=AttrWriteType.READ_WRITE, + ) + + scalar = attribute(dtype=bool, access=AttrWriteType.READ_WRITE) + + spectrum = attribute(dtype=(bool,), max_dim_x=2, access=AttrWriteType.READ_WRITE) + + image = attribute( + dtype=((bool,),), max_dim_x=2, max_dim_y=3, access=AttrWriteType.READ_WRITE + ) + + def init_device(self): + self.value_A = numpy.zeros((6, 4), dtype=bool) + self.value_scalar = False + self.value_spectrum = [False, False] + self.value_image = [[False, False]] * 3 + + def read_scalar(self): + return self.value_scalar + + def write_scalar(self, value): + self.value_scalar = value + + def read_spectrum(self): + return self.value_spectrum + + def write_spectrum(self, value): + self.value_spectrum = value + + def read_image(self): + return self.value_image + + def write_image(self, value): + self.value_image = value + + def read_A(self): + return self.value_A + + def write_A(self, value): + self.value_A = value + + +class LofarDeviceProxyTest(base.TestCase): + TEST_DEVICE_INFO = [ + { + "class": MyDevice, + "devices": [{"name": "STAT/MyDevice/1", "properties": {}, "memorized": {}}], + } + ] + + @classmethod + def setUpClass(cls): + # setting up the TestContext takes ~1 second, so do it only once + cls.context = MultiDeviceTestContext( + cls.TEST_DEVICE_INFO, + process=True, + ) + + cls.context.start() + cls.proxy = LofarDeviceProxy(cls.context.get_device_access("STAT/MyDevice/1")) + + @classmethod + def tearDownClass(cls): + # In Python3.8+, we can use addClassCleanup instead + cls.context.stop() + + def test_read_scalar(self): + value = self.proxy.scalar + + self.assertEqual(bool, type(value)) + + def test_write_scalar(self): + self.proxy.scalar = True + + def test_read_spectrum(self): + value = self.proxy.spectrum + + self.assertEqual((2,), value.shape) + self.assertEqual(numpy.bool_, type(value[0])) + + def test_write_spectrum(self): + self.proxy.spectrum = [True, False] + + def test_read_image(self): + value = self.proxy.image + + self.assertEqual((3, 2), value.shape) + self.assertEqual(numpy.bool_, type(value[0, 0])) + + def test_write_image(self): + self.proxy.image = [[True, False]] * 3 + + def test_write_3D_attribute_lists(self): + self.proxy.A = [ + [True, True, True, True], + [True, True, True, True], + [True, True, True, True], + ], [ + [False, False, False, False], + [False, False, False, False], + [False, False, False, False], + ] + + def test_write_3D_attribute_numpy(self): + self.proxy.A = numpy.zeros((2, 3, 4), dtype=bool) + + def test_write_3D_attribute_numpy_with_too_few_dimensions(self): + # write a 2D shape + with self.assertRaises(ValueError): + self.proxy.A = numpy.zeros((2, 12), dtype=bool) + + def test_write_3D_attribute_numpy_with_too_many_dimension(self): + # write a 4D shape + with self.assertRaises(ValueError): + self.proxy.A = numpy.zeros((2, 3, 2, 2), dtype=bool) + + def test_read_3D_attribute(self): + value = self.proxy.A + + self.assertEqual((2, 3, 4), value.shape) diff --git a/tox.ini b/tox.ini index c340ad3df01be8fcabc18a8f217e1d6f1e2d7b89..a358069440d42dfbfe9790a122a5cca328be7acf 100644 --- a/tox.ini +++ b/tox.ini @@ -14,6 +14,10 @@ setenv = deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt +commands_pre = + # required until https://gitlab.com/tango-controls/pytango/-/issues/468 is resolved + pip install 'numpy>=1.21.0' + pip install --no-cache 'PyTango>=9.3.5' commands = {envpython} --version stestr run {posargs}