diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 000922275dcfd0b49e617152ae51237b3f87ada4..64e26f5b2b5659db05a6cd975e3d2ac30be5ef64 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -42,6 +42,11 @@ stages: - . bootstrap/etc/lofar20rc.sh || true ## Allow docker image script to execute # - chmod u+x $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh +# This suffers from only refs changes not working as expected: +# https://gitlab.com/gitlab-org/gitlab-foss/-/issues/55012 +# Therefore we have to add `only: refs: - merge_requests` to all jobs that are +# only supposed to run on merge requests with file changes. However, +# two pipelines will spawn instead of one of which one tagged with 'detached`. .base_docker_images_except: extends: .base_docker_images except: @@ -62,6 +67,8 @@ docker_store_images_master_tag: docker_store_images_changes: extends: .base_docker_store_images only: + refs: + - merge_requests changes: - docker-compose/.env except: @@ -101,6 +108,8 @@ docker_build_image_all: docker_build_image_elk: extends: .base_docker_images_except only: + refs: + - merge_requests changes: - docker-compose/elk.yml - docker-compose/elk/* @@ -112,6 +121,8 @@ docker_build_image_elk: docker_build_image_lofar_device_base: extends: .base_docker_images_except only: + refs: + - merge_requests changes: - docker-compose/lofar-device-base.yml - docker-compose/lofar-device-base/* @@ -121,27 +132,30 @@ docker_build_image_lofar_device_base: docker_build_image_prometheus: extends: .base_docker_images_except only: + refs: + - merge_requests changes: - docker-compose/prometheus.yml - docker-compose/prometheus/* - except: - refs: - - tags - - master script: # Do not remove 'bash' or statement will be ignored by primitive docker shell - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh prometheus $tag docker_build_image_itango: extends: .base_docker_images_except only: + refs: + - merge_requests changes: - docker-compose/itango.yml + - docker-compose/itango/* script: # Do not remove 'bash' or statement will be ignored by primitive docker shell - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh itango $tag docker_build_image_grafana: extends: .base_docker_images_except only: + refs: + - merge_requests changes: - docker-compose/grafana.yml - docker-compose/grafana/* @@ -151,6 +165,8 @@ docker_build_image_grafana: docker_build_image_jupyter: extends: .base_docker_images_except only: + refs: + - merge_requests changes: - docker-compose/jupyter.yml - docker-compose/jupyter/* @@ -160,6 +176,8 @@ docker_build_image_jupyter: docker_build_image_apsct_sim: extends: .base_docker_images_except only: + refs: + - merge_requests changes: - docker-compose/aspct-sim.yml - docker-compose/pypcc-sim-base/* @@ -169,6 +187,8 @@ docker_build_image_apsct_sim: docker_build_image_apspu_sim: extends: .base_docker_images_except only: + refs: + - merge_requests changes: - docker-compose/apspu-sim.yml - docker-compose/pypcc-sim-base/* @@ -178,6 +198,8 @@ docker_build_image_apspu_sim: docker_build_image_recv_sim: extends: .base_docker_images_except only: + refs: + - merge_requests changes: - docker-compose/recv-sim.yml - docker-compose/pypcc-sim-base/* @@ -187,6 +209,8 @@ docker_build_image_recv_sim: docker_build_image_sdptr_sim: extends: .base_docker_images_except only: + refs: + - merge_requests changes: - docker-compose/sdptr-sim.yml - docker-compose/sdptr-sim/* @@ -196,6 +220,8 @@ docker_build_image_sdptr_sim: docker_build_image_unb2_sim: extends: .base_docker_images_except only: + refs: + - merge_requests changes: - docker-compose/unb2-sim.yml - docker-compose/pypcc-sim-base/* @@ -205,6 +231,8 @@ docker_build_image_unb2_sim: docker_build_image_device_apsct: extends: .base_docker_images_except only: + refs: + - merge_requests changes: - docker-compose/device-aspct.yml - docker-compose/lofar-device-base/* @@ -214,6 +242,8 @@ docker_build_image_device_apsct: docker_build_image_device_apspu: extends: .base_docker_images_except only: + refs: + - merge_requests changes: - docker-compose/device-apspu.yml - docker-compose/lofar-device-base/* @@ -223,6 +253,8 @@ docker_build_image_device_apspu: docker_build_image_device_boot: extends: .base_docker_images_except only: + refs: + - merge_requests changes: - docker-compose/device-boot.yml - docker-compose/lofar-device-base/* @@ -232,6 +264,8 @@ docker_build_image_device_boot: docker_build_image_device_docker: extends: .base_docker_images_except only: + refs: + - merge_requests changes: - docker-compose/device-docker.yml - docker-compose/lofar-device-base/* @@ -241,6 +275,8 @@ docker_build_image_device_docker: docker_build_image_device_ovservation_control: extends: .base_docker_images_except only: + refs: + - merge_requests changes: - docker-compose/device-observation_control.yml - docker-compose/lofar-device-base/* @@ -250,6 +286,8 @@ docker_build_image_device_ovservation_control: docker_build_image_device_recv: extends: .base_docker_images_except only: + refs: + - merge_requests changes: - docker-compose/device-recv.yml - docker-compose/lofar-device-base/* @@ -259,6 +297,8 @@ docker_build_image_device_recv: docker_build_image_device_sdp: extends: .base_docker_images_except only: + refs: + - merge_requests changes: - docker-compose/device-sdp.yml - docker-compose/lofar-device-base/* @@ -268,6 +308,8 @@ docker_build_image_device_sdp: docker_build_image_device_sst: extends: .base_docker_images_except only: + refs: + - merge_requests changes: - docker-compose/device-sst.yml - docker-compose/lofar-device-base/* @@ -277,6 +319,8 @@ docker_build_image_device_sst: docker_build_image_device_unb2: extends: .base_docker_images_except only: + refs: + - merge_requests changes: - docker-compose/device-unb2.yml - docker-compose/lofar-device-base/* @@ -286,6 +330,8 @@ docker_build_image_device_unb2: docker_build_image_device_xst: extends: .base_docker_images_except only: + refs: + - merge_requests changes: - docker-compose/device-xst.yml - docker-compose/lofar-device-base/* diff --git a/docker-compose/itango/Dockerfile b/docker-compose/itango/Dockerfile index 044ae985c4d1d45c26a6c00f6c3a455b93187cff..448afacfe687f6b75b163c159a779ff14ec07ee3 100644 --- a/docker-compose/itango/Dockerfile +++ b/docker-compose/itango/Dockerfile @@ -4,7 +4,8 @@ FROM ${SOURCE_IMAGE} RUN sudo apt-get -y update && \ sudo apt-get -y upgrade && \ sudo apt-get -y install apt-file apt-transport-https apt-utils aptitude && \ - sudo aptitude -y install htop iftop iproute2 mc most net-tools tcpdump telnet tmux traceroute vim xterm git && \ + sudo apt-get -y install htop iftop iproute2 mc most net-tools tcpdump && \ + sudo apt-get -y install telnet tmux traceroute vim xterm git && \ sudo aptitude clean && \ sudo aptitude autoclean diff --git a/tangostationcontrol/tangostationcontrol/devices/abstract_device.py b/tangostationcontrol/tangostationcontrol/devices/abstract_device.py deleted file mode 100644 index c3c6aea23d0af39f80dd733efb4c911847d93635..0000000000000000000000000000000000000000 --- a/tangostationcontrol/tangostationcontrol/devices/abstract_device.py +++ /dev/null @@ -1,32 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of the XXX project -# -# -# -# Distributed under the terms of the APACHE license. -# See LICENSE.txt for more info. - -"""Abstract Device Meta for LOFAR2.0 - -""" - -import logging - -from tango.server import DeviceMeta - -logger = logging.getLogger() - - -# TODO(Corne): Fix combining metaclasses by iterating over their variables and -# methods. https://support.astron.nl/jira/browse/L2SS-551 -# class AbstractDeviceMetas(DeviceMeta, ABCMeta): -class AbstractDeviceMetas(DeviceMeta): - """Collects meta classes to allow lofar_device to be both a Device and an ABC. """ - - def __new__(mcs, name, bases, namespace, **kwargs): - cls = DeviceMeta.__new__(mcs, name, bases, namespace, **kwargs) - # temp_cls = ABCMeta.__new__(mcs, name, bases, namespace) - # setattr(cls, '__abstractmethods__', temp_cls.__abstractmethods__) - # setattr(cls, '_abc_impl', temp_cls._abc_impl) - return cls diff --git a/tangostationcontrol/tangostationcontrol/devices/lofar_device.py b/tangostationcontrol/tangostationcontrol/devices/lofar_device.py index 690f65104b1ada278b7c09447df47de1817941ce..6fed370c4e714813b1a7ce6768cc93209cff1c5e 100644 --- a/tangostationcontrol/tangostationcontrol/devices/lofar_device.py +++ b/tangostationcontrol/tangostationcontrol/devices/lofar_device.py @@ -11,10 +11,8 @@ """ -from abc import abstractmethod - # PyTango imports -from tango.server import Device, command, attribute +from tango.server import attribute, command, Device, DeviceMeta from tango import AttrWriteType, DevState, DebugIt, Attribute, DeviceProxy import time import math @@ -23,7 +21,6 @@ import math from tangostationcontrol.clients.attribute_wrapper import attribute_wrapper from tangostationcontrol.common.lofar_logging import log_exceptions from tangostationcontrol.common.lofar_version import get_version -from tangostationcontrol.devices.abstract_device import AbstractDeviceMetas from tangostationcontrol.devices.device_decorators import only_in_states, fault_on_error @@ -32,7 +29,8 @@ __all__ = ["lofar_device"] import logging logger = logging.getLogger() -class lofar_device(Device, metaclass=AbstractDeviceMetas): + +class lofar_device(Device, metaclass=DeviceMeta): """ **Properties:** @@ -217,14 +215,12 @@ class lofar_device(Device, metaclass=AbstractDeviceMetas): def configure_for_fault(self): pass - @abstractmethod def configure_for_off(self): pass def configure_for_on(self): pass - @abstractmethod def configure_for_initialise(self): pass diff --git a/tangostationcontrol/tangostationcontrol/devices/opcua_device.py b/tangostationcontrol/tangostationcontrol/devices/opcua_device.py index 6da1fdf21d635e4f32cf848cb4c8c81287b896d6..eb2e508c35cf6923fb9d121f729465a4eca621e3 100644 --- a/tangostationcontrol/tangostationcontrol/devices/opcua_device.py +++ b/tangostationcontrol/tangostationcontrol/devices/opcua_device.py @@ -17,14 +17,15 @@ import numpy import asyncio # Additional import -from tangostationcontrol.common.lofar_logging import log_exceptions +from tangostationcontrol.common.lofar_logging import log_exceptions from tangostationcontrol.clients.opcua_client import OPCUAConnection from tangostationcontrol.devices.lofar_device import lofar_device import logging logger = logging.getLogger() -__all__ = ["opcua_device", "main"] +__all__ = ["opcua_device"] + class opcua_device(lofar_device): """ diff --git a/tangostationcontrol/tangostationcontrol/devices/sdp/statistics.py b/tangostationcontrol/tangostationcontrol/devices/sdp/statistics.py index ae8b663dc61b4eafa17a73acaa4efc548c3d4d95..21286acd44b5a83a8ac3fb904f55b2024bbf03fe 100644 --- a/tangostationcontrol/tangostationcontrol/devices/sdp/statistics.py +++ b/tangostationcontrol/tangostationcontrol/devices/sdp/statistics.py @@ -11,8 +11,6 @@ """ -from abc import abstractmethod - # PyTango imports from tango.server import device_property @@ -32,15 +30,13 @@ import numpy __all__ = ["Statistics"] -# TODO(Corne): Make Statistics use ABCMeta again when L2SS-551 is fixed -# https://support.astron.nl/jira/browse/L2SS-551 + class Statistics(opcua_device): # In derived classes, set this to a subclass of StatisticsCollector @property - @abstractmethod def STATISTICS_COLLECTOR_CLASS(self): - pass + raise NotImplementedError # ----------------- # Device Properties diff --git a/tangostationcontrol/tangostationcontrol/devices/sdp/xst.py b/tangostationcontrol/tangostationcontrol/devices/sdp/xst.py index adef57e3144e297b273ff5b33e77c9fde513ee82..c0b6f7a98894b7c7d660828231935b63c5b0f5da 100644 --- a/tangostationcontrol/tangostationcontrol/devices/sdp/xst.py +++ b/tangostationcontrol/tangostationcontrol/devices/sdp/xst.py @@ -28,6 +28,7 @@ import numpy __all__ = ["XST", "main"] + class XST(Statistics): STATISTICS_COLLECTOR_CLASS = XSTCollector diff --git a/tangostationcontrol/tangostationcontrol/test/devices/test_abstract_device.py b/tangostationcontrol/tangostationcontrol/test/devices/test_abstract_device.py deleted file mode 100644 index 742fa9e405d112444239ae30284e51b8452416e8..0000000000000000000000000000000000000000 --- a/tangostationcontrol/tangostationcontrol/test/devices/test_abstract_device.py +++ /dev/null @@ -1,91 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of the LOFAR 2.0 Station Software -# -# -# -# Distributed under the terms of the APACHE license. -# See LICENSE.txt for more info. - -import abc -from unittest import mock - -from tango import server -from tango.server import attribute - -from tangostationcontrol.devices.abstract_device import AbstractDeviceMetas - -from tangostationcontrol.test import base - - -class TestAbstractDevice(base.TestCase): - - class AbstractExample(object, metaclass=abc.ABCMeta): - """A pure abc.ABCMeta metaclass with an abstract method - - This is an abstract class that inherits object with the abc.ABCMeta as - metaclass - """ - - @abc.abstractmethod - def example_method(self): - raise NotImplementedError - - class TestHardwareDevice(server.Device, metaclass=AbstractDeviceMetas): - """This is your overarching abstract class with a combined metaclass - - Device is an object with DeviceMeta as metaclass - We use HardwareDeviceMetas as metaclass - - Our metaclass contract is now fulfilled. - """ - - @attribute(dtype=float) - def call_example_method(self): - return self.example_method() - - @abc.abstractmethod - def example_method(self): - raise NotImplementedError - - class ConcreteHardwareDevice(TestHardwareDevice): - - def example_method(self): - return 12 - - def setUp(self): - super(TestAbstractDevice, self).setUp() - - # def test_instance_tango(self): - # - # try: - # with DeviceTestContext(self.TestHardwareDevice, process=True) as proxy: - # # Calling this method raises the NotImplementedError exception - # proxy.call_example_method() - # except Exception as e: - # self.assertIsInstance(e, RuntimeError) - # - # with DeviceTestContext(self.ConcreteHardwareDevice, process=True) as proxy: - # self.assertEqual(12, proxy.call_example_method) - - @mock.patch.object(server, 'get_worker') - @mock.patch.object(server, 'LatestDeviceImpl') - def test_instance_error(self, m_worker, m_implement): - # Creating this object should raise a type error but it does not - # combining metaclasses in this way does not have the desired result. - # This is a known limitation of this approach - m_device = self.TestHardwareDevice(mock.Mock(), mock.Mock()) - - # Raising the NotImplementedError works as expected, however. - self.assertRaises(NotImplementedError, m_device.example_method) - - # Creating this object of a class that has a pure metaclass does raise - # the expected error. - self.assertRaises(TypeError, self.AbstractExample) - - @mock.patch.object(server, 'get_worker') - @mock.patch.object(server, 'LatestDeviceImpl') - def test_isinstance(self, m_worker, m_implement): - m_device = self.TestHardwareDevice(mock.Mock(), mock.Mock()) - - self.assertFalse(isinstance(m_device, AbstractDeviceMetas)) diff --git a/tangostationcontrol/tangostationcontrol/test/devices/test_statistics_device.py b/tangostationcontrol/tangostationcontrol/test/devices/test_statistics_device.py new file mode 100644 index 0000000000000000000000000000000000000000..29c2462fe95047bc6364acd0d4c6a03c945801f2 --- /dev/null +++ b/tangostationcontrol/tangostationcontrol/test/devices/test_statistics_device.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +# +# This file is part of the LOFAR 2.0 Station Software +# +# +# +# Distributed under the terms of the APACHE license. +# See LICENSE.txt for more info. + +import mock + +from tango import server +from tango.test_context import DeviceTestContext + +from tangostationcontrol.test import base + + +class TestStatisticsDevice(base.TestCase): + + def setUp(self): + super(TestStatisticsDevice, self).setUp() + + def test_python_bug(self): + """Python is bugged and super() calls in ctypes fail + + This is the same type of issues that prevents combining metaclasses + DeviceMeta and ABCMeta + + https://github.com/waveform80/picamera/issues/355 + https://bugs.python.org/issue29270 + """ + import ctypes as ct + + try: + class TestSubclass(ct.c_uint32): + def __repr__(self): + return super().__repr__() + except Exception as e: + self.assertIsInstance(e, TypeError) + + @mock.patch.object(server, 'get_worker') + def test_instance_statistics(self, m_worker): + """Test that we can import and create a statistics device + + The import of Statistics fails when both ABCMeta and DeviceMeta are + combined. Leave this test to prevent regressions when reattempting our + combined metaclass. + """ + + from tangostationcontrol.devices.sdp.statistics import Statistics + + with DeviceTestContext(Statistics, process=True) as proxy: + self.assertTrue(proxy.ping() > 0)