diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c642888588327b2369367a21720aa65f123704fc..b832d175e1cd6f44fff0694b9217b62c9c93a61b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -476,7 +476,7 @@ integration_test_docker: # Do not remove 'bash' or statement will be ignored by primitive docker shell - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh pull $tag # Do not remove 'bash' or statement will be ignored by primitive docker shell - - bash $CI_PROJECT_DIR/sbin/run_integration_test.sh + - bash -e $CI_PROJECT_DIR/sbin/run_integration_test.sh after_script: # Collect output of all containers - | diff --git a/CDB/stations/dummy_positions_ConfigDb.json b/CDB/stations/dummy_positions_ConfigDb.json index 926145a0c6a2f652c637ae4ab409b57b07c655d7..460f49d2e5c9af9d968c11be32f11819a4ab943e 100644 --- a/CDB/stations/dummy_positions_ConfigDb.json +++ b/CDB/stations/dummy_positions_ConfigDb.json @@ -5,6 +5,9 @@ "RECV": { "STAT/RECV/1": { "properties": { + "Antenna_Field_Reference_ETRS": [ + "3826896.631", "460979.131", "5064657.943" + ], "HBAT_reference_ETRS": [ "3826886.142", "460980.772", "5064665.668", "3826887.237", "460985.643", "5064664.406", diff --git a/docker-compose/Makefile b/docker-compose/Makefile index 1f859907bd6eb9038c569e24fd2a40a339145ca5..d329b9bbb5bd6cbaff6cd0d06f9bf1f93ed47ae5 100644 --- a/docker-compose/Makefile +++ b/docker-compose/Makefile @@ -114,7 +114,9 @@ ifeq ($(NETWORK_MODE),host) MYSQL_HOST := $(shell hostname):3306 else ifeq ($(TANGO_HOST),) - TANGO_HOST := $(CONTAINER_NAME_PREFIX)databaseds:10000 + # Use FQDN for TANGO_HOST to avoid confusion in the archiver, which also + # adds the domain. + TANGO_HOST := $(CONTAINER_NAME_PREFIX)databaseds.$(NETWORK_MODE):10000 else TANGO_HOST := $(TANGO_HOST) endif diff --git a/sbin/run_integration_test.sh b/sbin/run_integration_test.sh index acac6e21e1aeb0850d19aa420a4fe5c9a72970b9..aec2d1d4082c04460a52476ac506597c28aa45eb 100755 --- a/sbin/run_integration_test.sh +++ b/sbin/run_integration_test.sh @@ -60,6 +60,10 @@ make start archiver-timescale hdbppts-cm hdbppts-es # TODO(Corne Lukken): Use a nicer more reliable mechanism sleep 60 +# Give archiver-timescale time to start +# shellcheck disable=SC2016 +echo '/usr/local/bin/wait-for-it.sh archiver-timescale:5432 --strict --timeout=300 -- true' | make run dsconfig bash - + # Start the integration test cd "$LOFAR20_DIR/docker-compose" || exit 1 make up integration-test diff --git a/tangostationcontrol/tangostationcontrol/devices/sdp/beamlet.py b/tangostationcontrol/tangostationcontrol/devices/sdp/beamlet.py index e6db319a5b1838bf307e4f56df0d6922c1f17e1b..ae61d77bd51299d8dba44e77422d09d324d1ec53 100644 --- a/tangostationcontrol/tangostationcontrol/devices/sdp/beamlet.py +++ b/tangostationcontrol/tangostationcontrol/devices/sdp/beamlet.py @@ -8,10 +8,10 @@ """ # PyTango imports -from tango.server import device_property -from tango import AttrWriteType -# Additional import +from tango.server import device_property, command +from tango import AttrWriteType, DevVarFloatArray, DevVarULongArray +# Additional import from tangostationcontrol.common.entrypoint import entry from tangostationcontrol.clients.attribute_wrapper import attribute_wrapper from tangostationcontrol.devices.opcua_device import opcua_device @@ -140,9 +140,33 @@ class Beamlet(opcua_device): # Overloaded functions # -------- + # -------- + # internal functions + # -------- + def _calculate_bf_weights(self, phases: numpy.ndarray): + """ Helper function that converts a difference in phase (in radiants) + to a FPGA weight (in complex number) """ + + # Convert array values in complex numbers + unit = numpy.power(2,14) + real = numpy.array(unit * numpy.cos(phases), dtype=numpy.short) + imag = numpy.array(unit * numpy.sin(phases), dtype=numpy.short) + # join 16 bits of imaginary part (MSB) with 16 bits of real part (LSB) + bf_weights = numpy.array( numpy.frombuffer( b''.join([imag,real]), dtype=numpy.uint32 ) ) + + return bf_weights + # -------- # Commands # -------- + @command(dtype_in=DevVarFloatArray, dtype_out=DevVarULongArray) + def calculate_bf_weights(self, phases: numpy.ndarray): + """ converts a difference in phase (in radiants) to a FPGA weight (in complex number) """ + + # Calculate the FPGA weight array + bf_weights = self._calculate_bf_weights(phases) + + return bf_weights # ---------- # Run server diff --git a/tangostationcontrol/tangostationcontrol/devices/sdp/sdp.py b/tangostationcontrol/tangostationcontrol/devices/sdp/sdp.py index 44e89a6cb2c8f3c1c1ddba17a2b94e3e34236e7e..dd4179e47840488b1033d9612a4a5015f0ec2a49 100644 --- a/tangostationcontrol/tangostationcontrol/devices/sdp/sdp.py +++ b/tangostationcontrol/tangostationcontrol/devices/sdp/sdp.py @@ -167,7 +167,6 @@ class SDP(opcua_device): FPGA_signal_input_bsn_R = attribute_wrapper(comms_annotation=["FPGA_signal_input_bsn_R"], datatype=numpy.int64, dims=(N_pn,)) FPGA_signal_input_nof_blocks_R = attribute_wrapper(comms_annotation=["FPGA_signal_input_nof_blocks_R"], datatype=numpy.int32, dims=(N_pn,)) FPGA_signal_input_nof_samples_R = attribute_wrapper(comms_annotation=["FPGA_signal_input_nof_samples_R"], datatype=numpy.int32, dims=(N_pn,)) - FPGA_signal_input_nof_err_R = attribute_wrapper(comms_annotation=["FPGA_signal_input_nof_err_R"], datatype=numpy.int32, dims=(N_pn,)) FPGA_signal_input_samples_delay_R = attribute_wrapper(comms_annotation=["FPGA_signal_input_samples_delay_R"], datatype=numpy.uint32, dims=(S_pn, N_pn)) FPGA_signal_input_samples_delay_RW = attribute_wrapper(comms_annotation=["FPGA_signal_input_samples_delay_RW"], datatype=numpy.uint32, dims=(S_pn, N_pn), access=AttrWriteType.READ_WRITE) diff --git a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_lofar_device.py b/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_lofar_device.py similarity index 99% rename from tangostationcontrol/tangostationcontrol/integration_test/devices/test_lofar_device.py rename to tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_lofar_device.py index d5a935eb537a9031ee67b2252ba1b1cfd1eb6687..3a420bbd3c3f03b34d4664f39d631de154c69bd0 100644 --- a/tangostationcontrol/tangostationcontrol/integration_test/devices/test_lofar_device.py +++ b/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_lofar_device.py @@ -13,6 +13,7 @@ from tangostationcontrol.integration_test.device_proxy import TestDeviceProxy from tangostationcontrol.integration_test import base + class TestProxyAttributeAccess(base.IntegrationTestCase): """ Test whether DeviceProxy's can always access attributes immediately after turning them on. """ diff --git a/tangostationcontrol/tangostationcontrol/integration_test/toolkit/__init__.py b/tangostationcontrol/tangostationcontrol/integration_test/default/toolkit/__init__.py similarity index 100% rename from tangostationcontrol/tangostationcontrol/integration_test/toolkit/__init__.py rename to tangostationcontrol/tangostationcontrol/integration_test/default/toolkit/__init__.py diff --git a/tangostationcontrol/tangostationcontrol/integration_test/toolkit/test_archiver.py b/tangostationcontrol/tangostationcontrol/integration_test/default/toolkit/test_archiver.py similarity index 98% rename from tangostationcontrol/tangostationcontrol/integration_test/toolkit/test_archiver.py rename to tangostationcontrol/tangostationcontrol/integration_test/default/toolkit/test_archiver.py index a6c361435cd953616bbd5cdf9514b2550a9e824a..1ead9d257645eeb3934b57052cd3cbd89bce39d6 100644 --- a/tangostationcontrol/tangostationcontrol/integration_test/toolkit/test_archiver.py +++ b/tangostationcontrol/tangostationcontrol/integration_test/default/toolkit/test_archiver.py @@ -16,6 +16,7 @@ from tangostationcontrol.integration_test.device_proxy import TestDeviceProxy import time from datetime import datetime + class TestArchiver(BaseIntegrationTestCase): def setUp(self): @@ -54,10 +55,12 @@ class TestArchiver(BaseIntegrationTestCase): recv_proxy.set_defaults() recv_proxy.on() self.assertEqual(DevState.ON, recv_proxy.state()) - + + """ # Safety operation that prevents event subscriber to go in Fault state self.archiver.remove_attributes_in_error() time.sleep(3) + """ attr_fullname = 'stat/recv/1/recvtr_translator_busy_r' # boolean self.archiver.add_attribute_to_archiver(attr_fullname, polling_period=1000, archive_event_period=3000) time.sleep(3) @@ -75,11 +78,13 @@ class TestArchiver(BaseIntegrationTestCase): self.assertEqual(datetime,type(item.data_time)) # column datetime self.assertEqual(bool,type(item.value)) # column value + """ # Remove attribute at the end of the test self.archiver.remove_attribute_from_archiver(attr_fullname) time.sleep(3) # Test if the attribute has been correctly removed self.assertFalse(self.archiver.is_attribute_archived(attribute_fqdn(attr_fullname))) + """ recv_proxy.off() def test_archive_array_attribute(self): @@ -95,9 +100,11 @@ class TestArchiver(BaseIntegrationTestCase): sdp_proxy.on() self.assertEqual(DevState.ON, sdp_proxy.state()) + """ # Safety operation that prevents event subscriber to go in Fault state self.archiver.remove_attributes_in_error() time.sleep(3) + """ attr_fullname = 'stat/sdp/1/fpga_temp_r' # double self.archiver.add_attribute_to_archiver(attr_fullname, polling_period=1000, archive_event_period=3000) time.sleep(3) @@ -116,11 +123,13 @@ class TestArchiver(BaseIntegrationTestCase): self.assertEqual(int,type(item.x)) # column index self.assertEqual(float,type(item.value)) # column value + """ # Remove attribute at the end of the test self.archiver.remove_attribute_from_archiver(attr_fullname) time.sleep(3) # Test if the attribute has been correctly removed self.assertFalse(self.archiver.is_attribute_archived(attribute_fqdn(attr_fullname))) + """ sdp_proxy.off() def test_get_maximum_device_load(self): diff --git a/tangostationcontrol/tangostationcontrol/integration_test/toolkit/test_archiver_util.py b/tangostationcontrol/tangostationcontrol/integration_test/default/toolkit/test_archiver_util.py similarity index 99% rename from tangostationcontrol/tangostationcontrol/integration_test/toolkit/test_archiver_util.py rename to tangostationcontrol/tangostationcontrol/integration_test/default/toolkit/test_archiver_util.py index ad4595951d43c5cacdf26459e9cb982c1e32e142..952e665674d2bcb79e8a393a33c62a6144ec9125 100644 --- a/tangostationcontrol/tangostationcontrol/integration_test/toolkit/test_archiver_util.py +++ b/tangostationcontrol/tangostationcontrol/integration_test/default/toolkit/test_archiver_util.py @@ -14,6 +14,7 @@ from tango import DevState import json import pkg_resources + class TestArchiverUtil(BaseIntegrationTestCase): def setUp(self): diff --git a/tangostationcontrol/tangostationcontrol/test/toolkit/test_archiver_util.py b/tangostationcontrol/tangostationcontrol/test/toolkit/test_archiver_util.py index 4265c277554adb9d4e92c8491f392bffaddb3084..842aff8367ab78a30ba55eb31bd48fcb29193d8e 100644 --- a/tangostationcontrol/tangostationcontrol/test/toolkit/test_archiver_util.py +++ b/tangostationcontrol/tangostationcontrol/test/toolkit/test_archiver_util.py @@ -22,12 +22,12 @@ class TestArchiverUtil(base.TestCase): def test_device_fqdn(self): """Test if a device name is correctly converted in a Tango FQDN""" - self.assertEqual(f"tango://databaseds:10000/{self.device_name}".lower(), device_fqdn(self.device_name)) + self.assertEqual(f"tango://databaseds:10000/{self.device_name}".lower(), device_fqdn(self.device_name, "databaseds:10000")) def test_attribute_fqdn(self): """Test if an attribute name is correctly converted in a Tango FQDN""" self.assertEqual(f"tango://databaseds:10000/{self.device_name}/{self.attribute_name}".lower(), - attribute_fqdn(f"{self.device_name}/{self.attribute_name}")) + attribute_fqdn(f"{self.device_name}/{self.attribute_name}", "databaseds:10000")) self.assertRaises(ValueError, lambda: attribute_fqdn(self.attribute_name)) def test_split_tango_name(self): diff --git a/tangostationcontrol/tangostationcontrol/toolkit/archiver.py b/tangostationcontrol/tangostationcontrol/toolkit/archiver.py index 3b78af4ae47d05d8cbe143abc1b676ae4781d8c8..2e178083d2428d1b0706e283ded68e7ee0c5ea46 100644 --- a/tangostationcontrol/tangostationcontrol/toolkit/archiver.py +++ b/tangostationcontrol/tangostationcontrol/toolkit/archiver.py @@ -25,8 +25,8 @@ def warn_if_attribute_not_found(): try: return func(self, attribute_name, *args, **kwargs) except DevFailed as e: - if e.args[0].reason == 'Attribute not found' or 'NOT FOUND in signal list': - logger.warning(f"Attribute {attribute_name} not found!") + if e.args[0].reason in ['Attribute not found', 'BadSignalName']: + logger.warning(f"Attribute {attribute_name} not found: {e.args[0].desc}") else: raise @@ -59,7 +59,7 @@ class Archiver(): self.cm = DeviceProxy(cm_name) try: if self.cm.state() == DevState.FAULT: - raise Exception(f"Configuration Manager {cm_name} is in FAULT state") + raise Exception(f"Configuration Manager {cm_name} is in FAULT state: {self.cm.status()}") except Exception as e: raise Exception(f"Connection failed with Configuration Manager {cm_name}") from e self.es_list = [es_name for es_name in self.get_subscribers(from_db=False)] @@ -145,7 +145,8 @@ class Archiver(): dev_polling_time, dev_archive_abs_change, dev_archive_rel_change, dev_archive_period, dev_event_period, dev_strategy = get_global_env_parameters(config_dict, environment) # Attributes to be included in archiving stategy include_att_list = get_include_attribute_list(device, config_dict, environment) - self.remove_attributes_by_device(device, exclude=include_att_list) + # TODO Cleanup the subscriber + # self.remove_attributes_by_device(device, exclude=include_att_list) # Include attributes by custom configuration try: for att in include_att_list: @@ -281,6 +282,10 @@ class Archiver(): """ Stops the data archiving of the attribute passed as input, and remove it from the subscriber's list. """ + + # Removal of attributes leads to hdbpp-es freezing up, see https://github.com/tango-controls-hdbpp/hdbpp-es/issues/25 + raise NotImplementedError("Removing attributes is not supported yet") + attribute_name = attribute_fqdn(attribute_name) self.cm.AttributeStop(attribute_name) self.cm.AttributeRemove(attribute_name) @@ -341,6 +346,7 @@ class Archiver(): Starts the archiving of the attribute passed as input. The attribute must be already present in the subscriber's list """ + attribute_name = attribute_fqdn(attribute_name) self.cm.AttributeStart(attribute_name) @warn_if_attribute_not_found() @@ -349,6 +355,7 @@ class Archiver(): Stops the archiving of the attribute passed as input. The attribute must be already present in the subscriber's list """ + attribute_name = attribute_fqdn(attribute_name) self.cm.AttributeStop(attribute_name) def is_attribute_archived(self, attribute_name:str): diff --git a/tangostationcontrol/tangostationcontrol/toolkit/archiver_util.py b/tangostationcontrol/tangostationcontrol/toolkit/archiver_util.py index 0d5e8f4d992170016bae7405c204b785c8cb8573..c797c5267a517dbe00c566297b94cc176f9a9574 100644 --- a/tangostationcontrol/tangostationcontrol/toolkit/archiver_util.py +++ b/tangostationcontrol/tangostationcontrol/toolkit/archiver_util.py @@ -6,6 +6,7 @@ from tango import DeviceProxy, CmdArgType import re +import os """ A dictionary whose keys are the Tango datatypes mapping, and the values are the relative byte size @@ -21,6 +22,8 @@ DATATYPES_SIZE_DICT = {CmdArgType.DevBoolean:1, CmdArgType.DevShort:2, CmdArgTyp CmdArgType.DevULong64:8,CmdArgType.DevVarLong64Array:None,CmdArgType.DevVarULong64Array:None, CmdArgType.DevInt:4,CmdArgType.DevEncoded:None, CmdArgType.DevEnum:None, CmdArgType.DevPipeBlob:None} +TANGO_HOST = os.environ.get("TANGO_HOST", None) + def get_db_config(device_name:str) -> dict: """ Retrieve the DB credentials from the Tango properties of Configuration Manager or EventSubscribers @@ -46,7 +49,7 @@ def get_attribute_from_fqdn(attribute_name:str): return attribute_name -def device_fqdn(device_name:str, tango_host:str = 'databaseds:10000'): +def device_fqdn(device_name:str, tango_host:str = TANGO_HOST): """ For some operations Tango devices must be transformed from the form 'domain/family/name' to 'tango://db:port/domain/family/name' @@ -59,7 +62,7 @@ def device_fqdn(device_name:str, tango_host:str = 'databaseds:10000'): return f"tango://{tango_host}/{device_name}".lower() -def attribute_fqdn(attribute_name:str, tango_host:str = 'databaseds:10000'): +def attribute_fqdn(attribute_name:str, tango_host:str = TANGO_HOST): """ For some operations Tango devices must be transformed from the form 'domain/family/name/attribute' to 'tango://db:port/domain/family/name/attribute'