Merge branch 'L2SS-722' into 'master'

Resolve L2SS-722: Fix the integration tests

Closes L2SS-722

See merge request !299

.gitlab-ci.yml

@@ -476,7 +476,7 @@ integration_test_docker:
 #    Do not remove 'bash' or statement will be ignored by primitive docker shell
     - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh pull $tag
 #    Do not remove 'bash' or statement will be ignored by primitive docker shell
-    - bash $CI_PROJECT_DIR/sbin/run_integration_test.sh
+    - bash -e $CI_PROJECT_DIR/sbin/run_integration_test.sh
   after_script:
 #    Collect output of all containers
     - |

sbin/run_integration_test.sh

@@ -60,6 +60,10 @@ make start archiver-timescale hdbppts-cm hdbppts-es
 # TODO(Corne Lukken): Use a nicer more reliable mechanism
 sleep 60
 
+# Give archiver-timescale time to start
+# shellcheck disable=SC2016
+echo '/usr/local/bin/wait-for-it.sh archiver-timescale:5432 --strict --timeout=300 -- true' | make run dsconfig bash -
+
 # Start the integration test
 cd "$LOFAR20_DIR/docker-compose" || exit 1
 make up integration-test

tangostationcontrol/tangostationcontrol/integration_test/default/toolkit/test_archiver.py

@@ -55,10 +55,12 @@ class TestArchiver(BaseIntegrationTestCase):
         recv_proxy.set_defaults()
         recv_proxy.on()
         self.assertEqual(DevState.ON, recv_proxy.state())
-        
+
+        """
         # Safety operation that prevents event subscriber to go in Fault state
         self.archiver.remove_attributes_in_error()
         time.sleep(3)
+        """
         attr_fullname = 'stat/recv/1/recvtr_translator_busy_r'  # boolean
         self.archiver.add_attribute_to_archiver(attr_fullname, polling_period=1000, archive_event_period=3000)
         time.sleep(3)
@@ -76,11 +78,13 @@ class TestArchiver(BaseIntegrationTestCase):
         self.assertEqual(datetime,type(item.data_time))           # column datetime
         self.assertEqual(bool,type(item.value))                   # column value
 
+        """
         # Remove attribute at the end of the test
         self.archiver.remove_attribute_from_archiver(attr_fullname)
         time.sleep(3)
         # Test if the attribute has been correctly removed
         self.assertFalse(self.archiver.is_attribute_archived(attribute_fqdn(attr_fullname)))
+        """
         recv_proxy.off()
     
     def test_archive_array_attribute(self):
@@ -96,9 +100,11 @@ class TestArchiver(BaseIntegrationTestCase):
         sdp_proxy.on()
         self.assertEqual(DevState.ON, sdp_proxy.state())
 
+        """
         # Safety operation that prevents event subscriber to go in Fault state
         self.archiver.remove_attributes_in_error()
         time.sleep(3)
+        """
         attr_fullname = 'stat/sdp/1/fpga_temp_r'  # double
         self.archiver.add_attribute_to_archiver(attr_fullname, polling_period=1000, archive_event_period=3000)
         time.sleep(3)
@@ -117,11 +123,13 @@ class TestArchiver(BaseIntegrationTestCase):
         self.assertEqual(int,type(item.x))                                  # column index
         self.assertEqual(float,type(item.value))                            # column value
 
+        """
         # Remove attribute at the end of the test
         self.archiver.remove_attribute_from_archiver(attr_fullname)
         time.sleep(3)
         # Test if the attribute has been correctly removed
         self.assertFalse(self.archiver.is_attribute_archived(attribute_fqdn(attr_fullname)))
+        """
         sdp_proxy.off()
     
     def test_get_maximum_device_load(self):

tangostationcontrol/tangostationcontrol/toolkit/archiver.py

@@ -25,8 +25,8 @@ def warn_if_attribute_not_found():
             try:
                 return func(self, attribute_name, *args, **kwargs)
             except DevFailed as e:
-                if e.args[0].reason == 'Attribute not found' or 'NOT FOUND in signal list':
-                    logger.warning(f"Attribute {attribute_name} not found!")
+                if e.args[0].reason in ['Attribute not found', 'BadSignalName']:
+                    logger.warning(f"Attribute {attribute_name} not found: {e.args[0].desc}")
                 else:
                     raise
 
@@ -59,7 +59,7 @@ class Archiver():
         self.cm = DeviceProxy(cm_name)
         try: 
             if self.cm.state() == DevState.FAULT:
-                raise Exception(f"Configuration Manager {cm_name} is in FAULT state")
+                raise Exception(f"Configuration Manager {cm_name} is in FAULT state: {self.cm.status()}")
         except Exception as e:
             raise Exception(f"Connection failed with Configuration Manager {cm_name}") from e
         self.es_list = [es_name for es_name in self.get_subscribers(from_db=False)]
@@ -145,7 +145,8 @@ class Archiver():
         dev_polling_time, dev_archive_abs_change, dev_archive_rel_change, dev_archive_period, dev_event_period, dev_strategy = get_global_env_parameters(config_dict, environment)
         # Attributes to be included in archiving stategy
         include_att_list = get_include_attribute_list(device, config_dict, environment)
-        self.remove_attributes_by_device(device, exclude=include_att_list)
+        # TODO Cleanup the subscriber
+        # self.remove_attributes_by_device(device, exclude=include_att_list)
         # Include attributes by custom configuration
         try:        
             for att in include_att_list:
@@ -281,6 +282,10 @@ class Archiver():
         """
         Stops the data archiving of the attribute passed as input, and remove it from the subscriber's list. 
         """
+
+        # Removal of attributes leads to hdbpp-es freezing up, see https://github.com/tango-controls-hdbpp/hdbpp-es/issues/25
+        raise NotImplementedError("Removing attributes is not supported yet")
+
         attribute_name = attribute_fqdn(attribute_name)
         self.cm.AttributeStop(attribute_name)
         self.cm.AttributeRemove(attribute_name)
@@ -341,6 +346,7 @@ class Archiver():
         Starts the archiving of the attribute passed as input.
         The attribute must be already present in the subscriber's list
         """
+        attribute_name = attribute_fqdn(attribute_name)
         self.cm.AttributeStart(attribute_name)
     
     @warn_if_attribute_not_found()
@@ -349,6 +355,7 @@ class Archiver():
         Stops the archiving of the attribute passed as input.
         The attribute must be already present in the subscriber's list
         """
+        attribute_name = attribute_fqdn(attribute_name)
         self.cm.AttributeStop(attribute_name)
     
     def is_attribute_archived(self, attribute_name:str):