Merge branch 'L2SS-791-restore-attribute-removal' into 'master'

Resolve L2SS-791 "Restore attribute removal"

Closes L2SS-791

See merge request !339

tangostationcontrol/tangostationcontrol/integration_test/default/toolkit/test_archiver.py

@@ -82,13 +82,13 @@ class TestArchiver(BaseIntegrationTestCase):
         self.assertEqual(datetime,type(item.data_time))           # column datetime
         self.assertEqual(int,type(item.value))                    # column value
 
-        """
+        
         # Remove attribute at the end of the test
         self.archiver.remove_attribute_from_archiver(attr_fullname)
         time.sleep(3)
         # Test if the attribute has been correctly removed
         self.assertFalse(self.archiver.is_attribute_archived(attribute_fqdn(attr_fullname)))
-        """
+        
         recv_proxy.off()
     
     def test_archive_array_attribute(self):
@@ -104,11 +104,11 @@ class TestArchiver(BaseIntegrationTestCase):
         sdp_proxy.on()
         self.assertEqual(DevState.ON, sdp_proxy.state())
 
-        """
+        
         # Safety operation that prevents event subscriber to go in Fault state
         self.archiver.remove_attributes_in_error()
         time.sleep(3)
-        """
+        
         polling_period=1000
         archive_event_period=3000
         attr_fullname = 'stat/sdp/1/fpga_temp_r'  # double
@@ -129,13 +129,13 @@ class TestArchiver(BaseIntegrationTestCase):
         self.assertEqual(int,type(item.x))                                  # column index
         self.assertEqual(float,type(item.value))                            # column value
 
-        """
+        
         # Remove attribute at the end of the test
         self.archiver.remove_attribute_from_archiver(attr_fullname)
         time.sleep(3)
         # Test if the attribute has been correctly removed
         self.assertFalse(self.archiver.is_attribute_archived(attribute_fqdn(attr_fullname)))
-        """
+        
         sdp_proxy.off()
     
     def test_archive_image_boolean_attribute(self):
@@ -151,11 +151,11 @@ class TestArchiver(BaseIntegrationTestCase):
         recv_proxy.on()
         self.assertEqual(DevState.ON, recv_proxy.state())
 
-        """
+        
         # Safety operation that prevents event subscriber to go in Fault state
         self.archiver.remove_attributes_in_error()
         time.sleep(3)
-        """
+        
         polling_period=1000
         archive_event_period=5000
         attr_fullname = 'stat/recv/1/ant_mask_rw'  # boolean 3x32
@@ -178,13 +178,13 @@ class TestArchiver(BaseIntegrationTestCase):
         self.assertEqual(int,type(item.value))                              # column value (bool stored as int)
         self.assertLessEqual(item.value,1)                                  # column value (must be 0 or 1)
 
-        """
+        
         # Remove attribute at the end of the test
         self.archiver.remove_attribute_from_archiver(attr_fullname)
         time.sleep(3)
         # Test if the attribute has been correctly removed
         self.assertFalse(self.archiver.is_attribute_archived(attribute_fqdn(attr_fullname)))
-        """
+        
         recv_proxy.off()
     
     def test_get_maximum_device_load(self):

tangostationcontrol/tangostationcontrol/toolkit/archiver.py

@@ -25,13 +25,28 @@ def warn_if_attribute_not_found():
             try:
                 return func(self, attribute_name, *args, **kwargs)
             except DevFailed as e:
-                if e.args[0].reason in ['Attribute not found', 'BadSignalName']:
+                if e.args[0].reason in ['Attribute not found', 'BadSignalName', 'API_AttrNotFound']:
                     logger.warning(f"Attribute {attribute_name} not found: {e.args[0].desc}")
                 else:
                     raise
-
         return warn_wrapper
+    return inner
 
+def warn_if_device_not_connected():
+    """
+      Log a warning if an exception is thrown indicating access to an non-connected device
+    """
+    def inner(func):
+        @wraps(func)
+        def warn_wrapper(self, attribute_name, *args, **kwargs):
+            try:
+                return func(self, attribute_name, *args, **kwargs)
+            except DevFailed as e:
+                if 'API_CantConnectToDevice' in str(e):
+                    logger.warning(f"Attribute {attribute_name} not reachable: {e.args[0].desc}")
+                else:
+                    raise
+        return warn_wrapper
     return inner
 
 class Archiver():
@@ -150,21 +165,21 @@ class Archiver():
         # Retrieve global parameters
         dev_polling_time, dev_archive_abs_change, dev_archive_rel_change, dev_archive_period, dev_event_period, dev_strategy = get_global_env_parameters(config_dict, environment)
         # Attributes to be included in archiving stategy
-        include_att_list = get_include_attribute_list(device, config_dict, environment)
-        # TODO Cleanup the subscriber
-        # self.remove_attributes_by_device(device, exclude=include_att_list)
+        include_att_list = [f"{device}/{a}".lower() for a in get_include_attribute_list(device, config_dict, environment)]
+        # Cleanup the subscriber
+        self.remove_attributes_by_device(device, exclude=include_att_list)
         # Include attributes by custom configuration
         try:        
             for att in include_att_list:
                 # Retrieve specific attribute parameters from config file
                 archive_period, event_period, abs_change, rel_change = get_parameters_from_attribute(device,att,config_dict,environment)
-                att_fqname = attribute_fqdn(f"{device}/{att}")
+                att_fqname = attribute_fqdn(att)
                 # Add the attribute to the archiver setting either specific or global parameters
                 self.add_attribute_to_archiver(att_fqname, dev_polling_time, archive_period or dev_archive_period, dev_strategy,
                                                                 abs_change or dev_archive_abs_change, rel_change or dev_archive_rel_change)
         except DevFailed as e:
             if 'already subscribed' in str(e):
-                logger.warning(f"Multiple entries of Attribute {device}'/'{att} in config file")
+                logger.warning(f"Multiple entries of Attribute {att} in config file")
             else:
                 raise
     
@@ -177,12 +192,12 @@ class Archiver():
         prod_polling_time, prod_archive_abs_change, prod_archive_rel_change, prod_archive_period, prod_event_period, prod_strategy = get_global_env_parameters(config_dict, environment) 
         # TODO Cleanup the subscriber
         # self.remove_attributes_by_device(device)
-        attribute_list = DeviceProxy(device).get_attribute_list()
+        attribute_list = [f"{device}/{a}".lower() for a in DeviceProxy(device).get_attribute_list()]
         try: 
             # Add attributes in 'suffixes' and 'infixes' list which have different parameters
             for att in attribute_list:
                 archive_period, event_period, abs_change, rel_change = get_parameters_from_attribute(device,att,config_dict,environment)
-                att_fqname = attribute_fqdn(f"{device}/{att}")
+                att_fqname = attribute_fqdn(att)
                 self.add_attribute_to_archiver(att_fqname, prod_polling_time, archive_period or prod_archive_period, prod_strategy,
                                                                 abs_change or prod_archive_abs_change, rel_change or prod_archive_rel_change)
             exclude_att_list = get_exclude_attribute_list(device, config_dict)
@@ -193,11 +208,11 @@ class Archiver():
             # The following cycle is a security check in the special case that an attribute is in the
             # included list in DEV mode, and in the excluded list in PROD mode             
             for att in exclude_att_list:
-                att_fqname = attribute_fqdn(f"{device}/{att}")
+                att_fqname = attribute_fqdn(att)
                 self.remove_attribute_from_archiver(att_fqname)
         except DevFailed as e:
             if 'already subscribed' in str(e):
-                logger.warning(f"Multiple entries of Attribute {device}'/'{att} in config file")
+                logger.warning(f"Multiple entries of Attribute {att} in config file")
             else:
                 raise
 
@@ -222,6 +237,7 @@ class Archiver():
                 raise
 
     @warn_if_attribute_not_found()
+    @warn_if_device_not_connected()
     def add_attribute_to_archiver(self, attribute_name: str, polling_period: int, archive_event_period: int, strategy: str = 'RUN', 
                                         abs_change: int = 1, rel_change: int = None, es_name:str=None):
         """
@@ -284,36 +300,33 @@ class Archiver():
         """
         Stops the data archiving of the attribute passed as input, and remove it from the subscriber's list. 
         """
-
-        # Removal of attributes leads to hdbpp-es freezing up, see https://github.com/tango-controls-hdbpp/hdbpp-es/issues/25
-        raise NotImplementedError("Removing attributes is not supported yet")
-
         attribute_name = attribute_fqdn(attribute_name)
         self.cm.AttributeStop(attribute_name)
         self.cm.AttributeRemove(attribute_name)
         logger.warning(f"Attribute {attribute_name} removed!")
 
+    @warn_if_attribute_not_found()
     def remove_attributes_by_device(self, device_name:str, exclude:list = None):
         """
         Stops the data archiving of all the attributes of the selected device, and remove them from the
         subscriber's list
         """
-
         if not exclude:
             """ B006 Do not use mutable data structures for argument defaults.
             They are created during function definition time. All calls to the
             function reuse this one instance of that data structure,
             persisting changes between them"""
             exclude = []
-
-        attrs_list = filter_attribute_list(device_name, exclude)      
+        es_list = self.get_subscribers()
+        for es_name in es_list:
+            es = DeviceProxy(es_name)
+            archived_attrs = es.AttributeList or []
+            exclude_list = [attribute_fqdn(a.lower()) for a in exclude]
+            # Search the attributes in the EventSubscriber list from their device name
+            match = re.compile(f'.*{device_name}.*').match
+            attrs_list = [a.lower() for a in list(filter(match, archived_attrs)) if a.lower() not in exclude_list]
             for a in attrs_list:
-            try:
-                attr_fullname = attribute_fqdn(f"{device_name}/{a}")
-                if self.is_attribute_archived(attr_fullname):
-                    self.remove_attribute_from_archiver(attr_fullname)
-            except Exception as e:
-                raise Exception from e
+                self.remove_attribute_from_archiver(a)
 
     def remove_attributes_in_error(self, exclude:list = None, es_name:str=None):
         """