Merge branch 'L2SS-781-add-observation-device-to-archiver' into 'master'

Resolve L2SS-781-add-observation-device-to-archiver

Closes L2SS-781

See merge request !328

tangostationcontrol/tangostationcontrol/integration_test/default/toolkit/test_archiver.py

@@ -11,6 +11,7 @@ from tangostationcontrol.integration_test.base import BaseIntegrationTestCase
 from tangostationcontrol.toolkit.archiver import Archiver
 from tangostationcontrol.toolkit.retriever import RetrieverTimescale
 from tangostationcontrol.toolkit.archiver_util import attribute_fqdn
+from tangostationcontrol.toolkit.archiver_configurator import get_multimember_devices
 from tangostationcontrol.integration_test.device_proxy import TestDeviceProxy
 
 import time
@@ -214,3 +215,13 @@ class TestArchiver(BaseIntegrationTestCase):
             records = self.retriever.get_lofar_attribute(attr_fullname)
             wait+=1
         return records
+    
+    def test_get_multimember_devices(self):
+        """Test if multimember devices are correctly identified"""
+        config_dict = self.archiver.get_configuration()
+        self.assertIsNotNone(config_dict)
+        env_dict = config_dict['devices']
+        matched_devices_dict = get_multimember_devices(env_dict)
+        self.assertIsNotNone(matched_devices_dict)
+        # TODO add assertions when multi-member devices will be added to TangoDB
+        

tangostationcontrol/tangostationcontrol/toolkit/archiver.py

@@ -4,7 +4,7 @@ import logging
 
 from tango import DeviceProxy, AttributeProxy, DevState, DevFailed
 from tangostationcontrol.toolkit.archiver_util import get_db_config, device_fqdn, attribute_fqdn, get_size_from_datatype, filter_attribute_list
-from tangostationcontrol.toolkit.archiver_configurator import get_parameters_from_attribute, get_include_attribute_list, get_exclude_attribute_list, get_global_env_parameters
+from tangostationcontrol.toolkit.archiver_configurator import get_parameters_from_attribute, get_include_attribute_list, get_exclude_attribute_list, get_global_env_parameters, get_multimember_devices
 
 import time
 import re
@@ -119,13 +119,19 @@ class Archiver():
         self.prod_polling_time, self.prod_archive_abs_change, self.prod_archive_rel_change, self.prod_archive_period, self.prod_event_period, self.prod_strategy = get_global_env_parameters(config_dict, "production")
         # Set devices archiving
         env_dict = config_dict['devices']
-        for device in env_dict:
+        # Check if device has more than one member (domain/family/*)
+        multimember_devices_dict = get_multimember_devices(env_dict)
+        # Merge the two configuration dictionaries
+        extended_env_dict = {**env_dict, **multimember_devices_dict}
+        extended_config_dict = config_dict.copy()   # Copy to preserve original dict
+        extended_config_dict['devices'] = extended_env_dict
+        for device in extended_env_dict:
             try:
-                dev_env = str(env_dict[device]['environment']) # Get device environment
+                dev_env = str(extended_env_dict[device]['environment']) # Get device environment
                 if dev_env == 'development':    # DEV environment -> all attributes are excluded by default
-                    self.configure_for_development(config_dict, device)             
+                    self.configure_for_development(extended_config_dict, device)             
                 elif dev_env == 'production':   # PROD environment -> all attributes are included by default
-                    self.configure_for_production(config_dict, device)
+                    self.configure_for_production(extended_config_dict, device)
             except Exception as e:
                 if 'API_DeviceNotExported' in str(e):   # ignore if device is offline
                     logger.warning(f"Device {device} offline")

tangostationcontrol/tangostationcontrol/toolkit/archiver_config/lofar2.json

@@ -41,6 +41,11 @@
             ],
             "include": []
         },
+        "STAT/Observation/*": {
+            "environment": "development",
+            "exclude": [],
+            "include": []
+        },
         "STAT/RECV/1": {
             "environment": "development",
             "exclude": [],

tangostationcontrol/tangostationcontrol/toolkit/archiver_configurator.py

@@ -7,6 +7,8 @@ Functions related to the managing of the archiver configuration JSON file
 """
 import logging
 import re
+from tango import Database
+
 from tangostationcontrol.toolkit.archiver_util import get_attributes_from_suffix, retrieve_attributes_from_wildcards
 
 logger = logging.getLogger()
@@ -81,3 +83,22 @@ def get_global_env_parameters(config_dict:dict, environment:str):
     event_period = int(var_dict[environment]['event_period'])
     strategy = var_dict[environment]['strategy']
     return polling_time, archive_abs_change, archive_rel_change, archive_period, event_period, strategy
+
+def get_multimember_devices(env_dict:dict):
+    """Given a regular expression, return multi-member device configuration if they are stored in TangoDB"""
+    # Get a Tango DB reference
+    tangodb = Database()
+    # Scan configuration dictionary for possible multi-member devices
+    matched_devices_dict = {}
+    for device in env_dict:
+        # Search for asterisk in device names 
+        if re.match('.*/.*/\*', device):
+            # Return a list of string with the members of the matched device name (f.e. ['1','2'])
+            members = tangodb.get_device_member(device)         
+            # Add to matched devices list the device name in the form 'domain/family/member'
+            retrieved_devices = [f"{device[:-1]}{m}" for m in members]
+            # Append device-names and relative configuration to dictionary
+            for d in retrieved_devices:
+                matched_devices_dict[d] = env_dict[device]
+    return matched_devices_dict
+