Merge branch 'L2SS-362-Switch-Env-Mode' into 'master'

Resolve L2SS-362 "Switch env mode"

Closes L2SS-362

See merge request !133

devices/common/lofar_environment.py

-#
-#   Change manually the method to switch between modes
-#
-
-def isProduction():
-    return False

devices/toolkit/README.md

@@ -10,7 +10,8 @@ The main components (and the relative Docker containers) are:
 - (Optional) HDB++ Viewer (container: hdbpp-viewer): Standalone JAVA application designed to monitor signals coming from database
 
 ## Archiver creation
-When an Archiver object is created, we can define three of its properties:
+When an Archiver object is created, we can define four of its properties:
+- the Selector configuration file-name, a JSON file in which environment properties are defined
 - the ConfigurationManager name (Tango namespace)
 - at least one EventSubscriber name (Tango namespace)
 - the default context archiving for the subscribers. This means that a default archiving strategy will be applied to
@@ -21,6 +22,17 @@ Archiving strategies are ['ALWAYS','RUN','SHUTDOWN','SERVICE']
 - SHUTDOWN:stored during shutdown
 - SERVICE:stored during maintenance activities
 
+## Select environment configuration
+The Selector object creates a dictionary from a JSON configuration file (if not defined by user, a default lofar2.json is retrieved)
+in order to allow a custom starting configuration of the archiving procedure.
+In the JSON file, for each Tango device, three variables are defined:
+- Environment, which defines the general behaviour of archiving framework, in particular:
+    - "Development" -> none of the attributes are archived by default
+    - "Production" -> all the attributes are archived by default
+- Include, which defines a list of the attributes that must be added to archiving (to be used in "Development" mode)
+- Exclude, which defines a list of the attributes that must be removed from the archiving (to be used in "Production" mode)
+The advantages of using such a configuration selection is that every user can load a custom configuration following its necessities.
+
 ## Add an attribute
 When adding an attribute to the archiving framework, we must define the following properties:
 - the EventSubscriber name that will take charge of the attribute

devices/toolkit/archiver.py

 #! /usr/bin/env python3
 
-from logging import raiseExceptions
+#from logging import raiseExceptions
+import logging
 import traceback
 from clients.attribute_wrapper import attribute_wrapper
 from tango import DeviceProxy, AttributeProxy
 from datetime import datetime, timedelta
 
 import time
+import json, os
 from sqlalchemy import create_engine, and_
 from sqlalchemy.orm import sessionmaker
+from sqlalchemy.orm.exc import NoResultFound
 from .archiver_base import *
 
+logger = logging.getLogger()
+
 class Archiver():
     """
     The Archiver class implements the basic operations to perform attributes archiving
     """
-    def __init__(self, cm_name: str = 'archiving/hdbpp/confmanager01', es_name: str = 'archiving/hdbpp/eventsubscriber01', context: str = 'RUN'):
+
+    # Global 'DEVELOPMENT' environment variables set by configuration file 
+    dev_polling_time = None
+    dev_archive_time = None
+
+    def __init__(self, selector_filename:str = None, cm_name: str = 'archiving/hdbpp/confmanager01', es_name: str = 'archiving/hdbpp/eventsubscriber01', context: str = 'RUN'):
         self.cm_name = cm_name
         self.cm = DeviceProxy(cm_name)
         try: 
             cm_state = self.cm.state() # ping the device server
-            if cm_state is 'FAULT':
-                print('Configuration Manager is in FAULT state')
-                print(self.cm.status())
-                return
-        except:
-            print(traceback.format_exc())
-            return
+            if 'FAULT' in str(cm_state):
+                raise Exception("Configuration Manager is in FAULT state")
+        except Exception as e:
+            raise Exception("Connection failed with Configuration Manager device") from e
         self.es_name = es_name
         self.es = DeviceProxy(es_name)
         self.cm.write_attribute('Context',context)    # Set default Context Archiving for all the subscribers
+        self.selector = Selector() if selector_filename is None else Selector(selector_filename)  # Create selector for customized strategies
+        try:
+            self.apply_selector()
+        except Exception as e:
+            raise Exception("Error in selecting configuration! Archiving framework will not be updated!") from e
 
-    def add_attribute_to_archiver(self, attribute_name: str, polling_period: int = 1000, event_period: int = 1000, strategy: str = 'RUN'):
+    def apply_selector(self):
+        """
+        Apply the customized strategy defined by the selector
+        """
+        config_dict = self.selector.get_dict()
+        # Set global development env variables
+        var_dict = config_dict.get('global_variables')
+        self.dev_polling_time = int(var_dict.get('development_polling_time')) or 10000
+        self.dev_archive_time = int(var_dict.get('development_archive_time')) or 60000
+        # Set devices archiving
+        env_dict = config_dict.get('devices')
+        for device in env_dict:
+            try:
+                dev_env = str(env_dict[device].get('environment')) # Get device environment
+                if dev_env == 'development':    # DEV environment -> all attributes are excluded by default
+                    include_att_list = env_dict[device].get('include',[])
+                    self.remove_attributes_by_device(device, exclude=include_att_list)
+                    # Include attributes by custom configuration              
+                    for att in include_att_list:
+                        att_fqname = f"{device}/{att}".lower()
+                        self.add_attribute_to_archiver(att_fqname,self.dev_polling_time,self.dev_archive_time)                 
+                elif dev_env == 'production':   # PROD environment -> all attributes are included by default
+                    exclude_att_list = env_dict[device].get('exclude', [])
+                    self.add_attributes_by_device(device, exclude=exclude_att_list)
+                    # Remove attributes by custom configuration if already present
+                    # The following cycle is a security check in the special case that an attribute is in the
+                    # included list in DEV mode, and in the excluded list in PROD mode             
+                    for att in exclude_att_list:
+                        att_fqname = f"{device}/{att}".lower()
+                        self.remove_attribute_from_archiver(att_fqname)    
+            except Exception as e:
+                if 'API_DeviceNotExported' in str(e):   # ignore if device is offline
+                    logger.warning(f"Device {device} offline")
+                elif 'API_CantConnectToDevice' in str(e):
+                    logger.warning(f"Device {device} not found")
+                elif 'DB_DeviceNotDefined' in str(e):
+                    logger.warning(f"Device {device} not defined in TangoDB")
+                else:
+                    raise Exception from e                              
+        return env_dict
+    
+    def add_attribute_to_archiver(self, attribute_name: str, polling_period: int, event_period: int, strategy: str = 'RUN'):
         """
         Takes as input the attribute name, polling period (ms), event period (ms) and archiving strategy, 
         and adds the selected attribute to the subscriber's list of archiving attributes.
@@ -47,36 +100,38 @@ class Archiver():
             self.cm.write_attribute('SetPollingPeriod', polling_period)
             self.cm.write_attribute('SetPeriodEvent', event_period)
             self.cm.AttributeAdd()
-            print('Attribute %s added to archiving list!' % attribute_name)
+            logger.info(f"Attribute {attribute_name} added to archiving list!")
         except Exception as e:
             if 'already archived' not in str(e).lower():
-                traceback.format_exc()
+                raise Exception from e
             else:
-                print('Attribute %s already in archiving list!' % attribute_name)
+                logger.warning(f"Attribute {attribute_name} already in archiving list!")
 
-    def add_attributes_to_archiver(self,device_name,global_archive_period:int = None, exclude:list = ['Status','State']):
+    def add_attributes_by_device(self,device_name,global_archive_period:int = None, exclude:list = []):
         """
         Add sequentially all the attributes of the selected device in the event subscriber list, if not already present
         """
         d = DeviceProxy(device_name)
-        attrs_list = list(d.get_attribute_list()) # cast to list otherwise removal is not allowed
-        try: 
-            for a in exclude: attrs_list.remove(a)
-        except:
-            pass
+        dev_attrs_list = d.get_attribute_list()
+        exclude_list = [a.lower() for a in exclude]
+        attrs_list = [a.lower() for a in list(dev_attrs_list) if a.lower() not in exclude_list]    # transform list for string comparison
         for a in attrs_list:
-            attr_fullname = str(device_name+'/'+a).lower()
+            attr_fullname = f"{device_name}/{a}".lower()
             attr_proxy = AttributeProxy(attr_fullname)
             if attr_proxy.is_polled() is True:   # if not polled attribute is also not archived
                 try:
                     if self.es.AttributeList is None or not(self.cm.AttributeSearch(a)):
-                        polling_period = attr_proxy.get_poll_period()  
-                        archive_period = global_archive_period or int(attr_proxy.get_property('archive_period')['archive_period'][0])                 
+                        polling_period = attr_proxy.get_poll_period() or self.dev_polling_time  
+                        archive_period = global_archive_period or int(attr_proxy.get_property('archive_period')['archive_period'][0]) or self.dev_archive_time                 
                         self.add_attribute_to_archiver(attr_fullname,polling_period=polling_period,
                             event_period=archive_period)
                         #time.sleep(0.5)
-                except:
-                    print(traceback.format_exc())
+                except IndexError as e:
+                    logger.warning(f"Attribute {attr_fullname} will not be archived because archive event period is not defined!")
+                except Exception as e:
+                    raise Exception from e                        
+            else:
+                logger.warning(f"Attribute {attr_fullname} will not be archived because polling is set to FALSE!")
     
     def remove_attribute_from_archiver(self, attribute_name:str):
         """
@@ -87,26 +142,28 @@ class Archiver():
         try:
             self.cm.AttributeStop(attribute_name)
             self.cm.AttributeRemove(attribute_name)
-            print('Attribute %s removed!' % attribute_name)
+            logger.warning(f"Attribute {attribute_name} removed!")
         except Exception as e:
             if 'attribute not found' not in str(e).lower():
-                traceback.format_exc()
+                raise Exception from e
             else:
-                print('Attribute %s not found!' % attribute_name)      
+                logger.warning(f"Attribute {attribute_name} not found in archiving list!")     
     
-    def remove_attributes_by_device(self,device_name:str):
+    def remove_attributes_by_device(self,device_name:str,exclude:list=[]):
         """
         Stops the data archiving of all the attributes of the selected device, and remove them from the
         subscriber's list
         """
         d = DeviceProxy(device_name)
-        attrs_list = d.get_attribute_list()
+        dev_attrs_list = d.get_attribute_list()
+        exclude_list = [a.lower() for a in exclude]
+        attrs_list = [a.lower() for a in list(dev_attrs_list) if a.lower() not in exclude_list]    # transform list for string comparison      
         for a in attrs_list:
             try:
-                attr_fullname = str(device_name+'/'+a).lower()
+                attr_fullname = f"{device_name}/{a}".lower()
                 self.remove_attribute_from_archiver(attr_fullname)
-            except:
-                print(traceback.format_exc())
+            except Exception as e:
+                raise Exception from e
 
     def start_archiving_attribute(self, attribute_name:str):
         """
@@ -119,9 +176,9 @@ class Archiver():
             self.cm.AttributeStart(attribute_name)
         except Exception as e:
             if 'attribute not found' not in str(e).lower():
-                traceback.format_exc()
+                raise Exception from e
             else: 
-                print('Attribute %s not found!' % attribute_name)
+                logger.warning(f"Attribute {attribute_name} not found!")
     
     def stop_archiving_attribute(self, attribute_name:str):
         """
@@ -134,36 +191,37 @@ class Archiver():
             self.cm.AttributeStop(attribute_name)
         except Exception as e:
             if 'attribute not found' not in str(e).lower():
-                traceback.format_exc()
+                raise Exception from e
             else: 
-                print('Attribute %s not found!' % attribute_name)
+                logger.warning(f"Attribute {attribute_name} not found!")
     
-    def check_and_add_attribute_in_archiving_list(self, attribute_name:str):
+    def is_attribute_archived(self,attribute_name:str):
         """
         Check if an attribute is in the archiving list
         """
         if (len(attribute_name.split('/'))!=4): 
             raise AttributeFormatException
-        # Add attribute if not present in event subscriber list
         try: 
-            if self.es.AttributeList is None or not(self.cm.AttributeSearch(attribute_name)):
-                self.add_attribute_to_archiver(attribute_name)
-        except:
-            print(traceback.format_exc())
-        return attribute_name
+            attributes = self.cm.AttributeSearch(attribute_name)
+            a = [a for a in attributes if a.lower().endswith(attribute_name.lower())]   # handle cases differences
+            if len(attributes)>1: 
+                raise Exception("MultipleAttributesMatched!")
+            if len(attributes)==1:
+                return True
+            else:
+                return False
+        except Exception as e:
+            raise Exception from e
     
-    def update_archiving_attribute(self, attribute_name: str, polling_period: int = 1000, event_period: int = 1000, strategy: str = 'RUN'):
+    def update_archiving_attribute(self, attribute_name: str, polling_period: int, event_period: int, strategy: str = 'RUN'):
         """
         Update the archiving properties of an attribute already in a subscriber list
         """
-        try:
         self.remove_attribute_from_archiver(attribute_name)
         time.sleep(1)
         self.add_attribute_to_archiver(attribute_name,polling_period,event_period,strategy)
         time.sleep(1)
         self.start_archiving_attribute(attribute_name)
-        except:
-            print(traceback.format_exc())
     
     def get_subscriber_attributes(self,es_name:str = None):
         """
@@ -226,6 +284,27 @@ class AttributeFormatException(Exception):
         self.message = message
         super().__init__(self.message)
 
+class Selector():
+    """
+    The Selector class implements operations on select customized retrieval strategies
+    """
+
+    def __init__(self, filename='lofar2.json'):
+        self.filename = filename
+    
+    def get_dict(self):
+        """
+        Create a dictionary from the JSON file
+        """
+        try:
+            filepath = os.path.join(os.path.dirname(__file__),'archiver_config',self.filename)
+            f = open(filepath)
+            data = json.load(f)
+            f.close()
+        except FileNotFoundError as e:
+            raise Exception("JSON configuration file not found!") from e
+        return data
+
 class Retriever():
     """
     The Retriever class implements retrieve operations on a given DBMS
@@ -271,8 +350,7 @@ class Retriever():
         try:
             [domain, family, member] = device_fqname.split('/')
         except:
-            print("Device name error. Use FQDN - eg: LTS/Device/1")
-            return
+            raise AttributeFormatException(f"Could not parse device name {device_fqname}. Please provide FQDN, e.g. LTS/Device/1")
         attrs = self.session.query(Attribute).filter(and_(Attribute.domain == domain, Attribute.family == family, \
                                 Attribute.member == member)).all()
         # Returns the representation as set in __repr__ method of the mapper class
@@ -285,15 +363,15 @@ class Retriever():
         try:
             [domain, family, member, name] = attribute_fqname.split('/')
         except:
-            print("Attribute name error. Use FQDN - eg: LTS/Device/1/Attribute")
-            return
+            raise AttributeFormatException(f"Could not parse attribute name {attribute_fqname}. Please provide FQDN, e.g. LTS/Device/1/Attribute")
         try:
             result = self.session.query(Attribute.att_conf_id).filter(and_(Attribute.domain == domain, Attribute.family == family, \
                                     Attribute.member == member, Attribute.name == name)).one()
             return result[0]
-        except TypeError:
-            print("Attribute not found!")
-            return
+        except TypeError as e:
+            raise Exception("Attribute not found!") from e
+        except NoResultFound as e:
+            raise Exception(f"No records of attribute {attribute_fqname} found in DB") from e
 
     def get_attribute_datatype(self,attribute_fqname: str):
         """
@@ -304,15 +382,15 @@ class Retriever():
         try:
             [domain, family, member, name] = attribute_fqname.split('/')
         except:
-            print("Attribute name error. Use FQDN - eg: LTS/Device/1/Attribute")
-            return
+            raise AttributeFormatException(f"Could not parse attribute name {attribute_fqname}. Please provide FQDN, e.g. LTS/Device/1/Attribute")
         try:
             result = self.session.query(DataType.data_type).join(Attribute,Attribute.att_conf_data_type_id==DataType.att_conf_data_type_id).\
                         filter(and_(Attribute.domain == domain, Attribute.family == family, Attribute.member == member, Attribute.name == name)).one()
             return result[0]
-        except TypeError:
-            print("Attribute not found!")
-            return
+        except TypeError as e:
+            raise Exception("Attribute not found!") from e
+        except NoResultFound as e:
+            raise Exception(f"No records of attribute {attribute_fqname} found in DB") from e
 
     def get_attribute_value_by_hours(self,attribute_fqname: str, hours: float = 1.0):
         """
@@ -331,10 +409,13 @@ class Retriever():
         # Converts the timestamps in the right format for the query
         time_now_db = str(time_now.strftime("%Y-%m-%d %X"))
         time_delta_db = str(time_delta.strftime("%Y-%m-%d %X"))
+        try:
             result = self.session.query(base_class).\
                     join(Attribute,Attribute.att_conf_id==base_class.att_conf_id).\
                     filter(and_(Attribute.att_conf_id == attr_id,base_class.data_time >= time_delta_db, \
                             base_class.data_time <= time_now_db)).order_by(base_class.data_time).all()
+        except AttributeError as e:
+            raise Exception(f"Empty result! Attribute {attribute_fqname} not found") from e
         return result
 
     def get_attribute_value_by_interval(self,attribute_fqname: str, start_time: datetime, stop_time: datetime):
@@ -348,8 +429,12 @@ class Retriever():
         attr_table_name = 'att_'+str(attr_datatype)
         # Retrieves the class that maps the DB table given the tablename
         base_class = get_class_by_tablename(attr_table_name)
+        try:
             result = self.session.query(base_class).\
                     join(Attribute,Attribute.att_conf_id==base_class.att_conf_id).\
                         filter(and_(Attribute.att_conf_id == attr_id,base_class.data_time >= str(start_time), \
                                 base_class.data_time <= str(stop_time))).order_by(base_class.data_time).all()
+        except AttributeError as e:
+            raise Exception(f"Empty result! Attribute {attribute_fqname} not found") from e
         return result
+        

devices/toolkit/archiver_base.py

@@ -77,7 +77,7 @@ class Scalar_Boolean(Scalar):
 
 class Scalar_Boolean_RO(Scalar_Boolean):
     """
-    Class that represents a Tango Scalar Read-Only Value mapped to table 'att_scalar_devdouble_ro'
+    Class that represents a Tango Scalar Read-Only Value mapped to table 'att_scalar_devboolean_ro'
     """
     __tablename__ = 'att_scalar_devboolean_ro'
     __table_args__ = {'extend_existing': True}
@@ -88,7 +88,7 @@ class Scalar_Boolean_RO(Scalar_Boolean):
 
 class Scalar_Boolean_RW(Scalar_Boolean):
     """
-    Class that represents a Tango Scalar Read-Write Value mapped to table 'att_scalar_devdouble_rw'
+    Class that represents a Tango Scalar Read-Write Value mapped to table 'att_scalar_devboolean_rw'
     """
     __tablename__ = 'att_scalar_devboolean_rw'
     __table_args__ = {'extend_existing': True}
@@ -98,6 +98,38 @@ class Scalar_Boolean_RW(Scalar_Boolean):
         return "<Scalar_Boolean_RW(att_conf_id='%s',data_time='%s',recv_time='%s',insert_time='%s',value_r='%s',value_w='%s',quality='%s',att_error_desc_id='%s')>" \
                 % (self.att_conf_id,self.data_time,self.recv_time,self.insert_time,self.value_r,self.value_w,self.quality,self.att_error_desc_id)
 
+class Scalar_UChar(Scalar):
+    """
+    Abstract class that represents Parent class of Scalar Uchar mapper classes
+    """
+    # In the concrete inheritance use case, it is common that the base class is not represented 
+    # within the database, only the subclasses. In other words, the base class is abstract.
+    __abstract__ = True
+    value_r = Column(Integer)
+
+class Scalar_UChar_RO(Scalar_UChar):
+    """
+    Class that represents a Tango Scalar Read-Only Value mapped to table 'att_scalar_devuchar_ro'
+    """
+    __tablename__ = 'att_scalar_devuchar_ro'
+    __table_args__ = {'extend_existing': True}
+    
+    def __repr__(self):
+        return "<Scalar_UChar_RO(att_conf_id='%s',data_time='%s',recv_time='%s',insert_time='%s',value_r='%s',quality='%s',att_error_desc_id='%s')>" \
+                % (self.att_conf_id,self.data_time,self.recv_time,self.insert_time,self.value_r,self.quality,self.att_error_desc_id)
+
+class Scalar_UChar_RW(Scalar_UChar):
+    """
+    Class that represents a Tango Scalar Read-Write Value mapped to table 'att_scalar_devuchar_rw'
+    """
+    __tablename__ = 'att_scalar_devuchar_rw'
+    __table_args__ = {'extend_existing': True}
+    value_w = Column(Integer)
+
+    def __repr__(self):
+        return "<Scalar_UChar_RW(att_conf_id='%s',data_time='%s',recv_time='%s',insert_time='%s',value_r='%s',value_w='%s',quality='%s',att_error_desc_id='%s')>" \
+                % (self.att_conf_id,self.data_time,self.recv_time,self.insert_time,self.value_r,self.value_w,self.quality,self.att_error_desc_id)
+
 class Scalar_Double(Scalar):
     """
     Abstract class that represents Parent class of Scalar Double mapper classes
@@ -258,6 +290,39 @@ class Scalar_Long64_RW(Scalar_Long64):
         return "<Scalar_Long64_RW(att_conf_id='%s',data_time='%s',recv_time='%s',insert_time='%s',value_r='%s',value_w='%s',quality='%s',att_error_desc_id='%s')>" \
                 % (self.att_conf_id,self.data_time,self.recv_time,self.insert_time,self.value_r,self.value_w,self.quality,self.att_error_desc_id)
 
+class Scalar_ULong64(Scalar):
+    """
+    Abstract class that represents Parent class of Scalar ULong64 mapper classes
+    """
+    # In the concrete inheritance use case, it is common that the base class is not represented 
+    # within the database, only the subclasses. In other words, the base class is abstract.
+    __abstract__ = True
+    value_r = Column(INTEGER)
+
+class Scalar_ULong64_RO(Scalar_ULong64):
+    """
+    Class that represents a Tango Scalar Read-Only Value mapped to table 'att_scalar_devulong64_ro'
+    """
+    __tablename__ = 'att_scalar_devulong64_ro'
+    __table_args__ = {'extend_existing': True}
+
+    def __repr__(self):
+        return "<Scalar_ULong64_RO(att_conf_id='%s',data_time='%s',recv_time='%s',insert_time='%s',value_r='%s',quality='%s',att_error_desc_id='%s')>" \
+                % (self.att_conf_id,self.data_time,self.recv_time,self.insert_time,self.value_r,self.quality,self.att_error_desc_id)
+
+class Scalar_ULong64_RW(Scalar_ULong64):
+    """
+    Class that represents a Tango Scalar Read-Write Value mapped to table 'att_scalar_devulong64_rw'
+    """
+    __tablename__ = 'att_scalar_devulong64_rw'
+    __table_args__ = {'extend_existing': True}
+    value_w = Column(INTEGER)
+
+    def __repr__(self):
+        return "<Scalar_ULong64_RW(att_conf_id='%s',data_time='%s',recv_time='%s',insert_time='%s',value_r='%s',value_w='%s',quality='%s',att_error_desc_id='%s')>" \
+                % (self.att_conf_id,self.data_time,self.recv_time,self.insert_time,self.value_r,self.value_w,self.quality,self.att_error_desc_id)
+
+
 class Scalar_Long(Scalar):
     """
     Abstract class that represents Parent class of Scalar Long mapper classes
@@ -290,6 +355,38 @@ class Scalar_Long_RW(Scalar_Long):
         return "<Scalar_Long_RW(att_conf_id='%s',data_time='%s',recv_time='%s',insert_time='%s',value_r='%s',value_w='%s',quality='%s',att_error_desc_id='%s')>" \
                 % (self.att_conf_id,self.data_time,self.recv_time,self.insert_time,self.value_r,self.value_w,self.quality,self.att_error_desc_id)
 
+class Scalar_ULong(Scalar):
+    """
+    Abstract class that represents Parent class of Scalar ULong mapper classes
+    """
+    # In the concrete inheritance use case, it is common that the base class is not represented 
+    # within the database, only the subclasses. In other words, the base class is abstract.
+    __abstract__ = True
+    value_r = Column(INTEGER)
+
+class Scalar_ULong_RO(Scalar_ULong):
+    """
+    Class that represents a Tango Scalar Read-Only Value mapped to table 'att_scalar_devulong_ro'
+    """
+    __tablename__ = 'att_scalar_devulong_ro'
+    __table_args__ = {'extend_existing': True}
+
+    def __repr__(self):
+        return "<Scalar_ULong_RO(att_conf_id='%s',data_time='%s',recv_time='%s',insert_time='%s',value_r='%s',quality='%s',att_error_desc_id='%s')>" \
+                % (self.att_conf_id,self.data_time,self.recv_time,self.insert_time,self.value_r,self.quality,self.att_error_desc_id)
+
+class Scalar_ULong_RW(Scalar_ULong):
+    """
+    Class that represents a Tango Scalar Read-Write Value mapped to table 'att_scalar_devulong_rw'
+    """
+    __tablename__ = 'att_scalar_devulong_rw'
+    __table_args__ = {'extend_existing': True}
+    value_w = Column(INTEGER)
+
+    def __repr__(self):
+        return "<Scalar_ULong_RW(att_conf_id='%s',data_time='%s',recv_time='%s',insert_time='%s',value_r='%s',value_w='%s',quality='%s',att_error_desc_id='%s')>" \
+                % (self.att_conf_id,self.data_time,self.recv_time,self.insert_time,self.value_r,self.value_w,self.quality,self.att_error_desc_id)
+
 class Scalar_Short(Scalar):
     """
     Abstract class that represents Parent class of Scalar Short mapper classes
@@ -438,6 +535,40 @@ class Array_Boolean_RW(Array_Boolean):
         return "<Array_Boolean_RW(att_conf_id='%s',data_time='%s',recv_time='%s',insert_time='%s',idx='%s',dim_x_r='%s',dim_y_r='%s',value_r='%s',dim_x_w='%s',dim_y_w='%s',value_w='%s',quality='%s',att_error_desc_id='%s')>" \
                 % (self.att_conf_id,self.data_time,self.recv_time,self.insert_time,self.idx,self.dim_x_r,self.dim_y_r,self.value_r,self.dim_x_w,self.dim_y_w,self.value_w,self.quality,self.att_error_desc_id)
 
+class Array_UChar(Array):
+    """
+    Abstract class that represents Parent class of Array UChar mapper classes
+    """
+    # In the concrete inheritance use case, it is common that the base class is not represented 
+    # within the database, only the subclasses. In other words, the base class is abstract.
+    __abstract__ = True
+    value_r = Column(Integer)
+
+class Array_UChar_RO(Array_UChar):
+    """
+    Class that represents a Tango Array Read-Only Value mapped to table 'att_array_devuchar_ro'
+    """
+    __tablename__ = 'att_array_devuchar_ro'
+    __table_args__ = {'extend_existing': True}
+    
+    def __repr__(self):
+        return "<Array_UChar_RO(att_conf_id='%s',data_time='%s',recv_time='%s',insert_time='%s',idx='%s',dim_x_r='%s',dim_y_r='%s',value_r='%s',quality='%s',att_error_desc_id='%s')>" \
+                % (self.att_conf_id,self.data_time,self.recv_time,self.insert_time,self.idx,self.dim_x_r,self.dim_y_r,self.value_r,self.quality,self.att_error_desc_id)
+
+class Array_UChar_RW(Array_Boolean):
+    """
+    Class that represents a Tango Array Read-Write Value mapped to table 'att_array_devuchar_rw'
+    """
+    __tablename__ = 'att_array_devuchar_rw'
+    __table_args__ = {'extend_existing': True}
+    dim_x_w = Column(Integer)
+    dim_y_w = Column(Integer)
+    value_w = Column(Integer)
+
+    def __repr__(self):
+        return "<Array_UChar_RW(att_conf_id='%s',data_time='%s',recv_time='%s',insert_time='%s',idx='%s',dim_x_r='%s',dim_y_r='%s',value_r='%s',dim_x_w='%s',dim_y_w='%s',value_w='%s',quality='%s',att_error_desc_id='%s')>" \
+                % (self.att_conf_id,self.data_time,self.recv_time,self.insert_time,self.idx,self.dim_x_r,self.dim_y_r,self.value_r,self.dim_x_w,self.dim_y_w,self.value_w,self.quality,self.att_error_desc_id)
+
 class Array_Double(Array):
     """
     Abstract class that represents Parent class of Array Double mapper classes
@@ -608,6 +739,41 @@ class Array_Long64_RW(Array_Long64):
         return "<Array_Long64_RW(att_conf_id='%s',data_time='%s',recv_time='%s',insert_time='%s',idx='%s',dim_x_r='%s',dim_y_r='%s',value_r='%s',dim_x_w='%s',dim_y_w='%s',value_w='%s',quality='%s',att_error_desc_id='%s')>" \
                 % (self.att_conf_id,self.data_time,self.recv_time,self.insert_time,self.idx,self.dim_x_r,self.dim_y_r,self.value_r,self.dim_x_w,self.dim_y_w,self.value_w,self.quality,self.att_error_desc_id)
 
+class Array_ULong64(Array):
+    """
+    Abstract class that represents Parent class of Array ULong64 mapper classes
+    """
+    # In the concrete inheritance use case, it is common that the base class is not represented 
+    # within the database, only the subclasses. In other words, the base class is abstract.
+    __abstract__ = True
+    value_r = Column(INTEGER)
+
+class Array_ULong64_RO(Array_ULong64):
+    """
+    Class that represents a Tango Array Read-Only Value mapped to table 'att_array_devulong64_ro'
+    """
+    __tablename__ = 'att_array_devulong64_ro'
+    __table_args__ = {'extend_existing': True}
+    
+    def __repr__(self):
+        return "<Array_ULong64_RO(att_conf_id='%s',data_time='%s',recv_time='%s',insert_time='%s',idx='%s',dim_x_r='%s',dim_y_r='%s',value_r='%s',quality='%s',att_error_desc_id='%s')>" \
+                % (self.att_conf_id,self.data_time,self.recv_time,self.insert_time,self.idx,self.dim_x_r,self.dim_y_r,self.value_r,self.quality,self.att_error_desc_id)
+
+class Array_ULong64_RW(Array_ULong64):
+    """
+    Class that represents a Tango Array Read-Write Value mapped to table 'att_array_devulong64_rw'
+    """
+    __tablename__ = 'att_array_devulong64_rw'
+    __table_args__ = {'extend_existing': True}
+    dim_x_w = Column(Integer)
+    dim_y_w = Column(Integer)
+    value_w = Column(INTEGER)
+
+    def __repr__(self):
+        return "<Array_ULong64_RW(att_conf_id='%s',data_time='%s',recv_time='%s',insert_time='%s',idx='%s',dim_x_r='%s',dim_y_r='%s',value_r='%s',dim_x_w='%s',dim_y_w='%s',value_w='%s',quality='%s',att_error_desc_id='%s')>" \
+                % (self.att_conf_id,self.data_time,self.recv_time,self.insert_time,self.idx,self.dim_x_r,self.dim_y_r,self.value_r,self.dim_x_w,self.dim_y_w,self.value_w,self.quality,self.att_error_desc_id)
+
+
 class Array_Long(Array):
     """
     Abstract class that represents Parent class of Array Long mapper classes
@@ -642,6 +808,40 @@ class Array_Long_RW(Array_Long):
         return "<Array_Long_RW(att_conf_id='%s',data_time='%s',recv_time='%s',insert_time='%s',idx='%s',dim_x_r='%s',dim_y_r='%s',value_r='%s',dim_x_w='%s',dim_y_w='%s',value_w='%s',quality='%s',att_error_desc_id='%s')>" \
                 % (self.att_conf_id,self.data_time,self.recv_time,self.insert_time,self.idx,self.dim_x_r,self.dim_y_r,self.value_r,self.dim_x_w,self.dim_y_w,self.value_w,self.quality,self.att_error_desc_id)
 
+class Array_ULong(Array):
+    """
+    Abstract class that represents Parent class of Array ULong mapper classes
+    """
+    # In the concrete inheritance use case, it is common that the base class is not represented 
+    # within the database, only the subclasses. In other words, the base class is abstract.
+    __abstract__ = True
+    value_r = Column(INTEGER)
+
+class Array_ULong_RO(Array_ULong):
+    """
+    Class that represents a Tango Array Read-Only Value mapped to table 'att_array_devulong_ro'
+    """
+    __tablename__ = 'att_array_devulong_ro'
+    __table_args__ = {'extend_existing': True}
+    
+    def __repr__(self):
+        return "<Array_ULong_RO(att_conf_id='%s',data_time='%s',recv_time='%s',insert_time='%s',idx='%s',dim_x_r='%s',dim_y_r='%s',value_r='%s',quality='%s',att_error_desc_id='%s')>" \
+                % (self.att_conf_id,self.data_time,self.recv_time,self.insert_time,self.idx,self.dim_x_r,self.dim_y_r,self.value_r,self.quality,self.att_error_desc_id)
+
+class Array_ULong_RW(Array_ULong):
+    """
+    Class that represents a Tango Array Read-Write Value mapped to table 'att_array_devulong_rw'
+    """
+    __tablename__ = 'att_array_devulong_rw'
+    __table_args__ = {'extend_existing': True}
+    dim_x_w = Column(Integer)
+    dim_y_w = Column(Integer)
+    value_w = Column(INTEGER)
+
+    def __repr__(self):
+        return "<Array_ULong_RW(att_conf_id='%s',data_time='%s',recv_time='%s',insert_time='%s',idx='%s',dim_x_r='%s',dim_y_r='%s',value_r='%s',dim_x_w='%s',dim_y_w='%s',value_w='%s',quality='%s',att_error_desc_id='%s')>" \
+                % (self.att_conf_id,self.data_time,self.recv_time,self.insert_time,self.idx,self.dim_x_r,self.dim_y_r,self.value_r,self.dim_x_w,self.dim_y_w,self.value_w,self.quality,self.att_error_desc_id)
+
 class Array_Short(Array):
     """
     Abstract class that represents Parent class of Array Short mapper classes

devices/toolkit/archiver_config/lofar2.json

+{
+    "global_variables": {
+        "development_polling_time": "10000",
+        "development_archive_time": "60000"
+    },
+    "devices":{
+        "LTS/RECV/1": {
+            "environment": "development",
+            "include": ["rcu_temperature_r"],
+            "exclude": ["CLK_Enable_PWR_R","CLK_I2C_STATUS_R","CLK_PLL_error_R","CLK_PLL_locked_R","CLK_translator_busy_R"]      
+        },
+        "LTS/SDP/1": {
+            "environment": "development",
+            "include": [],
+            "exclude": []      
+        },
+        "LTS/SST/1": {
+            "environment": "development",
+            "include": [],
+            "exclude": []      
+        },
+        "LTS/XST/1": {
+            "environment": "development",
+            "include": [],
+            "exclude": []      
+        },
+        "LTS/UNB2/1": {
+            "environment": "development",
+            "include": [],
+            "exclude": []      
+        }
+    }
+}

jupyter-notebooks/RECV_archive_attribute.ipynb

-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "42e7f25a",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import sys, time\n",
-    "import numpy as np\n",
-    "sys.path.append('/hosthome/tango/devices')\n",
-    "from toolkit.archiver import Archiver,Retriever\n",
-    "from toolkit.archiver_base import *\n",
-    "from matplotlib import pyplot as plt"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "1f025912",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from common.lofar_environment import isProduction\n",
-    "print(isProduction())"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "e0656e2d",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Define an attribute for archiving\n",
-    "device_name = 'LTS/RECV/1'\n",
-    "d=DeviceProxy(device_name) \n",
-    "state = str(d.state())\n",
-    "print(device_name,'is',state)\n",
-    "\n",
-    "archiver = Archiver()\n",
-    "\n",
-    "# Attribute chosen to be archived\n",
-    "attr_name = 'rcu_temperature_r'\n",
-    "attr_fq_name = str(device_name+'/'+attr_name).lower()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "153d9420",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Print the list of the attributes in the event subscriber\n",
-    "# If any attribute is present, its archiving will begin when device will reach ON state,\n",
-    "# Otherwise, attribute will be added to the list at the device initializing phase only in PRODUCTION mode\n",
-    "archiver.get_subscriber_attributes()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "2ebb00f8",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Start the device\n",
-    "if state == \"OFF\":\n",
-    "    if isProduction():\n",
-    "        archiver.check_and_add_attribute_in_archiving_list(attr_fq_name)\n",
-    "    else:\n",
-    "        archiver.remove_attribute_from_archiver(attr_fq_name)\n",
-    "    time.sleep(1)\n",
-    "    d.initialise()\n",
-    "    time.sleep(1)\n",
-    "state = str(d.state())\n",
-    "if state == \"STANDBY\":\n",
-    "    d.on()\n",
-    "state = str(d.state())\n",
-    "if state == \"ON\":\n",
-    "    print(\"Device is now in ON state\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "75163627",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Modify attribute archiving features\n",
-    "archiver.update_archiving_attribute(attr_fq_name,polling_period=1000,event_period=5000,strategy='RUN')"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "7814715e",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Add attribute to the archiving list (starts the archiving if device is running)\n",
-    "\n",
-    "# Archiving strategies are ['ALWAYS','RUN','SHUTDOWN','SERVICE']\n",
-    "#Read [0]\tALWAYS:always stored\n",
-    "#Read [1]\tRUN:stored during run\n",
-    "#Read [2]\tSHUTDOWN:stored during shutdown\n",
-    "#Read [3]\tSERVICE:stored during maintenance activities\n",
-    "\n",
-    "archiver.add_attribute_to_archiver(attr_fq_name, polling_period=1000, event_period=1000, strategy='RUN')"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "52a27abb",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Stop the attribute archiving but do not remove it from the list\n",
-    "# This means that archiving is stopped for the current session, but if the device is restarted, \n",
-    "# the attribute archiving will be restarted as well\n",
-    "# In order to definitely stop the archiving, the attribute must be removed from the attribute list (go to last cell)\n",
-    "archiver.stop_archiving_attribute(attr_fq_name)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "c064e337",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Starts the attribute archiving if it was stopped\n",
-    "archiver.start_archiving_attribute(attr_fq_name)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "d199916c",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Initialise the retriever object and print the archived attributes in the database\n",
-    "retriever = Retriever()\n",
-    "retriever.get_all_archived_attributes()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "80e2a560",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Retrieve records in the last n hours (works even with decimals)\n",
-    "\n",
-    "# Use alternatively one of the following two methods to retrieve data (last n hours or interval)\n",
-    "records= retriever.get_attribute_value_by_hours(attr_fq_name,hours=0.1)\n",
-    "#records = retriever.get_attribute_value_by_interval(attr_fq_name,'2021-09-01 16:00:00', '2021-09-01 16:03:00')\n",
-    "\n",
-    "if not records:\n",
-    "    print('Empty result!')\n",
-    "else:\n",
-    "    # Convert DB Array records into Python lists\n",
-    "    data = build_array_from_record(records,records[0].dim_x_r)\n",
-    "    # Extract only the value from the array \n",
-    "    array_values = get_values_from_record(data)\n",
-    "\n",
-    "#records\n",
-    "#data\n",
-    "#array_values"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "64c8e060",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Extract and process timestamps for plotting purposes\n",
-    "def get_timestamps(data,strformat):\n",
-    "    timestamps = []\n",
-    "    for i in range(len(data)):\n",
-    "        timestamps.append(data[i][0].recv_time.strftime(strformat))\n",
-    "    return timestamps\n",
-    "timestamps = get_timestamps(data,\"%Y-%m-%d %X\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "59a0c05c",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Plot of array values\n",
-    "\n",
-    "heatmap = np.array(array_values,dtype=np.float)\n",
-    "fig = plt.figure()\n",
-    "plt.rcParams['figure.figsize'] = [128, 64]\n",
-    "#plt.rcParams['figure.dpi'] = 128\n",
-    "ax = fig.add_subplot(111)\n",
-    "im = ax.imshow(heatmap, interpolation='nearest',cmap='coolwarm')\n",
-    "ax.set_xlabel('Array index')\n",
-    "ax.set_ylabel('Timestamp')\n",
-    "ax.set_xlim([0,(records[0].dim_x_r)-1])\n",
-    "ax.set_xticks(np.arange(0,records[0].dim_x_r))\n",
-    "\n",
-    "ax.set_yticks(range(0,len(timestamps)))\n",
-    "ax.set_yticklabels(timestamps,fontsize=4)\n",
-    "\n",
-    "# Comment the previous two lines and uncomment the following line if there are too many timestamp labels\n",
-    "#ax.set_yticks(range(0,len(timestamps),10))\n",
-    "\n",
-    "ax.set_title('Archived data for '+ attr_fq_name)\n",
-    "ax.grid()\n",
-    "cbar = fig.colorbar(ax=ax, mappable=im, orientation='horizontal')\n",
-    "plt.show()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "1c753ed9",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Count number of archive events per minute\n",
-    "archiver.get_subscriber_load()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "a0e8dcab",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Turn off the device\n",
-    "d.off()\n",
-    "# Remove attribute from archiving list\n",
-    "#archiver.remove_attribute_from_archiver(attr_fq_name)\n",
-    "#archiver.remove_attributes_by_device(device_name)"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "StationControl",
-   "language": "python",
-   "name": "stationcontrol"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.7.3"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}