Merge branch 'L2SS-541-remove-mysql-archiver' into 'master'

Resolve L2SS-541-remove-mysql-archiver

Closes L2SS-541

See merge request !258

CDB/default_ConfigDb.json

@@ -31,18 +31,19 @@
         "hdbppcm-srv": {
             "01": {
                 "HdbConfigurationManager": {
-                    "archiving/hdbpp/confmanager01": {
+                    "archiving/hdbppts/confmanager01": {
                         "properties": {
                             "ArchiverList": [
-                                "archiving/hdbpp/eventsubscriber01"
+                                "archiving/hdbppts/eventsubscriber01"
                             ], 
                             "LibConfiguration": [
-                                "host=archiver-maria-db", 
-                                "libname=libhdb++mysql.so.6", 
-                                "dbname=hdbpp", 
-                                "port=3306", 
-                                "user=tango", 
-                                "password=tango"
+                                "connect_string= user=postgres password=password host=archiver-timescale port=5432 dbname=hdb",
+                                "host=archiver-timescale",
+                                "libname=libhdb++timescale.so",
+                                "dbname=hdb",
+                                "port=5432", 
+                                "user=postgres", 
+                                "password=password"
                             ], 
                             "MaxSearchSize": [
                                 "1000"
@@ -55,18 +56,19 @@
         "hdbppes-srv": {
             "01": {
                 "HdbEventSubscriber": {
-                    "archiving/hdbpp/eventsubscriber01": {
+                    "archiving/hdbppts/eventsubscriber01": {
                         "properties": {
                             "CheckPeriodicTimeoutDelay": [
                                 "5"
                             ], 
                             "LibConfiguration": [
-                                "host=archiver-maria-db", 
-                                "libname=libhdb++mysql.so.6", 
-                                "dbname=hdbpp", 
-                                "port=3306", 
-                                "user=tango", 
-                                "password=tango"
+                                "connect_string= user=postgres password=password host=archiver-timescale port=5432 dbname=hdb",
+                                "host=archiver-timescale",
+                                "libname=libhdb++timescale.so",
+                                "dbname=hdb",
+                                "port=5432",
+                                "user=postgres",
+                                "password=password"
                             ], 
                             "PollingThreadPeriod": [
                                 "3"

CDB/tango-archiver-data/archiver-devices.json

@@ -2,19 +2,6 @@
     "servers": {
         "hdbppes-srv": {
             "01": {
-                "HdbEventSubscriber": {
-                    "archiving/hdbpp/eventsubscriber01": {
-                        "attribute_properties": {},
-                        "properties": {
-                            "CheckPeriodicTimeoutDelay": ["5"],
-                            "PollingThreadPeriod": ["3"],
-                            "LibConfiguration": ["host=archiver-maria-db","libname=libhdb++mysql.so.6","dbname=hdbpp","port=3306", "user=tango", "password=tango"],
-                            "polled_attr": []
-                        }
-                    }
-                }
-            },
-            "02": {
                 "HdbEventSubscriber": {
                     "archiving/hdbppts/eventsubscriber01": {
                         "attribute_properties": {},
@@ -26,36 +13,10 @@
                         }
                     }
                 }
-            },
-            "03": {
-                "HdbEventSubscriber": {
-                    "archiving/hdbpp/eventsubscriber02": {
-                        "attribute_properties": {},
-                        "properties": {
-                            "CheckPeriodicTimeoutDelay": ["5"],
-                            "PollingThreadPeriod": ["3"],
-                            "LibConfiguration": ["host=archiver-maria-db","libname=libhdb++mysql.so.6","dbname=hdbpp","port=3306", "user=tango", "password=tango"],
-                            "polled_attr": []
-                        }
-                    }
-                }
             }
         },
         "hdbppcm-srv": {
             "01": {
-                "HdbConfigurationManager": {
-                    "archiving/hdbpp/confmanager01": {
-                        "attribute_properties": {},
-                        "properties": {
-                            "ArchiverList": ["archiving/hdbpp/eventsubscriber01"],
-                            "MaxSearchSize": ["1000"],
-                            "LibConfiguration": ["host=archiver-maria-db","libname=libhdb++mysql.so.6","dbname=hdbpp","port=3306", "user=tango", "password=tango"],
-                            "polled_attr": []
-                        }
-                    }
-                }
-            },
-            "02": {
                 "HdbConfigurationManager": {
                     "archiving/hdbppts/confmanager01": {
                         "attribute_properties": {},

docker-compose/grafana/datasources/archiver-maria-db.yaml

-apiVersion: 1
-
-datasources:
-  # <string, required> name of the datasource. Required
-  - name: Archiver
-    # <string, required> datasource type. Required
-    type: mysql
-    # <string, required> access mode. proxy or direct (Server or Browser in the UI). Required
-    access: proxy
-    # <int> org id. will default to orgId 1 if not specified
-    orgId: 1
-    # <string> custom UID which can be used to reference this datasource in other parts of the configuration, if not specified will be generated automatically
-    uid: ZqAMHGN7z
-    # <string> url
-    url: archiver-maria-db
-    # <string> Deprecated, use secureJsonData.password
-    password:
-    # <string> database user, if used
-    user: tango
-    # <string> database name, if used
-    database: hdbpp
-    # <bool> enable/disable basic auth
-    basicAuth: false
-    # <string> basic auth username
-    basicAuthUser:
-    # <string> Deprecated, use secureJsonData.basicAuthPassword
-    basicAuthPassword:
-    # <bool> enable/disable with credentials headers
-    withCredentials:
-    # <bool> mark as default datasource. Max one per org
-    isDefault: true
-    # <map> fields that will be converted to json and stored in jsonData
-    jsonData:
-    # <string> json object of data that will be encrypted.
-    secureJsonData:
-      # <string> database password, if used
-      password: tango
-    version: 1
-    # <bool> allow users to edit datasources from the UI.
-    editable: false

tangostationcontrol/tangostationcontrol/toolkit/README.md

@@ -4,9 +4,9 @@ The Archiver class in archiver.py defines the methods to manage the device attri
 
 The main components (and the relative Docker containers) are:
 
-- Configuration Manager (container: hdbpp-cm): Device server that assists in adding, modifying, moving, deleting an Attribute to/from the archiving system
-- Event Subscriber (container: hdbpp-es): The EventSubscriber TANGO device server, is the archiving system engine. On typical usage, it will subscribe to archive events on request by the ConfigurationManager device. The EventSubscriber is designed to start archiving all the already configured Attributes, even if the ConfigurationManager is not running. Moreover, being a TANGO device, the EventSubscriber configuration can be managed with Jive.
-- Archiving DBMS (container: archiver-maria-db): Specific Database devoted to storing attribute values.
+- Configuration Manager (container: hdbppts-cm): Device server that assists in adding, modifying, moving, deleting an Attribute to/from the archiving system
+- Event Subscriber (container: hdbppts-es): The EventSubscriber TANGO device server, is the archiving system engine. On typical usage, it will subscribe to archive events on request by the ConfigurationManager device. The EventSubscriber is designed to start archiving all the already configured Attributes, even if the ConfigurationManager is not running.
+- Archiving DBMS (container: archiver-timescale): Specific Database devoted to storing attribute values.
 - (Optional) HDB++ Viewer (container: hdbpp-viewer): Standalone JAVA application designed to monitor signals coming from database
 
 ## Archiver creation

tangostationcontrol/tangostationcontrol/toolkit/retriever.py

@@ -8,7 +8,6 @@ from sqlalchemy import create_engine, and_
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy.orm.exc import NoResultFound
 import importlib
-import numpy
 
 class Retriever(ABC):
     """
@@ -119,102 +118,6 @@ class Retriever(ABC):
             raise ValueError(f"Attribute {attribute_fqname} not found!") from e
         return result
 
-class RetrieverMySQL(Retriever):
-    
-    def __init__(self, cm_name: str = 'archiving/hdbpp/confmanager01'):
-        self.cm_name = cm_name
-
-        super().__init__()
-    
-    def connect_to_archiving_db(self):
-        """
-        Returns a session to a MySQL DBMS using default credentials.
-        """
-        creds = get_db_config(self.cm_name)
-
-        # Set sqlalchemy library connection
-        if creds["host"] == 'archiver-maria-db':
-            creds["libname"] = 'mysql+pymysql'         
-        else:
-            raise ValueError(f"Invalid hostname: {creds['host']}, we only support 'archiver-maria-db'")
-
-        Session = self.create_session(creds)
-        return Session()
-    
-    def set_archiver_base(self):
-        """
-        Sets the right mapper class following the DBMS connection
-        """    
-        return importlib.import_module('.archiver_base_mysql', package=__package__)
-    
-    def get_attribute_datatype(self,attribute_fqname: str):
-        """
-        Takes as input the fully-qualified name of an attribute and returns its Data-Type.
-        Data Type name indicates the type (e.g. string, int, ...) and the read/write property. The name is used
-        as DB table name suffix in which values are stored.
-        """
-        domain, family, member, name = split_tango_name(attribute_fqname,"attribute")
-        try:
-            result = self.session.query(self.ab.DataType.data_type).join(self.ab.Attribute,self.ab.Attribute.att_conf_data_type_id==self.ab.DataType.att_conf_data_type_id).\
-                        filter(and_(self.ab.Attribute.domain == domain, self.ab.Attribute.family == family, self.ab.Attribute.member == member, self.ab.Attribute.name == name)).one()
-            return result[0]
-        except (AttributeError, TypeError, NoResultFound) as e:
-            raise ValueError(f"Attribute {attribute_fqname} not found!") from e
-    
-    def get_attribute_value_by_hours(self,attribute_fqname: str, hours: float = 1.0):
-        """
-        Takes as input the attribute fully-qualified name and the number of past hours since the actual time 
-        (e.g. hours=1 retrieves values in the last hour, hours=8.5 retrieves values in the last eight hours and half).
-        Returns a list of timestamps and a list of values
-        """
-        attr_datatype = self.get_attribute_datatype(attribute_fqname)
-        # Retrieves the class that maps the DB table given the tablename
-        tablename = f"att_{attr_datatype}"
-        return super().get_attribute_value_by_hours(attribute_fqname,hours,tablename) 
-
-
-    def get_attribute_value_by_interval(self,attribute_fqname: str, start_time: datetime, stop_time: datetime):
-        """
-        Takes as input the attribute name and a certain starting and ending point-time. 
-        The datetime format is pretty flexible (e.g. "YYYY-MM-dd hh:mm:ss").
-        Returns a list of timestamps and a list of values
-        """
-        attr_datatype = self.get_attribute_datatype(attribute_fqname)
-        # Retrieves the class that maps the DB table given the tablename
-        tablename = f"att_{attr_datatype}"           
-        return super().get_attribute_value_by_interval(attribute_fqname,start_time,stop_time,tablename)
-    
-    # DRAFT #
-    def get_masked_fpga_temp(self,start_time: datetime, stop_time: datetime,temp_attr_name:str='stat/sdp/1/fpga_temp_r',
-                    mask_attr_name:str='stat/sdp/1/tr_fpga_mask_r'):
-        """
-        Returns a list of SDP/fpga_temp_r values, but only if SDP/tr_fpga_mask_r values are TRUE
-        """
-        mask_values = self.get_attribute_value_by_interval(mask_attr_name,start_time,stop_time)
-        temp_values = self.get_attribute_value_by_interval(temp_attr_name,start_time,stop_time)
-        # Since timestamps can be not syncrhonized, remove first or last element from arrays
-        if len(mask_values)==len(temp_values):
-            first_mask_datatime = mask_values[0].data_time
-            first_temp_datatime = temp_values[0].data_time
-            if (first_mask_datatime>first_temp_datatime):
-                mask_values = mask_values[:-int(mask_values[0].dim_x_r)]
-                temp_values = temp_values[int(temp_values[0].dim_x_r):]
-            elif (first_mask_datatime<first_temp_datatime):
-                mask_values = mask_values[int(mask_values[0].dim_x_r)]
-                temp_values = temp_values[:-int(temp_values[0].dim_x_r):]
-        else:
-            raise Exception
-        # Convert DB Array records into Python lists
-        mask_data = self.ab.build_array_from_record(mask_values,mask_values[0].dim_x_r)
-        temp_data = self.ab.build_array_from_record(temp_values,temp_values[0].dim_x_r)
-        # Extract only the value from the array 
-        mask_array_values = self.ab.get_values_from_record(mask_data)
-        temp_array_values = self.ab.get_values_from_record(temp_data)
-        # Multiply the matrix
-        #masked_values = np.multiply(temp_array_values,mask_array_values)
-        masked_values = numpy.ma.masked_array(temp_array_values,mask=numpy.invert(mask_array_values.astype(bool)))
-        return masked_values, mask_values, temp_values
-
 class RetrieverTimescale(Retriever):
     
     def __init__(self, cm_name: str = 'archiving/hdbppts/confmanager01'):