From 4de99cc9b17b4bacbafe6a59c5232576adab6a67 Mon Sep 17 00:00:00 2001
From: stedif <stefano.difrischia@inaf.it>
Date: Wed, 26 Oct 2022 15:17:08 +0200
Subject: [PATCH] L2SS-1030: rewrite configuration file
---
.../common/configuration.py | 394 +++++-------------
1 file changed, 101 insertions(+), 293 deletions(-)
diff --git a/tangostationcontrol/tangostationcontrol/common/configuration.py b/tangostationcontrol/tangostationcontrol/common/configuration.py
index 58de15909..ad3752528 100644
--- a/tangostationcontrol/tangostationcontrol/common/configuration.py
+++ b/tangostationcontrol/tangostationcontrol/common/configuration.py
@@ -1,300 +1,108 @@
-#
-# Code re-adapted from dsconfig python package in DSConfig container
-# See: https://gitlab.com/MaxIV/lib-maxiv-dsconfig
-# License: GPLv3
-#
-
from tango import DeviceProxy
-from collections import defaultdict
-import six
-try:
- from collections.abc import MutableMapping
-except ImportError:
- from collections import MutableMapping
-try:
- from collections.abc import Mapping
-except ImportError:
- from collections import Mapping
from itertools import islice
-def get_db_data(db):
- # dump TANGO database into JSON. Optionally filter which things to include
- # (currently only "positive" filters are possible; you can say which
- # servers/classes/devices to include, but you can't exclude selectively)
- # By default, dserver devices aren't included!
- dbproxy = DeviceProxy(db.dev_name())
- data = SetterDict()
- # the user did not specify a pattern, so we will dump *everything*
- servers = get_servers_with_filters(dbproxy)
- data.servers.update(servers)
- return data.to_dict()
-
-def get_servers_with_filters(dbproxy, server="*", clss="*", device="*",
- properties=True, attribute_properties=True,
- aliases=True, dservers=False,
- subdevices=False, uppercase_devices=False,
- timeout=10):
- """
- A performant way to get servers and devices in bulk from the DB
- by direct SQL statements and joins, instead of e.g. using one
- query to get the properties of each device.
-
- TODO: are there any length restrictions on the query results? In
- that case, use limit and offset to get page by page.
- """
-
- server = server.replace("*", "%") # mysql wildcards
- clss = clss.replace("*", "%")
- device = device.replace("*", "%")
-
- # Collect all info about each device in this dict
- devices = CaselessDictionary()
-
- # Queries can sometimes take more than de default 3 s, so it's
- # good to increase the timeout a bit.
- dbproxy.set_timeout_millis(timeout * 1000)
-
- if properties:
- # Get all relevant device properties
- query = (
- "SELECT device, property_device.name, property_device.value"
- " FROM property_device"
- " INNER JOIN device ON property_device.device = device.name"
- " WHERE server LIKE '%s' AND class LIKE '%s' AND device LIKE '%s'")
- if not dservers:
- query += " AND class != 'DServer'"
- if not subdevices:
- query += " AND property_device.name != '__SubDevices'"
- query += " ORDER BY property_device.count ASC"
- _, result = dbproxy.command_inout("DbMySqlSelect",
- query % (server, clss, device))
- for d, p, v in nwise(result, 3):
- dev = devices.setdefault(d, {})
- # Make sure to use caseless dicts, since in principle property names
- # should be caseles too.
- props = dev.setdefault("properties", CaselessDictionary())
- value = props.setdefault(p, [])
- value.append(v)
-
- if attribute_properties:
- # Get all relevant attribute properties
- query = (
- "SELECT device, attribute, property_attribute_device.name,"
- " property_attribute_device.value"
- " FROM property_attribute_device"
- " INNER JOIN device ON property_attribute_device.device ="
- " device.name"
- " WHERE server LIKE '%s' AND class LIKE '%s' AND device LIKE '%s'")
- if not dservers:
- query += " AND class != 'DServer'"
- query += " ORDER BY property_attribute_device.count ASC"
- _, result = dbproxy.command_inout("DbMySqlSelect", query % (server, clss, device))
- for d, a, p, v in nwise(result, 4):
- dev = devices.setdefault(d, {})
- props = dev.setdefault("attribute_properties", CaselessDictionary())
- attr = props.setdefault(a, CaselessDictionary())
- value = attr.setdefault(p, [])
- value.append(v)
-
- # dump relevant servers
- query = (
- "SELECT server, class, name, alias FROM device"
- " WHERE server LIKE '%s' AND class LIKE '%s' AND name LIKE '%s'")
-
- if not dservers:
- query += " AND class != 'DServer'"
- _, result = dbproxy.command_inout("DbMySqlSelect", query % (server, clss, device))
-
- # combine all the information we have
- servers = SetterDict()
- for s, c, d, a in nwise(result, 4):
- try:
- srv, inst = s.split("/")
- except ValueError:
- # Malformed server name? It can happen!
- continue
- device = devices.get(d, {})
- if a and aliases:
- device["alias"] = a
- devname = maybe_upper(d, uppercase_devices)
- servers[srv][inst][c][devname] = device
-
- return servers
-
-def nwise(it, n):
- # [s_0, s_1, ...] => [(s_0, ..., s_(n-1)), (s_n, ... s_(2n-1)), ...]
- return list(zip(*[islice(it, i, None, n) for i in range(n)]))
-
-def maybe_upper(s, upper=False):
- if upper:
- return s.upper()
- return s
-
-class CaselessString(object):
+DEVICE_PROPERTIES_QUERY = "SELECT device, property_device.name, property_device.value FROM property_device \
+ INNER JOIN device ON property_device.device = device.name \
+ WHERE server LIKE '%' AND class LIKE '%' AND device LIKE '%' \
+ AND class != 'DServer' \
+ AND property_device.name != '__SubDevices' \
+ ORDER BY device, property_device.name, property_device.count ASC"
+
+ATTRS_PROPERTIES_QUERY = "SELECT device, attribute, property_attribute_device.name, \
+ property_attribute_device.value \
+ FROM property_attribute_device \
+ INNER JOIN device ON property_attribute_device.device = device.name \
+ WHERE server LIKE '%' AND class LIKE '%' AND device LIKE '%' \
+ AND class != 'DServer' \
+ ORDER BY device, property_attribute_device.name, property_attribute_device.count ASC"
+
+SERVER_QUERY = "SELECT server, class, name FROM device \
+ WHERE server LIKE '%' AND class LIKE '%' AND name LIKE '%' \
+ AND class != 'DServer' \
+ ORDER BY server ASC"
+
+def get_db_data(db, tangodb_timeout:int = 10000):
+ """ Dump TANGO database into dictionary """
+ dbproxy = DeviceProxy(db.dev_name()) # TangoDB
+ dbproxy.set_timeout_millis(tangodb_timeout) # Set a security timeout (default is 3000ms)
+ # Create empty dictionaries to be populated
+ devices_dict = {}
+ server_dict = {}
+ # Query TangoDb with built-in function for devices data
+ _, raw_result = dbproxy.command_inout("DbMySqlSelect", DEVICE_PROPERTIES_QUERY)
+ # Remodel the query result
+ device_property_result = query_to_tuples(raw_result, 3)
+ # Populate devices dictionary from query data
+ devices_dict = model_devices_dict(devices_dict, device_property_result)
+ # Query TangoDb with built-in function for attributes data
+ _, raw_result = dbproxy.command_inout("DbMySqlSelect", ATTRS_PROPERTIES_QUERY)
+ # Remodel the query result
+ attrs_property_result = query_to_tuples(raw_result, 4)
+ # Populate devices dictionary from query data
+ devices_dict = model_attrs_dict(devices_dict, attrs_property_result)
+ # Query TangoDb with built-in function for server data
+ _, raw_result = dbproxy.command_inout("DbMySqlSelect", SERVER_QUERY)
+ # Remodel the query result
+ server_result = query_to_tuples(raw_result, 3)
+ # Populate server dictionary from query data and merge it with devices dict
+ server_dict = model_server_dict(server_dict, devices_dict, server_result)
+ return server_dict
+
+def model_devices_dict(devices_dict:dict, result:list):
+ """ Model a devices dictionary with the following structure:
+ 'device_name': { 'properties' : { 'property_name': ['property_value'] } }
"""
- A mixin to make a string subclass case-insensitive in dict lookups.
+ for device, property, value in result:
+ # lowercase data
+ device = device.lower()
+ property = property.lower()
+ # model dictionary
+ device_data = devices_dict.setdefault(device, {})
+ property_data = device_data.setdefault("properties", {})
+ value_data = property_data.setdefault(property, [])
+ value_data.append(value)
+ return devices_dict
+
+def model_attrs_dict(devices_dict:dict, result:list):
+ """ Model a device dictionary with the following structure :
+ 'device_name': { 'attribute_properties' : { 'attribute_name': {'property_name' : ['property_value'] } } }
"""
-
- def __hash__(self):
- return hash(self.lower())
-
- def __eq__(self, other):
- return self.lower() == other.lower()
-
- def __cmp__(self, other):
- return self.lower().__cmp__(other.lower())
-
- @classmethod
- def make_caseless(cls, string):
- if isinstance(string, six.text_type):
- return CaselessUnicode(string)
- return CaselessStr(string)
-
-class CaselessStr(CaselessString, str):
- pass
-
-
-class CaselessUnicode(CaselessString, six.text_type):
- pass
-
-class CaselessDictionary(MutableMapping):
+ for device, attribute, property, value in result:
+ # lowercase data
+ device = device.lower()
+ attribute = attribute.lower()
+ property = property.lower()
+ # model dictionary
+ device_data = devices_dict.setdefault(device, {})
+ property_data = device_data.setdefault("attribute_properties", {})
+ attr_data = property_data.setdefault(attribute, {})
+ value_data = attr_data.setdefault(property, [])
+ value_data.append(value)
+ return devices_dict
+
+def model_server_dict(server_dict:dict, devices_dict:dict, result:list):
+ """ Model the server dictionary and merge it with the devices dictionary.
+ At the end of the process, the dictionary will have the following structure :
+ 'server_name' : { 'server_instance' : { 'server_class' :
+ 'device_name': { 'properties' : { 'property_name': ['property_value'] } },
+ { 'attribute_properties' : { 'attribute_name': {'property_name' : ['property_value'] } } } } }
"""
- A dictionary-like object which ignores but preserves the case of strings.
-
- Example::
-
- >>> cdict = CaselessDictionary()
-
- Access is case-insensitive::
-
- >>> cdict['a'] = 1
- >>> cdict['A']
- 1
-
- As is writing::
-
- >>> cdict['key'] = 123
- >>> cdict['KeY'] = 456
- >>> cdict['key']
- 456
-
- And deletion::
-
- >>> del cdict['A']
- >>> 'a' in cdict
- False
- >>> 'A' in cdict
- False
-
- However, the case of keys is preserved (the case of overridden keys will be
- the first one which was set)::
-
- >>> cdict['aBcDeF'] = 1
- >>> sorted(list(cdict))
- ['aBcDeF', 'key']
- """
-
- def __init__(self, *args, **kwargs):
- self.__dict__["_dict"] = {}
- temp_dict = dict(*args, **kwargs)
- for key, value in list(temp_dict.items()):
- if isinstance(key, six.string_types):
- key = CaselessString.make_caseless(key)
-
- if isinstance(value, dict):
- self._dict[key] = CaselessDictionary(value)
- else:
- self._dict[key] = value
-
- def __getitem__(self, key):
- return self._dict[CaselessString.make_caseless(key)]
-
- def __setitem__(self, key, value):
- self._dict[CaselessString.make_caseless(key)] = value
-
- def __delitem__(self, key):
- del self._dict[CaselessString.make_caseless(key)]
-
- def __contains__(self, key):
- return CaselessString.make_caseless(key) in self._dict
-
- def __iter__(self):
- return iter(self._dict)
-
- def __len__(self):
- return len(self._dict)
-
- def keys(self):
- # convert back to ordinary strings
- return [str(k) for k in self._dict]
-
- def items(self):
- return list(zip(list(self.keys()), list(self.values())))
-
-class SetterDict(CaselessDictionary, defaultdict):
- """
- A recursive defaultdict with extra bells & whistles
-
- It enables you to set keys at any depth without first creating the
- intermediate levels, e.g.
-
- d = SetterDict()
- d["a"]["b"]["c"] = 1
-
- It also allows access using normal getattr syntax, interchangeably:
-
- d.a["b"].c == d.a.b.c == d["a"]["b"]["c"]
-
- Note: only allows string keys for now.
-
- Keys are caseless, meaning that the key "MyKey" is the same as
- "mykey", "MYKEY", etc. The case
-
- Note: this thing is dangerous! Accessing a non-existing key will
- result in creating it, which means that confusing behavior is
- likely. Please use it carefully and convert to an ordinary dict
- (using to_dict()) when you're done creating it.
- """
-
- def __init__(self, value=None, factory=None):
- factory = factory or SetterDict
- value = value or {}
- self.__dict__["_factory"] = factory
- CaselessDictionary.__init__(self)
- defaultdict.__init__(self, factory)
- for k, v in list(value.items()):
- self[k] = v
-
- def __getitem__(self, key):
- try:
- return CaselessDictionary.__getitem__(self, key)
- except KeyError:
- return self.__missing__(key)
-
- def __setitem__(self, key, value):
- if isinstance(value, SetterDict):
- CaselessDictionary.__setitem__(self, key, value)
- elif isinstance(value, Mapping):
- CaselessDictionary.__setitem__(self, key, self._factory(value))
- else:
- CaselessDictionary.__setitem__(self, key, value)
-
- def __getattr__(self, name):
- return self.__getitem__(name)
-
- def __setattr__(self, key, value):
- return self.__setitem__(key, value)
-
- def to_dict(self):
- """
- Returns a ordinary dict version of itself
- """
- result = {}
- for key, value in list(self.items()):
- if isinstance(value, SetterDict):
- result[key] = value.to_dict()
- else:
- result[key] = value
- return result
-
+ for server, sclass, device in result:
+ # lowercase data
+ device = device.lower()
+ server = server.lower()
+ sclass = sclass.lower()
+ # model dictionary
+ sname, instance = server.split('/')
+ device_data = devices_dict.get(device, {})
+ server_data = server_dict.setdefault(sname, {})
+ instance_data = server_data.setdefault(instance, {})
+ class_data = instance_data.setdefault(sclass, {})
+ # merge the two dictionaries
+ server_dict[sname][instance][sclass][device] = device_data
+ return server_dict
+
+def query_to_tuples(result, num_cols):
+ """ Given a query result and its number of columns, transforms the raw result in a list of tuples """
+ return list(zip(*[islice(result, i, None, num_cols) for i in range(num_cols)]))
--
GitLab