Skip to content
Snippets Groups Projects
Commit 4de99cc9 authored by Stefano Di Frischia's avatar Stefano Di Frischia
Browse files

L2SS-1030: rewrite configuration file

parent 0e7f75f0
No related branches found
No related tags found
1 merge request!468Resolve L2SS-1030 "Create configuration device"
Hide whitespace changes
Inline Side-by-side
tangostationcontrol/tangostationcontrol/common/configuration.py
+ 101
− 293
View file @ 4de99cc9
#
# Code re-adapted from dsconfig python package in DSConfig container
# See: https://gitlab.com/MaxIV/lib-maxiv-dsconfig
# License: GPLv3
#
from tango import DeviceProxy from tango import DeviceProxy
from collections import defaultdict
import six
try:
from collections.abc import MutableMapping
except ImportError:
from collections import MutableMapping
try:
from collections.abc import Mapping
except ImportError:
from collections import Mapping
from itertools import islice from itertools import islice
def get_db_data(db): DEVICE_PROPERTIES_QUERY = "SELECT device, property_device.name, property_device.value FROM property_device \
# dump TANGO database into JSON. Optionally filter which things to include INNER JOIN device ON property_device.device = device.name \
# (currently only "positive" filters are possible; you can say which WHERE server LIKE '%' AND class LIKE '%' AND device LIKE '%' \
# servers/classes/devices to include, but you can't exclude selectively) AND class != 'DServer' \
# By default, dserver devices aren't included! AND property_device.name != '__SubDevices' \
dbproxy = DeviceProxy(db.dev_name()) ORDER BY device, property_device.name, property_device.count ASC"
data = SetterDict()
# the user did not specify a pattern, so we will dump *everything* ATTRS_PROPERTIES_QUERY = "SELECT device, attribute, property_attribute_device.name, \
servers = get_servers_with_filters(dbproxy) property_attribute_device.value \
data.servers.update(servers) FROM property_attribute_device \
return data.to_dict() INNER JOIN device ON property_attribute_device.device = device.name \
WHERE server LIKE '%' AND class LIKE '%' AND device LIKE '%' \
def get_servers_with_filters(dbproxy, server="*", clss="*", device="*", AND class != 'DServer' \
properties=True, attribute_properties=True, ORDER BY device, property_attribute_device.name, property_attribute_device.count ASC"
aliases=True, dservers=False,
subdevices=False, uppercase_devices=False, SERVER_QUERY = "SELECT server, class, name FROM device \
timeout=10): WHERE server LIKE '%' AND class LIKE '%' AND name LIKE '%' \
""" AND class != 'DServer' \
A performant way to get servers and devices in bulk from the DB ORDER BY server ASC"
by direct SQL statements and joins, instead of e.g. using one
query to get the properties of each device. def get_db_data(db, tangodb_timeout:int = 10000):
""" Dump TANGO database into dictionary """
TODO: are there any length restrictions on the query results? In dbproxy = DeviceProxy(db.dev_name()) # TangoDB
that case, use limit and offset to get page by page. dbproxy.set_timeout_millis(tangodb_timeout) # Set a security timeout (default is 3000ms)
""" # Create empty dictionaries to be populated
devices_dict = {}
server = server.replace("*", "%") # mysql wildcards server_dict = {}
clss = clss.replace("*", "%") # Query TangoDb with built-in function for devices data
device = device.replace("*", "%") _, raw_result = dbproxy.command_inout("DbMySqlSelect", DEVICE_PROPERTIES_QUERY)
# Remodel the query result
# Collect all info about each device in this dict device_property_result = query_to_tuples(raw_result, 3)
devices = CaselessDictionary() # Populate devices dictionary from query data
devices_dict = model_devices_dict(devices_dict, device_property_result)
# Queries can sometimes take more than de default 3 s, so it's # Query TangoDb with built-in function for attributes data
# good to increase the timeout a bit. _, raw_result = dbproxy.command_inout("DbMySqlSelect", ATTRS_PROPERTIES_QUERY)
dbproxy.set_timeout_millis(timeout * 1000) # Remodel the query result
attrs_property_result = query_to_tuples(raw_result, 4)
if properties: # Populate devices dictionary from query data
# Get all relevant device properties devices_dict = model_attrs_dict(devices_dict, attrs_property_result)
query = ( # Query TangoDb with built-in function for server data
"SELECT device, property_device.name, property_device.value" _, raw_result = dbproxy.command_inout("DbMySqlSelect", SERVER_QUERY)
" FROM property_device" # Remodel the query result
" INNER JOIN device ON property_device.device = device.name" server_result = query_to_tuples(raw_result, 3)
" WHERE server LIKE '%s' AND class LIKE '%s' AND device LIKE '%s'") # Populate server dictionary from query data and merge it with devices dict
if not dservers: server_dict = model_server_dict(server_dict, devices_dict, server_result)
query += " AND class != 'DServer'" return server_dict
if not subdevices:
query += " AND property_device.name != '__SubDevices'" def model_devices_dict(devices_dict:dict, result:list):
query += " ORDER BY property_device.count ASC" """ Model a devices dictionary with the following structure:
_, result = dbproxy.command_inout("DbMySqlSelect", 'device_name': { 'properties' : { 'property_name': ['property_value'] } }
query % (server, clss, device))
for d, p, v in nwise(result, 3):
dev = devices.setdefault(d, {})
# Make sure to use caseless dicts, since in principle property names
# should be caseles too.
props = dev.setdefault("properties", CaselessDictionary())
value = props.setdefault(p, [])
value.append(v)
if attribute_properties:
# Get all relevant attribute properties
query = (
"SELECT device, attribute, property_attribute_device.name,"
" property_attribute_device.value"
" FROM property_attribute_device"
" INNER JOIN device ON property_attribute_device.device ="
" device.name"
" WHERE server LIKE '%s' AND class LIKE '%s' AND device LIKE '%s'")
if not dservers:
query += " AND class != 'DServer'"
query += " ORDER BY property_attribute_device.count ASC"
_, result = dbproxy.command_inout("DbMySqlSelect", query % (server, clss, device))
for d, a, p, v in nwise(result, 4):
dev = devices.setdefault(d, {})
props = dev.setdefault("attribute_properties", CaselessDictionary())
attr = props.setdefault(a, CaselessDictionary())
value = attr.setdefault(p, [])
value.append(v)
# dump relevant servers
query = (
"SELECT server, class, name, alias FROM device"
" WHERE server LIKE '%s' AND class LIKE '%s' AND name LIKE '%s'")
if not dservers:
query += " AND class != 'DServer'"
_, result = dbproxy.command_inout("DbMySqlSelect", query % (server, clss, device))
# combine all the information we have
servers = SetterDict()
for s, c, d, a in nwise(result, 4):
try:
srv, inst = s.split("/")
except ValueError:
# Malformed server name? It can happen!
continue
device = devices.get(d, {})
if a and aliases:
device["alias"] = a
devname = maybe_upper(d, uppercase_devices)
servers[srv][inst][c][devname] = device
return servers
def nwise(it, n):
# [s_0, s_1, ...] => [(s_0, ..., s_(n-1)), (s_n, ... s_(2n-1)), ...]
return list(zip(*[islice(it, i, None, n) for i in range(n)]))
def maybe_upper(s, upper=False):
if upper:
return s.upper()
return s
class CaselessString(object):
""" """
A mixin to make a string subclass case-insensitive in dict lookups. for device, property, value in result:
# lowercase data
device = device.lower()
property = property.lower()
# model dictionary
device_data = devices_dict.setdefault(device, {})
property_data = device_data.setdefault("properties", {})
value_data = property_data.setdefault(property, [])
value_data.append(value)
return devices_dict
def model_attrs_dict(devices_dict:dict, result:list):
""" Model a device dictionary with the following structure :
'device_name': { 'attribute_properties' : { 'attribute_name': {'property_name' : ['property_value'] } } }
""" """
for device, attribute, property, value in result:
def __hash__(self): # lowercase data
return hash(self.lower()) device = device.lower()
attribute = attribute.lower()
def __eq__(self, other): property = property.lower()
return self.lower() == other.lower() # model dictionary
device_data = devices_dict.setdefault(device, {})
def __cmp__(self, other): property_data = device_data.setdefault("attribute_properties", {})
return self.lower().__cmp__(other.lower()) attr_data = property_data.setdefault(attribute, {})
value_data = attr_data.setdefault(property, [])
@classmethod value_data.append(value)
def make_caseless(cls, string): return devices_dict
if isinstance(string, six.text_type):
return CaselessUnicode(string) def model_server_dict(server_dict:dict, devices_dict:dict, result:list):
return CaselessStr(string) """ Model the server dictionary and merge it with the devices dictionary.
At the end of the process, the dictionary will have the following structure :
class CaselessStr(CaselessString, str): 'server_name' : { 'server_instance' : { 'server_class' :
pass 'device_name': { 'properties' : { 'property_name': ['property_value'] } },
{ 'attribute_properties' : { 'attribute_name': {'property_name' : ['property_value'] } } } } }
class CaselessUnicode(CaselessString, six.text_type):
pass
class CaselessDictionary(MutableMapping):
""" """
A dictionary-like object which ignores but preserves the case of strings. for server, sclass, device in result:
# lowercase data
Example:: device = device.lower()
server = server.lower()
>>> cdict = CaselessDictionary() sclass = sclass.lower()
# model dictionary
Access is case-insensitive:: sname, instance = server.split('/')
device_data = devices_dict.get(device, {})
>>> cdict['a'] = 1 server_data = server_dict.setdefault(sname, {})
>>> cdict['A'] instance_data = server_data.setdefault(instance, {})
1 class_data = instance_data.setdefault(sclass, {})
# merge the two dictionaries
As is writing:: server_dict[sname][instance][sclass][device] = device_data
return server_dict
>>> cdict['key'] = 123
>>> cdict['KeY'] = 456 def query_to_tuples(result, num_cols):
>>> cdict['key'] """ Given a query result and its number of columns, transforms the raw result in a list of tuples """
456 return list(zip(*[islice(result, i, None, num_cols) for i in range(num_cols)]))
And deletion::
>>> del cdict['A']
>>> 'a' in cdict
False
>>> 'A' in cdict
False
However, the case of keys is preserved (the case of overridden keys will be
the first one which was set)::
>>> cdict['aBcDeF'] = 1
>>> sorted(list(cdict))
['aBcDeF', 'key']
"""
def __init__(self, *args, **kwargs):
self.__dict__["_dict"] = {}
temp_dict = dict(*args, **kwargs)
for key, value in list(temp_dict.items()):
if isinstance(key, six.string_types):
key = CaselessString.make_caseless(key)
if isinstance(value, dict):
self._dict[key] = CaselessDictionary(value)
else:
self._dict[key] = value
def __getitem__(self, key):
return self._dict[CaselessString.make_caseless(key)]
def __setitem__(self, key, value):
self._dict[CaselessString.make_caseless(key)] = value
def __delitem__(self, key):
del self._dict[CaselessString.make_caseless(key)]
def __contains__(self, key):
return CaselessString.make_caseless(key) in self._dict
def __iter__(self):
return iter(self._dict)
def __len__(self):
return len(self._dict)
def keys(self):
# convert back to ordinary strings
return [str(k) for k in self._dict]
def items(self):
return list(zip(list(self.keys()), list(self.values())))
class SetterDict(CaselessDictionary, defaultdict):
"""
A recursive defaultdict with extra bells & whistles
It enables you to set keys at any depth without first creating the
intermediate levels, e.g.
d = SetterDict()
d["a"]["b"]["c"] = 1
It also allows access using normal getattr syntax, interchangeably:
d.a["b"].c == d.a.b.c == d["a"]["b"]["c"]
Note: only allows string keys for now.
Keys are caseless, meaning that the key "MyKey" is the same as
"mykey", "MYKEY", etc. The case
Note: this thing is dangerous! Accessing a non-existing key will
result in creating it, which means that confusing behavior is
likely. Please use it carefully and convert to an ordinary dict
(using to_dict()) when you're done creating it.
"""
def __init__(self, value=None, factory=None):
factory = factory or SetterDict
value = value or {}
self.__dict__["_factory"] = factory
CaselessDictionary.__init__(self)
defaultdict.__init__(self, factory)
for k, v in list(value.items()):
self[k] = v
def __getitem__(self, key):
try:
return CaselessDictionary.__getitem__(self, key)
except KeyError:
return self.__missing__(key)
def __setitem__(self, key, value):
if isinstance(value, SetterDict):
CaselessDictionary.__setitem__(self, key, value)
elif isinstance(value, Mapping):
CaselessDictionary.__setitem__(self, key, self._factory(value))
else:
CaselessDictionary.__setitem__(self, key, value)
def __getattr__(self, name):
return self.__getitem__(name)
def __setattr__(self, key, value):
return self.__setitem__(key, value)
def to_dict(self):
"""
Returns a ordinary dict version of itself
"""
result = {}
for key, value in list(self.items()):
if isinstance(value, SetterDict):
result[key] = value.to_dict()
else:
result[key] = value
return result
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment