From b5953afec314e29c38630b6045b59827a483d396 Mon Sep 17 00:00:00 2001
From: stedif <stefano.difrischia@inaf.it>
Date: Mon, 24 Oct 2022 17:13:47 +0200
Subject: [PATCH] L2SS-1030: implement dump ConfigDB
---
.../common/configuration.py | 281 ++++++++++++++++++
.../devices/configuration_device.py | 13 +-
.../devices/test_device_configuration.py | 8 +
3 files changed, 300 insertions(+), 2 deletions(-)
create mode 100644 tangostationcontrol/tangostationcontrol/common/configuration.py
diff --git a/tangostationcontrol/tangostationcontrol/common/configuration.py b/tangostationcontrol/tangostationcontrol/common/configuration.py
new file mode 100644
index 000000000..bc7a6e07e
--- /dev/null
+++ b/tangostationcontrol/tangostationcontrol/common/configuration.py
@@ -0,0 +1,281 @@
+from collections import defaultdict
+import six
+try:
+ from collections.abc import MutableMapping
+except ImportError:
+ from collections import MutableMapping
+try:
+ from collections.abc import Mapping
+except ImportError:
+ from collections import Mapping
+from itertools import islice
+
+def get_servers_with_filters(dbproxy, server="*", clss="*", device="*",
+ properties=True, attribute_properties=True,
+ aliases=True, dservers=False,
+ subdevices=False, uppercase_devices=False,
+ timeout=10):
+ """
+ A performant way to get servers and devices in bulk from the DB
+ by direct SQL statements and joins, instead of e.g. using one
+ query to get the properties of each device.
+
+ TODO: are there any length restrictions on the query results? In
+ that case, use limit and offset to get page by page.
+ """
+
+ server = server.replace("*", "%") # mysql wildcards
+ clss = clss.replace("*", "%")
+ device = device.replace("*", "%")
+
+ # Collect all info about each device in this dict
+ devices = CaselessDictionary()
+
+ # Queries can sometimes take more than de default 3 s, so it's
+ # good to increase the timeout a bit.
+ # TODO: maybe instead use automatic retry and increase timeout
+ # each time?
+ dbproxy.set_timeout_millis(timeout * 1000)
+
+ if properties:
+ # Get all relevant device properties
+ query = (
+ "SELECT device, property_device.name, property_device.value"
+ " FROM property_device"
+ " INNER JOIN device ON property_device.device = device.name"
+ " WHERE server LIKE '%s' AND class LIKE '%s' AND device LIKE '%s'")
+ if not dservers:
+ query += " AND class != 'DServer'"
+ if not subdevices:
+ query += " AND property_device.name != '__SubDevices'"
+ query += " ORDER BY property_device.count ASC"
+ _, result = dbproxy.command_inout("DbMySqlSelect",
+ query % (server, clss, device))
+ for d, p, v in nwise(result, 3):
+ dev = devices.setdefault(d, {})
+ # Make sure to use caseless dicts, since in principle property names
+ # should be caseles too.
+ props = dev.setdefault("properties", CaselessDictionary())
+ value = props.setdefault(p, [])
+ value.append(v)
+
+ if attribute_properties:
+ # Get all relevant attribute properties
+ query = (
+ "SELECT device, attribute, property_attribute_device.name,"
+ " property_attribute_device.value"
+ " FROM property_attribute_device"
+ " INNER JOIN device ON property_attribute_device.device ="
+ " device.name"
+ " WHERE server LIKE '%s' AND class LIKE '%s' AND device LIKE '%s'")
+ if not dservers:
+ query += " AND class != 'DServer'"
+ query += " ORDER BY property_attribute_device.count ASC"
+ _, result = dbproxy.command_inout("DbMySqlSelect", query % (server, clss, device))
+ for d, a, p, v in nwise(result, 4):
+ dev = devices.setdefault(d, {})
+ props = dev.setdefault("attribute_properties", CaselessDictionary())
+ attr = props.setdefault(a, CaselessDictionary())
+ value = attr.setdefault(p, [])
+ value.append(v)
+
+ # dump relevant servers
+ query = (
+ "SELECT server, class, name, alias FROM device"
+ " WHERE server LIKE '%s' AND class LIKE '%s' AND name LIKE '%s'")
+
+ if not dservers:
+ query += " AND class != 'DServer'"
+ _, result = dbproxy.command_inout("DbMySqlSelect", query % (server, clss, device))
+
+ # combine all the information we have
+ servers = SetterDict()
+ for s, c, d, a in nwise(result, 4):
+ try:
+ srv, inst = s.split("/")
+ except ValueError:
+ # Malformed server name? It can happen!
+ continue
+ device = devices.get(d, {})
+ if a and aliases:
+ device["alias"] = a
+ devname = maybe_upper(d, uppercase_devices)
+ servers[srv][inst][c][devname] = device
+
+ return servers
+
+def nwise(it, n):
+ # [s_0, s_1, ...] => [(s_0, ..., s_(n-1)), (s_n, ... s_(2n-1)), ...]
+ return list(zip(*[islice(it, i, None, n) for i in range(n)]))
+
+def maybe_upper(s, upper=False):
+ if upper:
+ return s.upper()
+ return s
+
+class CaselessString(object):
+ """
+ A mixin to make a string subclass case-insensitive in dict lookups.
+ """
+
+ def __hash__(self):
+ return hash(self.lower())
+
+ def __eq__(self, other):
+ return self.lower() == other.lower()
+
+ def __cmp__(self, other):
+ return self.lower().__cmp__(other.lower())
+
+ @classmethod
+ def make_caseless(cls, string):
+ if isinstance(string, six.text_type):
+ return CaselessUnicode(string)
+ return CaselessStr(string)
+
+class CaselessStr(CaselessString, str):
+ pass
+
+
+class CaselessUnicode(CaselessString, six.text_type):
+ pass
+
+class CaselessDictionary(MutableMapping):
+ """
+ A dictionary-like object which ignores but preserves the case of strings.
+
+ Example::
+
+ >>> cdict = CaselessDictionary()
+
+ Access is case-insensitive::
+
+ >>> cdict['a'] = 1
+ >>> cdict['A']
+ 1
+
+ As is writing::
+
+ >>> cdict['key'] = 123
+ >>> cdict['KeY'] = 456
+ >>> cdict['key']
+ 456
+
+ And deletion::
+
+ >>> del cdict['A']
+ >>> 'a' in cdict
+ False
+ >>> 'A' in cdict
+ False
+
+ However, the case of keys is preserved (the case of overridden keys will be
+ the first one which was set)::
+
+ >>> cdict['aBcDeF'] = 1
+ >>> sorted(list(cdict))
+ ['aBcDeF', 'key']
+ """
+
+ def __init__(self, *args, **kwargs):
+ self.__dict__["_dict"] = {}
+ temp_dict = dict(*args, **kwargs)
+ for key, value in list(temp_dict.items()):
+ if isinstance(key, six.string_types):
+ key = CaselessString.make_caseless(key)
+
+ if isinstance(value, dict):
+ self._dict[key] = CaselessDictionary(value)
+ else:
+ self._dict[key] = value
+
+ def __getitem__(self, key):
+ return self._dict[CaselessString.make_caseless(key)]
+
+ def __setitem__(self, key, value):
+ self._dict[CaselessString.make_caseless(key)] = value
+
+ def __delitem__(self, key):
+ del self._dict[CaselessString.make_caseless(key)]
+
+ def __contains__(self, key):
+ return CaselessString.make_caseless(key) in self._dict
+
+ def __iter__(self):
+ return iter(self._dict)
+
+ def __len__(self):
+ return len(self._dict)
+
+ def keys(self):
+ # convert back to ordinary strings
+ return [str(k) for k in self._dict]
+
+ def items(self):
+ return list(zip(list(self.keys()), list(self.values())))
+
+class SetterDict(CaselessDictionary, defaultdict):
+ """
+ A recursive defaultdict with extra bells & whistles
+
+ It enables you to set keys at any depth without first creating the
+ intermediate levels, e.g.
+
+ d = SetterDict()
+ d["a"]["b"]["c"] = 1
+
+ It also allows access using normal getattr syntax, interchangeably:
+
+ d.a["b"].c == d.a.b.c == d["a"]["b"]["c"]
+
+ Note: only allows string keys for now.
+
+ Keys are caseless, meaning that the key "MyKey" is the same as
+ "mykey", "MYKEY", etc. The case
+
+ Note: this thing is dangerous! Accessing a non-existing key will
+ result in creating it, which means that confusing behavior is
+ likely. Please use it carefully and convert to an ordinary dict
+ (using to_dict()) when you're done creating it.
+ """
+
+ def __init__(self, value={}, factory=None):
+ factory = factory or SetterDict
+ self.__dict__["_factory"] = factory
+ CaselessDictionary.__init__(self)
+ defaultdict.__init__(self, factory)
+ for k, v in list(value.items()):
+ self[k] = v
+
+ def __getitem__(self, key):
+ try:
+ return CaselessDictionary.__getitem__(self, key)
+ except KeyError:
+ return self.__missing__(key)
+
+ def __setitem__(self, key, value):
+ if isinstance(value, SetterDict):
+ CaselessDictionary.__setitem__(self, key, value)
+ elif isinstance(value, Mapping):
+ CaselessDictionary.__setitem__(self, key, self._factory(value))
+ else:
+ CaselessDictionary.__setitem__(self, key, value)
+
+ def __getattr__(self, name):
+ return self.__getitem__(name)
+
+ def __setattr__(self, key, value):
+ return self.__setitem__(key, value)
+
+ def to_dict(self):
+ """
+ Returns a ordinary dict version of itself
+ """
+ result = {}
+ for key, value in list(self.items()):
+ if isinstance(value, SetterDict):
+ result[key] = value.to_dict()
+ else:
+ result[key] = value
+ return result
+
diff --git a/tangostationcontrol/tangostationcontrol/devices/configuration_device.py b/tangostationcontrol/tangostationcontrol/devices/configuration_device.py
index bc8222cd3..7f5092354 100644
--- a/tangostationcontrol/tangostationcontrol/devices/configuration_device.py
+++ b/tangostationcontrol/tangostationcontrol/devices/configuration_device.py
@@ -13,14 +13,16 @@ Handles and exposes the station configuration
"""
# PyTango imports
-from tango import AttrWriteType
+from tango import AttrWriteType, Database, DeviceProxy
from tango.server import attribute
# Additional import
+from tangostationcontrol.common.configuration import SetterDict, get_servers_with_filters
from tangostationcontrol.common.entrypoint import entry
from tangostationcontrol.devices.lofar_device import lofar_device
from tangostationcontrol.common.lofar_logging import device_logging_to_python
+import json
import logging
logger = logging.getLogger()
@@ -46,11 +48,18 @@ class Configuration(lofar_device):
N.B. it does not update, it loads a full new configuration.
"""
+ # TODO(Stefano): implement load configuration
self.proxy.tangodb_properties_RW = tangodb_properties
def _dump_configdb(self):
""" Returns the TangoDB station configuration as a JSON string """
- return 'Configuration'
+ dbproxy = DeviceProxy(Database().dev_name())
+ # Create a special dictionary to prevent Tango case errors
+ data = SetterDict()
+ servers = get_servers_with_filters(dbproxy)
+ data.servers.update(servers)
+ return json.dumps(data.to_dict(), ensure_ascii=False, indent=4, sort_keys=True)
+
# ----------
# Run server
diff --git a/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_configuration.py b/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_configuration.py
index 4a34cad03..71731810a 100644
--- a/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_configuration.py
+++ b/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_configuration.py
@@ -9,8 +9,16 @@
from .base import AbstractTestBases
+import json
class TestDeviceConfiguration(AbstractTestBases.TestDeviceBase):
def setUp(self):
super().setUp("STAT/Configuration/1")
+
+ def test_read_tangodb_properties(self):
+ """ Test whether the station control configuration is correctly retrieved as a JSON string """
+ tangodb_properties = self.proxy.tangodb_properties_RW
+ dbdata = json.loads(tangodb_properties)
+ self.assertTrue(type(dbdata), dict)
+ self.assertGreater(len(dbdata['servers']), 0)
--
GitLab