L2SS-1030: implement dump ConfigDB

tangostationcontrol/tangostationcontrol/common/configuration.py

+from collections import defaultdict
+import six
+try:
+    from collections.abc import MutableMapping
+except ImportError:
+    from collections import MutableMapping
+try:
+    from collections.abc import Mapping
+except ImportError:
+    from collections import Mapping
+from itertools import islice
+
+def get_servers_with_filters(dbproxy, server="*", clss="*", device="*",
+                             properties=True, attribute_properties=True,
+                             aliases=True, dservers=False,
+                             subdevices=False, uppercase_devices=False,
+                             timeout=10):
+    """
+    A performant way to get servers and devices in bulk from the DB
+    by direct SQL statements and joins, instead of e.g. using one
+    query to get the properties of each device.
+
+    TODO: are there any length restrictions on the query results? In
+    that case, use limit and offset to get page by page.
+    """
+
+    server = server.replace("*", "%")  # mysql wildcards
+    clss = clss.replace("*", "%")
+    device = device.replace("*", "%")
+
+    # Collect all info about each device in this dict
+    devices = CaselessDictionary()
+
+    # Queries can sometimes take more than de default 3 s, so it's
+    # good to increase the timeout a bit.
+    # TODO: maybe instead use automatic retry and increase timeout
+    # each time?
+    dbproxy.set_timeout_millis(timeout * 1000)
+
+    if properties:
+        # Get all relevant device properties
+        query = (
+            "SELECT device, property_device.name, property_device.value"
+            " FROM property_device"
+            " INNER JOIN device ON property_device.device = device.name"
+            " WHERE server LIKE '%s' AND class LIKE '%s' AND device LIKE '%s'")
+        if not dservers:
+            query += " AND class != 'DServer'"
+        if not subdevices:
+            query += " AND property_device.name != '__SubDevices'"
+        query += " ORDER BY property_device.count ASC"
+        _, result = dbproxy.command_inout("DbMySqlSelect",
+                                          query % (server, clss, device))
+        for d, p, v in nwise(result, 3):
+            dev = devices.setdefault(d, {})
+            # Make sure to use caseless dicts, since in principle property names
+            # should be caseles too.
+            props = dev.setdefault("properties", CaselessDictionary())
+            value = props.setdefault(p, [])
+            value.append(v)
+
+    if attribute_properties:
+        # Get all relevant attribute properties
+        query = (
+            "SELECT device, attribute, property_attribute_device.name,"
+            " property_attribute_device.value"
+            " FROM property_attribute_device"
+            " INNER JOIN device ON property_attribute_device.device ="
+            " device.name"
+            " WHERE server LIKE '%s' AND class LIKE '%s' AND device LIKE '%s'")
+        if not dservers:
+            query += " AND class != 'DServer'"
+        query += " ORDER BY property_attribute_device.count ASC"
+        _, result = dbproxy.command_inout("DbMySqlSelect", query % (server, clss, device))
+        for d, a, p, v in nwise(result, 4):
+            dev = devices.setdefault(d, {})
+            props = dev.setdefault("attribute_properties", CaselessDictionary())
+            attr = props.setdefault(a, CaselessDictionary())
+            value = attr.setdefault(p, [])
+            value.append(v)
+
+    # dump relevant servers
+    query = (
+        "SELECT server, class, name, alias FROM device"
+        " WHERE server LIKE '%s' AND class LIKE '%s' AND name LIKE '%s'")
+
+    if not dservers:
+        query += " AND class != 'DServer'"
+    _, result = dbproxy.command_inout("DbMySqlSelect", query % (server, clss, device))
+
+    # combine all the information we have
+    servers = SetterDict()
+    for s, c, d, a in nwise(result, 4):
+        try:
+            srv, inst = s.split("/")
+        except ValueError:
+            # Malformed server name? It can happen!
+            continue
+        device = devices.get(d, {})
+        if a and aliases:
+            device["alias"] = a
+        devname = maybe_upper(d, uppercase_devices)
+        servers[srv][inst][c][devname] = device
+
+    return servers
+
+def nwise(it, n):
+    # [s_0, s_1, ...] => [(s_0, ..., s_(n-1)), (s_n, ... s_(2n-1)), ...]
+    return list(zip(*[islice(it, i, None, n) for i in range(n)]))
+
+def maybe_upper(s, upper=False):
+    if upper:
+        return s.upper()
+    return s
+
+class CaselessString(object):
+    """
+    A mixin to make a string subclass case-insensitive in dict lookups.
+    """
+
+    def __hash__(self):
+        return hash(self.lower())
+
+    def __eq__(self, other):
+        return self.lower() == other.lower()
+
+    def __cmp__(self, other):
+        return self.lower().__cmp__(other.lower())
+
+    @classmethod
+    def make_caseless(cls, string):
+        if isinstance(string, six.text_type):
+            return CaselessUnicode(string)
+        return CaselessStr(string)
+
+class CaselessStr(CaselessString, str):
+    pass
+
+
+class CaselessUnicode(CaselessString, six.text_type):
+    pass
+
+class CaselessDictionary(MutableMapping):
+    """
+    A dictionary-like object which ignores but preserves the case of strings.
+
+    Example::
+
+        >>> cdict = CaselessDictionary()
+
+    Access is case-insensitive::
+
+        >>> cdict['a'] = 1
+        >>> cdict['A']
+        1
+
+    As is writing::
+
+        >>> cdict['key'] = 123
+        >>> cdict['KeY'] = 456
+        >>> cdict['key']
+        456
+
+    And deletion::
+
+        >>> del cdict['A']
+        >>> 'a' in cdict
+        False
+        >>> 'A' in cdict
+        False
+
+    However, the case of keys is preserved (the case of overridden keys will be
+    the first one which was set)::
+
+        >>> cdict['aBcDeF'] = 1
+        >>> sorted(list(cdict))
+        ['aBcDeF', 'key']
+    """
+
+    def __init__(self, *args, **kwargs):
+        self.__dict__["_dict"] = {}
+        temp_dict = dict(*args, **kwargs)
+        for key, value in list(temp_dict.items()):
+            if isinstance(key, six.string_types):
+                key = CaselessString.make_caseless(key)
+
+            if isinstance(value, dict):
+                self._dict[key] = CaselessDictionary(value)
+            else:
+                self._dict[key] = value
+
+    def __getitem__(self, key):
+        return self._dict[CaselessString.make_caseless(key)]
+
+    def __setitem__(self, key, value):
+        self._dict[CaselessString.make_caseless(key)] = value
+
+    def __delitem__(self, key):
+        del self._dict[CaselessString.make_caseless(key)]
+
+    def __contains__(self, key):
+        return CaselessString.make_caseless(key) in self._dict
+
+    def __iter__(self):
+        return iter(self._dict)
+
+    def __len__(self):
+        return len(self._dict)
+
+    def keys(self):
+        # convert back to ordinary strings
+        return [str(k) for k in self._dict]
+
+    def items(self):
+        return list(zip(list(self.keys()), list(self.values())))
+
+class SetterDict(CaselessDictionary, defaultdict):
+    """
+    A recursive defaultdict with extra bells & whistles
+
+    It enables you to set keys at any depth without first creating the
+    intermediate levels, e.g.
+
+      d = SetterDict()
+      d["a"]["b"]["c"] = 1
+
+    It also allows access using normal getattr syntax, interchangeably:
+
+      d.a["b"].c == d.a.b.c == d["a"]["b"]["c"]
+
+    Note: only allows string keys for now.
+
+    Keys are caseless, meaning that the key "MyKey" is the same as
+    "mykey", "MYKEY", etc. The case
+
+    Note: this thing is dangerous! Accessing a non-existing key will
+    result in creating it, which means that confusing behavior is
+    likely. Please use it carefully and convert to an ordinary dict
+    (using to_dict()) when you're done creating it.
+    """
+
+    def __init__(self, value={}, factory=None):
+        factory = factory or SetterDict
+        self.__dict__["_factory"] = factory
+        CaselessDictionary.__init__(self)
+        defaultdict.__init__(self, factory)
+        for k, v in list(value.items()):
+            self[k] = v
+
+    def __getitem__(self, key):
+        try:
+            return CaselessDictionary.__getitem__(self, key)
+        except KeyError:
+            return self.__missing__(key)
+
+    def __setitem__(self, key, value):
+        if isinstance(value, SetterDict):
+            CaselessDictionary.__setitem__(self, key, value)
+        elif isinstance(value, Mapping):
+            CaselessDictionary.__setitem__(self, key, self._factory(value))
+        else:
+            CaselessDictionary.__setitem__(self, key, value)
+
+    def __getattr__(self, name):
+        return self.__getitem__(name)
+
+    def __setattr__(self, key, value):
+        return self.__setitem__(key, value)
+
+    def to_dict(self):
+        """
+        Returns a ordinary dict version of itself
+        """
+        result = {}
+        for key, value in list(self.items()):
+            if isinstance(value, SetterDict):
+                result[key] = value.to_dict()
+            else:
+                result[key] = value
+        return result
+

tangostationcontrol/tangostationcontrol/devices/configuration_device.py

@@ -13,14 +13,16 @@ Handles and exposes the station configuration
 
 """
 # PyTango imports
-from tango import AttrWriteType
+from tango import AttrWriteType, Database, DeviceProxy
 from tango.server import attribute
 
 # Additional import
+from tangostationcontrol.common.configuration import SetterDict, get_servers_with_filters
 from tangostationcontrol.common.entrypoint import entry
 from tangostationcontrol.devices.lofar_device import lofar_device
 from tangostationcontrol.common.lofar_logging import device_logging_to_python
 
+import json
 import logging
 logger = logging.getLogger()
 
@@ -46,11 +48,18 @@ class Configuration(lofar_device):
         
         N.B. it does not update, it loads a full new configuration. 
         """
+        # TODO(Stefano): implement load configuration 
         self.proxy.tangodb_properties_RW = tangodb_properties
     
     def _dump_configdb(self):
         """ Returns the TangoDB station configuration as a JSON string """
-        return 'Configuration'
+        dbproxy = DeviceProxy(Database().dev_name())
+        # Create a special dictionary to prevent Tango case errors
+        data = SetterDict()
+        servers = get_servers_with_filters(dbproxy)
+        data.servers.update(servers)
+        return json.dumps(data.to_dict(), ensure_ascii=False, indent=4, sort_keys=True)
+         
 
 # ----------
 # Run server

tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_configuration.py

@@ -9,8 +9,16 @@
 
 from .base import AbstractTestBases
 
+import json
 
 class TestDeviceConfiguration(AbstractTestBases.TestDeviceBase):
 
     def setUp(self):
         super().setUp("STAT/Configuration/1")
+    
+    def test_read_tangodb_properties(self):
+        """ Test whether the station control configuration is correctly retrieved as a JSON string """
+        tangodb_properties = self.proxy.tangodb_properties_RW
+        dbdata = json.loads(tangodb_properties)
+        self.assertTrue(type(dbdata), dict)
+        self.assertGreater(len(dbdata['servers']), 0)