L2SS-863: Update refactoring to match inherited LSC parts

tangostationcontrol/requirements.txt

@@ -2,7 +2,7 @@
 # order of appearance. Changing the order has an impact on the overall
 # integration process, which may cause wedges in the gate later.
 
-lofar-station-client@git+https://git.astron.nl/lofar2.0/lofar-station-client@0.3.0
+lofar-station-client@git+https://git.astron.nl/lofar2.0/lofar-station-client@0.4.0
 asyncua >= 0.9.90 # LGPLv3
 PyMySQL[rsa] >= 1.0.2 # MIT
 psycopg2-binary >= 2.9.2 # LGPL

tangostationcontrol/tangostationcontrol/integration_test/default/statistics/test_writer_sst.py

@@ -25,38 +25,57 @@ import numpy
 
 class TestStatisticsWriterSST(BaseIntegrationTestCase):
 
+    RECV_PROXY_STRING = "STAT/RECV/1"
+
     def setUp(self):
         self.recv_proxy = self.setup_recv_proxy()
         return super().setUp()
 
-    def setup_recv_proxy(self):
+    @staticmethod
+    def setup_recv_proxy():
         # setup RECV
-        recv_proxy = TestDeviceProxy("STAT/RECV/1")
+        recv_proxy = TestDeviceProxy(TestStatisticsWriterSST.RECV_PROXY_STRING)
         recv_proxy.off()
         recv_proxy.warm_boot()
         recv_proxy.set_defaults()
         return recv_proxy
     
     def test_retrieve_data_from_RECV(self):
-        recv_proxy = self.setup_recv_proxy()
-        self.assertEqual(DevState.ON, recv_proxy.state())
-        self.assertIsNotNone(recv_proxy.RCU_Attenuator_dB_R)
-        self.assertIsNotNone(recv_proxy.RCU_Band_Select_R)
-        self.assertIsNotNone(recv_proxy.RCU_DTH_on_R)
+        self.assertEqual(DevState.ON, self.recv_proxy.state())
+        self.assertIsNotNone(self.recv_proxy.RCU_Attenuator_dB_R)
+        self.assertIsNotNone(self.recv_proxy.RCU_Band_Select_R)
+        self.assertIsNotNone(self.recv_proxy.RCU_DTH_on_R)
     
     def test_insert_tango_SST_statistics(self):
-        self.setup_recv_proxy()
         self.assertEqual(DevState.ON, self.recv_proxy.state())
         collector = StationSSTCollector(device=self.recv_proxy)
 
         # Test attribute values retrieval 
         collector.parse_device_attributes()
-        numpy.testing.assert_equal(collector.parameters["rcu_attenuator_dB"].flatten(), self.recv_proxy.rcu_attenuator_dB_r)
-        numpy.testing.assert_equal(collector.parameters["rcu_band_select"].flatten(), self.recv_proxy.rcu_band_select_r.tolist())
-        numpy.testing.assert_equal(collector.parameters["rcu_dth_on"].flatten(), self.recv_proxy.rcu_dth_on_r.tolist())
+        numpy.testing.assert_equal(
+            collector.parameters["rcu_attenuator_dB"].flatten(),
+            self.recv_proxy.rcu_attenuator_dB_r
+        )
+        numpy.testing.assert_equal(
+            collector.parameters["rcu_band_select"].flatten(),
+            self.recv_proxy.rcu_band_select_r.tolist()
+        )
+        numpy.testing.assert_equal(
+            collector.parameters["rcu_dth_on"].flatten(),
+            self.recv_proxy.rcu_dth_on_r.tolist()
+        )
 
         with TemporaryDirectory() as tmpdir:
-            new_sys_argv = [sys.argv[0], "--mode", "SST", "--file", join(dirname(dirname(dirname(dirname(__file__)))), "test/statistics", "SDP_SST_statistics_packets.bin"), "--output_dir", tmpdir]
+            new_sys_argv = [
+                sys.argv[0],
+                "--mode", "SST",
+                "--file", join(
+                    dirname(dirname(dirname(dirname(__file__)))),
+                    "test/statistics", "SDP_SST_statistics_packets.bin"
+                ),
+                ""
+                "--output_dir", tmpdir
+            ]
             with mock.patch.object(writer.sys, 'argv', new_sys_argv):
                 with self.assertRaises(SystemExit):
                     writer.main()
@@ -65,12 +84,20 @@ class TestStatisticsWriterSST(BaseIntegrationTestCase):
             self.assertTrue(isfile(f"{tmpdir}/SST_2021-09-20-12-17-40.h5"))
         
             # test statistics reader
-            new_sys_argv = [sys.argv[0], "--files", f"{tmpdir}/SST_2021-09-20-12-17-40.h5", "--start_time", "2021-09-20#07:40:08.937+00:00", "--end_time", "2021-10-04#07:50:08.937+00:00"]
+            new_sys_argv = [
+                sys.argv[0],
+                "--files", f"{tmpdir}/SST_2021-09-20-12-17-40.h5",
+                "--start_time", "2021-09-20#07:40:08.937+00:00",
+                "--end_time", "2021-10-04#07:50:08.937+00:00"
+            ]
             with mock.patch.object(statistics_reader.sys, 'argv', new_sys_argv):           
                 stat_parser = statistics_reader.setup_stat_parser()
                 SSTstatistics = stat_parser.list_statistics()
                 self.assertIsNotNone(SSTstatistics)
-                stat = stat_parser.get_statistic('2021-09-20T12:17:40.000+00:00') # same as stat_parser.statistics[0] 
+                # same as stat_parser.statistics[0]
+                stat = stat_parser.get_statistic(
+                    '2021-09-20T12:17:40.000+00:00'
+                )
                 self.assertIsNotNone(stat)
                 self.assertEqual(121, stat.data_id_signal_input_index)
                 # Test RECV attributes
@@ -80,7 +107,17 @@ class TestStatisticsWriterSST(BaseIntegrationTestCase):
 
     def test_no_tango_SST_statistics(self):
         with TemporaryDirectory() as tmpdir:
-            new_sys_argv = [sys.argv[0], "--mode", "SST", "--no-tango", "--file", join(dirname(dirname(dirname(dirname(__file__)))), "test/statistics", "SDP_SST_statistics_packets.bin"), "--output_dir", tmpdir]
+            new_sys_argv = [
+                sys.argv[0],
+                "--mode", "SST",
+                "--no-tango",
+                "--file", join(
+                    dirname(dirname(dirname(dirname(__file__)))),
+                    "test/statistics", "SDP_SST_statistics_packets.bin"
+                ),
+                "--output_dir", tmpdir
+            ]
+
             with mock.patch.object(writer.sys, 'argv', new_sys_argv):
                 with self.assertRaises(SystemExit):
                     writer.main()
@@ -89,12 +126,20 @@ class TestStatisticsWriterSST(BaseIntegrationTestCase):
             self.assertTrue(isfile(f"{tmpdir}/SST_2021-09-20-12-17-40.h5"))
 
             # test statistics reader
-            new_sys_argv = [sys.argv[0], "--files", f"{tmpdir}/SST_2021-09-20-12-17-40.h5", "--start_time", "2021-09-20#07:40:08.937+00:00", "--end_time", "2021-10-04#07:50:08.937+00:00"]
+            new_sys_argv = [
+                sys.argv[0],
+                "--files", f"{tmpdir}/SST_2021-09-20-12-17-40.h5",
+                "--start_time", "2021-09-20#07:40:08.937+00:00",
+                "--end_time", "2021-10-04#07:50:08.937+00:00"
+            ]
             with mock.patch.object(statistics_reader.sys, 'argv', new_sys_argv):           
                 stat_parser = statistics_reader.setup_stat_parser()
                 SSTstatistics = stat_parser.list_statistics()
                 self.assertIsNotNone(SSTstatistics)
-                stat = stat_parser.get_statistic('2021-09-20T12:17:40.000+00:00') # same as stat_parser.statistics[0] 
+                # same as stat_parser.statistics[0]
+                stat = stat_parser.get_statistic(
+                    '2021-09-20T12:17:40.000+00:00'
+                )
                 self.assertIsNotNone(stat)
                 self.assertEqual(121, stat.data_id_signal_input_index)
                 # Test RECV attributes      

tangostationcontrol/tangostationcontrol/statistics/collector.py

@@ -19,37 +19,51 @@ from tango import DevState
 logger = logging.getLogger()
 
 
-class StationStatisticsCollectorInterface(abc.ABC):
+class DeviceCollectorInterface(abc.ABC):
+    """Small interface for deviceproxy enabled collectors"""
+
+    def __init__(self, device: DeviceProxy = None):
+        self.device = device
 
     @abc.abstractmethod
-    def parse_device_attributes(self, device: DeviceProxy):
+    def parse_device_attributes(self):
         """Update information based on device attributes"""
         raise NotImplementedError
 
 
-class StationSSTCollector(StationStatisticsCollectorInterface, SSTCollector):
+class StationSSTCollector(DeviceCollectorInterface, SSTCollector):
+
+    def __init__(self, device: DeviceProxy = None):
+        """Manually combine the constructors with appropriate arguments"""
+
+        DeviceCollectorInterface.__init__(self, device=device)
+        SSTCollector.__init__(self)
 
-    def parse_packet(self, packet, obj):
-        super(StationSSTCollector, self).parse_packet(packet, obj)
+    def _parse_packet(self, packet):
+        super()._parse_packet(packet)
 
         # add tango values to packet
-        self.parse_device_attributes(obj)
+        self.parse_device_attributes()
 
-    def parse_device_attributes(self, device: DeviceProxy):
+    def parse_device_attributes(self):
 
         # If Tango connection has been disabled, set explicitly to None,
         # because otherwise default_values are inserted
-        if device is None or device.state() != DevState.ON:
+        if not self.device or self.device.state() != DevState.ON:
             self.parameters["rcu_attenuator_dB"] = None
             self.parameters["rcu_band_select"] = None
             self.parameters["rcu_dth_on"] = None
         else:
             try:
-                self.parameters["rcu_attenuator_dB"] = device.RCU_Attenuator_dB_R
-                self.parameters["rcu_band_select"] = device.RCU_Band_Select_R
-                self.parameters["rcu_dth_on"] = device.RCU_DTH_on_R
+                self.parameters[
+                    "rcu_attenuator_dB"
+                ] = self.device.RCU_Attenuator_dB_R
+                self.parameters[
+                    "rcu_band_select"
+                ] = self.device.RCU_Band_Select_R
+                self.parameters["rcu_dth_on"] = self.device.RCU_DTH_on_R
             except DevFailed as e:
-                logger.warning(f"Device {device.name()} not responding.")
+                logger.warning("Device: %s not responding.", self.device.name())
                 self.parameters["rcu_attenuator_dB"] = None
                 self.parameters["rcu_band_select"] = None
                 self.parameters["rcu_dth_on"] = None

tangostationcontrol/tangostationcontrol/statistics/writer.py

@@ -13,10 +13,16 @@ import sys
 from tango import DeviceProxy
 
 from tangostationcontrol.statistics_writer.receiver import tcp_receiver, file_receiver
-from tangostationcontrol.statistics_writer.hdf5_writer import sst_hdf5_writer, parallel_xst_hdf5_writer, bst_hdf5_writer
+from tangostationcontrol.statistics_writer.hdf5_writer import BstHdf5Writer
+from tangostationcontrol.statistics_writer.hdf5_writer import SstHdf5Writer
+from tangostationcontrol.statistics_writer.hdf5_writer import ParallelXstHdf5Writer
 
 import logging
-logging.basicConfig(level=logging.INFO, format = '%(asctime)s:%(levelname)s: %(message)s')
+
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s:%(levelname)s: %(message)s'
+)
 logger = logging.getLogger("statistics_writer")
 
 
@@ -77,16 +83,31 @@ def _create_receiver(filename, host, port):
         sys.exit(1)
 
 
-def _create_writer(mode, interval, output_dir, decimation):
+def _create_writer(
+    mode, interval, output_dir, decimation, device: DeviceProxy = None
+):
     """Create the writer"""
     if mode == "XST":
-        return parallel_xst_hdf5_writer(new_file_time_interval=interval, file_location=output_dir, decimation_factor=decimation)
+        return ParallelXstHdf5Writer(
+            new_file_time_interval=interval,
+            file_location=output_dir,
+            decimation_factor=decimation,
+        )
     elif mode == "SST":
-        return sst_hdf5_writer(new_file_time_interval=interval, file_location=output_dir, decimation_factor=decimation)
+        return SstHdf5Writer(
+            new_file_time_interval=interval,
+            file_location=output_dir,
+            decimation_factor=decimation,
+            device=device
+        )
     elif mode == "BST":
-        return bst_hdf5_writer(new_file_time_interval=interval, file_location=output_dir, decimation_factor=decimation)
+        return BstHdf5Writer(
+            new_file_time_interval=interval,
+            file_location=output_dir,
+            decimation_factor=decimation,
+        )
     else:
-        logger.fatal(f"Invalid mode: {mode}")
+        logger.fatal("Invalid mode: %s", mode)
         sys.exit(1)
 
 
@@ -168,7 +189,7 @@ def main():
     receiver = _create_receiver(filename, host, port)
 
     # create the writer
-    writer = _create_writer(mode, interval, output_dir, decimation)
+    writer = _create_writer(mode, interval, output_dir, decimation, device)
 
     # start looping
     _start_loop(receiver, writer, reconnect, filename)

tangostationcontrol/tangostationcontrol/statistics_writer/README.md

 # TCP to HDF5 statistics writer
-The TCP to HDF5 statistics writer can be started with `statistics_writer.py` This script imports the receiver script and `hdf5_writer.py`. `receiver.py` only takes care of receiving packets. 
-`hdf5_writer.py` takes the receive function from the receiver and uses it to obtain packets. 
+The TCP to HDF5 statistics writer can be started with `statistics_writer.py` This script imports the receiver script and `HDF5Writer.py`. `receiver.py` only takes care of receiving packets. 
+`HDF5Writer.py` takes the receive function from the receiver and uses it to obtain packets. 
 Any function that can deliver statistics packets can be used by this code. 
-`hdf5_writer.py` takes care of processing the packets it receives, filling statistics matrices
+`HDF5Writer.py` takes care of processing the packets it receives, filling statistics matrices
 and writing those matrices (as well as a bunch of metadata) to hdf5.