Merge branch 'L2SS-871-update-hdf5-writer' into 'master'

Resolve L2SS-871 "update-hdf5-writer"

Closes L2SS-871

See merge request !407

tangostationcontrol/tangostationcontrol/integration_test/default/statistics/test_writer_sst.py

@@ -46,6 +46,45 @@ class TestStatisticsWriterSST(BaseIntegrationTestCase):
         self.assertIsNotNone(self.recv_proxy.RCU_Band_Select_R)
         self.assertIsNotNone(self.recv_proxy.RCU_DTH_on_R)
     
+    def test_header_info(self):
+        """ Test whether the header info are inserted and collected in the proper way"""
+        with TemporaryDirectory() as tmpdir:
+            new_sys_argv = [
+                sys.argv[0],
+                "--mode", "SST",
+                "--no-tango",
+                "--file", join(
+                    dirname(dirname(dirname(dirname(__file__)))),
+                    "test/statistics", "SDP_SST_statistics_packets.bin"
+                ),
+                "--output_dir", tmpdir
+            ]
+
+            with mock.patch.object(entry.sys, 'argv', new_sys_argv):
+                with self.assertRaises(SystemExit):
+                    entry.main()
+            
+            # check if file was written
+            self.assertTrue(isfile(f"{tmpdir}/SST_2021-09-20-12-17-40.h5"))
+
+            # test statistics reader
+            new_sys_argv = [
+                sys.argv[0],
+                "--files", f"{tmpdir}/SST_2021-09-20-12-17-40.h5",
+                "--start_time", "2021-09-20#07:40:08.937+00:00",
+                "--end_time", "2021-10-04#07:50:08.937+00:00"
+            ]
+            with mock.patch.object(reader.sys, 'argv', new_sys_argv):
+                stat_parser = reader.setup_stat_parser()
+                SSTstatistics = stat_parser.list_statistics()
+                self.assertIsNotNone(SSTstatistics)
+                stat = stat_parser.get_statistic(
+                    '2021-09-20T12:17:40.000+00:00'
+                )
+                self.assertIsNotNone(stat)
+                self.assertEqual("0.1", stat.station_version_id)
+                self.assertEqual("0.1", stat.writer_version_id)
+    
     def test_insert_tango_SST_statistics(self):
         self.assertEqual(DevState.ON, self.recv_proxy.state())
         collector = StationSSTCollector(device=self.recv_proxy)

tangostationcontrol/tangostationcontrol/statistics/reader.py

@@ -156,7 +156,7 @@ class statistics_data:
 
     # we will be creating potentially tens of thousands of these object. Using __slots__ makes them faster and uses less memory. At the cost of
     # having to list all self attributes here.
-    __slots__ = ("version_id", "timestamp", "station_id", "source_info_t_adc", "source_info_subband_calibrated_flag", "source_info_payload_error",
+    __slots__ = ("version_id", "station_version_id", "writer_version_id", "timestamp", "station_id", "source_info_t_adc", "source_info_subband_calibrated_flag", "source_info_payload_error",
                 "source_info_payload_error", "source_info_payload_error", "source_info_nyquist_zone_index", "source_info_gn_index",
                 "source_info_fsub_type", "source_info_beam_repositioning_flag", "source_info_antenna_band_index", "source_info__raw",
                 "observation_id", "nof_statistics_per_packet", "nof_signal_inputs", "nof_bytes_per_statistic", "marker", "integration_interval_raw",
@@ -170,6 +170,10 @@ class statistics_data:
         self.version_id = file[group_key].attrs["version_id"]
         self.station_id = file[group_key].attrs["station_id"]
         
+        # get software version info
+        self.station_version_id = file[group_key].attrs["station_version_id"]
+        self.writer_version_id = file[group_key].attrs["writer_version_id"]
+
         # convert string timestamp to datetime object
         self.timestamp = datetime.datetime.fromisoformat(file[group_key].attrs["timestamp"])
 

tangostationcontrol/tangostationcontrol/statistics/writer/VERSION

+0.1

tangostationcontrol/tangostationcontrol/statistics/writer/hdf5.py

@@ -10,6 +10,7 @@
 from abc import ABC, abstractmethod
 from datetime import datetime, timedelta
 import logging
+import os
 
 # python hdf5
 import h5py
@@ -33,6 +34,19 @@ __all__ = [
     "SstHdf5Writer", "BstHdf5Writer",
 ]
 
+def _get_lsc_version() -> str:
+    """ Retrieve the Lofar Station Control software version """
+    filepath = os.path.join(os.path.realpath('../'), 'tangostationcontrol/VERSION')
+    with open(filepath) as f:
+        version = f.readline().strip()
+    return version or ""
+
+def _get_writer_version() -> str:
+    """ Retrieve the Statistics Writer software version """
+    filepath = os.path.join(os.path.dirname(__file__), 'VERSION')
+    with open(filepath) as f:
+        version = f.readline().strip()
+    return version or ""
 
 class HDF5Writer(ABC):
     SST_MODE = "SST"
@@ -67,6 +81,10 @@ class HDF5Writer(ABC):
         # writer is in (SST,XST,BST)
         self.mode = statistics_mode.upper()
 
+        # software version
+        self.station_version_id = _get_lsc_version()
+        self.writer_version_id = _get_writer_version()
+
         # Set device if any, defaults to None
         self.device = device
 
@@ -207,6 +225,10 @@ class HDF5Writer(ABC):
             timespec="milliseconds"
         )
 
+        # store software version entries
+        header["station_version_id"] = self.station_version_id
+        header["writer_version_id"] = self.writer_version_id
+
         # Stores the header of the packet received for this matrix as a list of
         # attributes
         for k, v in header.items():