Merge branch 'L2SS-634_adjust_statistics_writer_for_BSTs' into 'master'

Resolve L2SS-634 "Adjust statistics writer for bsts"

Closes L2SS-634

See merge request !353

tangostationcontrol/tangostationcontrol/statistics_writer/README.md

 # TCP to HDF5 statistics writer
-The TCP to HDF5 statistics writer can be started with `tcp_hdf5_writer.py` This script imports
-`tcp_receiver.py` and `statistics_writer.py`. `tcp_receiver.py` only takes care of receiving packets. 
-`statistics_writer.py` takes the receive function from the tcp_receiver and uses it to obtain packets. 
+The TCP to HDF5 statistics writer can be started with `statistics_writer.py` This script imports the receiver script and `hdf5_writer.py`. `receiver.py` only takes care of receiving packets. 
+`hdf5_writer.py` takes the receive function from the receiver and uses it to obtain packets. 
 Any function that can deliver statistics packets can be used by this code. 
-`statistics_writer.py` takes care of processing the packets it receives, filling statistics matrices
+`hdf5_writer.py` takes care of processing the packets it receives, filling statistics matrices
 and writing those matrices (as well as a bunch of metadata) to hdf5.
 
 
 ### TCP Statistics writer
 
-The TCP statistics writer can be called with the `tcp_hdf5_writer.py` script.
+The statistics writer can be called with the `statistics_writer.py` script.
 This script can be called with the following arguments:
   ```
   -a  --host        the address to connect to
-  -p  --port        the port to use
-  -f  --file        file to read from (as opposed to host and port)
-  -i  --interval    The time between creating new files in hours
-  -o  --output_dir  specifies the folder to write all the files
-  -m  --mode        sets the statistics type to be decoded options: "SST", "XST", "BST"
-  -v  --debug       takes no arguments, when used prints a lot of extra data to help with debugging
-  -d  --decimation  Configure the writer to only store one every n samples. Saves storage space
+  -p  --port        the port to use (default: 0)
+  -f  --file        file to read from (alternative to using host and port)
+  -i  --interval    The time between creating new files in seconds (default: 3600)
+  -o  --output_dir  specifies the folder to write all the files (default: current)
+  -m  --mode        sets the statistics type to be decoded options: "SST", "XST", "BST" (default: SST)
+  -v  --debug       takes no arguments, when used prints a lot of extra data to help with debugging (default: off)
+  -d  --decimation  Configure the writer to only store one every n samples. Saves storage space (default: 1 (everything))
+  -r  --reconnect   Set the writer to keep trying to reconnect whenever connection is lost. (default: off)
   ```
+An example call could be: `l2ss-statistics-writer statistics_writer.py --host localhost --port 1234 --mode XST --debug`
+This starts the script up to listen on localhost:1234 for XST's with debug mode on. 
 
 ## HFD5 structure
 Statistics packets are collected by the StatisticsCollector in to a matrix. Once the matrix is done or a newer 

tangostationcontrol/tangostationcontrol/statistics_writer/hdf5_writer.py

@@ -12,13 +12,13 @@ from abc import ABC, abstractmethod
 # import statistics classes with workaround
 import sys
 sys.path.append("..")
-from tangostationcontrol.devices.sdp.packet import SSTPacket, XSTPacket
+from tangostationcontrol.devices.sdp.packet import SSTPacket, XSTPacket, BSTPacket
 import tangostationcontrol.devices.sdp.statistics_collector as statistics_collector
 
 
 logger = logging.getLogger("statistics_writer")
 
-__all__ = ["hdf5_writer", "parallel_xst_hdf5_writer", "xst_hdf5_writer", "sst_hdf5_writer"]
+__all__ = ["hdf5_writer", "parallel_xst_hdf5_writer", "xst_hdf5_writer", "sst_hdf5_writer", "bst_hdf5_writer"]
 
 
 class hdf5_writer(ABC):
@@ -235,6 +235,20 @@ class sst_hdf5_writer(hdf5_writer):
         # store the SST values
         current_group.create_dataset(name="values", data=self.current_matrix.parameters["sst_values"].astype(numpy.float32), compression="gzip")
 
+class bst_hdf5_writer(hdf5_writer):
+    def __init__(self, new_file_time_interval, file_location, decimation_factor):
+        super().__init__(new_file_time_interval, file_location, hdf5_writer.BST_MODE, decimation_factor)
+
+    def decoder(self, packet):
+        return BSTPacket(packet)
+
+    def new_collector(self):
+        return statistics_collector.BSTCollector()
+
+    def write_values_matrix(self, current_group):
+        # store the BST values
+        current_group.create_dataset(name="values", data=self.current_matrix.parameters["bst_values"].astype(numpy.float32), compression="gzip")
+
 
 class xst_hdf5_writer(hdf5_writer):
     def __init__(self, new_file_time_interval, file_location, decimation_factor, subband_index):

tangostationcontrol/tangostationcontrol/statistics_writer/statistics_writer.py

@@ -3,7 +3,7 @@ import time
 import sys
 
 from tangostationcontrol.statistics_writer.receiver import tcp_receiver, file_receiver
-from tangostationcontrol.statistics_writer.hdf5_writer import sst_hdf5_writer, parallel_xst_hdf5_writer
+from tangostationcontrol.statistics_writer.hdf5_writer import sst_hdf5_writer, parallel_xst_hdf5_writer, bst_hdf5_writer
 
 import logging
 logging.basicConfig(level=logging.INFO, format = '%(asctime)s:%(levelname)s: %(message)s')
@@ -63,8 +63,7 @@ def _create_writer(mode, interval, output_dir, decimation):
     elif mode == "SST":
         return sst_hdf5_writer(new_file_time_interval=interval, file_location=output_dir, decimation_factor=decimation)
     elif mode == "BST":
-        logger.fatal(f"BST mode not supported")
-        sys.exit(1)
+        return bst_hdf5_writer(new_file_time_interval=interval, file_location=output_dir, decimation_factor=decimation)
     else:
         logger.fatal(f"Invalid mode: {mode}")
         sys.exit(1)

tangostationcontrol/tangostationcontrol/test/test_statistics_writer.py

@@ -46,3 +46,12 @@ class TestStatisticsWriter(base.TestCase):
             self.assertTrue(isfile(f"{tmpdir}/XST_SB102_2021-09-13-13-21-32.h5"))
             self.assertTrue(isfile(f"{tmpdir}/XST_SB103_2021-09-13-13-21-32.h5"))
 
+    def test_bst(self):
+        with TemporaryDirectory() as tmpdir:
+            new_sys_argv = [sys.argv[0], "--mode", "BST", "--file", dirname(__file__) + "/SDP_BST_statistics_packets.bin", "--output_dir", tmpdir]
+            with mock.patch.object(statistics_writer.sys, 'argv', new_sys_argv):
+                with self.assertRaises(SystemExit):
+                    statistics_writer.main()
+
+            # check if file was written
+            self.assertTrue(isfile(f"{tmpdir}/BST_2022-05-20-11-08-44.h5"))