Merge branch...

Merge branch 'L2SS-414-413_2021-09-30-branched_from_master-add_writer_skip_parameter' of https://git.astron.nl/lofar2.0/tango

devices/statistics_writer/README.md

@@ -19,6 +19,7 @@ This script can be called with the following arguments:
   --output_dir  specifies the folder to write all the files
   --mode        sets the statistics type to be decoded options: "SST", "XST", "BST"
   --debug       takes no arguments, when used prints a lot of extra data to help with debugging
+  --fraction        Configure the writer to only write one every here specified number of statistics. Saves file space
   ```
 
 

devices/statistics_writer/hdf5_writer.py

@@ -26,19 +26,22 @@ class hdf5_writer:
     XST_MODE = "XST"
     BST_MODE = "BST"
 
-
-    def __init__(self, new_file_time_interval, file_location, statistics_mode):
+    def __init__(self, new_file_time_interval, file_location, statistics_mode, store_fraction):
 
         # all variables that deal with the matrix that's currently being decoded
         self.current_matrix = None
         self.current_timestamp = datetime.min.replace(tzinfo=pytz.UTC)
 
+        # counter that tracks how many statistics have been received
+        self.statistics_counter = -1
+
         # the header of the first packet of a new matrix is written as metadata.
         # Assumes all subsequent headers of the same matrix are identical (minus index)
         self.statistics_header = None
 
         # file handing
         self.file_location = file_location
+        self.store_fraction = store_fraction
         self.new_file_time_interval = timedelta(seconds=new_file_time_interval)
         self.last_file_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.file = None
@@ -88,13 +91,22 @@ class hdf5_writer:
         self.process_packet(packet)
 
     def start_new_matrix(self, timestamp):
-        logger.info(f"starting new matrix with timestamp: {timestamp}")
         """
         is called when a statistics packet with a newer timestamp is received.
         Writes the matrix to the hdf5 file
         Creates a new hdf5 file if needed
         updates current timestamp and statistics matrix collector
         """
+        # received new statistic, so increment counter
+        self.statistics_counter += 1
+
+        # only write the specified fraction of statistics, skip the rest
+        if self.statistics_counter % self.store_fraction != 0:
+            logger.info(f"Skipping statistic with timestamp: {timestamp}. Only writing 1/{self.store_fraction} statistics")
+            return
+
+        logger.info(f"starting new matrix with timestamp: {timestamp}")
+
 
         # write the finished (and checks if its the first matrix)
         if self.current_matrix is not None:
@@ -159,6 +171,10 @@ class hdf5_writer:
         """
         Adds the newly received statistics packet to the statistics matrix
         """
+        # only process the packets of the wanted fraction
+        if self.statistics_counter % self.store_fraction != 0:
+            return
+
         self.current_matrix.process_packet(packet)
 
     def start_new_hdf5(self, timestamp):

devices/statistics_writer/statistics_writer.py

@@ -18,6 +18,7 @@ parser.add_argument('--mode', type=str, choices=['SST', 'XST', 'BST'], default='
 parser.add_argument('--interval', type=float, default=3600, nargs="?", help='The time between creating new files in seconds (default: %(default)s)')
 parser.add_argument('--output_dir', type=str, default=".", nargs="?", help='specifies the folder to write all the files (default: %(default)s)')
 parser.add_argument('--debug', dest='debug', action='store_true', default=False, help='increase log output')
+parser.add_argument('--fraction', type=int, default=1, help='Fraction of statistics written to file to save space. When used only writes 1/n statistics')
 
 
 # create a data dumper that creates a new file every 10s (for testing)
@@ -32,6 +33,7 @@ if __name__ == "__main__":
     interval = args.interval
     mode = args.mode
     debug = args.debug
+    fraction = args.fraction
 
     if port == 0:
         default_ports = { "SST": 5101, "XST": 5102, "BST": 5103 }
@@ -51,7 +53,7 @@ if __name__ == "__main__":
         sys.exit(1)
 
     # create the writer
-    writer = hdf5_writer(new_file_time_interval=interval, file_location=output_dir, statistics_mode=mode)
+    writer = hdf5_writer(new_file_time_interval=interval, file_location=output_dir, statistics_mode=mode, store_fraction=fraction)
 
     # start looping
     try: