diff --git a/devices/statistics_writer/README.md b/devices/statistics_writer/README.md index 62e940bacb6512eb702cc4fdd816b8ba61153958..1f25f29336b98f365a0f20e78f65c2956c6f1a50 100644 --- a/devices/statistics_writer/README.md +++ b/devices/statistics_writer/README.md @@ -19,6 +19,7 @@ This script can be called with the following arguments: --output_dir specifies the folder to write all the files --mode sets the statistics type to be decoded options: "SST", "XST", "BST" --debug takes no arguments, when used prints a lot of extra data to help with debugging + --fraction Configure the writer to only write one every here specified number of statistics. Saves file space ``` diff --git a/devices/statistics_writer/hdf5_writer.py b/devices/statistics_writer/hdf5_writer.py index 5a5eb5ee3fe4d5d05094874d65293ea7fbc68314..35cef886f866647430493581c8d883ef60234a84 100644 --- a/devices/statistics_writer/hdf5_writer.py +++ b/devices/statistics_writer/hdf5_writer.py @@ -26,19 +26,22 @@ class hdf5_writer: XST_MODE = "XST" BST_MODE = "BST" - - def __init__(self, new_file_time_interval, file_location, statistics_mode): + def __init__(self, new_file_time_interval, file_location, statistics_mode, store_fraction): # all variables that deal with the matrix that's currently being decoded self.current_matrix = None self.current_timestamp = datetime.min.replace(tzinfo=pytz.UTC) + # counter that tracks how many statistics have been received + self.statistics_counter = -1 + # the header of the first packet of a new matrix is written as metadata. # Assumes all subsequent headers of the same matrix are identical (minus index) self.statistics_header = None # file handing self.file_location = file_location + self.store_fraction = store_fraction self.new_file_time_interval = timedelta(seconds=new_file_time_interval) self.last_file_time = datetime.min.replace(tzinfo=pytz.UTC) self.file = None @@ -88,13 +91,22 @@ class hdf5_writer: self.process_packet(packet) def start_new_matrix(self, timestamp): - logger.info(f"starting new matrix with timestamp: {timestamp}") """ is called when a statistics packet with a newer timestamp is received. Writes the matrix to the hdf5 file Creates a new hdf5 file if needed updates current timestamp and statistics matrix collector """ + # received new statistic, so increment counter + self.statistics_counter += 1 + + # only write the specified fraction of statistics, skip the rest + if self.statistics_counter % self.store_fraction != 0: + logger.info(f"Skipping statistic with timestamp: {timestamp}. Only writing 1/{self.store_fraction} statistics") + return + + logger.info(f"starting new matrix with timestamp: {timestamp}") + # write the finished (and checks if its the first matrix) if self.current_matrix is not None: @@ -159,6 +171,10 @@ class hdf5_writer: """ Adds the newly received statistics packet to the statistics matrix """ + # only process the packets of the wanted fraction + if self.statistics_counter % self.store_fraction != 0: + return + self.current_matrix.process_packet(packet) def start_new_hdf5(self, timestamp): diff --git a/devices/statistics_writer/statistics_writer.py b/devices/statistics_writer/statistics_writer.py index 444ee2323e950a0428513cb4506d8b2b2376fc27..cbe77ec10fa2273c431459b5caa229cdec4991a7 100644 --- a/devices/statistics_writer/statistics_writer.py +++ b/devices/statistics_writer/statistics_writer.py @@ -18,6 +18,7 @@ parser.add_argument('--mode', type=str, choices=['SST', 'XST', 'BST'], default=' parser.add_argument('--interval', type=float, default=3600, nargs="?", help='The time between creating new files in seconds (default: %(default)s)') parser.add_argument('--output_dir', type=str, default=".", nargs="?", help='specifies the folder to write all the files (default: %(default)s)') parser.add_argument('--debug', dest='debug', action='store_true', default=False, help='increase log output') +parser.add_argument('--fraction', type=int, default=1, help='Fraction of statistics written to file to save space. When used only writes 1/n statistics') # create a data dumper that creates a new file every 10s (for testing) @@ -32,6 +33,7 @@ if __name__ == "__main__": interval = args.interval mode = args.mode debug = args.debug + fraction = args.fraction if port == 0: default_ports = { "SST": 5101, "XST": 5102, "BST": 5103 } @@ -51,7 +53,7 @@ if __name__ == "__main__": sys.exit(1) # create the writer - writer = hdf5_writer(new_file_time_interval=interval, file_location=output_dir, statistics_mode=mode) + writer = hdf5_writer(new_file_time_interval=interval, file_location=output_dir, statistics_mode=mode, store_fraction=fraction) # start looping try: