diff --git a/devices/statistics_writer/README.md b/devices/statistics_writer/README.md index 1f25f29336b98f365a0f20e78f65c2956c6f1a50..7887e9f471afa90dc28690aa47f93b0440beb200 100644 --- a/devices/statistics_writer/README.md +++ b/devices/statistics_writer/README.md @@ -19,10 +19,9 @@ This script can be called with the following arguments: --output_dir specifies the folder to write all the files --mode sets the statistics type to be decoded options: "SST", "XST", "BST" --debug takes no arguments, when used prints a lot of extra data to help with debugging - --fraction Configure the writer to only write one every here specified number of statistics. Saves file space + --Decimation Configure the writer to only store one every n samples. Saves storage space ``` - ##HFD5 structure Statistics packets are collected by the StatisticsCollector in to a matrix. Once the matrix is done or a newer timestamp arrives this matrix along with the header of first packet header, nof_payload_errors and nof_valid_payloads. diff --git a/devices/statistics_writer/hdf5_writer.py b/devices/statistics_writer/hdf5_writer.py index 387a1ad8acf4ab418b65868a10648c987627aa11..55c05cf1e0931845489364307aaa0d0fc4bb8759 100644 --- a/devices/statistics_writer/hdf5_writer.py +++ b/devices/statistics_writer/hdf5_writer.py @@ -26,7 +26,7 @@ class hdf5_writer: XST_MODE = "XST" BST_MODE = "BST" - def __init__(self, new_file_time_interval, file_location, statistics_mode, store_fraction): + def __init__(self, new_file_time_interval, file_location, statistics_mode, decimation_factor): # all variables that deal with the matrix that's currently being decoded self.current_matrix = None @@ -34,7 +34,7 @@ class hdf5_writer: # counter that tracks how many statistics have been received # starts at -1 to simplify fraction logic. - self.statistics_counter = -1 + self.statistics_counter = 0 # the header of the first packet of a new matrix is written as metadata. # Assumes all subsequent headers of the same matrix are identical (minus index) @@ -42,7 +42,7 @@ class hdf5_writer: # file handing self.file_location = file_location - self.store_fraction = store_fraction + self.decimation_factor = decimation_factor self.new_file_time_interval = timedelta(seconds=new_file_time_interval) self.last_file_time = datetime.min.replace(tzinfo=pytz.UTC) self.file = None @@ -98,16 +98,19 @@ class hdf5_writer: Creates a new hdf5 file if needed updates current timestamp and statistics matrix collector """ - # received new statistic, so increment counter - self.statistics_counter += 1 # only write the specified fraction of statistics, skip the rest - if self.statistics_counter % self.store_fraction != 0: - logger.info(f"Skipping statistic with timestamp: {timestamp}. Only writing 1/{self.store_fraction} statistics") + if self.statistics_counter % self.decimation_factor != 0: + logger.info(f"Skipping statistic with timestamp: {timestamp}. Only writing 1/{self.decimation_factor} statistics") + + # increment even though its skipped + self.statistics_counter += 1 return - logger.info(f"starting new matrix with timestamp: {timestamp}") + # received new statistic, so increment counter + self.statistics_counter += 1 + logger.info(f"starting new matrix with timestamp: {timestamp}") # write the finished (and checks if its the first matrix) if self.current_matrix is not None: @@ -173,7 +176,7 @@ class hdf5_writer: Adds the newly received statistics packet to the statistics matrix """ # only process the packets of the wanted fraction - if self.statistics_counter % self.store_fraction != 0: + if self.statistics_counter % self.decimation_factor != 0: return self.current_matrix.process_packet(packet) diff --git a/devices/statistics_writer/statistics_writer.py b/devices/statistics_writer/statistics_writer.py index 31f8262ab52a9000e32c70032ece7b69b58edbac..e2d4666fd581b01cdb99e9ad717fbccd32cfa33c 100644 --- a/devices/statistics_writer/statistics_writer.py +++ b/devices/statistics_writer/statistics_writer.py @@ -10,15 +10,15 @@ logging.basicConfig(level=logging.INFO) logger = logging.getLogger("statistics_writer") parser = argparse.ArgumentParser(description='Converts a stream of statistics packets into HDF5 files.') -parser.add_argument('--host', type=str, help='the host to connect to') -parser.add_argument('--port', type=int, default=0, help='the port to connect to, or 0 to use default port for the selected mode (default: %(default)s)') -parser.add_argument('--file', type=str, help='the file to read from') +parser.add_argument('-a', '--host', type=str, help='the host to connect to') +parser.add_argument('-p', '--port', type=int, default=0, help='the port to connect to, or 0 to use default port for the selected mode (default: %(default)s)') +parser.add_argument('-f', '--file', type=str, help='the file to read from') -parser.add_argument('--mode', type=str, choices=['SST', 'XST', 'BST'], default='SST', help='sets the statistics type to be decoded options (default: %(default)s)') -parser.add_argument('--interval', type=float, default=3600, nargs="?", help='The time between creating new files in seconds (default: %(default)s)') -parser.add_argument('--output_dir', type=str, default=".", nargs="?", help='specifies the folder to write all the files (default: %(default)s)') -parser.add_argument('--debug', dest='debug', action='store_true', default=False, help='increase log output') -parser.add_argument('--fraction', type=int, default=1, help='Fraction of statistics written to file to save space. When used only writes 1/n statistics') +parser.add_argument('-m', '--mode', type=str, choices=['SST', 'XST', 'BST'], default='SST', help='sets the statistics type to be decoded options (default: %(default)s)') +parser.add_argument('-i', '--interval', type=float, default=3600, nargs="?", help='The time between creating new files in seconds (default: %(default)s)') +parser.add_argument('-o', '--output_dir', type=str, default=".", nargs="?", help='specifies the folder to write all the files (default: %(default)s)') +parser.add_argument('-v', '--debug', dest='debug', action='store_true', default=False, help='increase log output') +parser.add_argument('-d', '--decimation', type=int, default=1, help='Configure the writer to only store one every n samples. Saves storage space') if __name__ == "__main__": @@ -32,10 +32,10 @@ if __name__ == "__main__": interval = args.interval mode = args.mode debug = args.debug - fraction = args.fraction + decimation = args.decimation - if fraction < 1: - raise ValueError("Please use an integer --fraction value 1 or higher. The fraction format is 1/'--fraction' ") + if decimation < 1: + raise ValueError("Please use an integer --Decimation value 1 or higher to only store one every n statistics' ") if port == 0: default_ports = { "SST": 5101, "XST": 5102, "BST": 5103 } @@ -55,7 +55,7 @@ if __name__ == "__main__": sys.exit(1) # create the writer - writer = hdf5_writer(new_file_time_interval=interval, file_location=output_dir, statistics_mode=mode, store_fraction=fraction) + writer = hdf5_writer(new_file_time_interval=interval, file_location=output_dir, statistics_mode=mode, decimation_factor=decimation) # start looping try: