SW-43: convenience script to do the pre averaging of the holography data set.

CAL/CalibrationProcessing/lib/process_holography_dataset.py

@@ -3,6 +3,7 @@
 import argparse
 import logging
 from lofar.calibration.common.utils import *
+import lofar.calibration.processing as processing
 import sys
 DEFAULT_SLEEP_TIME = 1
 
@@ -12,9 +13,10 @@ logger = logging.getLogger('mshologextract')
 def main():
     cla_parser = specify_command_line_arguments()
     arguments = parse_command_line_arguments_and_set_verbose_logging(cla_parser)
+
     process_holography_dataset(input_path=arguments.input_path,
                                output_path=arguments.output_path,
-                               average_window=arguments.average_window,
+                               average_window=arguments.average_samples,
                                time_average_step=arguments.time_average_step)
 
 
@@ -29,16 +31,19 @@ def parse_command_line_arguments_and_set_verbose_logging(parser):
     """
     arguments=parser.parse_args()
     if arguments.v:
-        logging.basicConfig(logging=logging.DEBUG)
+        logging.basicConfig(level=logging.DEBUG)
     else:
-        logging.basicConfig(logging=logging.INFO)
+        logging.basicConfig(level=logging.INFO)
+
+    logger.debug('The selected arguments are %s', arguments)
 
-    if arguments.average_window is not None and arguments.time_average_step is not None:
+
+    if arguments.average_samples is not None and arguments.time_average_step is not None:
         logger.error('Please specify either the average window in samples'
                      ' or the time average step in seconds')
         sys.exit(1)
 
-    if arguments.average_window is None and arguments.time_average_step is None:
+    if arguments.average_samples is None and arguments.time_average_step is None:
         logger.error('Please specify at least one between the average window in samples'
                      ' or the time average step in seconds')
         sys.exit(1)
@@ -60,7 +65,7 @@ def specify_command_line_arguments():
                         help='average window for the crosscorrelation in seconds', default=None)
     parser.add_argument('--average_samples',
                         help='number of samples to perform one average step of the'
-                             'cross correlations', default=None)
+                             'cross correlations', default=None, type=int)
 
     parser.add_argument('--num_proc', help='number of processes used to convert the holography'
                                            ' observation', type=int)
@@ -77,4 +82,17 @@ def process_holography_dataset(input_path, output_path, average_window, time_ave
     :param average_window: averaging sample step
     :param time_average_step: averaging time window
     :return:
-    """
+    """
\ No newline at end of file
+    if average_window is not None:
+        logger.info('Averaging %s with a time sample window of %s', input_path, average_window)
+        output_hds = processing.average_dataset_by_sample(input_path, average_window)
+        logger.info('Averaged %s with a time sample window of %s', input_path, output_path)
+
+        logger.info('Storing processed file %s in %s', input_path, output_path)
+        output_hds.store_to_file(output_path)
+        logger.info('Stored processed file %s in %s', input_path, output_path)
+
+
+if __name__ == '__main__':
+    logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.DEBUG)
+    main()
\ No newline at end of file

CAL/CalibrationProcessing/lib/processing.py

@@ -110,7 +110,7 @@ def average_dataset_by_sample(input_path, window_size):
                 the beamlet
                     mean : -> containing the averaged results
                     std : -> containing the standard deviation of the averaged results
-    :rtype: dict(str:dict(str:dict(str:dict(str:numpy.ndarray))))
+    :rtype: HolographyDataset
     """
     dataset = HolographyDataset.load_from_file(input_path)
 
@@ -135,7 +135,11 @@ def average_dataset_by_sample(input_path, window_size):
                 outdata_per_beamlet[beamlet]['mean'] = averaged_datatable['mean']
                 outdata_per_beamlet[beamlet]['std'] = averaged_datatable['std']
 
-    return out_data
+    if dataset.derived_data is None:
+        dataset.derived_data = dict()
+    dataset.derived_data['AVERAGE_STEP1'] = out_data
+
+    return dataset
 
 
 def _round_up_size(array_size, window_size):

CMake/LofarPackageList.cmake

 # - Create for each LOFAR package a variable containing the absolute path to
 # its source directory. 
 #
-# Generated by gen_LofarPackageList_cmake.sh at Fri Sep  7 11:53:48 CEST 2018
+# Generated by gen_LofarPackageList_cmake.sh at Mon Oct  1 13:57:51 CEST 2018
 #
 #                      ---- DO NOT EDIT ----
 #
@@ -15,6 +15,7 @@
 if(NOT DEFINED LOFAR_PACKAGE_LIST_INCLUDED)
   set(LOFAR_PACKAGE_LIST_INCLUDED TRUE)
   set(CalibrationCommon_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CAL/CalibrationCommon)
+  set(CalibrationProcessing_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CAL/CalibrationProcessing)
   set(Calibration_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/Calibration)
   set(DP3_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/DP3)
   set(GSM_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/GSM)