From d6645da67d444c67d37d6e9771b461f32d1f762f Mon Sep 17 00:00:00 2001
From: Mattia Mancini <mancini@astron.nl>
Date: Wed, 3 Oct 2018 14:52:52 +0000
Subject: [PATCH] SW-43: convenience script to do the pre averaging of the
holography data set.
---
.../lib/process_holography_dataset.py | 32 +++++++++++++++----
CAL/CalibrationProcessing/lib/processing.py | 8 +++--
CMake/LofarPackageList.cmake | 3 +-
3 files changed, 33 insertions(+), 10 deletions(-)
diff --git a/CAL/CalibrationProcessing/lib/process_holography_dataset.py b/CAL/CalibrationProcessing/lib/process_holography_dataset.py
index 8347465d5e8..dc1d37a9874 100644
--- a/CAL/CalibrationProcessing/lib/process_holography_dataset.py
+++ b/CAL/CalibrationProcessing/lib/process_holography_dataset.py
@@ -3,6 +3,7 @@
import argparse
import logging
from lofar.calibration.common.utils import *
+import lofar.calibration.processing as processing
import sys
DEFAULT_SLEEP_TIME = 1
@@ -12,9 +13,10 @@ logger = logging.getLogger('mshologextract')
def main():
cla_parser = specify_command_line_arguments()
arguments = parse_command_line_arguments_and_set_verbose_logging(cla_parser)
+
process_holography_dataset(input_path=arguments.input_path,
output_path=arguments.output_path,
- average_window=arguments.average_window,
+ average_window=arguments.average_samples,
time_average_step=arguments.time_average_step)
@@ -29,16 +31,19 @@ def parse_command_line_arguments_and_set_verbose_logging(parser):
"""
arguments=parser.parse_args()
if arguments.v:
- logging.basicConfig(logging=logging.DEBUG)
+ logging.basicConfig(level=logging.DEBUG)
else:
- logging.basicConfig(logging=logging.INFO)
+ logging.basicConfig(level=logging.INFO)
+
+ logger.debug('The selected arguments are %s', arguments)
- if arguments.average_window is not None and arguments.time_average_step is not None:
+
+ if arguments.average_samples is not None and arguments.time_average_step is not None:
logger.error('Please specify either the average window in samples'
' or the time average step in seconds')
sys.exit(1)
- if arguments.average_window is None and arguments.time_average_step is None:
+ if arguments.average_samples is None and arguments.time_average_step is None:
logger.error('Please specify at least one between the average window in samples'
' or the time average step in seconds')
sys.exit(1)
@@ -60,7 +65,7 @@ def specify_command_line_arguments():
help='average window for the crosscorrelation in seconds', default=None)
parser.add_argument('--average_samples',
help='number of samples to perform one average step of the'
- 'cross correlations', default=None)
+ 'cross correlations', default=None, type=int)
parser.add_argument('--num_proc', help='number of processes used to convert the holography'
' observation', type=int)
@@ -77,4 +82,17 @@ def process_holography_dataset(input_path, output_path, average_window, time_ave
:param average_window: averaging sample step
:param time_average_step: averaging time window
:return:
- """
\ No newline at end of file
+ """
+ if average_window is not None:
+ logger.info('Averaging %s with a time sample window of %s', input_path, average_window)
+ output_hds = processing.average_dataset_by_sample(input_path, average_window)
+ logger.info('Averaged %s with a time sample window of %s', input_path, output_path)
+
+ logger.info('Storing processed file %s in %s', input_path, output_path)
+ output_hds.store_to_file(output_path)
+ logger.info('Stored processed file %s in %s', input_path, output_path)
+
+
+if __name__ == '__main__':
+ logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.DEBUG)
+ main()
\ No newline at end of file
diff --git a/CAL/CalibrationProcessing/lib/processing.py b/CAL/CalibrationProcessing/lib/processing.py
index 24c697580e9..6a7c3186c2c 100644
--- a/CAL/CalibrationProcessing/lib/processing.py
+++ b/CAL/CalibrationProcessing/lib/processing.py
@@ -110,7 +110,7 @@ def average_dataset_by_sample(input_path, window_size):
the beamlet
mean : -> containing the averaged results
std : -> containing the standard deviation of the averaged results
- :rtype: dict(str:dict(str:dict(str:dict(str:numpy.ndarray))))
+ :rtype: HolographyDataset
"""
dataset = HolographyDataset.load_from_file(input_path)
@@ -135,7 +135,11 @@ def average_dataset_by_sample(input_path, window_size):
outdata_per_beamlet[beamlet]['mean'] = averaged_datatable['mean']
outdata_per_beamlet[beamlet]['std'] = averaged_datatable['std']
- return out_data
+ if dataset.derived_data is None:
+ dataset.derived_data = dict()
+ dataset.derived_data['AVERAGE_STEP1'] = out_data
+
+ return dataset
def _round_up_size(array_size, window_size):
diff --git a/CMake/LofarPackageList.cmake b/CMake/LofarPackageList.cmake
index c95369895c1..fdde5e923fc 100644
--- a/CMake/LofarPackageList.cmake
+++ b/CMake/LofarPackageList.cmake
@@ -1,7 +1,7 @@
# - Create for each LOFAR package a variable containing the absolute path to
# its source directory.
#
-# Generated by gen_LofarPackageList_cmake.sh at Fri Sep 7 11:53:48 CEST 2018
+# Generated by gen_LofarPackageList_cmake.sh at Mon Oct 1 13:57:51 CEST 2018
#
# ---- DO NOT EDIT ----
#
@@ -15,6 +15,7 @@
if(NOT DEFINED LOFAR_PACKAGE_LIST_INCLUDED)
set(LOFAR_PACKAGE_LIST_INCLUDED TRUE)
set(CalibrationCommon_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CAL/CalibrationCommon)
+ set(CalibrationProcessing_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CAL/CalibrationProcessing)
set(Calibration_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/Calibration)
set(DP3_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/DP3)
set(GSM_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/GSM)
--
GitLab