From a83cd8a6b87478feea45ac43bb69b4c49d0da38a Mon Sep 17 00:00:00 2001 From: Wouter Klijn <klijn@astron.nl> Date: Mon, 6 Oct 2014 13:58:25 +0000 Subject: [PATCH] Task #5961: Add additional documentation --- .../recipes/sip/bin/long_baseline_pipeline.py | 35 +++++++------------ .../recipes/sip/master/long_baseline.py | 6 ++-- .../recipes/sip/nodes/long_baseline.py | 2 +- 3 files changed, 17 insertions(+), 26 deletions(-) diff --git a/CEP/Pipeline/recipes/sip/bin/long_baseline_pipeline.py b/CEP/Pipeline/recipes/sip/bin/long_baseline_pipeline.py index 4faec44bbff..6350c58fc48 100644 --- a/CEP/Pipeline/recipes/sip/bin/long_baseline_pipeline.py +++ b/CEP/Pipeline/recipes/sip/bin/long_baseline_pipeline.py @@ -26,43 +26,38 @@ from lofar.parameterset import parameterset class msss_imager_pipeline(control): """ - The Automatic MSSS imager pipeline is used to generate MSSS images and find - sources in the generated images. Generated images and lists of found - sources are complemented with meta data and thus ready for consumption by - the Long Term Storage (LTA) + The Automatic MSSS long baselione pipeline is used to generate MSSS + measurement sets combining information of multiple subbands and or + observations into measurements sets. They are are complemented with meta + data and thus ready for consumption by the Long Term Storage (LTA) *subband groups* - The imager_pipeline is able to generate images on the frequency range of + The pipeline is able to generate measurementssets on the frequency range of LOFAR in parallel. Combining the frequency subbands together in so called - subbandgroups. Each subband group will result in an image and sourcelist, - (typically 8, because ten subband groups are combined). + subbandgroups. *Time Slices* - MSSS images are compiled from a number of so-called (time) slices. Each - slice comprises a short (approx. 10 min) observation of a field (an area on + the measurmentsets are compiled from a number of so-called (time) slices. Each + slice comprises an observation of a field (an area on the sky) containing typically 80 subbands. The number of slices will be different for LBA observations (typically 9) and HBA observations (typically 2), due to differences in sensitivity. - Each image will be compiled on a different cluster node to balance the - processing load. The input- and output- files and locations are determined - by the scheduler and specified in the parset-file. - + **This pipeline performs the following operations:** - 1. Prepare Phase. Copy the preprocessed MS's from the different compute + 1. Long baseline . Copy the preprocessed MS's from the different compute nodes to the nodes where the images will be compiled (the prepare phase) Combine the subbands in subband groups, concattenate the timeslice in a single large measurement set and perform flagging, RFI and bad station exclusion. + 2. Generate meta information feedback files based on dataproduct information + and parset/configuration data **Per subband-group, the following output products will be delivered:** - a. An image - b. A source list - c. (Calibration solutions and corrected visibilities) - + a. An measurement set """ def __init__(self): """ @@ -165,8 +160,6 @@ class msss_imager_pipeline(control): processed_ms_dir = self._long_baseline(input_mapfile, target_mapfile, add_beam_tables) - - # ********************************************************************* # (7) Get metadata # create a parset with information that is available on the toplevel @@ -205,8 +198,6 @@ class msss_imager_pipeline(control): toplevel_meta_data_path=toplevel_meta_data_path, product_type = "Correlated") - - return 0 def _get_io_product_specs(self): diff --git a/CEP/Pipeline/recipes/sip/master/long_baseline.py b/CEP/Pipeline/recipes/sip/master/long_baseline.py index 637933d7f97..a1456ac808a 100644 --- a/CEP/Pipeline/recipes/sip/master/long_baseline.py +++ b/CEP/Pipeline/recipes/sip/master/long_baseline.py @@ -1,5 +1,5 @@ # LOFAR IMAGING PIPELINE -# Prepare phase master +# long basseline master # # 1. Create input files for individual nodes based on the input mapfile # 2. Perform basic input parsing and input validation @@ -7,7 +7,7 @@ # 4. validate performance # # Wouter Klijn -# 2012 +# 2014 # klijn@astron.nl # ------------------------------------------------------------------------------ from __future__ import with_statement @@ -22,7 +22,7 @@ from lofarpipe.support.data_map import DataMap, MultiDataMap class long_baseline(BaseRecipe, RemoteCommandRecipeMixIn): """ - Prepare phase master: + Long baseline master: 1. Validate input 2. Create mapfiles with input for work to be perform on the individual nodes diff --git a/CEP/Pipeline/recipes/sip/nodes/long_baseline.py b/CEP/Pipeline/recipes/sip/nodes/long_baseline.py index d897ed562aa..da84684044c 100644 --- a/CEP/Pipeline/recipes/sip/nodes/long_baseline.py +++ b/CEP/Pipeline/recipes/sip/nodes/long_baseline.py @@ -1,7 +1,7 @@ # LOFAR IMAGING PIPELINE # long_baseline node # Wouter Klijn -# 2012 +# 2014 # klijn@astron.nl # ----------------------------------------------------------------------------- from __future__ import with_statement -- GitLab