diff --git a/.gitattributes b/.gitattributes index 477822fd5550a8d878039e4a9a565513e684280e..b4aea62eb3721517f5e3022dec0028dcc97abc89 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1113,6 +1113,11 @@ CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/bbs.rst eol=lf CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/cimager.rst eol=lf CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/datamapper.rst eol=lf CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/dppp.rst eol=lf +CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/imager_awimager.rst -text +CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/imager_bbs.rst -text +CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/imager_create_dbs.rst -text +CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/imager_prepare.rst -text +CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/imager_source_finding.rst -text CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/index.rst eol=lf CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/parmdb.rst eol=lf CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/rficonsole.rst eol=lf diff --git a/CEP/Pipeline/docs/sphinx/source/conf.py b/CEP/Pipeline/docs/sphinx/source/conf.py index d5738750ca6df3db8dd981b5cb5f9f7747151079..c79318463d2d7f04f3c7e3477d5ded864a6211b8 100644 --- a/CEP/Pipeline/docs/sphinx/source/conf.py +++ b/CEP/Pipeline/docs/sphinx/source/conf.py @@ -37,11 +37,19 @@ def add_recipe_inputs(app, what_, name, obj, options, lines): extra = "; optional" else: extra = "" - lines.append("``%s`` (:class:`%s`%s)" % (name, type(field).__name__, extra)) - if field.help: - lines.append(" %s" % field.help) - lines.append("") + + parameter_line = ":param {0}: ``({2})`` {1} ({2})".format(name, field.help, + type(field).__name__) + lines.append(parameter_line) + #lines.append("``%s`` (:class:`%s`%s)" % (name, type(field).__name__, extra)) + #if field.help: + # lines.append(" %s" % field.help) + lines.append("") if what_ == "class" and issubclass(obj, RecipeIngredients): + # Skip printing of input and output of both are not present + # This is normaly the toplevel recipe + if (not obj.inputs) and (not obj.outputs): + return lines.append("**Recipe inputs**") lines.append("") if obj.inputs: @@ -49,7 +57,7 @@ def add_recipe_inputs(app, what_, name, obj, options, lines): else: lines.append("None defined -- defaults apply (see :class:`~lofarpipe.support.lofaringredient.RecipeIngredients`).") lines.append("") - lines.append("**Recipe outputs**") + lines.append("**Recipe outputs (job.results[parameter])**") lines.append("") if obj.outputs: format_ingredient_dict(obj.outputs) @@ -95,7 +103,7 @@ master_doc = 'index' # General information about the project. project = u'LOFAR Pipeline System' -copyright = u'2009—11, John Swinbank' +copyright = u'2009—12, John Swinbank, Wouter Klijn' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/CEP/Pipeline/docs/sphinx/source/developer/lofarpipe/tests/ingredients.rst b/CEP/Pipeline/docs/sphinx/source/developer/lofarpipe/tests/ingredients.rst index cf713cf0ee635a479f12e566ff55876c0075e149..d888845decd9425557c39938be95bba843a52113 100644 --- a/CEP/Pipeline/docs/sphinx/source/developer/lofarpipe/tests/ingredients.rst +++ b/CEP/Pipeline/docs/sphinx/source/developer/lofarpipe/tests/ingredients.rst @@ -2,5 +2,5 @@ The :mod:`lofarpipe.tests.lofaringredient` module ************************************************* -.. automodule:: lofarpipe.tests.lofaringredient +.. dsautomodule :: lofarpipe.tests.lofaringredient :members: diff --git a/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/bbs.rst b/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/bbs.rst index aeaa1e985252ea3b5312638cf189b73ddcbdc55d..2916be1493d8e608706065ea792efeed98dbad58 100644 --- a/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/bbs.rst +++ b/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/bbs.rst @@ -1,8 +1,8 @@ -.. _recipe-bbs: +.. _recipe-new_bbs: -=== -BBS -=== +========= +NEW_BBS +========= -.. autoclass:: bbs.bbs +.. autoclass:: lofarpipe.recipes.master.new_bbs.new_bbs :show-inheritance: diff --git a/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/cimager.rst b/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/cimager.rst index 6cb2cc99b5c419d967142d68ec9783b987185515..3c1c3218b889e2e1892cb75efc9c1abeb6467456 100644 --- a/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/cimager.rst +++ b/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/cimager.rst @@ -4,8 +4,8 @@ cimager ======= -.. autoclass:: cimager.cimager +.. autoclass:: lofarpipe.recipes.master.cimager.cimager :show-inheritance: -.. autoclass:: cimager.ParsetTypeField +.. autoclass:: lofarpipe.recipes.master.cimager.ParsetTypeField :show-inheritance: diff --git a/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/datamapper.rst b/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/datamapper.rst index 986665153806b995533ee6f1fbd45c2aebd92ec6..e077a91cb4948e38c0e433add24f729c427d7d8d 100644 --- a/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/datamapper.rst +++ b/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/datamapper.rst @@ -4,5 +4,5 @@ datamapper ========== -.. autoclass:: datamapper.datamapper +.. autoclass:: lofarpipe.recipes.master.datamapper.datamapper :show-inheritance: diff --git a/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/dppp.rst b/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/dppp.rst index 37814fe44efc302f27b011a6a7e300169cdb137d..b6c47d8ec6d7198a996676034245a71edba4166b 100644 --- a/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/dppp.rst +++ b/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/dppp.rst @@ -4,5 +4,5 @@ DPPP ==== -.. autoclass:: dppp.dppp +.. autoclass:: lofarpipe.recipes.master.dppp.dppp :show-inheritance: diff --git a/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/imager_awimager.rst b/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/imager_awimager.rst new file mode 100644 index 0000000000000000000000000000000000000000..8019f72816a0c1117a19c313bcf9c81312069b76 --- /dev/null +++ b/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/imager_awimager.rst @@ -0,0 +1,17 @@ +.. _imager_awimager-recipe: + +================ +imager_awimager +================ + +***Master Side of the recipe *** + + +.. autoclass:: lofarpipe.recipes.master.imager_awimager.imager_awimager + :members: go + +***Node Side of the recipe*** + +.. autoclass:: lofarpipe.recipes.nodes.imager_awimager.imager_awimager + :members: run, _calc_par_from_measurement, _create_mask, _msss_mask + diff --git a/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/imager_bbs.rst b/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/imager_bbs.rst new file mode 100644 index 0000000000000000000000000000000000000000..1c2b8d4ad60002469ee1c4c8dfc5876fbd274b80 --- /dev/null +++ b/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/imager_bbs.rst @@ -0,0 +1,18 @@ +.. _imager_bbs-recipe: + +================= +imager_bbs +================= + +***Master Side of the recipe *** + + +.. autoclass:: lofarpipe.recipes.master.imager_bbs.imager_bbs + :members: + +***Node Side of the recipe*** + +.. autoclass:: lofarpipe.recipes.nodes.imager_bbs.imager_bbs + :members: + + \ No newline at end of file diff --git a/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/imager_create_dbs.rst b/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/imager_create_dbs.rst new file mode 100644 index 0000000000000000000000000000000000000000..be4db3a88e59d4ccec30c33c2784418c6259bf80 --- /dev/null +++ b/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/imager_create_dbs.rst @@ -0,0 +1,18 @@ +.. _imager_create_dbs-recipe: + +================= +imager_create_dbs +================= + +***Master Side of the recipe *** + + +.. autoclass:: lofarpipe.recipes.master.imager_create_dbs.imager_create_dbs + :members: _validate_input_data, _run_create_dbs_node, _collect_and_assign_outputs, go + +***Node Side of the recipe*** + +.. autoclass:: lofarpipe.recipes.nodes.imager_create_dbs.imager_create_dbs + :members: _create_source_list, _create_source_db, _field_of_view, _create_parmdb, _create_parmdb_for_timeslices, _create_monet_db_connection, _get_ra_and_decl_from_ms, _get_soucelist_from_gsm, run + + \ No newline at end of file diff --git a/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/imager_prepare.rst b/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/imager_prepare.rst new file mode 100644 index 0000000000000000000000000000000000000000..7b6ffd5369ebd381d710ffb7784865d2671065e7 --- /dev/null +++ b/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/imager_prepare.rst @@ -0,0 +1,16 @@ +.. _imager_prepare-recipe: + +================ +imager_prepare +================ + +***Master Side of the recipe *** + + +.. autoclass:: lofarpipe.recipes.master.imager_prepare.imager_prepare + :members: _create_input_map_for_sbgroup, _validate_input_map, go + +***Node Side of the recipe*** + +.. autoclass:: lofarpipe.recipes.nodes.imager_prepare.imager_prepare + :members: _copy_input_files, _run_dppp, _concat_timeslices, _run_rficonsole, _filter_bad_stations, run diff --git a/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/imager_source_finding.rst b/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/imager_source_finding.rst new file mode 100644 index 0000000000000000000000000000000000000000..2102107a0ce2c5abb71de33cd7aca82146b3b258 --- /dev/null +++ b/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/imager_source_finding.rst @@ -0,0 +1,16 @@ +.. _imager_source_finding-recipe: + +================================ +imager_source_finding +================================ + +**Master Side of the recipe** + + +.. autoclass:: lofarpipe.recipes.master.imager_source_finding.imager_source_finding + +**Node Side of the recipe** + +.. autoclass:: lofarpipe.recipes.nodes.imager_source_finding.imager_source_finding + :members: run, _combine_source_lists, _create_source_db + diff --git a/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/index.rst b/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/index.rst index 55a89e09812837b4aa9af1cb7e1d1c3aba914b3e..174f1d54db07eec48b2cfb02b812ecb532c47e17 100644 --- a/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/index.rst +++ b/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/index.rst @@ -2,14 +2,20 @@ Standard Imaging Pipeline recipes ================================= -Here we outline the various components which make up LOFAR's Standard Imaging -Pipeline and how they can be combined to form a coherent whole. These -components are made available as pipeline recipes; the reader is encouraged to -be familiar with the :ref:`recipe-docs` section. +On this page the three toplevel recipes of the LOFAR Automatic Imaging Pipeline +for MSSS type observations. +The Calibrator pipeline creates an instrument model based on a calibration +observation. +The instrument model, the calibration solution, is applied to the actual measurements +in the target pipeline. These Measurement sets are then used by the imaging pipeline +to produce, sky images and a list of sources found in this image. +Each of these steps will get more details in each of the chapters +Calibrator Pipeline +------------------------------------ .. toctree:: :maxdepth: 1 - + sip datamapper storagemapper @@ -21,3 +27,33 @@ be familiar with the :ref:`recipe-docs` section. cimager vdsmaker vdsreader + +Target Pipeline +------------------------------------ +.. toctree:: + :maxdepth: 1 + + missing recipe<imager_bbs> + + + +Imager Pipeline +------------------------------------ + +.. autoclass:: msss_imager_pipeline.msss_imager_pipeline + +**Recipes of the Imager Pipeline** + +.. toctree:: + :maxdepth: 1 + + 1. imager_prepare <imager_prepare> + 2. imager_create_dbs <imager_create_dbs> + 3. imager_bbs <imager_bbs> + 4. imager_awimager <imager_awimager> + 5. imager_source_finding <imager_source_finding> + 6. imager_finalize <imager_finalize> + 7. get_metadata + + + \ No newline at end of file diff --git a/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/parmdb.rst b/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/parmdb.rst index 4b7ecd066ac09f475603fc3f82c646060f20fabe..5014ab16271bab7572c904c7a7cb2060db754e78 100644 --- a/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/parmdb.rst +++ b/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/parmdb.rst @@ -4,5 +4,5 @@ parmdb ====== -.. autoclass:: parmdb.parmdb +.. autoclass:: lofarpipe.recipes.master.setupparmdb.setupparmdb :show-inheritance: diff --git a/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/rficonsole.rst b/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/rficonsole.rst index 6197b4a97467f6a347fe9ca5251baca02231b7b0..808023d2e1ade7d39a51aed5c4d424cd85ce1bb7 100644 --- a/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/rficonsole.rst +++ b/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/rficonsole.rst @@ -4,5 +4,5 @@ rficonsole ========== -.. autoclass:: rficonsole.rficonsole +.. autoclass:: lofarpipe.recipes.master.rficonsole.rficonsole :show-inheritance: diff --git a/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/sourcedb.rst b/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/sourcedb.rst index 3561c7290a12cb6d685031b21464ee5c5284124b..5385d753ba2d793b0fa20c31b62d042ae29d9776 100644 --- a/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/sourcedb.rst +++ b/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/sourcedb.rst @@ -4,5 +4,5 @@ sourcedb ======== -.. autoclass:: sourcedb.sourcedb +.. autoclass:: lofarpipe.recipes.master.setupsourcedb.setupsourcedb :show-inheritance: diff --git a/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/storagemapper.rst b/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/storagemapper.rst index c70bbed346175bbe063b248c1d0b91195427ec08..b098e05696e824337ad0930123f2621b0c9497a9 100644 --- a/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/storagemapper.rst +++ b/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/storagemapper.rst @@ -4,5 +4,5 @@ storagemapper ============= -.. autoclass:: storagemapper.storagemapper +.. autoclass:: lofarpipe.recipes.master.storagemapper.storagemapper :show-inheritance: diff --git a/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/vdsmaker.rst b/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/vdsmaker.rst index 1a55828d1799679c42a51b279ae6b2a346b9339a..a88c4f8cf7d71a9e3036b5ba360d2a912c02dc22 100644 --- a/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/vdsmaker.rst +++ b/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/vdsmaker.rst @@ -2,5 +2,5 @@ vdsmaker ======== -.. autoclass:: vdsmaker.vdsmaker +.. autoclass:: lofarpipe.recipes.master.vdsmaker.vdsmaker :show-inheritance: diff --git a/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/vdsreader.rst b/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/vdsreader.rst index eb4e200b982d38543021dd0c3b7a970af3d279e9..b42a513b21849f6cee7f4c87887a7e360cf97a74 100644 --- a/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/vdsreader.rst +++ b/CEP/Pipeline/docs/sphinx/source/pipelines/sip/recipes/vdsreader.rst @@ -4,5 +4,5 @@ vdsreader ========= -.. autoclass:: vdsreader.vdsreader +.. autoclass:: lofarpipe.recipes.master.vdsreader.vdsreader :show-inheritance: diff --git a/CEP/Pipeline/recipes/sip/bin/msss_imager_pipeline.py b/CEP/Pipeline/recipes/sip/bin/msss_imager_pipeline.py index e5a0491f98e4bd7421d74d7d6a935546044b70da..8e2b4afbef05f43f62ca689806deb272979056ed 100755 --- a/CEP/Pipeline/recipes/sip/bin/msss_imager_pipeline.py +++ b/CEP/Pipeline/recipes/sip/bin/msss_imager_pipeline.py @@ -19,45 +19,72 @@ from lofar.parameterset import parameterset class msss_imager_pipeline(control): - """ - The MSSS imager pipeline can be used to generate MSSS images. - - MSSS images are compiled from a number of so-called slices. Each slice + """ + The Automatic MSSS imager pipeline is used to generate MSSS images and find + sources in the generated images. Generated images and lists of found sources + are complemented with meta data and thus ready for consumption by the + Long Term Storage (LTA) + + *subband groups* + The imager_pipeline is able to generate images on the frequency range of + LOFAR in parallel. Combining the frequency subbands together in so called + subbandgroups. Each subband group will result in an image and sourcelist, + (typically 8, because ten subband groups are combined). + + *Time Slices* + MSSS images are compiled from a number of so-called (time) slices. Each slice comprises a short (approx. 10 min) observation of a field (an area on the sky) containing typically 80 subbands. The number of slices will be different for LBA observations (typically 9) and HBA observations (typically 2), due to differences in sensitivity. - - One MSSS observation will produce a number of images (typically 8), one for - each so-called subband-group (SBG). Each SBG consists of the same number - of consecutive subbands (typically 10). Each image will be compiled on a different cluster node to balance the processing load. The input- and output- files and locations are determined by the scheduler and specified in the parset-file. - - This pipeline will perform the following operations: - 1. Copy the preprocessed MS's from the different compute nodes to the nodes - where the images will be compiled (the prepare phase). - 2. Flag the long baselines using DPPP - Concatenate the MS's of the different slices as one virtual MS for - imaging. - 3. Generate a local sky model (LSM) from the global sky model (GSM) for the - sources that are in the field-of-view (FoV). - 4. Repeat until convergence (3 times for the time being): - Per slice: solve and correct for phases using BBS with TEC enabled - Run the awimager. - Run the source finder (PyBDSM) and update the local sky model (LSM). - - Per subband-group, the following output products will be delivered: + **steps:** + + This pipeline performs the following operations: - a. Calibration solutions and corrected visibilities - b. An image - c. A source list + 1. Prepare Phase: Copy the preprocessed MS's from the different compute + nodes to the nodes where the images will be compiled (the prepare phase). + Combine the subbands in subband groups, concattenate the timeslice in a + single large measurement set and perform flagging, RFI and bad station + exclusion. + 2. Create db: Generate a local sky model (LSM) from the global sky model + (GSM) for the sources that are in the field-of-view (FoV). The LSM + is stored as sourcedb. + In step 3 calibration of the measurement sets is performed on these + sources and in step 4 to create a mask for the awimager. The calibration + solution will be placed in an instrument table/db also created in this + step. + 3. BBS: Calibrate the measurement set with the sourcedb from the gsm. + In later iterations sourced found in the created images will be added + to this list. Resulting in a selfcalibration cycle. + 4. Awimager: The combined measurement sets are now imaged. The imaging + is performed using a mask: The sources in the sourcedb are used to create + an casa image masking known sources. Together with the measurement set + an image is created. + 5. Sourcefinding: The images created in step 4 are fed to pyBDSM to find and + describe sources. In multiple itterations substracting the found sources, + all sources are collectedin a sourcelist. + I. The sources found in step 5 are fed back into step 2. This allows the + Measurement sets to be calibrated with sources currently found in the + image. This loop will continue until convergence (3 times for the time + being). + 6. Finalize: Meta data with regards to the input, computations performed and + results are collected an added to the casa image. The images created are + converted from casa to HDF5 and copied to the correct output location. + 7. Export meta data: An outputfile with meta data is generated ready for + consumption by the LTA and/or the LOFAR framework + | + Per subband-group, the following output products will be delivered: + + a. An image + b. A source list + c. (Calibration solutions and corrected visibilities) """ - def __init__(self): """ Initialize member variables and call superclass init function @@ -110,7 +137,7 @@ class msss_imager_pipeline(control): # Define scratch directory to be used by the compute nodes. self.scratch_directory = os.path.join( self.inputs['working_directory'], self.inputs['job_name']) - # Get input/output-data products specifications. + # Get input/output-data products specifications. self._get_io_product_specs() # remove prepending parset identifiers, leave only pipelinecontrol @@ -310,7 +337,7 @@ class msss_imager_pipeline(control): mapfile=source_list_map, sourcedb_target_path=sourcedb_path, sourcedb_map_path=sourcedb_map_path - ) + ) return source_list_map, sourcedb_map_path @@ -361,7 +388,7 @@ class msss_imager_pipeline(control): #save the new mapfile store_data_map(converted_sourcedb_map_path, converted_sourcedb_map) self.logger.error("Wrote converted sourcedb datamap: {0}".format( - converted_sourcedb_map_path)) + converted_sourcedb_map_path)) self.run_task("imager_bbs", timeslice_map_path,