Restructure output

Former-commit-id: bf8da57c [formerly d73065ba]
Former-commit-id: 80f16b43
Former-commit-id: 45f128cb

.gitlab-ci.yml

@@ -70,7 +70,7 @@ run_hba_calibrator:
   allow_failure: true
   needs: ["download_data"]
   script:
-    - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH workflows/HBA_calibrator.cwl test_jobs/HBA_calibrator.json
+    - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH --outdir results workflows/HBA_calibrator.cwl test_jobs/HBA_calibrator.json
     - test_jobs/check_workflow_results.py results /builds/eosc/prefactor3-cwl/data/results_calibrator
 
 run_hba_target:
@@ -78,7 +78,7 @@ run_hba_target:
   allow_failure: true
   needs: ["download_data"]
   script:
-    - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH workflows/HBA_target.cwl test_jobs/HBA_target.json
+    - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH --outdir results workflows/HBA_target.cwl test_jobs/HBA_target.json
     - test_jobs/check_workflow_results.py results /builds/eosc/prefactor3-cwl/data/results_target
 
 build-doc:

README.md

-# Prefactor3-CWL
+# prefactor
+## The LOFAR pre-facet calibration pipeline
 
-This is a CWL implementation of the Prefactor 3 pipeline, which lives at https://github.com/lofar-astron/prefactor.
+**prefactor** is a pipeline to correct for various instrumental and ionospheric effects in both **LOFAR HBA** and **LOFAR LBA** observations.
+It will prepare your data so that you will be able to use any direction-dependent calibration software, like [Rapthor](https://github.com/darafferty/rapthor), [factor](https://github.com/lofar-astron/factor), or [killMS](https://github.com/saopicc/killMS/).
 
+It includes:
+* removal of clock offsets between core and remote stations (using clock-TEC separation)
+* correction of the polarization alignment between XX and YY
+* robust time-independent bandpass correction
+* ionospheric RM corrections with [RMextract](https://github.com/lofar-astron/RMextract/)
+* removal of the element beam
+* advanced flagging and removal of bad data
+* mitigation of broad-band RFI and bad stations
+* direction-independent phase correction of the target, using a global sky model from [TGSS ADR](https://http://tgssadr.strw.leidenuniv.nl/)  or the new Global Sky Model [GSM](http://172.104.228.177/)
+* detailled diagnostics
+
+The full documentation can be found at the [prefactor webpage](https://www.astron.nl/citt/prefactor/).
+
+**WARNING**: The current skymodel used for 3C295 is not using the Scaife&Heald flux density scale.
+
+### Software requirements
+* [DP3](https://github.com/lofar-astron/DP3) (v5.1 or later)
+* [LoSoTo](https://github.com/revoltek/losoto) (v2.2.1 or later)
+* [LSMTool](https://github.com/darafferty/LSMTool) (v1.4.3 or later)
+* [EveryBeam](https://git.astron.nl/RD/EveryBeam) (v0.2.0 or later)
+* [RMextract](https://github.com/maaijke/RMextract) (v0.4.4 or later)
+* [AOFlagger](https://gitlab.com/aroffringa/aoflagger) (v3.1 or later)
+* [WSClean](https://gitlab.com/aroffringa/wsclean) (v3.0 or later)
+* [IDG](https://gitlab.com/astron-idg/idg) (v0.8 or later)
+* [LofarStMan](https://github.com/lofar-astron/LofarStMan)
+* [Dysco](https://github.com/aroffringa/dysco.git) (v1.2 or later)
+* casacore
+* Python3 (including matplotlib, scipy, and astropy)
+* [cwltool](https://github.com/common-workflow-language/cwltool) or [toil-cwl-runner](https://github.com/DataBiosphere/toil)
+
+### Installation
+Detailled installation instructions for prefactor are available at the [prefactor documentation](https://www.astron.nl/citt/prefactor/installation.html).
+
+It is recommended to use [Docker](https://docs.docker.com/engine/install/ubuntu/) or [Singularity](https://sylabs.io/guides/3.0/user-guide/installation.html) for the use of software containers.
+
+To download and install the repository run the following commands:
+
+    git clone https://git.astron.nl/eosc/prefactor3-cwl.git prefactor --depth 1
+    cd prefactor
+    pip3 install --upgrade $PWD
+
+### Running prefactor
+Instructions how to [setup](https://www.astron.nl/citt/prefactor/parset.html) and [run](https://www.astron.nl/citt/prefactor/running.html) prefactor are also available at the prefactor documentation.
+
+### Directory structure
+prefactor contains the following sub-directories:
+* **Docker**: contains the Dockerfile for creating the Docker image which contains all necessary software packages
+* **docs**: contains the documentation for prefactor
+* **rfistrategies**: strategies for statistical RFI mitigation using [AOFlagger](https://sourceforge.net/p/aoflagger/wiki/Home/)
+* **scripts**: scripts that the pipeline calls to process data, generate plots, etc.
+* **skymodels**: skymodels that are used by the pipeline (e.g. for demixing or calibrating the calibrator)
+* **solutions**: template solutions for the use of non-supported calibrator sources
+* **steps**: steps of the pipeline described in the [Common Workflow Language (CWL)](https://www.commonwl.org/)
+* **test_jobs**: descriptions for running continouos integration (CI)
+* **workflows**: pipeline workflow desriptions in the [Common Workflow Language (CWL)](https://www.commonwl.org/)
+
+**prefactor** and its scripts were developed by:
+* Alexander Drabent
+* David Rafferty
+* Mattia Mancini
+* Marcel Loose
+* Andreas Horneffer
+* Francesco de Gasperin
+* Marco Iacobelli
+* Emanuela Orru
+* Björn Adebahr
+* Martin Hardcastle
+* George Heald
+* Soumyajit Mandal
+* Carole Roskowinski
+* Jose Sabater Montes
+* Timothy Shimwell
+* Sarrvesh Sridhar
+* Reinout van Weeren
+* Wendy Williams
+
+### Acknowledgements
 <img src="https://www.egi.eu/wp-content/uploads/2020/01/eu-logo.jpeg" alt="EU Flag" width="80">
 <img src="https://www.egi.eu/wp-content/uploads/2020/01/eosc-hub-v-web.png" alt="EOSC-hub logo" height="60">
-This work is co-funded by the EOSC-hub project (Horizon 2020) under Grant number 777536. 
+This work is co-funded by the EOSC-hub project (Horizon 2020) under Grant number 777536.
+
+This work was supported by the BMBF Verbundforschung under the grant 05A20STA.
+
+
+The Prefactor procedure is described in this paper:
+* de Gasperin, F.; Dijkema, T. J.; Drabent, A.; Mevius, M.; Rafferty, van Weeren, R., et al. 2019, [A&A, 662, A5](http://adsabs.harvard.edu/abs/2018arXiv181107954D)

docs/source/calibrator.rst

@@ -5,7 +5,7 @@ Calibrator pipeline
 
 .. note::
 
-   If you are running the deprecated genericpipeline version of the pipeline (**prefactor** 3.2 or older), please check the :doc:`old instrunctions page<calibrator_old>`.
+   If you are running the deprecated genericpipeline version of the pipeline (**prefactor** 3.2 or older), please check the :doc:`old instructions page<calibrator_old>`.
 
 This pipeline processes the calibrator data in order to derive direction-independent corrections.
 It will take into account the correct order of distortions to be calibrated for.
@@ -48,7 +48,7 @@ The workflow ``prep`` consists of:
 Calibration of the polarization alignment (``PA``)
 --------------------------------------------------
 The phase solutions derived from the preparation step are now collected and loaded into **LoSoTo**.
-**LoSoTo** will derive the polarizion alignment and provide diagnostic plots under ``results/inspection/``:
+**LoSoTo** will derive the polarizion alignment and provide diagnostic plots under ``inspection/``:
 
 - ``polalign_ph_pol??``: matrix plot of the phase solutions for the XX and YY polarization
     .. image:: polalign_ph_polXX.png
@@ -127,24 +127,24 @@ The workflow ``ion`` consists of:
 .. note::
     All solutions are written in the h5parm file format via the steps ``H5parm_collector`` and called during all the workflows.
     
-    The solutions are stored in the final calibrator solution set ``results/cal_values/cal_solutions.h5``.
+    The solutions are stored in the final calibrator solution set ``cal_solutions.h5``.
 
 Further diagnostics
 -------------------
 
-The ``results`` directory will contain all relevant outputs of the current **prefactor** run, once the pipeline has finished:
-    - logfiles in ``results/logs``
-    - summary file (JSON format) in ``results/summary``
-    - calibration solutions in ``results/cal_values/cal_solutions.h5``
-    - inspection plots in ``results/inspection``
+The results directory will contain all relevant outputs of the current **prefactor** run, once the pipeline has finished:
+    - logfiles in ``logs``
+    - summary file (JSON format) in ``??_prefactor_calibrator_summary.json``
+    - calibration solutions in ``cal_solutions.h5``
+    - inspection plots in ``inspection``
 
 ``Ateam_separation.png`` shows the distance and the elevation of A-Team sources with respect to the analyzed observation.
 
 You can also check the calibration solutions for more details::
 
-    $ losoto -i results/cal_values/cal_solutions.h5
+    $ losoto -i cal_solutions.h5
     
-    Summary of results/cal_values/cal_solutions.h5
+    Summary of cal_solutions.h5
 
     Solution set 'calibrator':
     ==========================
@@ -181,7 +181,7 @@ You can also check the calibration solutions for more details::
 
 For an overall summary it is advised to check the summary logfile::
 
-    $ cat results/logs/3c286_summary.log
+    $ cat logs/3c286_summary.log
     
     *********************************************
     *** prefactor calibrator pipeline summary ***

docs/source/calibrator_old.rst

@@ -5,7 +5,7 @@ Calibrator pipeline
 
 .. note::
 
-   These instructions are outdated and only valid for **prefactor** 3.2 or older. Please check the :doc:`recent instrunctions page<calibrator>`.
+   These instructions are outdated and only valid for **prefactor** 3.2 or older. Please check the :doc:`recent instructions page<calibrator>`.
 
 
 This pipeline processes the calibrator data in order to derive direction-independent corrections.

docs/source/parset.rst

@@ -4,7 +4,7 @@ Configuring prefactor
 =====================
 .. note::
 
-   If you are running the deprecated genericpipeline version of the pipeline (**prefactor** 3.2 or older), please check the :doc:`old instrunctions page<parset_old>`.
+   If you are running the deprecated genericpipeline version of the pipeline (**prefactor** 3.2 or older), please check the :doc:`old instructions page<parset_old>`.
     
 Preparing the configuration file
 --------------------------------
@@ -114,7 +114,7 @@ If you run the target pipeline you also need to provide the calibrator solution
                     {"class": "Directory", "path": "?????/L228163_SB008_uv.dppp.MS"},
                     {"class": "Directory", "path": "?????/L228163_SB009_uv.dppp.MS"}
                 ],
-        "cal_solutions": {"class": "File", "path": "results/cal_values/cal_solutions.h5"}
+        "cal_solutions": {"class": "File", "path": "cal_solutions.h5"}
     }
 
 A detailed description of the input parameters can be found in the :doc:`calibrator` and :doc:`target` section.

docs/source/parset_old.rst

@@ -4,7 +4,7 @@ Configuring prefactor
 =====================
 .. note::
 
-   These instructions are outdated and only valid for **prefactor** 3.2 or older. Please check the :doc:`recent instrunctions page<parset>`.
+   These instructions are outdated and only valid for **prefactor** 3.2 or older. Please check the :doc:`recent instructions page<parset>`.
    
 Preparing the configuration file
 --------------------------------

docs/source/pipelineoverview.rst

@@ -4,7 +4,7 @@ Pipeline overview
 =================
 .. note::
 
-   If you are running the deprecated genericpipeline version of the pipeline (**prefactor** 3.2 or older), please check the :doc:`old instrunctions page<pipelineoverview_old>`.
+   If you are running the deprecated genericpipeline version of the pipeline (**prefactor** 3.2 or older), please check the :doc:`old instructions page<pipelineoverview_old>`.
 
 **Prefactor** is organized in three major parts to process **LOFAR** data:
 

docs/source/pipelineoverview_old.rst

@@ -5,7 +5,7 @@ Pipeline overview
 
 .. note::
 
-   These instructions are outdated and only valid for **prefactor** 3.2 or older. Please check the :doc:`recent instrunctions page<pipelineoverview>`.
+   These instructions are outdated and only valid for **prefactor** 3.2 or older. Please check the :doc:`recent instructions page<pipelineoverview>`.
 
 **Prefactor** is organized in three major parts to process **LOFAR** data:
 

docs/source/running.rst

@@ -5,7 +5,7 @@ Starting a pipeline
 
 .. note::
 
-   If you are running the deprecated genericpipeline version of the pipeline (**prefactor** 3.2 or older), please check the :doc:`old instrunctions page<running_old>`.
+   If you are running the deprecated genericpipeline version of the pipeline (**prefactor** 3.2 or older), please check the :doc:`old instructions page<running_old>`.
 
 
 Once you have the data and the input JSON file ready, you can run the pipeline, e.g., with ``cwltool`` or ``toil`` for the HBA calibrator pipeline::

docs/source/running_old.rst

@@ -4,7 +4,7 @@ Starting a pipeline
 
 .. note::
 
-   These instructions are outdated and only valid for **prefactor** 3.2 or older. Please check the :doc:`recent instrunctions page<running>`.
+   These instructions are outdated and only valid for **prefactor** 3.2 or older. Please check the :doc:`recent instructions page<running>`.
 
 
 Once you have the data and the parsets ready, you can run the pipeline using the

docs/source/target.rst

@@ -5,7 +5,7 @@ Target pipeline
 
 .. note::
 
-   If you are running the deprecated genericpipeline version of the pipeline (**prefactor** 3.2 or older), please check the :doc:`old instrunctions page<target_old>`.
+   If you are running the deprecated genericpipeline version of the pipeline (**prefactor** 3.2 or older), please check the :doc:`old instructions page<target_old>`.
 
 This pipeline processes the target data in order to apply the direction-independent corrections from the calibrator pipeline. A first initial direction-independent self-calibration of the target field is performed, using a global sky model based on the `TGSS ADR`_ or the new `Global Sky Model`_ (GSM), and applied to the data.
 
@@ -89,18 +89,18 @@ The last step also incorporates full `Dysco`_ compression to save disk space. Th
 
 .. note::
     All solutions are written in the h5parm file format via the steps ``H5parm_collector`` and called during all the workflows.
-
-    The solutions are stored in the final calibrator solution set ``results/cal_values/cal_solutions.h5``.
+    
+    The solutions are stored in the final calibrator solution set ``cal_solutions.h5``.
 
 Further diagnostics
 -------------------
 
-The ``results`` directory will contain all relevant outputs of the current **prefactor** run, once the pipeline has finished:
-    - fully calibrated datasets, concatenated with ``num_SBs_per_group`` subbands per MS file and averaged, if desired (see averaging options below). The DATA column of each MS contains the calibrated data (with the direction-independent solutions applied).
-    - logfiles in ``results/logs``
-    - summary file (JSON format) in ``results/summary``
-    - calibration solutions in ``results/cal_values/cal_solutions.h5``
-    - inspection plots in ``results/inspection``
+The output directory will contain all relevant outputs of the current **prefactor** run, once the pipeline has finished:
+    - fully calibrated datasets in ``results``, concatenated with ``num_SBs_per_group`` subbands per MS file and averaged, if desired (see averaging options below). The DATA column of each MS contains the calibrated data (with the direction-independent solutions applied).
+    - logfiles in ``logs``
+    - summary file (JSON format) in ``??_prefactor_target_summary.json``
+    - calibration solutions in ``cal_solutions.h5``
+    - inspection plots in ``inspection``
 
 The following diagnostic help to assess the quality of the data reduction:
     - ``Ateam_separation.png``: shows the distance and the elevation of A-Team sources with respect to the analyzed observation
@@ -120,9 +120,10 @@ The following diagnostic help to assess the quality of the data reduction:
 
 You can also check the calibration solutions for more details::
 
-    $ losoto -i results/cal_values/cal_solutions.h5
+    $ losoto -i cal_solutions.h5
+    
+    Summary of cal_solutions.h5
 
-    Summary of results/cal_values/cal_solutions.h5
 
     Solution set 'calibrator':
     ==========================
@@ -188,8 +189,8 @@ You can also check the calibration solutions for more details::
 
 For an overall summary it is advised to check the summary logfile::
 
-    $ cat results/logs/???_summary.log
-
+    $ cat logs/???_summary.log
+    
     *****************************************
     *** prefactor target pipeline summary ***
     *****************************************

docs/source/target_old.rst

@@ -5,7 +5,7 @@ Target pipeline
 
 .. note::
 
-   These instructions are outdated and only valid for **prefactor** 3.2 or older. Please check the :doc:`recent instrunctions page<target>`.
+   These instructions are outdated and only valid for **prefactor** 3.2 or older. Please check the :doc:`recent instructions page<target>`.
 
 
 This pipeline processes the target data in order to apply the direction-independent corrections from the calibrator pipeline (line 26). A first initial direction-independent self-calibration of the target field is performed, using a global sky model based on the `TGSS ADR`_ or the new Global Sky Model (GSM), and applied to the data.

steps/collectlog.cwl → steps/collectfiles.cwl

 class: CommandLineTool
 cwlVersion: v1.2
-id: collectlog
+id: collectfiles
 baseCommand:
   - bash
-  - collect_logs.sh
+  - collect_files.sh
 inputs:
   - id: start_directory
     type: Directory?
-  - id: log_files
+  - id: files
     type:
       - File
       - type: array
@@ -19,16 +19,16 @@ inputs:
   - id: sub_directory_name
     type: string
 outputs: 
-  - id: log_dir
+  - id: dir
     type: Directory
     outputBinding:
         glob: |
           $(inputs.start_directory === null ? inputs.sub_directory_name: inputs.start_directory.basename)
-label: CollectLog
+label: CollectFiles
 requirements:
   - class: InitialWorkDirRequirement
     listing:
-      - entryname: collect_logs.sh
+      - entryname: collect_files.sh
         entry: |
           #!/bin/bash
           set -e

test_jobs/HBA_target.json

@@ -85,7 +85,7 @@
     "ncores": 12,
     "cal_solutions": {
         "class": "File",
-        "path": "/builds/eosc/prefactor3-cwl/data/results_calibrator/cal_values/cal_solutions.h5"
+        "path": "/builds/eosc/prefactor3-cwl/data/results_calibrator/cal_solutions.h5"
     },
     "A-Team_skymodel": {
         "class": "File",

test_jobs/check_workflow_results.py

@@ -8,6 +8,7 @@ from filecmp import dircmp
 import h5py
 import sys
 import numpy as np
+import os
 
 
 def check_all_files_present(dcmp, leftname='left', rightname='right'):
@@ -62,8 +63,8 @@ def main(check, control):
     if not all_present:
         error = True
 
-    check_path = check + '/cal_values/cal_solutions.h5'
-    control_path = control + '/cal_values/cal_solutions.h5'
+    check_path = os.path.join(check, 'cal_solutions.h5')
+    control_path = os.path.join(control, 'cal_solutions.h5')
     check_h5 = h5py.File(check_path, 'r')
     control_h5 = h5py.File(control_path, 'r')
 

workflows/HBA_calibrator.cwl

@@ -121,10 +121,22 @@ inputs:
     type: string?
     default: 195.3125kHz
 outputs:
-  - id: results
+  - id: log_files
     outputSource:
-      - save_results/log_dir
+      - save_logfiles/dir
     type: Directory
+  - id: inspection_plots
+    outputSource:
+      - save_inspection/dir
+    type: Directory
+  - id: summary
+    outputSource:
+      - prefactor/summary_file
+    type: File
+  - id: solutions
+    outputSource:
+      - prefactor/solutions
+    type: File
 steps:
   - id: prefactor
     in:
@@ -219,65 +231,28 @@ steps:
     label: prefactor_calibrator
   - id: save_logfiles
     in:
-      - id: log_files
+      - id: files
         linkMerge: merge_flattened
         source:
           - prefactor/logfiles
       - id: sub_directory_name
         default: logs
     out:
-      - id: log_dir
-    run: ./../steps/collectlog.cwl
+      - id: dir
+    run: ./../steps/collectfiles.cwl
     label: save_logfiles
   - id: save_inspection
     in:
-      - id: log_files
+      - id: files
         linkMerge: merge_flattened
         source:
           - prefactor/inspection
       - id: sub_directory_name
         default: inspection
     out:
-      - id: log_dir
-    run: ./../steps/collectlog.cwl
+      - id: dir
+    run: ./../steps/collectfiles.cwl
     label: save_inspection
-  - id: save_solutions
-    in:
-      - id: log_files
-        source:
-          - prefactor/solutions
-      - id: sub_directory_name
-        default: cal_values
-    out:
-      - id: log_dir
-    run: ./../steps/collectlog.cwl
-    label: save_solutions
-  - id: save_summary
-    in:
-      - id: log_files
-        source:
-          - prefactor/summary_file
-      - id: sub_directory_name
-        default: summary
-    out:
-      - id: log_dir
-    run: ./../steps/collectlog.cwl
-    label: save_summary
-  - id: save_results
-    in:
-      - id: log_files
-        linkMerge: merge_flattened
-        source:
-          - save_solutions/log_dir
-          - save_inspection/log_dir
-          - save_logfiles/log_dir
-          - save_summary/log_dir
-      - id: sub_directory_name
-        default: results
-    out:
-      - id: log_dir
-    run: ./../steps/collectlog.cwl
-    label: save_results
 requirements:
   - class: SubworkflowFeatureRequirement
-  - class: MultipleInputFeatureRequirement
\ No newline at end of file
+  - class: MultipleInputFeatureRequirement

workflows/HBA_target.cwl

@@ -153,10 +153,26 @@ inputs:
     type: int?
     default: 16
 outputs:
-  - id: results
+  - id: calibrated_data
     outputSource:
-      - save_results/log_dir
+      - save_results/dir
     type: Directory
+  - id: log_files
+    outputSource:
+      - save_logfiles/dir
+    type: Directory
+  - id: inspection_plots
+    outputSource:
+      - save_inspection/dir
+    type: Directory
+  - id: summary
+    outputSource:
+      - prefactor/summary_file
+    type: File
+  - id: solutions
+    outputSource:
+      - prefactor/solutions
+    type: File
 steps:
   - id: prefactor
     in:
@@ -269,66 +285,40 @@ steps:
     label: prefactor_target
   - id: save_logfiles
     in:
-      - id: log_files
+      - id: files
         linkMerge: merge_flattened
         source:
           - prefactor/logfiles
       - id: sub_directory_name
         default: logs
     out:
-      - id: log_dir
-    run: ./../steps/collectlog.cwl
+      - id: dir
+    run: ./../steps/collectfiles.cwl
     label: save_logfiles
   - id: save_inspection
     in:
-      - id: log_files
+      - id: files
         linkMerge: merge_flattened
         source:
           - prefactor/inspection
       - id: sub_directory_name
         default: inspection
     out:
-      - id: log_dir
-    run: ./../steps/collectlog.cwl
+      - id: dir
+    run: ./../steps/collectfiles.cwl
     label: save_inspection
-  - id: save_solutions
-    in:
-      - id: log_files
-        source:
-          - prefactor/solutions
-      - id: sub_directory_name
-        default: cal_values
-    out:
-      - id: log_dir
-    run: ./../steps/collectlog.cwl
-    label: save_solutions
-  - id: save_summary
-    in:
-      - id: log_files
-        source:
-          - prefactor/summary_file
-      - id: sub_directory_name
-        default: summary
-    out:
-      - id: log_dir
-    run: ./../steps/collectlog.cwl
-    label: save_summary
   - id: save_results
     in:
-      - id: log_files
+      - id: files
         linkMerge: merge_flattened
         source:
-          - save_solutions/log_dir
-          - save_inspection/log_dir
-          - save_logfiles/log_dir
-          - save_summary/log_dir
           - prefactor/msout
       - id: sub_directory_name
         default: results
     out:
-      - id: log_dir
-    run: ./../steps/collectlog.cwl
+      - id: dir
+    run: ./../steps/collectfiles.cwl
     label: save_results
 requirements:
   - class: SubworkflowFeatureRequirement
-  - class: MultipleInputFeatureRequirement
\ No newline at end of file
+  - class: MultipleInputFeatureRequirement