diff --git a/docs/.keep b/docs/.keep
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/docs/Makefile b/docs/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..a82495350584590f53b00ceedbe14bad0a971681
--- /dev/null
+++ b/docs/Makefile
@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS =
+SPHINXBUILD = sphinx-build
+SPHINXPROJ = Prefactor
+SOURCEDIR = source
+BUILDDIR = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+ @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+ @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
\ No newline at end of file
diff --git a/docs/source/Ateam_separation.png b/docs/source/Ateam_separation.png
new file mode 100644
index 0000000000000000000000000000000000000000..73193f763cc06f4c7db28e24b751030de64adc58
Binary files /dev/null and b/docs/source/Ateam_separation.png differ
diff --git a/docs/source/Ateamclipper.png b/docs/source/Ateamclipper.png
new file mode 100644
index 0000000000000000000000000000000000000000..f8bf8ec3583d457ab3e20a86f51b164398ae767d
Binary files /dev/null and b/docs/source/Ateamclipper.png differ
diff --git a/docs/source/MFS-I-image-pb.plot_im_high_i.png.REMOVED.git-id b/docs/source/MFS-I-image-pb.plot_im_high_i.png.REMOVED.git-id
new file mode 100644
index 0000000000000000000000000000000000000000..77765ca1eb3d36f48e1d11e5160ffbd52791a7ab
--- /dev/null
+++ b/docs/source/MFS-I-image-pb.plot_im_high_i.png.REMOVED.git-id
@@ -0,0 +1 @@
+4e67d4cbcae86474b49270b3b8f0994336edc3fd
\ No newline at end of file
diff --git a/docs/source/MFS-V-image-pb.plot_im_high_v.png.REMOVED.git-id b/docs/source/MFS-V-image-pb.plot_im_high_v.png.REMOVED.git-id
new file mode 100644
index 0000000000000000000000000000000000000000..4cb7aec3aed1c9b73da6fe88cb91851c9fde6dbb
--- /dev/null
+++ b/docs/source/MFS-V-image-pb.plot_im_high_v.png.REMOVED.git-id
@@ -0,0 +1 @@
+171e3e127ccac94e51586d8dd00d867e0c577073
\ No newline at end of file
diff --git a/docs/source/RMextract.png b/docs/source/RMextract.png
new file mode 100644
index 0000000000000000000000000000000000000000..4aa85ed122a59bc4ce8853406446dcce289e7a59
Binary files /dev/null and b/docs/source/RMextract.png differ
diff --git a/docs/source/acknowledgements.rst b/docs/source/acknowledgements.rst
new file mode 100644
index 0000000000000000000000000000000000000000..c6e5a2d55f44ce8b6f62fc59d4256ba676fd107e
--- /dev/null
+++ b/docs/source/acknowledgements.rst
@@ -0,0 +1,31 @@
+.. _acknowledgements:
+
+Acknowledgements
+================
+
+The **prefactor** pipeline and its scripts were developed by:
+
+ * Alexander Drabent
+ * David Rafferty
+ * Andreas Horneffer
+ * Francesco de Gasperin
+ * Marco Iacobelli
+ * Emanuela Orru
+ * Björn Adebahr
+ * Martin Hardcastle
+ * George Heald
+ * Soumyajit Mandal
+ * Carole Roskowinski
+ * Jose Sabater Montes
+ * Timothy Shimwell
+ * Sarrvesh Sridhar
+ * Reinout van Weeren
+ * Wendy Williams
+
+With special thanks to Stefan Fröhlich for developing the genericpipeline.
+
+The procedure is also mostly described in these papers:
+
+ * de Gasperin, F.; Dijkema, T. J.; Drabent, A.; Mevius, M.; Rafferty, van Weeren, R., et al. 2018, arXiv:1811.07954
+ * van Weeren, R. J., Williams, W. L., Hardcastle, M. J., et al. 2016, ApJS, 223, 2
+ * Williams, W. L., van Weeren, R. J., Röttgering, H. J. A., et al. 2016, MNRAS, 460, 2385
diff --git a/docs/source/ampAFlag_polXX.png.REMOVED.git-id b/docs/source/ampAFlag_polXX.png.REMOVED.git-id
new file mode 100644
index 0000000000000000000000000000000000000000..1a71ff3a59156f4f83cfd3992fad7c9794285813
--- /dev/null
+++ b/docs/source/ampAFlag_polXX.png.REMOVED.git-id
@@ -0,0 +1 @@
+ec6c1d1a16fcb48c7a8a1bd4d67ca4120747fbc1
\ No newline at end of file
diff --git a/docs/source/ampBFlag_polXX.png.REMOVED.git-id b/docs/source/ampBFlag_polXX.png.REMOVED.git-id
new file mode 100644
index 0000000000000000000000000000000000000000..de22225bd074d2442091eda3747487084b57c0ac
--- /dev/null
+++ b/docs/source/ampBFlag_polXX.png.REMOVED.git-id
@@ -0,0 +1 @@
+73483c9d0c198f7c47f60082baae6995a9572700
\ No newline at end of file
diff --git a/docs/source/bandpass.png b/docs/source/bandpass.png
new file mode 100644
index 0000000000000000000000000000000000000000..40e7c9774094748105361734c01a5feded41fa41
Binary files /dev/null and b/docs/source/bandpass.png differ
diff --git a/docs/source/bandpass_polXX.png b/docs/source/bandpass_polXX.png
new file mode 100644
index 0000000000000000000000000000000000000000..45562607134ca7a1543eede803dd122b5b0c49ad
Binary files /dev/null and b/docs/source/bandpass_polXX.png differ
diff --git a/docs/source/calibrator.rst b/docs/source/calibrator.rst
new file mode 100644
index 0000000000000000000000000000000000000000..5491e2e6f830a180940a86bca6bb3c0074aeaa45
--- /dev/null
+++ b/docs/source/calibrator.rst
@@ -0,0 +1,293 @@
+.. _calibrator_pipeline:
+
+Calibrator pipeline
+===================
+
+.. note::
+
+ If you are running the deprecated genericpipeline version of the pipeline (**prefactor** 3.2 or older), please check the :doc:`old instrunctions page<calibrator_old>`.
+
+This pipeline processes the calibrator data in order to derive direction-independent corrections.
+It will take into account the correct order of distortions to be calibrated for.
+This chapter will present the specific steps of the calibrator pipeline in more detail.
+
+All results (diagnostic plots and calibration solutions) will be stored usually in the ``--outdir`` directory specified with your ``cwltool`` or ``toil`` command.
+
+ .. image:: calibscheme.png
+
+Prepare calibrator, incl. "demixing" (``prep``)
+-----------------------------------------------
+
+This part of the pipeline prepares the calibrator data in order to be calibration-ready.
+This mainly includes mitigation of bad data (RFI, bad antennas, contaminations from A-Team sources), selection of the data to be calibrated, and some averaging to reduce data size and enhance the signal-to-noise ratio.
+The user can specify whether to do raw data or pre-processed data flagging and whether demixing should be performed.
+
+The basic workflows are:
+
+- preparation of data (``prep``)
+- correcting for polarization alignment (``PA``)
+- correcting for Faraday Rotation (``FR``)
+- correcting for bandpass (``BP``)
+- correcting for ionospheric disturbances (``ion``)
+
+The workflow ``prep`` consists of:
+ - determining suitable calibrator skymodel (steps ``find_skymodel_cal``, ``make_sourcedb``)
+ - checking for nearby A-Team sources (step ``check_Ateam_separation``)
+ - creating a model of A-Team sources to be subtracted (step ``make_sourcedb_ateam``)
+ - basic flagging and averaging (subworkflow ``ndppp_prep_cal``)
+ - edges of the band (``flagedge``) -- only used if ``raw_data : true``
+ - statistical flagging (``aoflag``) -- only used in ``raw_data : true``
+ - baseline flagging (``flagbaseline``)
+ - low elevation flagging (below 15 degress elevation) (``flagelev``)
+ - low amplitude flagging (below 1e-30) (``flagamp``)
+ - demix A-Team sources (``demix``) -- only used if specified ``demix : true``
+ - averaging of the data in time and frequency
+ - wide-band statistical flagging (steps ``ms_concat`` and ``aoflag``)
+ - write the calibrator skymodel into the ``MODEL_DATA`` column (step ``predict``) and perform direction-independent phase-only calibration (diagonal terms + common rotation angle, step ``calib_cal``) (subworkflow ``predict_cal``, baseline-dependent smoothing (step ``BLsmooth``) if specified ``do_smooth : true``)
+
+Calibration of the polarization alignment (``PA``)
+--------------------------------------------------
+The phase solutions derived from the preparation step are now collected and loaded into **LoSoTo**.
+**LoSoTo** will derive the polarizion alignment and provide diagnostic plots under ``results/inspection/``:
+
+- ``polalign_ph_pol??``: matrix plot of the phase solutions for the XX and YY polarization
+ .. image:: polalign_ph_polXX.png
+- ``polalign_ph_poldif``: matrix plot of the phase solutions from XX-YY
+ .. image:: polalign_ph_poldif.png
+- ``polalign_rotange``: matrix plot of the common rotation angle solutions
+ .. image:: polalign_rotangle.png
+- ``polalign_amp_pol??``: matrix plot of the amplitude solutions for the XX and YY polarization
+ .. image:: polalign_amp_polXX.png
+- ``polalign``: matrix plot of the derived polarization alignment between XX and YY
+ .. image:: polalign.png
+- ``polalign_ph-res_pol??``: matrix plot of the residual phase solutions for the XX and YY polarization after subtraction the derived polarization alignment
+- ``polalign_ph-res_poldif``: matrix plot of the residual phase solutions for XX-YY after subtraction of the derived polarization alignment
+
+The workflow ``PA`` consists of:
+ - deriving the polarization alignment from the calibration solutions (subworkflow ``PolAlign``)
+ - creating diagnostic plots (steps ``losoto_plot``)
+ - applying polarization alignment solutions (step ``applyPA``) and the element beam correction (step ``applybeam``) to the original data and re-calibrate (diagonal terms + common rotation angle, step ``calib_cal``) (subworkflow ``apply_calibrate_pa``, baseline-dependend smoothing (step ``BLsmooth``) if specified ``do_smooth : true``)
+
+Calibration of the Faraday Rotation (``FR``)
+--------------------------------------------
+The outcome of the re-calibration **after** correcting for the polarization alignment is again loaded into **LoSoTo** in order to derive corrections for Faraday Rotation.
+The following diagnostic plots are created:
+
+- ``fr_ph_pol??``: matrix plot of the phase solutions for the XX and YY polarization
+- ``fr_ph_poldif``: matrix plot of the phase solutions from XX-YY
+- ``fr_rotange``: matrix plot of the common rotation angle solutions
+- ``fr_amp_pol??``: matrix plot of the amplitude solutions for the XX and YY polarization
+- ``fr``: matrix plot of the derived differential Rotation Measure from Faraday Rotation
+ .. image:: fr.png
+
+The workflow ``FR`` consists of:
+ - deriving the Faraday Rotation from the calibration solutions (subworkflow ``FaradayRot``)
+ - creating diagnostic plots (steps ``losoto_plot``)
+ - applying Faraday Rotation solutions (step ``applyFR``) and re-calibrate (diagonal terms + common rotation angle, step ``calib_cal``) (subworkflow ``apply_calibrate_fr``, baseline-dependend smoothing (step ``BLsmooth``) if specified ``do_smooth : true``)
+
+Calibration of the Bandpass (``BP``)
+------------------------------------
+The outcome of the re-calibration **after** correcting for the polarization alignment and Faraday Rotation is loaded into **LoSoTo** in order to derive corrections for the bandpass. A robust flagging on the amplitude solutions as well as a Savitzky-Golay filter is applied in order to reject bad solutions and smooth the outcome. Frequency regimes up to a certain maximum width (parameter ``max2interpolate``) will be interpolated if flagged.
+The following diagnostic plots are created:
+
+- ``ampBFlag__??``: matrix plot of the amplitude solutions for the XX and YY polarization ``before`` flagging
+ .. image:: ampBFlag_polXX.png
+- ``ampAFlag__??``: matrix plot of the amplitude solutions for the XX and YY polarization ``after`` flagging
+ .. image:: ampAFlag_polXX.png
+- ``bandpass_pol??``: the derived bandpass of all stations in the XX and YY polarization
+- ``bandpass_time??``: matrix plot of the derived bandpass, where both polarizations are colorcoded
+ .. image:: bandpass.png
+- ``bandpass_time??_pol??``: plot of the derived bandpass of the XX and YY polarization, where all stations are colorcoded
+ .. image:: bandpass_polXX.png
+
+The workflow ``BP`` consists of:
+ - deriving the bandpass from the calibration solutions (subworkflow ``bandpass``)
+ - creating diagnostic plots (steps ``losoto_plot``)
+ - transfer solutions for international stations for non-trusted calibrator sources (step ``transfer_solutions``), see parameters ``do_transfer`` and ``trusted_sources``
+ - applying polarization alignment solutions (step ``applyPA``), the bandpass (step ``applyBP``), the element beam correction (step ``applybeam``) and the Faraday Rotation solutions (``applyFR``) to the original data and calibrate (only single scalar phase, step ``calib_cal``) (subworkflow ``apply_calibrate_bp``, baseline-dependend smoothing (step ``BLsmooth``) if specified ``do_smooth : true``)
+ - deriving final amount of flags applied to the data (step ``final_flags``)
+
+Calibration of the instrumental and ionospheric delays (``ion``)
+----------------------------------------------------------------
+The outcome of the re-calibration **after** correcting for the polarization alignment, the bandpass and the Faraday Rotation is loaded into **LoSoTo** in order to derive corrections for the instrumental and ionospheric delays. A robust flagging on the amplitude solutions is applied in order to reject bad solutions. These flags are applied to the phase solutions. These phase solutions should be mainly affected by instrumental (clock) and ionospheric (TEC) delays. This **LoSoTo** step will aim to separate both effects (clock-TEC separation).
+The following diagnostic plots are created:
+
+- ``ion_ph``: matrix plot of the phase solutions
+- ``clock``: matrix plot of the derived (instrumental) clock offsets in seconds
+ .. image:: clock.png
+- ``tec``: matrix plot of the derived differential TEC in TECU
+ .. image:: tec.png
+- ``ion_ph-res``: matrix plot of the residual phase solutions after the subtraction of the derived instrumental and ionospheric delays
+
+The workflow ``ion`` consists of:
+ - deriving the bandpass from the calibration solutions (subworkflow ``clocktec``)
+ - creating diagnostic plots (steps ``losoto_plot``)
+ - create a summary file (step ``summary``)
+
+.. note::
+ All solutions are written in the h5parm file format via the steps ``H5parm_collector`` and called during all the workflows.
+
+ The solutions are stored in the final calibrator solution set ``results/cal_values/cal_solutions.h5``.
+
+Further diagnostics
+-------------------
+
+The ``results`` directory will contain all relevant outputs of the current **prefactor** run, once the pipeline has finished:
+ - logfiles in ``results/logs``
+ - summary file (JSON format) in ``results/summary``
+ - calibration solutions in ``results/cal_values/cal_solutions.h5``
+ - inspection plots in ``results/inspection``
+
+``Ateam_separation.png`` shows the distance and the elevation of A-Team sources with respect to the analyzed observation.
+
+You can also check the calibration solutions for more details::
+
+ $ losoto -i results/cal_values/cal_solutions.h5
+
+ Summary of results/cal_values/cal_solutions.h5
+
+ Solution set 'calibrator':
+ ==========================
+
+ Directions: 3c286
+
+ Stations: CS001HBA0 CS001HBA1 CS002HBA0 CS002HBA1
+ CS003HBA0 CS003HBA1 CS004HBA0 CS004HBA1
+ CS005HBA0 CS005HBA1 CS006HBA0 CS006HBA1
+ CS007HBA0 CS007HBA1 CS011HBA0 CS011HBA1
+ CS017HBA0 CS017HBA1 CS021HBA0 CS021HBA1
+ CS024HBA0 CS024HBA1 CS026HBA0 CS026HBA1
+ CS028HBA0 CS028HBA1 CS030HBA0 CS030HBA1
+ CS031HBA0 CS031HBA1 CS032HBA0 CS032HBA1
+ CS101HBA0 CS101HBA1 CS103HBA0 CS103HBA1
+ CS201HBA0 CS201HBA1 CS301HBA0 CS301HBA1
+ CS302HBA0 CS302HBA1 CS401HBA0 CS401HBA1
+ CS501HBA0 CS501HBA1 RS106HBA RS205HBA
+ RS208HBA RS210HBA RS305HBA RS306HBA
+ RS307HBA RS310HBA RS406HBA RS407HBA
+ RS409HBA RS503HBA RS508HBA RS509HBA
+
+ Solution table 'bandpass' (type: amplitude): 120 times, 11 freqs, 60 ants, 2 pols
+ Flagged data: 0.000%
+
+ Solution table 'clock' (type: clock): 120 times, 60 ants
+ Flagged data: 0.000%
+
+ Solution table 'faraday' (type: rotationmeasure): 60 ants, 120 times
+ Flagged data: 0.222%
+
+ Solution table 'polalign' (type: phase): 120 times, 60 ants, 40 freqs, 2 pols
+ Flagged data: 0.000%
+
+For an overall summary it is advised to check the summary logfile::
+
+ $ cat results/logs/3c286_summary.log
+
+ *********************************************
+ *** prefactor calibrator pipeline summary ***
+ *********************************************
+
+ Field name: 3c286
+
+ User-specified baseline filter: *&
+ Additional antennas removed from the data: NONE
+ A-Team sources close to the phase reference center: VirA
+ Of which were demixed: NONE
+ Of which were clipped: NONE
+
+ Amount of flagged solutions per station and solution table:
+ Station bandpass clock faraday polalign
+ CS001HBA0 9.08% 100.00% 0.00% 0.00%
+ CS001HBA1 9.08% 100.00% 0.00% 0.00%
+ CS002HBA0 9.08% 100.00% 0.00% 0.00%
+ CS002HBA1 9.08% 100.00% 0.00% 0.00%
+ CS003HBA0 9.08% 100.00% 0.00% 0.00%
+ CS003HBA1 9.08% 100.00% 0.00% 0.00%
+
+ Amount of flagged data per station at a given state:
+ Station initial final
+ CS001HBA0 3.18% 3.96%
+ CS001HBA1 2.98% 3.97%
+ CS002HBA0 3.18% 4.42%
+ CS002HBA1 2.95% 3.67%
+ CS003HBA0 2.96% 3.94%
+ CS003HBA1 3.10% 4.21%
+
+ **********
+ Summary file is written to: 3c286_prefactor_calibrator_summary.json
+ Summary has been created.
+
+User-defined parameter configuration
+------------------------------------
+
+**Parameters you will need to adjust**
+
+*Location of the calibrator solutions*
+
+- ``msin``: location of the input calibrator data, for instructions look at the :doc:`configuration instructions<parset>` page
+
+**Parameters you may need to adjust**
+
+*Data selection and calibration options*
+
+- ``refant``: regular expression of the stations that are allowed to be selected as a reference antenna by the pipeline (default: ``CS00.*``)
+- ``flag_baselines``: DP3-compatible pattern for baselines or stations to be flagged (may be an empty list, i.e.: ``[]`` )
+- ``process_baselines_cal``: performs A-Team-clipping/demixing and direction-independent phase-only self-calibration only on these baselines. Choose ``[CR]S*&`` if you want to process only cross-correlations and remove international stations (default: ``*&``)
+- ``filter_baselines``: selects only this set of baselines to be processed. Choose ``[CR]S*&`` if you want to process only cross-correlations and remove international stations (default: ``*&``)
+- ``do_smooth``: enable or disable baseline-based smoothing (default: ``false``)
+- ``rfistrategy``: strategy to be applied with the statistical flagger (`AOFlagger`_, default: ``HBAdefault.rfis``)
+- ``max2interpolate``: amount of channels in which interpolation should be performed for deriving the bandpass (default: 30)
+- ``fit_offset_PA``: assume that together with a delay each station has also a differential phase offset (important for old LBA observations, default: ``false``)
+- ``interp_windowsize``: size of the window over which a value is interpolated. Should be odd. (default: 15)
+- ``ampRange``: range of median amplitudes accepted per station (default: ``[0,0]``)
+- ``skip_international``: skip fitting the bandpass for international stations (this avoids flagging them in many cases, default: ``true``)
+- ``raw_data``: use autoweight, set to True in case you are using raw data (default: ``false``)
+- ``propagatesolutions``: use already derived solutions as initial guess for the upcoming time slot (default: ``true``)
+- ``flagunconverged`` : flag solutions for solves that did not converge (if they were also detected to diverge, default: ``false``)
+- ``maxStddev``: maximum allowable standard deviation when outlier clipping is done. For phases, this should value should be in radians, for amplitudes in log(amp). If None (or negative), a value of 0.1 rad is used for phases and 0.01 for amplitudes (default: ``-1.0``)
+- ``solutions2transfer``: provide own solutions from a reference calibrator observation in case calibrator source is not trusted
+- ``antennas2transfer``: DP3-compatible baseline pattern for those stations who should get calibration solutions from a reference solution set in case calibrator source is not trusted (default: ``[FUSPID].*``)
+- ``do_transfer``: enable solutions transfer for non-trusted calibrator sources (default: ``false``)
+- ``trusted_sources``: comma-separated list of trusted calibrator sources. Solutions are only transferred from a reference solution set in case the observed calibrator is not among them (default: ``3C48,3C147,3C196,3C295,3C380``)
+- ``ion_3rd``: take into account also 3rd-order effects for the clock-TEC separation (ionospheric calibration, default: ``false``)
+- ``clock_smooth``: only take the median of the derived clock solutions (enable this in case of non-joint observations, default: ``true``)
+
+A comprehensive explanation of the baseline selection syntax can be found `here`_.
+
+
+*Demixing options* (only used if demix step is added to the ``prep_cal_strategy`` variable)
+
+- ``demix_sources``: choose sources to demix (provided as list), e.g., ``[CasA,CygA]``
+- ``demix_target``: if given, the target source model (its patch in the SourceDB) is taken into account when solving (default: ``""``)
+- ``demix_freqstep``: number of channels to average when demixing (default: 16)
+- ``demix_timestep`` : number of time slots to average when demixing (default: 10)
+- ``demix``: enable demixing (default: ``false``)
+
+*Further pipeline options*
+
+- ``min_separation``: minimum accepted distance to an A-team source on the sky in degrees (will raise a WARNING, default: ``30``)
+- ``tables2export``: choose which tables to export to the solutions file after the ionospheric calibration (default: ``clock``)
+
+**Parameters for pipeline performance**
+
+- ``max_dppp_threads``: number of threads per process for DP3 (default: 10)
+- ``memoryperc``: maximum of memory used for aoflagger in raw_flagging mode in percent (default: 20)
+- ``min_length``: minimum amount of subbands to concatenate in frequency necessary to perform the wide-band flagging in the RAM. It data is too big aoflag will use indirect-read (default: 50)
+- ``overhead``: only use this fraction of the available memory for deriving the amount of data to be concatenated (default: 0.8)
+
+**Parameters you may want to adjust**
+
+*Skymodel directory*
+
+- ``calibrator_path_skymodel``: location of the prefactor calibrator skymodels
+- ``max_separation_arcmin``: maximum separation between phase center of the observation and the patch of a calibrator skymodel which is accepted to be chosen as a skymodel (default: 1.0)
+- ``A-Team_skymodel``: location of the prefactor A-Team skymodels
+
+*Averaging for the calibrator data*
+
+- ``avg_timeresolution``: final time resolution of the data in seconds after averaging (default: 4)
+- ``avg_freqresolution`` : final frequency resolution of the data after averaging (default: 48.82kHz, which translates to 4 channels per subband)
+- ``bandpass_freqresolution``: frequency resolution of the bandpass solution table (default: 195.3125kHz, which translates to 1 channel per subband)
+
+.. _here: https://www.astron.nl/lofarwiki/doku.php?id=public:user_software:documentation:ndppp#description_of_baseline_selection_parameters
+.. _AOFlagger: https://gitlab.com/aroffringa/aoflagger.git
diff --git a/docs/source/calibrator_old.rst b/docs/source/calibrator_old.rst
new file mode 100644
index 0000000000000000000000000000000000000000..305554f9aa0f3c7d5c5ab78b14e97091c9d4bd4c
--- /dev/null
+++ b/docs/source/calibrator_old.rst
@@ -0,0 +1,234 @@
+.. _calibrator_pipeline_old:
+
+Calibrator pipeline
+===================
+
+.. note::
+
+ These instructions are outdated and only valid for **prefactor** 3.2 or older. Please check the :doc:`recent instrunctions page<calibrator>`.
+
+
+This pipeline processes the calibrator data in order to derive direction-independent corrections.
+It will take into account the correct order of distortions to be calibrated for.
+This chapter will present the specific steps of the calibrator pipeline in more detail.
+You will find the single steps in the parameter ``pipeline.steps`` in line 98.
+All results (diagnostic plots and calibration solutions) are usually stored in a subfolder of the results directory, see ``inspection_directory`` (line 79) and ``cal_values_directory`` (line 80), respectively.
+
+ .. image:: calibscheme.png
+
+Prepare calibrator (incl. "demixing")
+-------------------------------------
+
+This part of the pipeline prepares the calibrator data in order to be calibration-ready.
+This mainly includes mitigation of bad data (RFI, bad antennas, contaminations from A-Team sources), selection of the data to be calibrated (usually Dutch stations only), and some averaging to reduce data size and enhance the signal-to-noise ratio.
+The user can specify whether to do raw data or pre-processed data flagging and whether demixing should be performed.
+
+The basic steps are:
+
+- mapping of data to be used (``createmap_cal``)
+- creating a model of A-Team sources to be subtracted (``make_sourcedb_ateam``)
+- basic flagging and averaging (``ndppp_prep_cal``)
+ - edges of the band (``flagedge``) -- only used in ``raw_flagging`` mode
+ - statistical flagging (``aoflag``) -- only used in ``raw_flagging`` mode
+ - baseline flagging (``flag``)
+ - low elevation flagging (below 20 degress elevation) (``elev``)
+ - demix A-Team sources (``demix``) -- only used if specified
+ - interpolation of flagged data (``interp``)
+ - averaging of the data to 4 sec and 4 channels per subband (``avg``)
+- wide-band statistical flagging (``aoflag``)
+- find needed skymodel of calibrator automatically (``sky_cal``)
+- write the calibrator skymodel into the MODEL_DATA column (``predict_cal``)
+- interpolate flagged data from the wide-band statistical flagging step (``interp_cal``)
+- baseline-dependent smoothing of the data (``smooth_data``)
+- perform direction-independent phase-only calibration (diagonal terms + common rotation angle) (``calib_cal``)
+
+The solutions are stored in the h5parm file format.
+
+Calibration of the polarization alignment (PA)
+----------------------------------------------
+The phase solutions derived from the preparation step are now collected and loaded into **LoSoTo**.
+**LoSoTo** will derive the polarizion alignment and provide diagnostic plots:
+
+- ``polalign_ph_pol??``: matrix plot of the phase solutions for the XX and YY polarization
+ .. image:: polalign_ph_polXX.png
+- ``polalign_ph_poldif``: matrix plot of the phase solutions from XX-YY
+ .. image:: polalign_ph_poldif.png
+- ``polalign_rotange``: matrix plot of the common rotation angle solutions
+ .. image:: polalign_rotangle.png
+- ``polalign_amp_pol??``: matrix plot of the amplitude solutions for the XX and YY polarization
+ .. image:: polalign_amp_polXX.png
+- ``polalign``: matrix plot of the derived polarization alignment between XX and YY
+ .. image:: polalign.png
+- ``polalign_ph-res_pol??``: matrix plot of the residual phase solutions for the XX and YY polarization after subtraction the derived polarization alignment
+- ``polalign_ph-res_poldif``: matrix plot of the residual phase solutions for XX-YY after subtraction of the derived polarization alignment
+
+The solutions are then stored in the final calibrator solution set ``cal_solutions`` (line 83) and applied to the interpolated data (``apply_PA``), together with the LOFAR beam correction (``apply_beam``)
+The calibration (``calib_cal``) is then repeated on the corrected and re-smoothed data (``smooth_corrected``).
+
+Calibration of the Faraday Rotation (FR)
+----------------------------------------
+The outcome of the re-calibration **after** correcting for the polarization alignment is again loaded into **LoSoTo** in order to derive corrections for Faraday Rotation.
+The following diagnostic plots are created:
+
+- ``fr_ph_pol??``: matrix plot of the phase solutions for the XX and YY polarization
+- ``fr_ph_poldif``: matrix plot of the phase solutions from XX-YY
+- ``fr_rotange``: matrix plot of the common rotation angle solutions
+- ``fr_amp_pol??``: matrix plot of the amplitude solutions for the XX and YY polarization
+- ``fr``: matrix plot of the derived differential Rotation Measure from Faraday Rotation
+ .. image:: fr.png
+- ``fr_ph-res_pol??``: matrix plot of the residual phase solutions for the XX and YY polarization after subtraction the derived Rotation Measure
+- ``fr_ph-res_poldif``: matrix plot of the residual phase solutions for XX-YY after subtraction of the derived Rotation Measure
+
+The solutions are then stored in the final calibrator solution set ``cal_solutions`` (line 83) and applied, together with the polarization alignment and the LOFAR beam correction, to the interpolated data (``apply_PA`` + ``apply_beam`` + ``apply_FR``).
+The calibration (``calib_cal``) is then repeated on the corrected and re-smoothed data (``smooth_corrected``).
+
+Calibration of the Bandpass (bandpass)
+----------------------------------------
+The outcome of the re-calibration **after** correcting for the polarization alignment and Faraday Rotation is loaded into **LoSoTo** in order to derive corrections for the bandpass. A robust flagging on the amplitude solutions as well as a Savitzky-Golay filter is applied in order to reject bad solutions and smooth the outcome. Frequency regimes up to a certain maximum width (``maxFlaggedWidth``) will be interpolated if flagged.
+The following diagnostic plots are created:
+
+- ``ampBFlag__??``: matrix plot of the amplitude solutions for the XX and YY polarization ``before`` flagging
+ .. image:: ampBFlag_polXX.png
+- ``ampAFlag__??``: matrix plot of the amplitude solutions for the XX and YY polarization ``after`` flagging
+ .. image:: ampAFlag_polXX.png
+- ``bandpass_pol??``: the derived bandpass of all stations in the XX and YY polarization
+- ``bandpass_time??``: matrix plot of the derived bandpass, where both polarizations are colorcoded
+ .. image:: bandpass.png
+- ``bandpass_time??_pol??``: plot of the derived bandpass of the XX and YY polarization, where all stations are colorcoded
+ .. image:: bandpass_polXX.png
+
+The solutions are then stored in the final calibrator solution set ``cal_solutions`` (line 83) and applied, together with the polarization alignment, the LOFAR beam correction and the Faraday Rotation corrections to the interpolated data in the correct order (``apply_PA`` + ``apply_bandpass`` + ``apply_beam`` + ``apply_FR`` ).
+The calibration (``calib_cal``) is then repeated on the corrected and re-smoothed data (``smooth_corrected``).
+
+Calibration of the instrumental and ionospheric delays (ion)
+------------------------------------------------------------
+The outcome of the re-calibration **after** correcting for the polarization alignment, the bandpass and the Faraday Rotation is loaded into **LoSoTo** in order to derive corrections for the instrumental and ionospheric delays. A robust flagging on the amplitude solutions is applied in order to reject bad solutions. These flags are applied to the phase solutions. These phase solutions should be mainly affected by instrumental (clock) and ionospheric (TEC) delays. This **LoSoTo** step will aim for seperating both effects (clock-TEC separation).
+The following diagnostic plots are created:
+
+- ``ion_ampBFlag__??``: matrix plot of the amplitude solutions for the XX and YY polarization **before** flagging
+- ``ion_ampAFlag__??``: matrix plot of the amplitude solutions for the XX and YY polarization **after** flagging
+- ``ion_ph_pol??``: matrix plot of the phase solutions for the XX and YY polarization
+- ``ion_ph_poldif``: matrix plot of the phase solutions from XX-YY
+- ``clock``: matrix plot of the derived (instrumental) clock offsets in seconds
+ .. image:: clock.png
+- ``tec``: matrix plot of the derived differential TEC in TECU
+ .. image:: tec.png
+- ``ion_ph-res_pol??``: matrix plot of the residual phase solutions for the XX and YY polarization after subtraction the derived instrumental and ionospheric delays
+- ``ion_ph-res_poldif``: matrix plot of the residual phase solutions for XX-YY after subtraction of the derived instrumental and ionospheric delays
+ .. image:: ion_ph-res_poldif.png
+
+The solutions are then stored in the final calibrator solution set ``cal_solutions`` (line 83).
+
+User-defined parameter configuration
+------------------------------------
+**Parameters you will need to adjust**
+
+*Information about the input data*
+
+- ``cal_input_path``: specify the directory where your calibrator data is stored (a full UNIX-compatible directory is required)
+- ``cal_input_pattern``: regular expression pattern of all your calibrator files (e.g. ``L72318*.MS``)
+
+*Location of the software*
+
+- ``prefactor_directory``: full path to your prefactor copy
+- ``losoto_directory``: full path to your local LoSoTo installation
+- ``aoflagger``: full path to your aoflagger executable
+
+
+
+**Parameters you may need to adjust**
+
+*Data selection and calibration options*
+
+- ``refant``:name of the station that will be used as a reference for the phase-plots
+- ``flag_baselines``: NDPPP-compatible pattern for baselines or stations to be flagged (may be an empty list, i.e.: ``[]`` )
+- ``process_baselines_cal``: performs A-Team-clipping/demixing and direction-independent phase-only self-calibration only on these baselines. Choose [CR]S*& if you want to process only cross-correlations and remove international stations.
+- ``filter_baselines``: selects only this set of baselines to be processed. Choose [CR]S*& if you want to process only cross-correlations and remove international stations.
+- ``do_smooth``: enable or disable baseline-based smoothing
+- ``rfistrategy``: strategy to be applied with the statistical flagger (AOFlagger), default: ``HBAdefault.rfis``
+- ``max_length``: amount of subbands to concatenate for full-bandwidth flagging (for an HBA calibrator, you can take all SBs if memory allows)
+- ``max2interpolate``: amount of channels in which interpolation should be performed for deriving the bandpass (default: 30)
+- ``interp_windowsize``: size of the window over which a value is interpolated. Should be odd. (default: 15)
+- ``ampRange``: range of median amplitudes accepted per station
+- ``skip_international``: skip fitting the bandpass for international stations (this avoids flagging them in many cases)
+- ``raw_data``: use autoweight, set to True in case you are using raw data (default: False)
+- ``propagatesolutions``: use already derived solutions as initial guess for the upcoming time slot
+- ``maxStddev``: maximum allowable standard deviation when outlier clipping is done. For phases, this should value should be in radians, for amplitudes in log(amp). If None (or negative), a value of 0.1 rad is used for phases and 0.01 for amplitudes
+
+A comprehensive explanation of the baseline selection syntax can be found `here`_.
+
+
+*Demixing options* (only used if demix step is added to the ``prep_cal_strategy`` variable)
+
+- ``demix_sources``: choose sources to demix (provided as list), e.g., ``[CasA,CygA]``
+- ``demix_target``: if given, the target source model (its patch in the SourceDB) is taken into account when solving (default: ``""``)
+- ``demix_freqstep``: number of channels to average when demixing (default: 16)
+- ``demix_timestep`` : number of time slots to average when demixing (default: 10)
+
+*Definitions for pipeline options*
+
+- ``default_flagging``: regular flagging steps after pre-processing by the observatory pipelines (default: ``flag,elev,flagamp``)
+- ``raw_flagging``: full set flagging steps (usually only necessary for raw data, default: ``flagedge,aoflag,{{ default_flagging }}``)
+- ``1st_order``: steps for first order clock-TEC separation (Do not change! Only ``cal_ion`` should be edited if needed, default: ``ct,plotTEC,residuals``)
+- ``3rd_order``: steps for third order clock-TEC separation (Do not change! Only ``cal_ion`` should be edited if needed, default: ``ct3,plotTEC3,residuals3``)
+- ``prep_cal_strategy``: steps to be performed for the preparation of the calibrator data. Add ``,demix`` if you want to enable demixing. (default: ``{{ default_flagging }}``)
+- ``cal_ion``: choose whether you want to perform 1st or 3rd order ionospheric effects during clock-TEC separation (default: ``{{ 1st_order }}``)
+
+
+**Parameters for pipeline performance**
+
+- ``num_proc_per_node``: number of processes to use per step per node (default: ``input.output.max_per_node``, reads the parameter ``max_per_node`` from the ``pipeline.cfg``)
+- ``num_proc_per_node_limit``: number of processes to use per step per node for tasks with high I/O (DPPP or cp) or memory (e.g. calibration) (default: 4)
+- ``max_dppp_threads``: number of threads per process for NDPPP (default: 10)
+- ``memoryperc``: maximum of memory used for aoflagger in raw_flagging mode in percent
+- ``min_length``: minimum amount of subbands to concatenate in frequency necessary to perform the wide-band flagging in the RAM. It data is too big aoflag will use indirect-read.
+- ``overhead``: Only use this fraction of the available memory for deriving the amount of data to be concatenated.
+- ``min_separation``: minimal accepted distance to an A-team source on the sky in degrees (will raise a WARNING)
+- ``error_tolerance``: defines whether pipeline run will continue if single bands fail (default: False)
+
+**Parameters you may want to adjust**
+
+*Main directories*
+
+- ``lofar_directory``: base directory of your **LOFAR** installation (default: $LOFARROOT)
+- ``job_directory``: directory of the prefactor outputs (usually the ``job_directory`` as defined in the ``pipeline.cfg``, default: ``input.output.job_directory``)
+
+*Script and plugin directories*
+
+- ``scripts``: location of the prefactor scripts (default: ``{{ prefactor_directory }}/scripts``)
+- ``pipeline.pluginpath``: location of the prefactor plugins: (default: ``{{ prefactor_directory }}/plugins``)
+
+*Skymodel directory*
+
+- ``calibrator_path_skymodel``: location of the prefactor skymodels (default: ``{{ prefactor_directory }}/skymodels``)
+
+*Result directories*
+
+- ``results_directory``: location of the prefactor results (default: ``{{ job_directory }}/results``)
+- ``inspection_directory``: location of the inspection plots (default: ``{{ results_directory }}/inspection``)
+- ``cal_values_directory``: directory of the calibration solutions (h5parm file, default: ``{{ results_directory }}/cal_values``)
+
+*Location of calibrator solutions*
+
+- ``cal_solutions``: location of the calibration solutions (h5parm file, default: ``{{ cal_values_directory }}/cal_solutions.h5``)
+
+*Averaging for the calibrator data*
+
+- ``avg_timeresolution``: final time resolution of the data in seconds after averaging (default: 4)
+- ``avg_freqresolution`` : final frequency resolution of the data after averaging (default: 48.82kHz, which translates to 4 channels per subband)
+- ``bandpass_freqresolution``: frequency resolution of the bandpass solution table (default: 195.3125kHz, which translates to 1 channel per subband)
+
+Parameters for **HBA** and **LBA** observations
+-----------------------------------------------
+====================== =============== =======================
+**parameter** **HBA** **LBA**
+---------------------- --------------- -----------------------
+``do_smooth`` False True
+``rfistrategy`` HBAdefault.rifs LBAdefaultwideband.rfis
+``cal_ion`` {{ 1st_order }} {{ 3rd_order }}
+``tables2export`` clock phaseOrig
+====================== =============== =======================
+
+In case of **LBA** observation you might also want to enable demixing in the ``prep_cal_strategy`` variable.
+
+.. _here: https://www.astron.nl/lofarwiki/doku.php?id=public:user_software:documentation:ndppp#description_of_baseline_selection_parameters
diff --git a/docs/source/calibscheme.png b/docs/source/calibscheme.png
new file mode 100644
index 0000000000000000000000000000000000000000..f4e862df985f55944d3544dcb12c6c924d5f43d1
Binary files /dev/null and b/docs/source/calibscheme.png differ
diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
new file mode 100644
index 0000000000000000000000000000000000000000..e392a0ca8cbaaffa5a0c0e4cb99483b04a7393db
--- /dev/null
+++ b/docs/source/changelog.rst
@@ -0,0 +1,48 @@
+.. _changelog:
+
+Changelog
+=========
+
+
+Version 4.0
+-----------
+
+ * First re-written version of the pipeline based on the `Common Workflow Language`_ (CWL)
+ * Additional diagnostics (UV-plots and summary file) and improved logging
+ * Full software is based on python3
+ * Easy-to-use with integrated Docker support
+
+Version 3.2 (last version based on the `genericpipeline`_ framework)
+-----------
+
+ * Several bugfixes and minor improvements
+ * Added high-resolution models for increased compatibility if using data including long baselines
+ * Option to add back missing stations for the use with long-baseline data
+ * Automated selection of the reference antenna
+
+Version 3.0
+-----------
+
+ * Major overhaul of calibrator and target pipelines to support LBA and HBA data
+ * Addition of production versions of calibrator, target, and image pipelines
+ * Documentation moved to http://www.astron.nl/citt/prefactor
+
+Version 2.0
+-----------
+
+ * applying Ionospheric RM corrections
+ * grouping of subbands by actual frequency
+ * speed and disk usage improvements by optimized usage of NDPPP
+ * (optional) wide-band cleaning in Initial-Subtract
+ * more diagnostic plots
+ * documentation moved to GitHib wiki: https://github.com/lofar-astron/prefactor/wiki
+ * automatic selection of calibrator sky model
+ * automatic generation of TGSS sky model
+
+Version 1.0
+-----------
+
+ * First release of the Pre-Facet-Calibration pipeline
+
+.. _Common Workflow Language: https://www.commonwl.org/
+.. _genericpipeline: https://www.astron.nl/citt/genericpipeline/
diff --git a/docs/source/clock.png b/docs/source/clock.png
new file mode 100644
index 0000000000000000000000000000000000000000..59c5636080c2310dc3103ade92bdcd23c5578b30
Binary files /dev/null and b/docs/source/clock.png differ
diff --git a/docs/source/concatenate.rst b/docs/source/concatenate.rst
new file mode 100644
index 0000000000000000000000000000000000000000..a895fd9eb6ba1ed48e8f9f4bed50aff50870fca3
--- /dev/null
+++ b/docs/source/concatenate.rst
@@ -0,0 +1,80 @@
+.. _concatenate_pipeline:
+
+Concatenate pipeline
+=========================
+
+This pipeline concatenates single-subband target data produced by production
+runs and retrieved through the LTA. The resulting concatenated files are
+required for further processing with the initial-subtract pipeline.
+
+.. note::
+
+ If you processed the target data yourself instead of retrieving them from the
+ LTA, this pipeline is not required as the concatenated datasets are already
+ produced by the user version of the target pipeline.
+
+
+Prepare data
+------------
+This part of the pipeline prepares the target data in order to be concatenated. The steps are
+as follows:
+
+``createmap_target``
+ Generate a mapfile of all the target data (the single-subband datasets retrieved
+ from the LTA, with the direction-independent phase-only calibration applied).
+``combine_target_map``
+ Generate a mapfile with all files in a single entry. This mapfile is used as
+ input to the next step.
+``sortmap_target``
+ Compute frequency groupings
+``do_magic_maps``, ``do_sortmap_maps``
+ Convert the output of do_magic into usable mapfiles.
+
+
+Concatenation
+-------------
+Subbands are concatenated into "bands".
+
+``dpppconcat``
+ Concatenate the data, averaging to the specified frequency and time resolution.
+``make_results_mapfile``, ``move_results``
+ Move the concatenated files to the results directory.
+
+
+
+User-defined parameter configuration
+------------------------------------
+
+**Parameters you will need to adjust**
+
+*Information about the input data*
+
+``! target_input_path``
+ Directory where your single-subband target data are stored.
+``! target_input_pattern``
+ Regular expression pattern of all your target files.
+ .. note::
+
+ These files should have the direction-independent calibration applied to the DATA
+ column.
+
+*Location of the software*
+
+``! prefactor_directory``
+ Path to your prefactor copy
+
+**Parameters you may need to adjust**
+
+*Interpolation options*
+
+- ``interp_windowsize``: size of the window over which a value is interpolated. Should be odd. (default: 15)
+
+*Averaging options*
+
+- ``avg_timeresolution_concat``: final time resolution of the data in seconds after averaging and concatenation (default: 8)
+- ``avg_freqresolution_concat``: final frequency resolution of the data after averaging and concatenation (default: 97.64kHz, which translates to 2 channels per subband)
+
+*Concatenation options*
+
+- ``num_SBs_per_group``: make concatenated measurement-sets with that many subbands (default: 10)
+- ``reference_stationSB``: station-subband number to use as reference for grouping, (default: ``None`` -> use lowest frequency input data as reference)
diff --git a/docs/source/conf.py b/docs/source/conf.py
new file mode 100644
index 0000000000000000000000000000000000000000..f5b1485e27c1f7a7440ef233be6309f70c3ae15d
--- /dev/null
+++ b/docs/source/conf.py
@@ -0,0 +1,160 @@
+# -*- coding: utf-8 -*-
+#
+# Configuration file for the Sphinx documentation builder.
+#
+# This file does only contain a selection of the most common options. For a
+# full list see the documentation:
+# http://www.sphinx-doc.org/en/master/config
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'Prefactor'
+copyright = '2021, Alexander Drabent and David Rafferty'
+author = 'Alexander Drabent and David Rafferty'
+
+# The short X.Y version
+version = '4.0'
+# The full version, including alpha/beta/rc tags
+release = '4.0'
+
+
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+ 'sphinx.ext.autodoc',
+ 'sphinx.ext.mathjax'
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path .
+exclude_patterns = []
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages. See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'classic'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further. For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+html_logo = 'bandpass.png'
+html_favicon = 'favicon.ico'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# The default sidebars (for documents that don't match any pattern) are
+# defined by theme itself. Builtin themes are using these templates by
+# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
+# 'searchbox.html']``.
+#
+# html_sidebars = {}
+
+
+# -- Options for HTMLHelp output ---------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'Prefactordoc'
+
+
+# -- Options for LaTeX output ------------------------------------------------
+
+latex_elements = {
+ # The paper size ('letterpaper' or 'a4paper').
+ #
+ # 'papersize': 'letterpaper',
+
+ # The font size ('10pt', '11pt' or '12pt').
+ #
+ # 'pointsize': '10pt',
+
+ # Additional stuff for the LaTeX preamble.
+ #
+ # 'preamble': '',
+
+ # Latex figure (float) alignment
+ #
+ # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+# author, documentclass [howto, manual, or own class]).
+latex_documents = [
+ (master_doc, 'Prefactor.tex', 'Prefactor Documentation',
+ 'Alexander Drabent and David Rafferty', 'manual'),
+]
+
+
+# -- Options for manual page output ------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+ (master_doc, 'prefactor', 'Prefactor Documentation',
+ [author], 1)
+]
+
+
+# -- Options for Texinfo output ----------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+# dir menu entry, description, category)
+texinfo_documents = [
+ (master_doc, 'Prefactor', 'Prefactor Documentation',
+ author, 'Prefactor', 'One line description of project.',
+ 'Miscellaneous'),
+]
+
diff --git a/docs/source/favicon.ico b/docs/source/favicon.ico
new file mode 100644
index 0000000000000000000000000000000000000000..bb2ecc8ee2bf01291602d2cb93c2cca94051fe8d
Binary files /dev/null and b/docs/source/favicon.ico differ
diff --git a/docs/source/flux_ratio_sky.png b/docs/source/flux_ratio_sky.png
new file mode 100644
index 0000000000000000000000000000000000000000..c15d87627ec8c8e9f5e5ab266436f6a93b1a6d22
Binary files /dev/null and b/docs/source/flux_ratio_sky.png differ
diff --git a/docs/source/flux_ratio_vs_distance.png b/docs/source/flux_ratio_vs_distance.png
new file mode 100644
index 0000000000000000000000000000000000000000..3433888686b181cf519629146e14be36aa479cf9
Binary files /dev/null and b/docs/source/flux_ratio_vs_distance.png differ
diff --git a/docs/source/flux_ratio_vs_flux.png b/docs/source/flux_ratio_vs_flux.png
new file mode 100644
index 0000000000000000000000000000000000000000..2b740273d1ac5a824744cb54ae24d07661800bec
Binary files /dev/null and b/docs/source/flux_ratio_vs_flux.png differ
diff --git a/docs/source/fr.png b/docs/source/fr.png
new file mode 100644
index 0000000000000000000000000000000000000000..67c255ea055771437a49ee0dd215db291b4a2fbf
Binary files /dev/null and b/docs/source/fr.png differ
diff --git a/docs/source/help.rst b/docs/source/help.rst
new file mode 100644
index 0000000000000000000000000000000000000000..df5cd0c8840d9438640d1dca47e5bb502fdeee99
--- /dev/null
+++ b/docs/source/help.rst
@@ -0,0 +1,14 @@
+.. _help:
+
+Getting help
+============
+
+**prefactor** is a continuously maintained and developed software package.
+If you need help or want to report any issues or bugs, follow these links:
+
+- `Prefactor GitHub issues`_
+- Frequently Asked Questions (`FAQ`_)
+
+
+.. _Prefactor GitHub issues: https://github.com/lofar-astron/prefactor/issues
+.. _FAQ: https://github.com/lofar-astron/prefactor/wiki/Documentation%3A-Faq
\ No newline at end of file
diff --git a/docs/source/image.rst b/docs/source/image.rst
new file mode 100644
index 0000000000000000000000000000000000000000..22126874f770100f00b771b96769014dbc0a14b8
--- /dev/null
+++ b/docs/source/image.rst
@@ -0,0 +1,114 @@
+.. _image_pipeline:
+
+Image pipeline
+==============
+
+This pipeline produces a Stokes-I image (a Stokes-V image is also produced for quality-control purposes) of the full FOV of the target data, using the full bandwidth. The
+parset is named ``Pre-Facet-Image.parset``.
+
+
+Prepare target
+--------------
+
+The target data that result from the target pipeline are averaged and concatenated in preparation for imaging. The steps
+are as follows:
+
+``create_ms_map``
+ Generate a mapfile of all the target data.
+``combine_mapfile``
+ Generate a mapfile with all files in a single entry. This mapfile is used as
+ input to the next step.
+``do_magic``
+ Compute image sizes and the number of channels to use during imaging from the MS
+ files from the previous step. The image size is calculated from the FWHM of the
+ primary beam at the lowest frequency at the mean elevation of the observation. The
+ number of channels is set simply as the number of subbands / 40, to result in
+ enough channels to allow multi-frequency synthesis (MFS), but not so many that
+ performance is impacted. A minimum of 2 channels is used.
+``do_magic_maps``
+ Convert the output of do_magic into usable mapfiles.
+``average``
+ Average the data as appropriate for imaging of the FOV. The amount of averaging
+ depends on the size of the image (to limit bandwidth and time smearing). The
+ averaging currently adopted is 16 s per time slot and 0.2 MHz per channel. These
+ values result in low levels of bandwidth and time smearing for the target image
+ sizes and resolutions.
+``combine_mapfile_deep``
+ Generate a mapfile with all files in a single entry. This mapfile is used as
+ input to the next step.
+``dpppconcat``
+ Run DPPP to concatenate the data. Concatenating the data speeds up gridding
+ and degridding with IDG by factors of several.
+
+
+Imaging
+-------
+WSClean is used to produce the Stokes-I/V images. See the parset and the ``do_magic`` step above
+for details of the parameters used. The values are chosen to produce good results for most
+standard observations.
+
+``wsclean_high_deep``
+ Image the data with WSClean+IDG. Imaging is done in MFS mode, resulting in a
+ single image for the full bandwidth. Primary-beam corrected and uncorrected images are
+ made.
+``plot_im_high_i/v``
+ Make a png figure of the Stokes-I/V images, including estimates of the image rms and dynamic
+ range and the restoring beam size. Typical HBA images look like the ones below (Stokes-I image is shown first and the Stokes-V image second).
+
+ .. image:: MFS-I-image-pb.plot_im_high_i.png
+ .. image:: MFS-V-image-pb.plot_im_high_v.png
+
+``make_source_list``
+ Make a list of sources from the Stokes-I image using PyBDSF and compare their properties to
+ those of the TGSS and GSM catalogs for HBA and LBA data, respectively. A number of plots
+ are made to allow quick assessment of the flux scale and astrometry of the image:
+
+ .. image:: flux_ratio_sky.png
+ .. image:: flux_ratio_vs_distance.png
+ .. image:: flux_ratio_vs_flux.png
+ .. image:: positional_offsets_sky.png
+
+
+User-defined parameter configuration
+------------------------------------
+
+*Information about the input data*
+
+``! target_input_path``
+ Directory where your concatenated target data are stored.
+``! target_input_pattern``
+ Regular expression pattern of all your target files.
+
+ .. note::
+
+ These files should have the direction-independent calibration applied to the DATA
+ column (usually the ``*.pre-cal.ms`` files from the target pipeline).
+
+*Imaging parameters*
+
+- ``cellsize_highres_deg``
+ Cellsize in degrees (default: 0.00208).
+- ``fieldsize_highres``
+ Size of the image is this value times the FWHM of mean semi-major axis of
+ the station beam at the lowest observed frequency (default: 1.5).
+- ``maxlambda_highres``
+ Maximum uv-distance in lambda that will be used for imaging. A minimum uv-distance
+ of 80 lambda is used in all cases (default: 7000).
+- ``image_padding``
+ Amount of padding to add during the imaging (default: 1.4).
+- ``idg_mode``
+ IDG mode to use: cpu or hybrid (default: cpu).
+- ``local_scratch_dir``
+ Scratch directory for WSClean (default: ``{{ job_directory }}``).
+- ``image_rootname``
+ Output image root name (default: ``{{ job_directory }}/fullband``). The image will be named ``image_rootname-MFS-I-image.fits``.
+
+
+Parameters for **HBA** and **LBA** observations
+-----------------------------------------------
+======================== ======= =======
+**parameter** **HBA** **LBA**
+------------------------ ------- -------
+``cellsize_highres_deg`` 0.00208 0.00324
+``maxlambda_highres`` 7000 4000
+======================== ======= =======
diff --git a/docs/source/image_pipeline_example.png.REMOVED.git-id b/docs/source/image_pipeline_example.png.REMOVED.git-id
new file mode 100644
index 0000000000000000000000000000000000000000..845c450184606fa7bbd757144707100797c204eb
--- /dev/null
+++ b/docs/source/image_pipeline_example.png.REMOVED.git-id
@@ -0,0 +1 @@
+6b76262567b49b4603d5ae6b7576743a205f8cc1
\ No newline at end of file
diff --git a/docs/source/index.rst b/docs/source/index.rst
new file mode 100644
index 0000000000000000000000000000000000000000..1453b2a9aeb8596752f353e6863648dfea2f4db7
--- /dev/null
+++ b/docs/source/index.rst
@@ -0,0 +1,81 @@
+.. Prefactor documentation master file, created by
+ sphinx-quickstart on Tue Nov 27 11:30:20 2018.
+ You can adapt this file completely to your liking, but it should at least
+ contain the root `toctree` directive.
+
+Prefactor: Preprocessing for Facet Calibration for LOFAR
+========================================================
+
+**prefactor** is a pipeline to correct for various instrumental and ionospheric effects in both **LOFAR HBA** and **LOFAR LBA** observations.
+
+It includes:
+
+- removal of clock offsets between core and remote stations (using clock-TEC separation)
+- correction of the polarization alignment between XX and YY
+- robust time-independent bandpass correction
+- ionospheric RM corrections with `RMextract`_
+- removal of the element beam
+- advanced flagging and interpolation of bad data
+- mitigation of broad-band RFI and bad stations
+- direction-independent phase correction of the target, using a global sky model from `TGSS ADR`_ or the new `Global Sky Model`_ (GSM)
+- detailed diagnostics
+
+It will prepare your data to allow you continuing the processing with any direction-dependent calibration software, like `Rapthor`_, `factor`_ or `killMS`_.
+
+.. note::
+
+ If you intend to use **prefactor** for the processing of long baselines (international stations) the user is referred to the `LOFAR-VLBI documentation`_, since the compatibility of both pipelines may be limited to certain version or releases. It is recommended to stick to the instructions therein.
+
+ **Note:** The current version of **prefactor** does not yet support state-of-the-art calibration of target fields with the **LOFAR LBA**. An implementation is planned for upcoming releases. Please continue your processing using `LiLF`_.
+
+
+Introduction
+------------
+
+.. toctree::
+ :maxdepth: 2
+
+ acknowledgements
+
+
+Obtaining Prefactor
+-------------------
+
+.. toctree::
+ :maxdepth: 2
+
+ installation
+ changelog
+
+
+Setting Up and Running Prefactor
+--------------------------------
+
+.. toctree::
+ :maxdepth: 2
+
+ preparation
+ parset
+ running
+ help
+
+
+The Prefactor Pipelines
+-----------------------
+
+.. toctree::
+ :maxdepth: 2
+
+ pipelineoverview
+ calibrator
+ target
+
+
+.. _LOFAR-VLBI documentation: https://lofar-vlbi.readthedocs.io/en/latest/
+.. _Rapthor: https://github.com/darafferty/rapthor
+.. _Global Sky Model: https://lcs165.lofar.eu/
+.. _TGSS ADR: https://http://tgssadr.strw.leidenuniv.nl/
+.. _RMextract: https://github.com/lofar-astron/RMextract/
+.. _factor: https://github.com/lofar-astron/factor/
+.. _killMS: https://github.com/saopicc/killMS/
+.. _LiLF: https://github.com/revoltek/LiLF
diff --git a/docs/source/initsub_high_image.png.REMOVED.git-id b/docs/source/initsub_high_image.png.REMOVED.git-id
new file mode 100644
index 0000000000000000000000000000000000000000..cb7ecff67d98ea3ed08fe23889b9ed52fff6c0c1
--- /dev/null
+++ b/docs/source/initsub_high_image.png.REMOVED.git-id
@@ -0,0 +1 @@
+2f50c9e12d58d95df6f91b8a3e821c5ffe7b27fb
\ No newline at end of file
diff --git a/docs/source/initsub_low_image.png.REMOVED.git-id b/docs/source/initsub_low_image.png.REMOVED.git-id
new file mode 100644
index 0000000000000000000000000000000000000000..85e53ed1a578ebe4c305c2ea3277618828836dab
--- /dev/null
+++ b/docs/source/initsub_low_image.png.REMOVED.git-id
@@ -0,0 +1 @@
+fdac062c9b99abb4445f242cf9a4d526780e3173
\ No newline at end of file
diff --git a/docs/source/initsubtract.rst b/docs/source/initsubtract.rst
new file mode 100644
index 0000000000000000000000000000000000000000..c67e129583336b70a6c1326c85f853a41031b185
--- /dev/null
+++ b/docs/source/initsubtract.rst
@@ -0,0 +1,214 @@
+.. _initsubtract_pipeline:
+
+Initial-subtract pipeline
+=========================
+
+This pipeline images the full FoV (and first side lobe) at two resolutions and at
+multiple frequencies, generating a sky model and subtracting it from the
+visibilities. This pipeline need only be run if you want to use Factor to do the
+direction-dependent imaging. The parset is named one of ``Initial-Subtract.parset``,
+``Initial-Subtract-IDG.parset``, or ``Initial-Subtract-IDG-LowMemory.parset``,
+depending on whether one wants to use IDG with WSClean. IDG is generally much
+faster than the normal WSClean if you have GPUs.
+
+.. note::
+
+ At this time, only HBA data are supported.
+
+
+Prepare data
+------------
+This part of the pipeline prepares the target data in order to be imaged. The steps are
+as follows:
+
+``create_ms_map``
+ Generate a mapfile of all the target data (the concatenated datasets output by the
+ target pipeline, with the direction-independent phase-only calibration applied).
+``combine_mapfile``
+ Generate a mapfile with all files in a single entry. This mapfile is used as
+ input to the next step.
+``do_magic``
+ Compute frequency groupings, image sizes, and averaging values using the MS
+ files from the previous step. The image size is calculated from the FWHM of the
+ primary beam at the lowest frequency at the mean elevation of the observation.
+``do_magic_maps``
+ Convert the output of do_magic into usable mapfiles.
+``create_h5parm_map``
+ Create a mapfile with the direction independent h5parm.
+``expand_h5parm_mapfile``
+ Expand the h5parm mapfile so that there is one entry for every file.
+``select_imaging_bands``
+ Select bands spread over the full bandwidth for imaging.
+``select_high_size``
+ Adjust the high_size mapfile to match the selected bands.
+``select_high_nwavelengths``
+ Adjust the nwavelengths mapfile to match the selected bands.
+
+
+Imaging and subtraction
+-----------------------
+Imaging is done at two resolutions to fully cover the expected range of source structure.
+WSClean is used to produce the images. See the parset and the do_magic step above
+for details of the parameters used. They are chosen to produce good results for
+most standard observations.
+
+``wsclean_high``
+ Image the data with WSClean to make the high-resolution images. The images will
+ automatically be stretched along the y-axis to account for the elongation of the
+ primary beam as a function of average elevation. A typical image at
+ lower Declination (+7 degrees) looks like the one below.
+
+ .. image:: initsub_high_image.png
+
+``mask_high``
+ Make masks for the high-res images. Masks are used to exclude artifacts from
+ being included in the subtract steps.
+``mk_inspect_dir``
+ Create the inspection_directory if needed.
+``copy_mask``
+ Copy the mask images to where we want them.
+``plot_im_high``
+ Plot the high-res image and mask as png files. Such an image is show above.
+``move_high``
+ Move the high-res images to where we want them.
+``create_maxsize_high_map``
+ Make a mapfile with maximum image size.
+``pad_model_high``
+ Pad the model images to a uniform size.
+``pad_mask_high``
+ Pad the mask images to a uniform size.
+``combine_model_high_mapfile``
+ Compress the model_high mapfile.
+``expand_model_high``
+ Expand the model_high mapfile so that there is one entry for every band.
+``combine_mask_high_mapfile``
+ Compress the mask_high mapfile.
+``expand_mask_high``
+ Expand the mask high mapfile so that there is one entry for every band.
+``fits_to_bbs_high``
+ Convert high-res model images to sky models that are understood by DPPP.
+``make_sourcedb_high``
+ Make sourcedbs from the high-res sky models.
+``expand_sourcedb_high``
+ Expand the sourcedb mapfile so that there is one entry for every file.
+``subtract_high``
+ Predict, corrupt, and subtract the high-resolution model. The subtraction is
+ done from the DATA column to the SUBTRACTED_DATA_HIGH column. The SUBTRACTED_DATA_HIGH
+ column is imaged later in the ``wsclean_low`` step to pick up any emission missed in
+ the high-resolution image.
+``select_low_size``
+ Adjust the low size mapfile to match the selected bands.
+``select_low_nwavelengths``
+ Adjust the low nwavelengths mapfile to match the selected bands.
+``wsclean_low``
+ Image the data (after subtraction of the high-resolution model) with WSClean
+ to make the low-resolution images. The images will automatically be
+ stretched along the y-axis to account for the elongation of the primary beam
+ as a function of average elevation. A typical image at lower Declination (+7
+ degrees) looks like the one below.
+
+ .. image:: initsub_low_image.png
+
+``mask_low``
+ Make masks for the low-res images. Masks are used to exclude artifacts from
+ being included in the subtract steps.
+``plot_im_low``
+ Plot the low-res image and mask as png files. Such an image is show above.
+``move_low``
+ Move the low-res images to where we want them.
+``create_maxsize_low_map``
+ Make a mapfile with maximum image size.
+``pad_model_low``
+ Pad the model images to a uniform size.
+``pad_mask_low``
+ Pad the mask images to a uniform size.
+``combine_model_low_mapfile``
+ Compress the model_low mapfile.
+``expand_model_low``
+ Expand the model_low mapfile so that there is one entry for every band.
+``combine_mask_low_mapfile``
+ Compress the mask_low mapfile.
+``expand_mask_low``
+ Expand the mask low mapfile so that there is one entry for every band.
+``fits_to_bbs_low``
+ Convert low-res model images to sky models.
+``make_sourcedb_low``
+ Make sourcedbs from the low-res sky models.
+``expand_sourcedb_low``
+ Expand the sourcedb mapfile so that there is one entry for every file.
+``subtract_low``
+ Predict, corrupt, and subtract the low-resolution model. The subtraction is
+ done from the SUBTRACTED_DATA_HIGH column to the SUBTRACTED_DATA_ALL column.
+ Therefore, the SUBTRACTED_DATA_ALL column contains the final residual data needed
+ for Factor.
+``merge``
+ Merge the high-res and low-res sky models together. These sky models are used
+ by Factor to add sources back before calibration.
+``copy_skymodels``
+ Copy the merged sky models to the directory with the input data.
+``createmap_plots``
+ Create a map with the generated plots.
+``move_plots``
+ Move the plots to the inpection directory.
+
+
+
+User-defined parameter configuration
+------------------------------------
+
+**Parameters you will need to adjust**
+
+*Information about the input data*
+
+``! data_input_path``
+ Directory where your concatenated target data are stored.
+``! data_input_pattern``
+ Regular expression pattern of all your target files.
+ .. note::
+
+ These files should have the direction-independent calibration applied to the DATA
+ column (usually the ``*.pre-cal.ms`` files from the target pipeline).
+
+*Location of the software*
+
+``! prefactor_directory``
+ Path to your prefactor copy
+``! wsclean_executable``
+ Path to your local WSClean executable
+
+**Parameters you may need to adjust**
+
+*Imaging and subtraction options*
+
+``! cellsize_highres_deg``
+ Cellsize in degrees for high-resolution images.
+``! cellsize_lowres_deg``
+ Cellsize in degrees for low-resolution images.
+``! fieldsize_highres``
+ Size of the high-resolution image is this value times the FWHM of mean semi-major axis of
+ the station beam.
+``! fieldsize_lowres``
+ Size of the low-resolution image is this value times the FWHM of mean semi-major axis of
+ the station beam.
+``! maxlambda_highres``
+ Maximum uv-distance in lambda that will be used for the high-resolution imaging.
+``! maxlambda_lowres``
+ Maximum uv-distance in lambda that will be used for the low-resolution imaging.
+``! image_padding``
+ How much padding shall we add during the imaging?
+``! nbands_image``
+ Number of bands to image (spread over the full bandwidth). Larger values
+ result in better subtraction but longer runtimes.
+``! min_flux_jy``
+ Minimum flux density in Jy of clean components from the high-resolution
+ imaging to include in subtract_high step.
+``! idg_mode``
+ IDG mode to use: cpu or hybrid (= CPU + GPU).
+``! local_scratch_dir``
+ Scratch directory for wsclean (can be local to the processing nodes!).
+
+
+Parameters for **HBA** and **LBA** observations
+-----------------------------------------------
+
+At this time, only HBA data are supported.
diff --git a/docs/source/installation.rst b/docs/source/installation.rst
new file mode 100644
index 0000000000000000000000000000000000000000..e66ece1e2b658e3b9718466dad2653ba88597239
--- /dev/null
+++ b/docs/source/installation.rst
@@ -0,0 +1,105 @@
+.. _installation:
+
+Downloading and installing prefactor
+====================================
+
+You can choose between the manual installation of all software required by **prefactor** or the use of pre-compiled Docker images.
+
+Manual installation
+--------------------------
+
+For installing **prefactor** a Debian based operating system is recommended. In order to compile the required packages for running **prefactor** you may need to install the following packages::
+
+ $ apt-get update && \
+ $ apt-get install -y gfortran flex bison wcslib-dev libncurses5-dev casacore-data casacore-dev libboost-python-dev libcfitsio-dev python-dev \
+ python3-numpy libcasa* cmake build-essential liblua5.3-dev libhdf5-serial-dev libarmadillo-dev libboost-filesystem-dev libboost-system-dev \
+ libboost-date-time-dev libboost-numpy-dev libboost-signals-dev libboost-program-options-dev libboost-test-dev pybind11-dev libxml2-dev \
+ libpng-dev pkg-config libgtkmm-3.0-dev git wget libfftw3-dev libgsl-dev
+
+In order to run **prefactor** you need to get the following packages onto your system::
+
+ $ apt-get install -y vim wget casacore-tools casacore-data wcslib-dev libarmadillo8 bison libncurses5 flex libboost-date-time1.65.1 libboost-filesystem1.65.1 \
+ libboost-numpy1.65.1 libboost-python1.65.1 libboost-program-options1.65.1 libboost-system1.65.1 libboost-signals1.65.1 \
+ libboost-test1.65.1 libboost-python1.65-dev libstationresponse3 liblua5.3-dev libcasa-* pybind11-dev libcfitsio5 libcfitsio-dev \
+ libgtkmm-3.0 libfftw3-3 libhdf5-cpp-100 libpng16-16 libxml2 python3.7 python3-casacore dysco python3-numpy python3-scipy
+
+Then you need to install the following software:
+
+ * `LofarStMan`_
+ * `Dysco`_ (v2.1 or newer)
+ * `IDG`_ (v0.8 or newer)
+ * `aoflagger`_ (v3.1.0 or newer)
+ * `LOFARBeam`_ (v4.1.1 or newer)
+ * `EveryBeam`_ (v0.2.0 or newer)
+ * `DP3`_ (v5.1 or newer)
+ * `WSClean`_ (v2.10.1 or newer)
+ * `RMextract`_ (v0.4.2 or newer)
+ * `LoSoTo`_ (v2.2 or newer)
+ * `LSMTool`_ (v1.4.3 or newer)
+
+To install the **prefactor** software package call::
+
+ $ pip3 install --upgrade pip && \
+ $ git clone https://github.com/lofar-astron/prefactor.git <prefactor_dir> && \
+ $ cd <prefactor_dir> && \
+ $ git checkout <prefactor_version> && \
+ $ git pull && \
+ $ pip3 install --upgrade $PWD && \
+ $ cd .. && \
+ $ rm -rfv <prefactor_dir>
+
+and ::
+
+ $ python3 -m pip install matplotlib
+
+
+whereas ``<prefactor-dir>`` is the name of the temporary prefactor source directory from where you build the package and ``<prefactor_version>`` the `version number`_ you aim to install.
+
+
+Docker installation
+--------------------------
+There is no need to install all necessary software packages required for **prefactor** onto your system.
+You can also make use of pre-compiled Docker images.
+Instructions on how you can install Docker on your system can be found here:
+ * `CentOS`_
+ * `Debian`_
+ * `Fedora`_
+ * `Ubuntu`_
+
+
+
+Getting prefactor pipeline description
+------------------------------
+The **prefactor** pipeline is described in the `Common Workflow Language`_ (CWL).
+In order to retrieve the pipeline description call::
+
+ $ git clone https://git.astron.nl/eosc/prefactor3-cwl.git <install_dir>
+
+
+To start the pipeline you need to install an interpreter for the CWL description files.
+The most common ones are `cwltool`_ and `toil`_::
+
+ $ python3 -m pip install cwltool cwl-runner toil[cwl]
+
+
+.. _toil: https://toil.readthedocs.io/en/latest/index.html
+.. _cwltool: https://github.com/common-workflow-language/cwltool
+.. _Common Workflow Language: https://www.commonwl.org/
+.. _CentOS: https://docs.docker.com/engine/install/centos/
+.. _Debian: https://docs.docker.com/engine/install/debian/
+.. _Fedora: https://docs.docker.com/engine/install/fedora/
+.. _Ubuntu: https://docs.docker.com/engine/install/ubuntu/
+.. _version number: https://github.com/lofar-astron/prefactor/tags
+.. _LSMTool: https://github.com/darafferty/LSMTool
+.. _RMextract: https://github.com/lofar-astron/RMextract.git
+.. _LoSoTo: https://github.com/revoltek/losoto.git
+.. _WSClean: https://gitlab.com/aroffringa/wsclean.git
+.. _DP3: https://git.astron.nl/RD/DP3.git
+.. _EveryBeam: https://git.astron.nl/RD/EveryBeam.git
+.. _LOFARBeam: https://github.com/lofar-astron/LOFARBeam.git
+.. _aoflagger: https://gitlab.com/aroffringa/aoflagger.git
+.. _IDG: https://git.astron.nl/RD/idg.git
+.. _Dysco: https://github.com/aroffringa/dysco.git
+.. _LofarStMan: https://github.com/lofar-astron/LofarStMan
+.. _github page: https://github.com/lofar-astron/prefactor
+.. _ASTRON gitlab page: https://git.astron.nl/eosc/prefactor3-cwl
diff --git a/docs/source/ion_ph-res_poldif.png.REMOVED.git-id b/docs/source/ion_ph-res_poldif.png.REMOVED.git-id
new file mode 100644
index 0000000000000000000000000000000000000000..d37f721e1ea78c677bcfdf06eea91d40940987b7
--- /dev/null
+++ b/docs/source/ion_ph-res_poldif.png.REMOVED.git-id
@@ -0,0 +1 @@
+359dd6bf8913eba92a82c2c4b65c5c0aa70a1b01
\ No newline at end of file
diff --git a/docs/source/parset.rst b/docs/source/parset.rst
new file mode 100644
index 0000000000000000000000000000000000000000..1b77b6911fd7f5b3789fbf632f85b28da6bb3dbb
--- /dev/null
+++ b/docs/source/parset.rst
@@ -0,0 +1,122 @@
+.. _parset:
+
+Configuring prefactor
+=====================
+.. note::
+
+ If you are running the deprecated genericpipeline version of the pipeline (**prefactor** 3.2 or older), please check the :doc:`old instrunctions page<parset_old>`.
+
+Preparing the configuration file
+--------------------------------
+
+The inputs for **prefactor** are provided in the `JSON format`_. The only required input is the input data to process (for the calibrator pipeline).
+A minimum input file may look like this::
+
+ {
+ "msin": [
+ {"class": "Directory", "path": "3C286/L228161_SB000_uv.dppp.MS"},
+ {"class": "Directory", "path": "3C286/L228161_SB001_uv.dppp.MS"},
+ {"class": "Directory", "path": "3C286/L228161_SB002_uv.dppp.MS"},
+ {"class": "Directory", "path": "3C286/L228161_SB003_uv.dppp.MS"},
+ {"class": "Directory", "path": "3C286/L228161_SB004_uv.dppp.MS"},
+ {"class": "Directory", "path": "3C286/L228161_SB005_uv.dppp.MS"},
+ {"class": "Directory", "path": "3C286/L228161_SB006_uv.dppp.MS"},
+ {"class": "Directory", "path": "3C286/L228161_SB007_uv.dppp.MS"},
+ {"class": "Directory", "path": "3C286/L228161_SB008_uv.dppp.MS"},
+ {"class": "Directory", "path": "3C286/L228161_SB009_uv.dppp.MS"}
+ ],
+ }
+
+There are more parameters you may want to adjust that can be added to this input JSON file. This is how an input looks like that uses all defaults of the HBA calibrator pipeline::
+
+ {
+ "msin": [
+ {"class": "Directory", "path": "3C286/L228161_SB000_uv.dppp.MS"},
+ {"class": "Directory", "path": "3C286/L228161_SB001_uv.dppp.MS"},
+ {"class": "Directory", "path": "3C286/L228161_SB002_uv.dppp.MS"},
+ {"class": "Directory", "path": "3C286/L228161_SB003_uv.dppp.MS"},
+ {"class": "Directory", "path": "3C286/L228161_SB004_uv.dppp.MS"},
+ {"class": "Directory", "path": "3C286/L228161_SB005_uv.dppp.MS"},
+ {"class": "Directory", "path": "3C286/L228161_SB006_uv.dppp.MS"},
+ {"class": "Directory", "path": "3C286/L228161_SB007_uv.dppp.MS"},
+ {"class": "Directory", "path": "3C286/L228161_SB008_uv.dppp.MS"},
+ {"class": "Directory", "path": "3C286/L228161_SB009_uv.dppp.MS"}
+ ],
+ "refant": "CS00.*",
+ "flag_baselines": [],
+ "process_baselines_cal": "*&",
+ "filter_baselines": "*&",
+ "fit_offset_PA": false,
+ "do_smooth": false,
+ "rfi_strategy": "HBAdefault.rfis",
+ "max2interpolate": 30,
+ "ampRange": [0,0],
+ "skip_international": true,
+ "raw_data": false,
+ "propagatesolutions": true,
+ "flagunconverged": false,
+ "maxStddev": -1.0,
+ "solutions2transfer": null,
+ "antennas2transfer": "[FUSPID].*",
+ "do_transfer": false,
+ "trusted_sources": "3C48,3C147,3C196,3C295,3C380",
+ "demix_sources": ["CasA","CygA"],
+ "demix_target": "",
+ "demix_freqstep": 16,
+ "demix_timestep": 10,
+ "demix": false,
+ "ion_3rd": false,
+ "clock_smooth": true,
+ "tables2export": "clock",
+ "max_dppp_threads": 10,
+ "memoryperc": 20,
+ "min_length": 50,
+ "overhead": 0.8,
+ "min_separation": 30,
+ "max_separation_arcmin": 1.0,
+ "calibrator_path_skymodel": null,
+ "A-Team_skymodel": null,
+ "avg_timeresolution": 4,
+ "avg_freqresolution": "48.82kHz",
+ "bandpass_freqresolution": "195.3125kHz"
+ }
+
+If you just want to alter one of the defaults it is sufficient to override it by specifing its new value the JSON input file::
+
+ {
+ "msin": [
+ {"class": "Directory", "path": "3C286/L228161_SB000_uv.dppp.MS"},
+ {"class": "Directory", "path": "3C286/L228161_SB001_uv.dppp.MS"},
+ {"class": "Directory", "path": "3C286/L228161_SB002_uv.dppp.MS"},
+ {"class": "Directory", "path": "3C286/L228161_SB003_uv.dppp.MS"},
+ {"class": "Directory", "path": "3C286/L228161_SB004_uv.dppp.MS"},
+ {"class": "Directory", "path": "3C286/L228161_SB005_uv.dppp.MS"},
+ {"class": "Directory", "path": "3C286/L228161_SB006_uv.dppp.MS"},
+ {"class": "Directory", "path": "3C286/L228161_SB007_uv.dppp.MS"},
+ {"class": "Directory", "path": "3C286/L228161_SB008_uv.dppp.MS"},
+ {"class": "Directory", "path": "3C286/L228161_SB009_uv.dppp.MS"}
+ ],
+ "demix": true,
+ }
+
+If you run the target pipeline you also need to provide the calibrator solution set::
+
+ {
+ "msin": [
+ {"class": "Directory", "path": "?????/L228163_SB000_uv.dppp.MS"},
+ {"class": "Directory", "path": "?????/L228163_SB001_uv.dppp.MS"},
+ {"class": "Directory", "path": "?????/L228163_SB002_uv.dppp.MS"},
+ {"class": "Directory", "path": "?????/L228163_SB003_uv.dppp.MS"},
+ {"class": "Directory", "path": "?????/L228163_SB004_uv.dppp.MS"},
+ {"class": "Directory", "path": "?????/L228163_SB005_uv.dppp.MS"},
+ {"class": "Directory", "path": "?????/L228163_SB006_uv.dppp.MS"},
+ {"class": "Directory", "path": "?????/L228163_SB007_uv.dppp.MS"},
+ {"class": "Directory", "path": "?????/L228163_SB008_uv.dppp.MS"},
+ {"class": "Directory", "path": "?????/L228163_SB009_uv.dppp.MS"}
+ ],
+ "cal_solutions": {"class": "File", "path": "results/cal_values/cal_solutions.h5"}
+ }
+
+A detailed description of the input parameters can be found in the :doc:`calibrator` and :doc:`target` section.
+
+.. _JSON format: https://www.json.org/json-en.html
diff --git a/docs/source/parset_old.rst b/docs/source/parset_old.rst
new file mode 100644
index 0000000000000000000000000000000000000000..242c6f0335d27439174be8b37f74428c94ca2a75
--- /dev/null
+++ b/docs/source/parset_old.rst
@@ -0,0 +1,111 @@
+.. _parset_old:
+
+Configuring prefactor
+=====================
+.. note::
+
+ These instructions are outdated and only valid for **prefactor** 3.2 or older. Please check the :doc:`recent instrunctions page<parset>`.
+
+Preparing the configuration file
+--------------------------------
+To set up the genericpipeline for prefactor you need to customize the ``pipeline.cfg`` configuration file:
+
+- Copy ``$LOFARROOT/share/pipeline/pipeline.cfg`` to someplace in your ``$HOME`` and open it in an editor.
+
+- It starts with a section ``[DEFAULT]``, in there you need to edit three entries:
+
+ - ``runtime_directory``: This is the directory where the pipeline puts
+ logfiles, parsets, the status of successful steps etc. This can be set to a
+ directory in your ``$HOME``, but it is recommended to set this to the same
+ value as the `working_directory`
+
+ - ``working_directory``: This is the directory where the processed data of the
+ intermediate and final steps is put. Set this to a directory on your data
+ disk, e.g. ``/data/scratch/<username>/PipelineExample``
+
+ - ``recipe_directories``: This is a list of directories where the pipeline
+ searches for recipes and plugins. There should already be an entry there, but
+ another needs to be added so that the plugin scripts of prefactor are found.
+ You need to add the pre-facet calibration directory to this list (so that the
+ ``plugins`` directory is a subdirectory of one of the
+ ``recipe\_directories``). E.g.:
+ ``recipe_directories = [%(pythonpath)s/lofarpipe/recipes,/home/<username>/software/prefactor]``.
+
+- There may be empty entries in the ``[DEFAULT]`` section of the
+ ``pipeline.cfg`` file. This was set during the installation of the LOFAR
+ software and usually is no need to worry.
+
+- In case you do not run it on a cluster, you need to tell the pipeline to start the processes on the local machine:
+
+ - Search for the section ``[cluster]``, set the entry clusterdesc to ``(lofarroot)s/share/local.clusterdesc``.
+
+ - Add a section ``[remote]`` by adding the following lines::
+
+ [remote]
+ method = local
+ max_per_node = <NumCPUCores>
+
+ If there is already another section ``[remote]``, then remove that.
+
+- The pipeline framework contains a number of parallel processing schemes for
+ working on multi-node clusters. Ask you local sysadmin for advice.
+
+
+Preparing the pipeline parset
+-----------------------------
+
+The pipeline parsets available in the prefactor directory (those files ending
+with ``.parset``) are templates of genericpipeline parsets and need some small
+editing before they can be run. To avoid confusion you should make a copy of the
+parset that you want to run and give it a descriptive name. All parameters that
+need to be changed are defined at the top of the parset with lines beginning
+with ``!``. See the comments in the pipeline parsets and the notes below for some
+hints on setting these parameters.
+
+Some parameters depend on the observation to be processed and need to be
+modified for each new observation, others are more "machine-dependent" so they
+are the same for different observations that are processed on the same
+machine(s).
+
+See :ref:`pipeline_overview_old` for an overview of the pipeline parsets, and their
+respective pages for a more in-depth description. Below are some general guidelines
+for preparing the parsets:
+
+- Don't edit the original parset files directly. Make a copy with a descriptive
+ name (e.g. ``Pre-Facet-Cal-calibrator-3c295.parset``) and edit that copy.
+
+- The pipeline framework will use the filename of the pipeline parset as the
+ job-name if the latter is not explicitly given. That way there is no need to
+ change the ``runtime_directory`` and ``working_directory`` entries in the
+ ``pipeline.cfg`` for different pipeline runs. The pipeline framework will
+ generate sub-directories with the job-name in there.
+
+- The ``reference_station`` should be a station that is present in the
+ observation, and didn't do anything strange. It is worth checking the plots and
+ possibly changing the reference station if problems are found.
+
+- The ``num_proc_per_node`` parameter is the number of processes of "small"
+ programs (with little memory usage and which are not multi-threaded) that are
+ run in parallel on one node.
+
+- The ``num_proc_per_node_limit`` parameter is the number of processes of "big"
+ programs (with large memory usage and/or multi-threaded programs) that are run
+ in parallel on one node.
+
+- In addition to setting how many DPPP processes (DPPP is a "big" program) are
+ run in parallel, you can set how many threads each DPPP process may use with the
+ ``max_dppp_threads`` parameter.
+
+- So both: ``num_proc_per_node`` and (``num_proc_per_node_limit`` *
+ ``max_dppp_threads``) should be equal or smaller than the number of cores that
+ you have on your processing computers.
+
+- Similarly (``max_imagers_per_node`` * ``max_percent_mem_per_img``) should be
+ less that 100% and (``max_imagers_per_node`` * ``max_cpus_per_img``) should be
+ equal or smaller than the number of cores.
+
+- If your pipeline runs out of memory, then you can also lower these parameters
+ to make the pipeline use less memory.
+
+- Most of the actual processing is now done in DPPP, so the parameters that
+ control its behavior are the important ones.
diff --git a/docs/source/ph_freq.png.REMOVED.git-id b/docs/source/ph_freq.png.REMOVED.git-id
new file mode 100644
index 0000000000000000000000000000000000000000..b01ebe60eb2eb5764fcfc93da4383a6a2cd10e29
--- /dev/null
+++ b/docs/source/ph_freq.png.REMOVED.git-id
@@ -0,0 +1 @@
+f6637a5b3284603fe8d43e4fe6ba5ccd45545fb8
\ No newline at end of file
diff --git a/docs/source/ph_polXX.png b/docs/source/ph_polXX.png
new file mode 100644
index 0000000000000000000000000000000000000000..cffefd7e1bc857a83d169fc21bd6c15267a24539
Binary files /dev/null and b/docs/source/ph_polXX.png differ
diff --git a/docs/source/ph_poldif.png b/docs/source/ph_poldif.png
new file mode 100644
index 0000000000000000000000000000000000000000..c189827124c6f5d4331cc1e301bb01ca5dfe6305
Binary files /dev/null and b/docs/source/ph_poldif.png differ
diff --git a/docs/source/ph_poldif_freq.png b/docs/source/ph_poldif_freq.png
new file mode 100644
index 0000000000000000000000000000000000000000..111af632155ca4536d3f07640e1e5634f05a84ee
Binary files /dev/null and b/docs/source/ph_poldif_freq.png differ
diff --git a/docs/source/pipelineoverview.rst b/docs/source/pipelineoverview.rst
new file mode 100644
index 0000000000000000000000000000000000000000..e2947afe1f9de2a13283d545c46caab86ebb8c98
--- /dev/null
+++ b/docs/source/pipelineoverview.rst
@@ -0,0 +1,16 @@
+.. _pipeline_overview:
+
+Pipeline overview
+=================
+.. note::
+
+ If you are running the deprecated genericpipeline version of the pipeline (**prefactor** 3.2 or older), please check the :doc:`old instrunctions page<pipelineoverview_old>`.
+
+**Prefactor** is organized in three major parts to process **LOFAR** data:
+
+ .. image:: prefactor_CWL_workflow_sketch.png
+
+``prefactor_calibrator``
+ Processes the (amplitude-)calibrator to derive direction-independent corrections. See :ref:`calibrator_pipeline` for details.
+``prefactor_target``
+ Transfers the direction-independent corrections to the target and does direction-independent calibration of the target. See :ref:`target_pipeline` for details.
diff --git a/docs/source/pipelineoverview_old.rst b/docs/source/pipelineoverview_old.rst
new file mode 100644
index 0000000000000000000000000000000000000000..23fcdd858aa0075e2ee672b5dda4281fc9607adb
--- /dev/null
+++ b/docs/source/pipelineoverview_old.rst
@@ -0,0 +1,24 @@
+.. _pipeline_overview_old:
+
+Pipeline overview
+=================
+
+.. note::
+
+ These instructions are outdated and only valid for **prefactor** 3.2 or older. Please check the :doc:`recent instrunctions page<pipelineoverview>`.
+
+**Prefactor** is organized in three major parts to process **LOFAR** data:
+
+ .. image:: prefactor_workflow_sketch.png
+
+``Pre-Facet-Calibrator``
+ Processes the (amplitude-)calibrator to derive direction-independent corrections. See :ref:`calibrator_pipeline_old` for details.
+``Pre-Facet-Target``
+ Transfers the direction-independent corrections to the target and does direction-independent calibration of the target. See :ref:`target_pipeline_old` for details.
+``Concatenate``
+ Concatenates the single-subband target data retrieved from the LTA into bands suitable for further processing with the initial-subtract pipeline. See :ref:`concatenate_pipeline` for details.
+``Initial-Subtract``
+ Images the full FoV (and 1st side-lobe), generating a sky-model and subtracting it from the visibilities. See :ref:`initsubtract_pipeline` for details.
+``Pre-Facet-Image``
+ Images the full FoV using the full bandwidth. See :ref:`image_pipeline` for details.
+
diff --git a/docs/source/polalign.png b/docs/source/polalign.png
new file mode 100644
index 0000000000000000000000000000000000000000..135c2f3a5823332fd021bc9660d5e5b191101981
Binary files /dev/null and b/docs/source/polalign.png differ
diff --git a/docs/source/polalign_amp_polXX.png.REMOVED.git-id b/docs/source/polalign_amp_polXX.png.REMOVED.git-id
new file mode 100644
index 0000000000000000000000000000000000000000..c0f85c0be41a904696c05fff603ec81b88a1bbdc
--- /dev/null
+++ b/docs/source/polalign_amp_polXX.png.REMOVED.git-id
@@ -0,0 +1 @@
+598d381ba68be35a56e5ee8bbe0ce90c9f34cb1d
\ No newline at end of file
diff --git a/docs/source/polalign_ph_polXX.png.REMOVED.git-id b/docs/source/polalign_ph_polXX.png.REMOVED.git-id
new file mode 100644
index 0000000000000000000000000000000000000000..b7ca80e2570a44f0cc59ff7a4eadcb03124ca458
--- /dev/null
+++ b/docs/source/polalign_ph_polXX.png.REMOVED.git-id
@@ -0,0 +1 @@
+47e9728cbc118383209b6905b11f42452004a6ce
\ No newline at end of file
diff --git a/docs/source/polalign_ph_poldif.png.REMOVED.git-id b/docs/source/polalign_ph_poldif.png.REMOVED.git-id
new file mode 100644
index 0000000000000000000000000000000000000000..38d0e13779dfb1d11a25978381d6d50f39420bde
--- /dev/null
+++ b/docs/source/polalign_ph_poldif.png.REMOVED.git-id
@@ -0,0 +1 @@
+1bf6088377afb06425be9e86d48cfde4a07437ff
\ No newline at end of file
diff --git a/docs/source/polalign_rotangle.png.REMOVED.git-id b/docs/source/polalign_rotangle.png.REMOVED.git-id
new file mode 100644
index 0000000000000000000000000000000000000000..b9942c3fcb2e656447d6ac0b7646023c0468311e
--- /dev/null
+++ b/docs/source/polalign_rotangle.png.REMOVED.git-id
@@ -0,0 +1 @@
+5301fd8f9a03325a97a56f4c443595e93362762e
\ No newline at end of file
diff --git a/docs/source/positional_offsets_sky.png b/docs/source/positional_offsets_sky.png
new file mode 100644
index 0000000000000000000000000000000000000000..5653667b51bb4ae81fff11f7069216a38250a26a
Binary files /dev/null and b/docs/source/positional_offsets_sky.png differ
diff --git a/docs/source/prefactor_CWL_workflow_sketch.png b/docs/source/prefactor_CWL_workflow_sketch.png
new file mode 100644
index 0000000000000000000000000000000000000000..270d1a4fcf521c988893a0106d19cc9c00ad58f3
Binary files /dev/null and b/docs/source/prefactor_CWL_workflow_sketch.png differ
diff --git a/docs/source/prefactor_workflow_sketch.png b/docs/source/prefactor_workflow_sketch.png
new file mode 100644
index 0000000000000000000000000000000000000000..0993d7f8aa3dedc1999d20ed5e8681eb952653ff
Binary files /dev/null and b/docs/source/prefactor_workflow_sketch.png differ
diff --git a/docs/source/preparation.rst b/docs/source/preparation.rst
new file mode 100644
index 0000000000000000000000000000000000000000..6153cbda0b9252e2716a8918732586ac1f2abe11
--- /dev/null
+++ b/docs/source/preparation.rst
@@ -0,0 +1,31 @@
+.. _data_preparation:
+
+Preparing the data
+------------------
+
+**Prefactor** requires **LOFAR LBA** or **HBA** raw or pre-processed data. These data are
+typically obtained from the LOFAR `Long-Term Archive`_.
+
+- The calibrator and target data have to match, i.e., be observed close enough
+ in time that calibration values can be transferred.
+
+- For each observation you should process all the calibrator data at once
+ together. Clock/TEC separation and flagging of bad amplitudes work better with
+ the full bandwidth.
+
+- For the target pipeline you will need to have internet access from the machine you are running **prefactor**.
+ It is required in order to retrieve RM values from `CODE`_ and a global sky model (`TGSS`_ or `GSM`_). Both are hosted as online services.
+ It is also possible to provide an own target skymodel to **prefactor** (using the parameters ``target_skymodel``, and ``use_target``, see the :doc:`target<target>` pipeline parameter information).
+
+.. note::
+
+ Processing of interleaved datasets is not currently supported.
+
+ **prefactor** can not handle multi-epoch observations at once.
+
+ Older versions of **prefactor**: All input measurement-sets for one pipeline run need to be in the same directory.
+
+.. _CODE: ftp://ftp.aiub.unibe.ch/CODE/
+.. _TGSS: http://tgssadr.strw.leidenuniv.nl/doku.php
+.. _GSM: http://172.104.228.177/
+.. _Long-Term Archive: https://lta.lofar.eu
diff --git a/docs/source/running.rst b/docs/source/running.rst
new file mode 100644
index 0000000000000000000000000000000000000000..1e746da656864d02930f3f012d96ca212da007a4
--- /dev/null
+++ b/docs/source/running.rst
@@ -0,0 +1,126 @@
+.. _runprefactor:
+
+Starting a pipeline
+=====================
+
+.. note::
+
+ If you are running the deprecated genericpipeline version of the pipeline (**prefactor** 3.2 or older), please check the :doc:`old instrunctions page<running_old>`.
+
+
+Once you have the data and the input JSON file ready, you can run the pipeline, e.g., with ``cwltool`` or ``toil`` for the HBA calibrator pipeline::
+
+ $ cwltool <cwl_options> <install_dir>/workflows/HBA_calibrator.cwl prefactor.json
+
+ $ toil-cwl-runner <cwl_options> <install_dir>/workflows/HBA_calibrator.cwl prefactor.json
+
+where ``prefactor.json`` is the input JSON file as described in the chapter :doc:`parset` and ``<install_dir>`` the location of the prefactor CWL description files.
+
+.. note::
+
+ Instead of specifying all options in ``prefactor.json`` the user can also use command line options to override the defaults.
+
+
+For standard LOFAR observations there are workflows available with pre-defined parameters (defaults) for **HBA** and **LBA** observations:
+
+============================ ====================== =======================
+**prefactor workflow** **HBA** **LBA**
+---------------------------- ---------------------- -----------------------
+``prefactor_calibrator.cwl`` ``HBA_calibrator.cwl`` ``LBA_calibrator.cwl``
+``prefactor_target.cwl`` ``HBA_target.cwl`` ``LBA_target.cwl``
+============================ ====================== =======================
+
+.. note::
+
+ The **LBA** workflows are not (yet) available.
+
+
+
+Pipeline options for ``cwltool``
+--------------------------------
+
+The following ``<cwl_options>`` are recommended to use for running **prefactor** with ``cwltool``:
+
+ * **---outdir**: specifies the location of the pipeline outputs
+ * **---tmpdir-prefix**: specifies the location of the intermediate data products
+ * **---parallel**: jobs will run in parallel
+ * **---no-container**: don't use Docker container (only for manual installation)
+ * **---preserve-entire-environment**: use system environment variables (only for manual installation)
+ * **---debug**: more verbose output (use only for debugging the pipeline)
+
+.. note::
+
+ ``cwltool`` has no option to resume a failed/crashed run. If you need this option have a look at ``toil``.
+
+
+While the pipeline runs, the terminal will output the current state of the pipeline. It is also possible to pipe this output into a runtime logfile::
+
+ $ cwltool <cwl_options> <install_dir>/workflows/HBA_calibrator.cwl prefactor.json > logfile 2>&1
+
+
+In the specified ``--tmpdir_prefix`` all temporary folders and files are generated. At the end of the run those files can be deleted.
+
+Pipeline options for ``toil``
+--------------------------------
+The following ``<cwl_options>`` are recommended to use for running **prefactor** with ``toil``:
+
+ * **---workDir**: specifies the location of the intermediate data products
+ * **---outDir**: specificies the lcoation of the final data products
+ * **---jobStore**: location of the jobStore ("statefile")
+ * **---writeLogs**: location of the pipeline job logfiles
+ * **---logFile**: location of the main pipeline logfile
+ * **---logLevel**: can be **CRITICAL**, **ERROR**, **WARNING**, **INFO** or **DEBUG**
+ * **---batchSystem**: use speficic batch system of an HPC cluster or similar, e.g. ``slurm`` or ``single_machine``
+ * **---stats**: creates runtime statistics
+ * **---maxLocalJobs**: amount of local jobs to be run at the same time ("max_per_node")
+ * **---retryCount**: amount of retries for each failed pipeline job
+ * **---preserve-entire-environment**: use system environment variables (only for manual installation)
+ * **---no-container**: don't use Docker container (only for manual installation)
+
+
+Stopping and restarting the pipeline
+------------------------------------
+
+You can stop a pipeline run anytime by terminating the CWL process (typically by pressing CRTL-C in the terminal where you started it).
+
+Sometimes some of
+the processes that the pipeline started don't get properly terminated, so if the
+genericpipeline process doesn't terminate you should look for its child
+processes and terminate them too.
+
+.. note::
+
+ If you stop and re-start pipelines a number of time then you should also
+ check occasionally if there are orphaned children that are eating up
+ resources on you computer.
+
+You can restart a pipeline if using ``toil`` through adding the parameter ``--restart`` on the terminal. If you want to start from scratch you should delete the directory created via ``jobStore`` and all intermediate data products (usually specified via the ``--workDir`` parameter). After that you will start with a clean run. As mentioned earlier, you can re-start the pipeline by running the same command with which you started it.
+
+Pipeline crashes
+----------------
+
+With ``cwltool`` a pipeline crash is a reported with this message::
+
+ WARNING Final process status is permanentFail
+
+In order to figure out at which step the pipeline failed you can search for the term ``permanentFail`` in the ``toil`` or ``cwltool`` logfile::
+
+ $ more logfile | grep "Permanent Fail"
+
+ WARNING [job compare_station_list] completed permanentFail
+ WARNING [step compare_station_list] completed permanentFail
+ INFO [workflow prep] completed permanentFail
+ WARNING [step prep] completed permanentFail
+ INFO [workflow prefactor] completed permanentFail
+ WARNING [step prefactor] completed permanentFail
+ INFO [workflow ] completed permanentFail
+ WARNING Final process status is permanentFail
+
+With that information it is possible to identify the failed step ``compare_station_list``. To find the corresponding part of the logfile where the step was launched you search for ``[job compare_station_list]``.
+
+It is usually best to also check all lines leading with ``ERROR`` to get additional information about the possible cause of the crash or diagnostic messages that tell you what exactly went wrong. See :ref:`help` for tips on interpreting the error messages.
+
+If you identify the problem and it does not affect the products that have been
+already produced, you can launch the pipeline again, after correcting the issue
+causing the process to stop. But in most cases it might be necessary to fully re-start the pipeline.
+
diff --git a/docs/source/running_old.rst b/docs/source/running_old.rst
new file mode 100644
index 0000000000000000000000000000000000000000..b3d6f007fc37e978993b6613fe1b4b8a8327aa24
--- /dev/null
+++ b/docs/source/running_old.rst
@@ -0,0 +1,99 @@
+.. _runprefactor_old:
+Starting a pipeline
+-------------------
+
+.. note::
+
+ These instructions are outdated and only valid for **prefactor** 3.2 or older. Please check the :doc:`recent instrunctions page<running>`.
+
+
+Once you have the data and the parsets ready, you can run the pipeline using the
+genericpipeline script, e.g.::
+
+ $ genericpipeline.py -d -c pipeline.cfg My_prefactor_calibrator.parset
+
+.. note::
+
+ The -d option is recommended: it does make the log-files extremely large
+ (many megabytes), but without it, often the important information as to why a
+ pipeline run fails is not included.
+
+While the pipeline runs, in the specified ``runtime_directory`` (see previous
+section) new files are generated in a directory named after the parset name (e.g.,
+if you are running ``My_prefactor_calibrator.parset`` a directory named
+``My_prefactor_calibrator`` will appear in your ``runtime_directory``)::
+
+ $ ls My_prefactor_calibrator/
+ logs mapfiles parsets statefile
+
+The logs dir contains all the logs of the pipeline runs, identified by the date
+and time of execution, e.g.::
+
+ $ ls My_prefactor_calibrator/logs/2016-06-30T15:07:21/pipeline.log
+
+These contain all the output from the processes the pipeline called and
+diagnostic information about the pipeline. So they are useful to follow the
+status of the process and possibly identify reasons why a process crashed.
+
+While running the pipeline writes a statefile in the ``runtime_directory``, with all
+the step which were successfully executed. If the pipeline stops for whatever
+reason, you can re-run the same command and it will skip all the steps that are
+already done and only work on those which are still missing.
+
+The intermediate data files of the pipeline are written in the ``working_directory``
+specified in the ``pipeline.cfg``.
+
+
+Stopping and restarting the pipeline
+------------------------------------
+
+You can stop a pipeline run anytime by terminating the genericpipeline process
+(typically by pressing CRTL-C in the terminal where you started it). Sometimes some of
+the processes that the pipeline started don't get properly terminated, so if the
+genericpipeline process doesn't terminate you should look for its child
+processes and terminate them too.
+
+.. note::
+
+ If you stop and re-start pipelines a number of time then you should also
+ check occasionally if there are orphaned children that are eating up
+ resources on you computer.
+
+As mentioned earlier, you can re-start the pipeline by running the same command
+with which you started it.
+
+
+Pipeline crashes
+----------------
+
+It can happen that the pipeline stops with a message like this::
+
+ ERROR genericpipeline: LOFAR Pipeline finished unsuccesfully.
+ WARNING genericpipeline: recipe genericpipeline completed with errors
+
+You need to read the log of that run to identify the reason why it stopped, e.g.::
+
+ > less My_prefactor_calibrator/logs/2016-06-30T15:07:21/pipeline.log
+
+It is usually best to first check at the end of the file for what ended the
+pipeline and then search from the beginning of the file for error or diagnostic
+messages that tell you what exactly went wrong. See :ref:`help` for tips on
+interpreting the error messages.
+
+If you identify the problem and it does not affect the products that have been
+already produced, you can launch the pipeline again, after correcting the issue
+causing the process to stop.
+
+
+Rerunning parts of the pipeline
+--------------------------------
+
+You can fully rerun a pipeline by deleting the runtime and working directories and restarting the pipeline.
+
+To rerun parts of the pipeline that were (allegedly) already executed
+successfully, you need to modify the ``statefile`` of the pipeline. To do this
+there is a ``statefile_manipulation.py`` script as part of prefactor::
+
+ python prefactor/bin/statefile_manipulation.py My_Workdir/My_calibrator_job/statefile
+
+If you then run the pipeline again, it will start at the step that you removed with the statefile manipulation tool.
diff --git a/docs/source/structure.png b/docs/source/structure.png
new file mode 100644
index 0000000000000000000000000000000000000000..d15431b70e8e2a5cf811cd99283c815b2877794b
Binary files /dev/null and b/docs/source/structure.png differ
diff --git a/docs/source/target.rst b/docs/source/target.rst
new file mode 100644
index 0000000000000000000000000000000000000000..28f488478c63c500e6375c2b55aa2845bc2e61a6
--- /dev/null
+++ b/docs/source/target.rst
@@ -0,0 +1,338 @@
+.. _target_pipeline:
+
+Target pipeline
+===============
+
+.. note::
+
+ If you are running the deprecated genericpipeline version of the pipeline (**prefactor** 3.2 or older), please check the :doc:`old instrunctions page<target_old>`.
+
+This pipeline processes the target data in order to apply the direction-independent corrections from the calibrator pipeline. A first initial direction-independent self-calibration of the target field is performed, using a global sky model based on the `TGSS ADR`_ or the new `Global Sky Model`_ (GSM), and applied to the data.
+
+This chapter will present the specific steps of the target pipeline in more detail.
+
+All results (diagnostic plots and calibration solutions) will be stored usually in the ``--outdir`` directory specified with your ``cwltool`` or ``toil`` command.
+
+ .. image:: targetscheme.png
+
+
+Prepare target, incl. "demixing" (``prep``)
+-------------------------------------------
+This part of the pipeline prepares the target data in order to be calibration-ready for the first direction-independent phase-only self-calibration against a global sky model.
+This mainly includes mitigation of bad data (RFI, bad antennas, contaminations from A-Team sources), selection of the data to be calibrated (usually Dutch stations only), and some averaging to reduce data size and enhance the signal-to-noise ratio.
+Furthermore, ionospheric Rotation Measure corrections are applied, using `RMextract`_
+The user can specify whether to do raw data or pre-processed data flagging and whether demixing should be performed.
+
+The basic workflows are:
+
+- preparation of data (``prep``)
+- concatenating and phase-only self-calibration against a global sky model (``gsmcal``)
+- creating the finally calibrated data set, via applying the self-calibration solutions and compressing the data (``finalize``)
+
+The workflow ``prep`` consists of:
+ - check for a potential station mismatch between calibrator solutions and the target data (step ``compare_station_list``)
+ - checking for nearby A-Team sources (step ``check_Ateam_separation``)
+ - creating a model of A-Team sources to be subtracted (step ``make_sourcedb_ateam``)
+ - getting ionospheric Rotation Measure corrections and adding it to the solutions (step ``createRMh5parm``)
+ .. image:: RMextract.png
+ - basic flagging, applying solutions, and averaging (subworkflow ``ndppp_prep_target``)
+ - edges of the band (``flagedge``) -- only used if ``raw_data : true``
+ - statistical flagging (``aoflag``) -- only used in ``raw_data : true``
+ - baseline flagging (``flagbaseline``)
+ - low elevation flagging (below 15 degress elevation) (``flagelev``)
+ - low amplitude flagging (below 1e-30) (``flagamp``)
+ - demix A-Team sources (``demix``) -- only used if specified ``demix : true``
+ - applying calibrator solutions (steps ``applyPA``, ``applyBandpass``, ``prep_target_applycal``)
+ - averaging of the data in time and frequency
+ - predicting impact of A-Team sources and write it to the ``MODEL_DATA`` column (step ``predict``)
+ - clipping time- and frequency chunks that are likely to be affected by A-Team sources (step ``Ateamclipper``)
+
+
+Phase-only self-calibration (``gsmcal``)
+-----------------------------------------
+These steps aim for deriving a good first guess for the phase correction in the direction of the phase center (direction-independent phase correction).
+
+Once this is done, the data is ready for further processing with direction-dependent calibration techniques, using software like `Rapthor`_, `factor`_ or `killMS`_.
+
+The phase solutions derived from the ``gsmcal`` workflow are collected and loaded into **LoSoTo** to provide diagnostic plots:
+
+- ``ph_freq??``: matrix plot of the phase solutions with time for a particular chunk of target data, where both polarizations are colorcoded
+ .. image:: ph_freq.png
+- ``ph_poldif_freq??``: matrix plot of the XX-YY phase solutions with time for a particular chunk of target data
+ .. image:: ph_poldif_freq.png
+- ``ph_pol??``: matrix plot of the phase solutions for the XX and YY polarization
+ .. image:: ph_polXX.png
+- ``ph_poldif``: matrix plot of the phase solutions for the XX-YY polarization
+ .. image:: ph_poldif.png
+
+The workflow ``gsmcal`` consists of:
+ - retrieving and creating a global sky model (steps ``find_skymodel_target``, ``make_sourcedb_target``)
+ - identification of fully flagged antennas (step ``identify_bad_antennas``)
+ - concatenating the data into chunks (subworkflow ``concat``)
+ - wide-band statistical flagging (steps ``ms_concat`` and ``aoflag``)
+ - checking for bad data chunks (step ``check_unflagged_fraction``)
+ - perform the self-calibration against the global skymodel (subworkflow ``calibrate_target``, baseline-dependend smoothing (step ``BLsmooth``) if specified ``do_smooth : true``))
+
+Finalizing the prefactor output (``finalize``)
+----------------------------------------------
+These steps produce the final data output and many helpful diagnostics.
+
+The workflow ``finalize`` consists of:
+ - adding missing stations to the solution set with zero phase and unit amplitude (for international stations, step ``add_missing_stations``)
+ - applying the phase-only self-calibration solutions to the data and compress them (step ``apply_gsmcal``)
+ - derive the structure function of the phases (step ``structure_function``)
+ - make a fast image of the target field (steps ``average`` and ``wsclean``)
+ - create plots of the ``uv``-coverage of the final data set (step ``uvplot``)
+ - create a summary file (step ``summary``)
+
+The last step also incorporates full `Dysco`_ compression to save disk space. The fully calibrated data is stored in the DATA column of the final data set.
+
+.. note::
+ All solutions are written in the h5parm file format via the steps ``H5parm_collector`` and called during all the workflows.
+
+ The solutions are stored in the final calibrator solution set ``results/cal_values/cal_solutions.h5``.
+
+Further diagnostics
+-------------------
+
+The ``results`` directory will contain all relevant outputs of the current **prefactor** run, once the pipeline has finished:
+ - fully calibrated data sets
+ - logfiles in ``results/logs``
+ - summary file (JSON format) in ``results/summary``
+ - calibration solutions in ``results/cal_values/cal_solutions.h5``
+ - inspection plots in ``results/inspection``
+
+The following diagnostic help to assess the quality of the data reduction:
+ - ``Ateam_separation.png``: shows the distance and the elevation of A-Team sources with respect to the analyzed observation
+ .. image:: Ateam_separation.png
+ - ``Ateamclipper.png``: fraction of flagged data due to their potential contamination from A-Team sources versus frequency
+ .. image:: Ateamclipper.png
+ - ``unflagged_fraction.png``: fraction of remaining unflagged data versus frequency
+ .. image:: unflagged_fraction.png
+ - ``??_uv-coverage_uvdist.png``: fraction of remaining unflagged data versus ``uv``-distance
+ .. image:: uv-coverage_uvdist.png
+ - ``??_uv_coverage.png``: the ``uv``-coverage of the final data set
+ .. image:: uv-coverage.png
+ - ``??_structure.png``: plot of the ionospheric `structure function`_ of the processed target field
+ .. image:: structure.png
+ - ``??-MFS-image.fits``: FITS image of the target field
+ .. image:: target_field.png
+
+You can also check the calibration solutions for more details::
+
+ $ losoto -i results/cal_values/cal_solutions.h5
+
+ Summary of results/cal_values/cal_solutions.h5
+
+ Solution set 'calibrator':
+ ==========================
+
+ Directions: 3c286
+
+ Stations: CS001HBA0 CS001HBA1 CS002HBA0 CS002HBA1
+ CS003HBA0 CS003HBA1 CS004HBA0 CS004HBA1
+ CS005HBA0 CS005HBA1 CS006HBA0 CS006HBA1
+ CS007HBA0 CS007HBA1 CS011HBA0 CS011HBA1
+ CS017HBA0 CS017HBA1 CS021HBA0 CS021HBA1
+ CS024HBA0 CS024HBA1 CS026HBA0 CS026HBA1
+ CS028HBA0 CS028HBA1 CS030HBA0 CS030HBA1
+ CS031HBA0 CS031HBA1 CS032HBA0 CS032HBA1
+ CS101HBA0 CS101HBA1 CS103HBA0 CS103HBA1
+ CS201HBA0 CS201HBA1 CS301HBA0 CS301HBA1
+ CS302HBA0 CS302HBA1 CS401HBA0 CS401HBA1
+ CS501HBA0 CS501HBA1 RS106HBA RS205HBA
+ RS208HBA RS210HBA RS305HBA RS306HBA
+ RS307HBA RS310HBA RS406HBA RS407HBA
+ RS409HBA RS503HBA RS508HBA RS509HBA
+
+ Solution table 'bandpass' (type: amplitude): 120 times, 372 freqs, 60 ants, 2 pols
+ Flagged data: 0.000%
+
+ Solution table 'clock' (type: clock): 120 times, 60 ants
+ Flagged data: 0.000%
+
+ Solution table 'faraday' (type: rotationmeasure): 60 ants, 120 times
+ Flagged data: 0.014%
+
+ Solution table 'polalign' (type: phase): 120 times, 60 ants, 1484 freqs, 2 pols
+ Flagged data: 0.000%
+
+ Solution set 'target':
+ ======================
+
+ Directions: P000+00
+
+ Stations: CS001HBA0 CS001HBA1 CS002HBA0 CS002HBA1
+ CS003HBA0 CS003HBA1 CS004HBA0 CS004HBA1
+ CS005HBA0 CS005HBA1 CS006HBA0 CS006HBA1
+ CS007HBA0 CS007HBA1 CS011HBA0 CS011HBA1
+ CS017HBA0 CS017HBA1 CS021HBA0 CS021HBA1
+ CS024HBA0 CS024HBA1 CS026HBA0 CS026HBA1
+ CS028HBA0 CS028HBA1 CS030HBA0 CS030HBA1
+ CS031HBA0 CS031HBA1 CS032HBA0 CS032HBA1
+ CS101HBA0 CS101HBA1 CS103HBA0 CS103HBA1
+ CS201HBA0 CS201HBA1 CS301HBA0 CS301HBA1
+ CS302HBA0 CS302HBA1 CS401HBA0 CS401HBA1
+ CS501HBA0 CS501HBA1 RS106HBA RS205HBA
+ RS208HBA RS210HBA RS305HBA RS306HBA
+ RS307HBA RS310HBA RS406HBA RS407HBA
+ RS409HBA RS503HBA RS508HBA RS509HBA
+
+ Solution table 'RMextract' (type: rotationmeasure): 60 ants, 119 times
+ Flagged data: 0.000%
+
+ Solution table 'TGSSphase' (type: phase): 3446 times, 58 ants, 1 freq, 2 pols
+ Flagged data: 0.000%
+ History: 2021-07-30 11:25:44: Bad stations 'CS006HBA1', 'CS006HBA0' have not been added
+ back.
+
+For an overall summary it is advised to check the summary logfile::
+
+ $ cat results/logs/???_summary.log
+
+ *****************************************
+ *** prefactor target pipeline summary ***
+ *****************************************
+
+ Field name: P000+00
+
+ User-specified baseline filter: [CR]S*&
+ Additional antennas removed from the data: CS006HBA1, CS006HBA0
+ A-Team sources close to the phase reference center: NONE
+
+ XX diffractive scale: 4.4 km
+ YY diffractive scale: 4.0 km
+
+ Changes applied to cal_solutions.h5:
+ 2021-07-30 11:25:44: Bad stations 'CS006HBA1', 'CS006HBA0' have not been added back.
+
+ Amount of flagged solutions per station and solution table:
+ Station bandpass clock faraday polalign RMextract TGSSphase
+ CS001HBA0 0.29% 0.00% 0.00% 0.00% 0.00% 0.00%
+ CS001HBA1 0.29% 0.00% 0.00% 0.00% 0.00% 0.00%
+ CS002HBA0 0.29% 0.00% 0.00% 0.00% 0.00% 0.05%
+ CS002HBA1 0.29% 0.00% 0.00% 0.00% 0.00% 0.00%
+ CS003HBA0 0.29% 0.00% 0.00% 0.00% 0.00% 0.00%
+ CS003HBA1 0.29% 0.00% 0.00% 0.00% 0.00% 0.05%
+ CS004HBA0 0.29% 0.00% 0.00% 0.00% 0.00% 0.05%
+ CS004HBA1 6.05% 0.00% 0.00% 0.00% 0.00% 0.05%
+ CS005HBA0 0.29% 0.00% 0.00% 0.00% 0.00% 0.05%
+ CS005HBA1 0.39% 0.00% 0.00% 0.00% 0.00% 0.00%
+ CS006HBA0 0.29% 0.00% 0.00% 0.00% 0.00%
+ CS006HBA1 0.29% 0.00% 0.00% 0.00% 0.00%
+
+ Amount of flagged data per station at a given state:
+ Station initial prep Ateam final
+ CS001HBA0 5.13% 5.41% 11.12% 22.74%
+ CS001HBA1 5.13% 5.41% 11.03% 22.51%
+ CS002HBA0 5.12% 5.39% 11.39% 23.18%
+ CS002HBA1 5.12% 5.40% 21.09% 29.95%
+ CS003HBA0 5.12% 5.39% 9.92% 22.58%
+ CS003HBA1 5.12% 5.40% 11.37% 23.95%
+ CS004HBA0 5.12% 5.40% 13.27% 24.62%
+ CS004HBA1 5.12% 5.40% 12.24% 23.53%
+ CS005HBA0 5.12% 5.40% 11.59% 23.38%
+ CS005HBA1 5.12% 15.36% 20.07% 30.09%
+ CS006HBA0 100.00% 100.00% 100.00%
+ CS006HBA1 100.00% 100.00% 100.00%
+
+ **********
+ Summary file is written to: ???_prefactor_target_summary.json
+ Summary has been created.
+
+
+User-defined parameter configuration
+------------------------------------
+**Parameters you will need to adjust**
+
+*Location of the target data and calibrator solutions*
+
+- ``msin``: location of the input target data, for instructions look at the :doc:`configuration instructions<parset>` page
+- ``cal_solutions``: location of the calibrator solutions, for instructions look at the :doc:`configuration instructions<parset>` page.
+
+**Parameters you may need to adjust**
+
+*Data selection and calibration options*
+
+- ``refant``: regular expression of the stations that are allowed to be selected as a reference antenna by the pipeline (default: ``CS00.*``)
+- ``flag_baselines``: DP3-compatible pattern for baselines or stations to be flagged (may be an empty list, i.e.: ``[]`` )
+- ``process_baselines_target``: performs A-Team-clipping/demixing and direction-independent phase-only self-calibration only on these baselines. Choose ``[CR]S*&`` if you want to process only cross-correlations and remove international stations (default: ``[CR]S*&``)
+- ``filter_baselines``: selects only this set of baselines to be processed. Choose ``[CR]S*&`` if you want to process only cross-correlations and remove international stations (default: ``[CR]S*&``)
+- ``do_smooth``: enable or disable baseline-based smoothing (default: ``false``)
+- ``rfistrategy``: strategy to be applied with the statistical flagger (`AOFlagger`_, default: ``HBAdefault.rfis``)
+- ``min_unflagged_fraction``: minimal fraction of unflagged data to be accepted for further processing of the data chunk (default: 0.5)
+- ``raw_data``: use autoweight, set to True in case you are using raw data (default: ``false``)
+- ``compression_bitrate``: defines the bitrate of `Dysco`_ compression of the data after the final step, choose 0 if you do NOT want to compress the data
+- ``propagatesolutions``: use already derived solutions as initial guess for the upcoming time slot
+- ``apply_tec``: apply TEC solutions from the calibrator (default: ``false``)
+- ``apply_clock``: apply clock solutions from the calibrator (default: ``true``)
+- ``apply_phase``: apply full phase solutions from the calibrator (default: ``false``)
+- ``apply_RM``: apply ionospheric Rotation Measure from `RMextract`_ (default: ``true``)
+- ``apply_beam``: apply element beam corrections (default: ``true``)
+- ``gsmcal_step``: type of calibration to be performed in the self-calibration step (default: ``phase``)
+- ``updateweights``: update ``WEIGHT_SPECTRUM`` column in a way consistent with the weights being inverse proportional to the autocorrelations (default: ``true``)
+- ``use_target``: enable downloading of a target skymodel (default: ``true``)
+- ``skymodel_source``: choose the target skymodel from `TGSS ADR`_ or the new `Global Sky Model`_ (GSM) (default: ``TGSS``)
+
+A comprehensive explanation of the baseline selection syntax can be found `here`_.
+
+*Demixing and clipping options*
+
+- ``demix_sources``: choose sources to demix (provided as list), e.g., ``[CasA,CygA]``
+- ``demix_target``: if given, the target source model (its patch in the SourceDB) is taken into account when solving (default: ``""``)
+- ``demix_freqstep``: number of channels to average when demixing (default: 16)
+- ``demix_timestep`` : number of time slots to average when demixing (default: 10)
+- ``demix``: enable demixing (default: ``false``)
+- ``clip_sources``: list of the skymodel patches to be used for Ateamclipping (default: ``[VirA_4_patch,CygAGG,CasA_4_patch,TauAGG]``)
+
+*Further pipeline options*
+
+- ``min_separation``: minimal accepted distance to an A-team source on the sky in degrees (will raise a WARNING, default: ``30``)
+
+**Parameters for pipeline performance**
+
+- ``max_dppp_threads``: number of threads per process for DP3 (default: 10)
+- ``memoryperc``: maximum of memory used for aoflagger in raw_flagging mode in percent (default: 20)
+- ``min_length``: minimum amount of subbands to concatenate in frequency necessary to perform the wide-band flagging in the RAM. It data is too big aoflag will use indirect-read (default: 50)
+- ``overhead``: only use this fraction of the available memory for deriving the amount of data to be concatenated (default: 0.8)
+
+*Skymodel directory*
+
+- ``A-Team_skymodel``: location of the prefactor A-Team skymodels
+- ``target_skymodel``: location of a user-defined target skymodel used for the self-calibration
+
+*Averaging for the calibrator data*
+
+- ``avg_timeresolution``: intermediate time resolution of the data in seconds after averaging (default: 4)
+- ``avg_freqresolution`` : intermediate frequency resolution of the data after averaging (default: 48.82kHz, which translates to 4 channels per subband)
+- ``avg_timeresolution_concat``: final time resolution of the data in seconds after averaging and concatenation (default: 8)
+- ``avg_freqresolution_concat``: final frequency resolution of the data after averaging and concatenation (default: 97.64kHz, which translates to 2 channels per subband)
+- ``num_SBs_per_group``: make concatenated measurement-sets with that many subbands, choose a high number if running LBA (default: 10)
+
+*Concatenating of the target data*
+
+- ``num_SBs_per_group``: make concatenated measurement-sets with that many subbands (default: 10)
+- ``reference_stationSB``: station-subband number to use as reference for grouping, (default: ``None`` -> use lowest frequency input data as reference)
+
+*RMextract settings*
+
+- ``ionex_server``: URL of the *IONEX* server (default: ``"ftp://ftp.aiub.unibe.ch/CODE/"``)
+- ``ionex_prefix``: the prefix of the *IONEX* files (default: ``CODG``)
+- ``proxy_server``: specify URL or IP of proxy server if needed
+- ``proxy_port``: port of proxy server if needed
+- ``proxy_user``: user name of proxy server if needed
+- ``proxy_pass``: password of proxy server if needed
+
+In case of **LBA** observations you might also want to enable demixing (``demix: true``).
+If your **LBA** data has **not** been demixed before you may still want to keep the A-Team-clipping.
+
+.. _structure function: https://ui.adsabs.harvard.edu/abs/2016RaSc...51..927M/abstract
+.. _Rapthor: https://github.com/darafferty/rapthor
+.. _Global Sky Model: https://lcs165.lofar.eu/
+.. _RMextract: https://github.com/lofar-astron/RMextract/
+.. _factor: https://github.com/lofar-astron/factor/
+.. _killMS: https://github.com/saopicc/killMS/
+.. _TGSS ADR: https://http://tgssadr.strw.leidenuniv.nl/
+.. _Dysco: https://github.com/aroffringa/dysco/
+.. _AOFlagger: https://gitlab.com/aroffringa/aoflagger.git
+.. _here: https://www.astron.nl/lofarwiki/doku.php?id=public:user_software:documentation:ndppp#description_of_baseline_selection_parameters
diff --git a/docs/source/target_field.png b/docs/source/target_field.png
new file mode 100644
index 0000000000000000000000000000000000000000..6f24ae7ebb49aa65d6a5ea449194a367101451ab
Binary files /dev/null and b/docs/source/target_field.png differ
diff --git a/docs/source/target_old.rst b/docs/source/target_old.rst
new file mode 100644
index 0000000000000000000000000000000000000000..68d48329bde612fe57627bb06a6f2c2a27b1620e
--- /dev/null
+++ b/docs/source/target_old.rst
@@ -0,0 +1,210 @@
+.. _target_pipeline_old:
+
+Target pipeline
+===============
+
+.. note::
+
+ These instructions are outdated and only valid for **prefactor** 3.2 or older. Please check the :doc:`recent instrunctions page<target>`.
+
+
+This pipeline processes the target data in order to apply the direction-independent corrections from the calibrator pipeline (line 26). A first initial direction-independent self-calibration of the target field is performed, using a global sky model based on the `TGSS ADR`_ or the new Global Sky Model (GSM), and applied to the data.
+You will find the single steps in the parameter ``pipeline.steps`` in line 99.
+
+This chapter will present the specific steps of the target pipeline in more detail.
+
+All results (diagnostic plots and calibration solutions) are usually stored in a subfolder of the results directory, see ``inspection_directory`` (line 71) and ``cal_values_directory`` (line 72), respectively.
+
+
+Prepare target (incl. "demixing")
+---------------------------------
+This part of the pipeline prepares the target data in order to be calibration-ready for the first direction-independent phase-only self-calibration against a global sky model.
+This mainly includes mitigation of bad data (RFI, bad antennas, contaminations from A-Team sources), selection of the data to be calibrated (usually Dutch stations only), and some averaging to reduce data size and enhance the signal-to-noise ratio.
+Furthermore, ionospheric Rotation Measure corrections are applied, using `RMextract`_
+The user can specify whether to do raw data or pre-processed data flagging and whether demixing should be performed.
+
+The basic steps are:
+
+- mapping of data to be used (``createmap_target``)
+- copying h5parm solution set from the calibrator (``copy_cal_sols``)
+- gathering RM satellite information and writing it into h5parm (``h5imp_RMextract``)
+ .. image:: RMextract.png
+- creating a model of A-Team sources to be subtracted (``make_sourcedb_ateam``)
+- check of any missing solutions for the target data (``check_station_mismatch``)
+- basic flagging and averaging (``ndppp_prep_target``)
+ - edges of the band (``flagedge``) -- only used in ``raw_flagging`` mode
+ - statistical flagging (``aoflag``) -- only used in ``raw_flagging`` mode
+ - baseline flagging (``flag``)
+ - low elevation flagging (below 20 degress elevation) (``elev``)
+ - demix A-Team sources (``demix``) -- only used if specified
+ - applying clock offsets, polarization alignment, and bandpass correction derived from the calibrator (``applyclock``, ``applyPA``, ``applybandpass``)
+ - applying LOFAR beam and Rotation Measure correction from `RMextract`_ (``applybeam``, ``applyRM``)
+ - interpolation of flagged data (``interp``)
+ - averaging of the data to 4 seconds and 4 channels per subband (``avg``)
+- write A-Team skymodel into the MODEL_DATA column (``predict_ateam``)
+- clipping potentially A-Team affected data (``ateamcliptar``)
+- interpolate, average (to 8 seconds and 2 channels per subband), and concatenate target data into chunks of ten subbands (``dpppconcat``). These chunks are enforced to be equidistant in frequency. Missing data will be filled back and flagged.
+- wide-band statistical flagging (``aoflag``)
+- remove chunks with more than 50\% flagged data (``check_unflagged``)
+- identify fully flagged antennas (``check_bad_antennas``)
+
+Now the data is prepared and cleaned from the majority of bad data.
+
+Phase-only self-calibration
+---------------------------
+These steps aim for deriving a good first guess for the phase correction into the direction of the phase center (direction-independent phase correction).
+Once this is done, the data is ready for further processing with direction-dependent calibration techniques, using software like `factor`_ or `killMS`_.
+
+- download global sky model for the target field automatically (``sky_tar``)
+- interpolate flagged data and perform direction-independent phase-only calibration (diagonal terms) within a limited baseline range, using the filter (``gsmcal_dysco``)
+
+The phase solutions derived from the preparation step are now collected and loaded into **LoSoTo** to provide diagnostic plots:
+
+- ``ph_freq??``: matrix plot of the phase solutions with time for a particular chunk of target data, where both polarizations are colorcoded
+ .. image:: ph_freq.png
+- ``ph_poldif_freq??``: matrix plot of the XX-YY phase solutions with time for a particular chunk of target data
+ .. image:: ph_poldif_freq.png
+- ``ph_pol??``: matrix plot of the phase solutions for the XX and YY polarization
+ .. image:: ph_polXX.png
+- ``ph_poldif``: matrix plot of the phase solutions for the XX-YY polarization
+ .. image:: ph_poldif.png
+
+The solutions are stored in the h5parm file format.
+The last step also incorporates full `Dysco`_ compression to save disk space. The fully calibrated data is stored in the DATA column.
+In the results directory also the uncompressed and uncorrected data is stored. These data are used for the :ref:`initsubtract_pipeline`.
+
+User-defined parameter configuration
+------------------------------------
+**Parameters you will need to adjust**
+
+*Information about the input data*
+
+- ``target_input_path``: specify the directory where your target data is stored (a full UNIX-compatible directory is required)
+- ``target_input_pattern``: regular expression pattern of all your target files (e.g. ``L72319*.MS``)
+
+*Location of the software*
+
+- ``prefactor_directory``: full path to your prefactor copy
+- ``losoto_directory``: full path to your local LoSoTo installation
+- ``aoflagger``: full path to your aoflagger executable
+
+*Location of the calibrator solutions*
+
+- ``cal_solutions``: location of the calibrator solutions (default: ``input.output.job_directory/../Pre-Facet-Calibrator/results/cal_values/cal_solutions.h5``, if you stick to the defaults)
+
+**Parameters you may need to adjust**
+
+*Data selection and calibration options*
+
+- ``refant``:name of the station that will be used as a reference for the phase-plots
+- ``flag_baselines``: NDPPP-compatible pattern for baselines or stations to be flagged (may be an empty list, i.e.: ``[]`` )
+- ``process_baselines_target``: performs A-Team-clipping/demixing and direction-independent phase-only self-calibration only on these baselines. Choose [CR]S*& if you want to process only cross-correlations and remove international stations.
+- ``filter_baselines``: selects only this set of baselines to be processed. Choose [CR]S*& if you want to process only cross-correlations and remove international stations.
+- ``do_smooth``: enable or disable baseline-based smoothing (may enhance signal-to-noise for **LBA** data)
+- ``rfistrategy``: strategy to be applied with the statistical flagger (AOFlagger), default: ``HBAdefault.rfis``
+- ``interp_windowsize``: size of the window over which a value is interpolated. Should be odd. (default: 15)
+- ``raw_data``: use autoweight, set to True in case you are using raw data (default: False)
+- ``compression_bitrate``: defines the bitrate of Dysco compression of the data after the final step, choose 0 if you do NOT want to compress the data
+- ``min_unflagged_fraction``: minimal fraction of unflagged data to be accepted for further processing of the data chunk
+- ``propagatesolutions``: use already derived solutions as initial guess for the upcoming time slot
+
+A comprehensive explanation of the baseline selection syntax can be found `here`_.
+
+*Demixing options* (only used if demix step is added to the ``prep_targ_strategy`` variable)
+
+- ``demix_sources``: choose sources to demix (provided as list), e.g., ``[CasA,CygA]``
+- ``demix_target``: if given, the target source model (its patch in the SourceDB) is taken into account when solving (default: ``""``)
+- ``demix_freqstep``: number of channels to average when demixing (default: 16)
+- ``demix_timestep`` : number of time slots to average when demixing (default: 10)
+
+*Definitions for pipeline options*
+
+- ``initial_flagging``: choose {{ raw_flagging }} if you process raw data
+- ``demix_step``: choose {{ demix }} if you want to demix
+- ``apply_steps``: comma-separated list of apply_steps performed in the target preparation (NOTE: only use applyRM if you have performed RMextract before!)
+- ``clipAteam_step``: choose {{ none }} if you want to skip A-team-clipping
+- ``gsmcal_step``: choose tec if you want to fit TEC instead of self-calibrating for phases
+- ``updateweights``: update the weights column, in a way consistent with the weights being inverse proportional to the autocorrelations
+
+**Parameters for pipeline performance**
+
+- ``num_proc_per_node``: number of processes to use per step per node (default: ``input.output.max_per_node``, reads the parameter ``max_per_node`` from the ``pipeline.cfg``)
+- ``num_proc_per_node_limit``: number of processes to use per step per node for tasks with high I/O (DPPP or cp) or memory (e.g. calibration) (default: 4)
+- ``max_dppp_threads``: number of threads per process for NDPPP (default: 10)
+- ``min_length``: minimum amount of chunks to concatenate in frequency necessary to perform the wide-band flagging in the RAM. It data is too big aoflag will use indirect-read.
+- ``overhead``: Only use this fraction of the available memory for deriving the amount of data to be concatenated.
+- ``min_separation``: minimal accepted distance to an A-team source on the sky in degrees (will raise a WARNING)
+- ``error_tolerance``: defines whether pipeline run will continue if single bands fail (default: False)
+
+**Parameters you may want to adjust**
+
+*Main directories*
+
+- ``lofar_directory``: base directory of your **LOFAR** installation (default: $LOFARROOT)
+- ``job_directory``: directory of the prefactor outputs (usually the ``job_directory`` as defined in the ``pipeline.cfg``, default: ``input.output.job_directory``)
+
+*Script and plugin directories*
+
+- ``scripts``: location of the prefactor scripts (default: ``{{ prefactor_directory }}/scripts``)
+- ``pipeline.pluginpath``: location of the prefactor plugins: (default: ``{{ prefactor_directory }}/plugins``)
+
+*Sky model directory*
+
+- ``A-team_skymodel``: path to A-team skymodel (used for demixing and clipping)
+- ``target_skymodel``: path to the skymodel for the phase-only calibration of the target
+- ``use_target``: download the phase-only calibration skymodel from TGSS, "Force" : always download , "True" download if {{ target_skymodel }} does not exist , "False" : never download
+- ``skymodel_source``: use GSM if you want to use the experimental (!) GSM SkyModel creator using TGSS, NVSS, WENSS and VLSS
+
+*Result directories*
+
+- ``results_directory``: location of the prefactor results (default: ``{{ job_directory }}/results``)
+- ``inspection_directory``: location of the inspection plots (default: ``{{ results_directory }}/inspection``)
+- ``cal_values_directory``: directory of the calibration solutions (h5parm file, default: ``{{ results_directory }}/cal_values``)
+
+*Location of calibrator solutions*
+
+- ``solutions``: location of the calibration solutions (h5parm file, default: ``{{ cal_values_directory }}/cal_solutions.h5``)
+
+*Averaging for the calibrator data*
+
+- ``avg_timeresolution``: intermediate time resolution of the data in seconds after averaging (default: 4)
+- ``avg_freqresolution`` : intermediate frequency resolution of the data after averaging (default: 48.82kHz, which translates to 4 channels per subband)
+- ``avg_timeresolution_concat``: final time resolution of the data in seconds after averaging and concatenation (default: 8)
+- ``avg_freqresolution_concat``: final frequency resolution of the data after averaging and concatenation (default: 97.64kHz, which translates to 2 channels per subband)
+
+*Concatenating of the target data*
+
+- ``num_SBs_per_group``: make concatenated measurement-sets with that many subbands (default: 10)
+- ``reference_stationSB``: station-subband number to use as reference for grouping, (default: ``None`` -> use lowest frequency input data as reference)
+
+*RMextract settings*
+
+- ``ionex_server``: URL of the *IONEX* server (default: "ftp://ftp.aiub.unibe.ch/CODE/")
+- ``ionex_prefix``: the prefix of the *IONEX* files (default: CODG)
+- ``ionex_path``: location of the *IONEX* files after downloading (default: ``{{ job_directory }}/IONEX/``)
+
+Recommended parameters for **HBA** and **LBA** observations
+-----------------------------------------------------------
+============================= ============================ =======================
+**parameter** **HBA** **LBA**
+----------------------------- ---------------------------- -----------------------
+``do_smooth`` False True
+``rfistrategy`` HBAdefault LBAdefaultwideband.rfis
+``apply_steps`` applyclock,applybeam,applyRM applyphase,applybeam
+``gsmcal_step`` phase tec
+``skymodel_source`` TGSS GSM
+``clipATeam_step`` {{ clipATeam }} {{ none }}
+``avg_timeresolution_concat`` 8. 4.
+``avg_freqresolution_concat`` 97.64kHz 48.82kHz
+``num_SBs_per_group`` 10 -1
+============================= ============================ =======================
+
+In case of **LBA** observation you might also want to enable demixing in the ``prep_targ_strategy`` variable.
+If your **LBA** data has **not** been demixed before you may still want to keep the A-Team-clipping.
+
+.. _RMextract: https://github.com/lofar-astron/RMextract/
+.. _factor: https://github.com/lofar-astron/factor/
+.. _killMS: https://github.com/saopicc/killMS/
+.. _TGSS ADR: https://http://tgssadr.strw.leidenuniv.nl/
+.. _Dysco: https://github.com/aroffringa/dysco/
+.. _here: https://www.astron.nl/lofarwiki/doku.php?id=public:user_software:documentation:ndppp#description_of_baseline_selection_parameters
diff --git a/docs/source/targetscheme.png b/docs/source/targetscheme.png
new file mode 100644
index 0000000000000000000000000000000000000000..231e3a4555de23ffe540f266011c6f217c58f466
Binary files /dev/null and b/docs/source/targetscheme.png differ
diff --git a/docs/source/tec.png b/docs/source/tec.png
new file mode 100644
index 0000000000000000000000000000000000000000..c6de4e58b8ad00573a0eaea965926e25082532f2
Binary files /dev/null and b/docs/source/tec.png differ
diff --git a/docs/source/unflagged_fraction.png b/docs/source/unflagged_fraction.png
new file mode 100644
index 0000000000000000000000000000000000000000..59b01d4bf94ca9233b1d1be4016dae75288a6361
Binary files /dev/null and b/docs/source/unflagged_fraction.png differ
diff --git a/docs/source/uv-coverage.png b/docs/source/uv-coverage.png
new file mode 100644
index 0000000000000000000000000000000000000000..426e3eaa6083c500ad63c3ffd4ec028e49df2e6f
Binary files /dev/null and b/docs/source/uv-coverage.png differ
diff --git a/docs/source/uv-coverage_uvdist.png b/docs/source/uv-coverage_uvdist.png
new file mode 100644
index 0000000000000000000000000000000000000000..5365ae6c96e918170d0b1bdef3be92983b937199
Binary files /dev/null and b/docs/source/uv-coverage_uvdist.png differ
diff --git a/lofar-cwl/steps/ms_concat.cwl b/lofar-cwl/steps/ms_concat.cwl
index 93228fb0af80197b13e6f05ad3afda1a8845dea8..1ff03cbb02ace1d645d5787856dfb8281643bb6a 100644
--- a/lofar-cwl/steps/ms_concat.cwl
+++ b/lofar-cwl/steps/ms_concat.cwl
@@ -25,9 +25,6 @@ inputs:
of data to be concatenated.
- id: msin
type: 'Directory[]'
-# inputBinding:
-# position: 1
-# shellQuote: false
- default: out.MS
id: msout
type: string
@@ -48,7 +45,6 @@ outputs:
type: 'Directory[]'
outputBinding:
glob: 'workdir/out_*'
-# outputEval: $(inputs.msin)
- id: logfile
type: File?
outputBinding:
@@ -74,23 +70,4 @@ requirements:
- class: InplaceUpdateRequirement
inplaceUpdate: true
- class: InlineJavascriptRequirement
-stdout: ms_concat.log
-
-
-
-#
-# mkdir -pv dirs && cp -rv \$directories dirs/. && cd dirs
-# directories=\$(ls -d out_*)
-#
-
- #- entryname: script.sh
- # entry: |
- # #!/bin/bash
- #
- # ls
-
- #- python3
- #- /usr/local/bin/concat_MS_CWL.py
-# directories=$(ls out_*)
-# python3 /usr/local/bin/concat_MS_CWL.py $@ $directories
-
+stdout: ms_concat.log
\ No newline at end of file