diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..6ada2877b3453550a6cec11b68ffad30847f84bf
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,131 @@
+test_data/*.MS
+Docker/scripts
+*.sif
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+# However, in case of collaboration, if having platform-specific dependencies or dependencies
+# having no cross-platform support, pipenv may install dependencies that don't work, or not
+# install all needed dependencies.
+#Pipfile.lock
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# Not idea
+.idea
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 0000000000000000000000000000000000000000..8deb132b9ddce5168b0dfea3178158cad26bc6a0
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,174 @@
+image: lofareosc/lofar-pipeline-ci:latest
+
+variables:
+ TEST_DATASET_NAME: "L570745_uv_first10.MS.tar.xz"
+ PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+
+stages:
+ - build_docker
+ - push_docker
+ - validate_cwl_scripts
+ - test_steps
+ - integration_test
+
+before_script:
+ - mkdir workdir
+ - mkdir logs
+
+after_script:
+ - echo "All done"
+
+build_docker:
+ image: docker:latest
+ stage: build_docker
+ only:
+ - tags
+ script:
+ - apk add git subversion git-lfs bash
+ - cd Docker
+ - bash build_docker.sh
+
+push_docker_tags:
+ image: docker:latest
+ allow_failure: true
+ stage: push_docker
+ only:
+ - tags
+ script:
+ - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+ - docker tag lofareosc/prefactor-ci:latest $CI_REGISTRY/prefactor-ci:$CI_COMMIT_REF_NAME
+ - docker push $CI_REGISTRY/prefactor-ci:$CI_COMMIT_REF_NAME
+
+validate_scripts:
+ stage: validate_cwl_scripts
+ script:
+ - for file in `find . -name 'steps/*.cwl'`; do cwltool --validate $file; done
+
+average_step_generator:
+ stage: test_steps
+ allow_failure: true
+ script:
+ - cwl-runner steps/DP3.AveragerStepGenerator.cwl test_jobs/averager_step_generator.json
+ - ! cwl-runner steps/DP3.AveragerStepGenerator.cwl test_jobs/averager_step_generator_error.json
+
+aoflagger_step_generator:
+ stage: test_steps
+ allow_failure: true
+ script:
+ - cwl-runner steps/DP3.AOFlaggerStepGenerator.cwl test_jobs/aoflagger_step_generator.json
+
+preflagger_step_generator:
+ stage: test_steps
+ allow_failure: true
+ script:
+ - cwl-runner steps/DP3.PreflaggerStepGenerator.cwl test_jobs/preflagger_step_generator.json
+
+losoto_abs:
+ stage: test_steps
+ allow_failure: true
+ script:
+ - cwl-runner --no-container steps/LoSoTo.Abs.cwl test_jobs/losoto_abs.json
+
+losoto_duplicate:
+ stage: test_steps
+ allow_failure: true
+ script:
+ - cwl-runner --no-container steps/LoSoTo.Duplicate.cwl test_jobs/losoto_duplicate.json
+
+losoto_plot:
+ stage: test_steps
+ allow_failure: true
+ script:
+ - cwl-runner --no-container steps/LoSoTo.Plot.cwl test_jobs/losoto_plot.json
+
+losoto_norm:
+ stage: test_steps
+ allow_failure: true
+ script:
+ - cwl-runner --no-container steps/LoSoTo.Norm.cwl test_jobs/losoto_norm.json
+
+losoto_replicateonaxis:
+ stage: test_steps
+ allow_failure: true
+ script:
+ - cwl-runner --no-container steps/LoSoTo.Replicateonaxis.cwl test_jobs/losoto_replicateonaxis.json
+
+losoto_reset:
+ stage: test_steps
+ allow_failure: true
+ script:
+ - cwl-runner --no-container steps/LoSoTo.Reset.cwl test_jobs/losoto_reset.json
+
+losoto_residual:
+ stage: test_steps
+ allow_failure: true
+ script:
+ - cwl-runner --no-container steps/LoSoTo.Residual.cwl test_jobs/losoto_residual.json
+
+parset_selector:
+ stage: test_steps
+ allow_failure: true
+ script:
+ - cwl-runner steps/DP3.Selector.cwl test_jobs/selector.json
+
+file_selector:
+ stage: test_steps
+ allow_failure: true
+ script:
+ - cwl-runner steps/FileSelector.cwl test_jobs/file_selector.json
+
+NDPPP:
+ stage: test_steps
+ allow_failure: true
+ script:
+ - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH steps/DPPP.cwl test_jobs/NDPPP.json
+
+DP3.Execute:
+ stage: test_steps
+ allow_failure: true
+ script:
+ - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH steps/DP3.Execute.cwl test_jobs/dp3_execute.json
+
+h5parm_collector:
+ stage: test_steps
+ allow_failure: true
+ script:
+ - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/H5ParmCollector.cwl test_jobs/h5parm_collector.json
+
+aoflagger:
+ stage: test_steps
+ allow_failure: true
+ script:
+ - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH steps/AOFlagger.cwl test_jobs/aoflagger.json
+
+interpolate_visibilities:
+ stage: test_steps
+ allow_failure: true
+ script:
+ - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/interpolate_visibilities.cwl test_jobs/interpolate_visibilities.json
+
+h5parm_collector:
+ stage: test_steps
+ allow_failure: true
+ script:
+ - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/H5ParmCollector.cwl test_jobs/h5parm_collector.json
+
+blsmooth:
+ stage: test_steps
+ allow_failure: true
+ script:
+ - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/blsmooth.cwl test_jobs/blsmooth.json
+
+ddecal:
+ stage: test_steps
+ allow_failure: true
+ script:
+ - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/ddecal.cwl test_jobs/ddecal.json
+
+predict:
+ stage: test_steps
+ allow_failure: true
+ script:
+ - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/predict.cwl test_jobs/predict.json
+
+
diff --git a/Docker/.entrypoint b/Docker/.entrypoint
new file mode 100644
index 0000000000000000000000000000000000000000..cc2b666b7d5c93c2a7e7ea6715c17acd498f338d
--- /dev/null
+++ b/Docker/.entrypoint
@@ -0,0 +1,5 @@
+#!/bin/bash
+set -e
+
+exec "$@"
+
diff --git a/Docker/Dockerfile b/Docker/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..9ed60bf990c4867abc89abf5aadae69978d4365a
--- /dev/null
+++ b/Docker/Dockerfile
@@ -0,0 +1,152 @@
+FROM kernsuite/base:5 as builder
+SHELL ["/bin/bash", "-c"]
+
+RUN export DEBIAN_FRONTEND=noninteractive && \
+ apt-get update && \
+ apt-get install -y \
+ casacore-data \
+ casacore-dev \
+ libboost-python-dev \
+ libcfitsio-dev \
+ python-dev \
+ python3-numpy \
+ cmake \
+ build-essential \
+ libhdf5-serial-dev \
+ libarmadillo-dev \
+ libboost-filesystem-dev \
+ libboost-system-dev \
+ libboost-date-time-dev \
+ libboost-signals-dev \
+ libboost-program-options-dev \
+ libboost-test-dev \
+ libxml2-dev \
+ libpng-dev \
+ pkg-config \
+ aoflagger-dev \
+ libgtkmm-3.0-dev \
+ git \
+ wget \
+ libfftw3-dev \
+ libgsl-dev
+
+RUN mkdir -p /src/
+WORKDIR /src/
+
+
+# Build the IDG version from source
+
+
+ADD ./idg /src/idg/src
+RUN mkdir idg/build && cd idg/build && cmake -DCMAKE_INSTALL_PREFIX=/usr/local/ ../src && make install -j2
+
+ADD ./DP3 /src/dp3/src
+RUN cd /src/dp3/ && \
+ mkdir build && \
+ cd build && \
+ cmake ../src -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPORTABLE=True &&\
+ make -j2 && \
+ make install -j2
+
+ADD ./STMAN /src/STMAN/src
+RUN cd /src/STMAN/ && \
+ mkdir build && \
+ cd build && \
+ cmake ../src -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPORTABLE=True && \
+ make -j2 && \
+ make install -j2
+
+ADD ./Dysco /src/Dysco/src
+RUN cd /src/Dysco/ && \
+ mkdir build && \
+ cd build && \
+ cmake ../src -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPORTABLE=True && \
+ make -j2 && \
+ make install -j2
+
+#----------------------------------------------
+FROM kernsuite/base:5 as runner
+
+COPY --from=builder /usr/local/ /usr/local
+
+SHELL ["/bin/bash", "-c"]
+
+RUN export DEBIAN_FRONTEND=noninteractive && \
+ apt-get update && \
+ apt-get install -y \
+ aoflagger \
+ casacore-data \
+ libarmadillo8 \
+ libboost-date-time1.65.1 \
+ libboost-filesystem1.65.1 \
+ libboost-python1.65.1 \
+ libboost-program-options1.65.1 \
+ libboost-system1.65.1 \
+ libboost-signals1.65.1 \
+ libboost-test1.65.1 \
+ libboost-python1.65-dev \
+ libstationresponse2 \
+ libcasa-* \
+ libcfitsio5 \
+ libgtkmm-3.0 \
+ libfftw3-3 \
+ libhdf5-cpp-100 \
+ libpng16-16 \
+ libxml2 \
+ python3.7 \
+ python3-casacore \
+ python3-numpy \
+ python3-scipy && \
+ rm -rf /var/lib/apt/lists/*
+
+
+RUN mkdir /home/lofaruser/ && \
+ groupadd -r lofaruser && \
+ useradd --no-log-init -r -g lofaruser lofaruser && \
+ chown lofaruser:lofaruser /home/lofaruser
+
+ARG TOIL_VERSION=3.20.0
+LABEL toil.version=${TOIL_VERSION}
+
+ARG LSMTool_TAG=master
+LABEL LSMTool.version.tag=${LSMTool_TAG}
+
+ARG LoSoTo_TAG=master
+LABEL LoSoTo.version.tag=${LoSoTo_TAG}
+
+ARG STMAN_TAG=master
+LABEL STMAN.version.tag=${STMAN_TAG}
+
+ARG Dysco_TAG=master
+LABEL Dysco.version.tag=${Dysco_TAG}
+
+
+RUN apt update && apt install -y nodejs python3-pip git wsclean
+RUN python3 -m pip install cwltool cwl-runner -e "git://github.com/darafferty/LSMTool.git@${LSMTool_TAG}#egg=LSMTool" \
+ "toil[cwl]"==${TOIL_VERSION} \
+ matplotlib
+
+RUN python3 -m pip install -e "git+https://github.com/revoltek/losoto.git@${LoSoTo_TAG}#egg=LoSoTo"
+
+ADD losoto.patch /home/lofaruser/losoto.patch
+RUN patch /src/losoto/losoto/operations/__init__.py /home/lofaruser/losoto.patch
+ADD .entrypoint /home/lofaruser/.entrypoint
+RUN chown lofaruser:lofaruser /home/lofaruser/.entrypoint && \
+ chmod +rx /home/lofaruser/.entrypoint
+
+WORKDIR /home/lofaruser
+
+COPY prefactor/skymodels/* /data/skymodels/
+COPY prefactor/rfistrategies/* /data/rfistrategies/
+COPY prefactor/solutions/ /data/solutions/*
+
+
+RUN chmod +rx /usr/local/bin/*
+ENTRYPOINT ["/home/lofaruser/.entrypoint"]
+USER lofaruser
+
+ARG DP3_TAG=master
+ARG IDG_TAG=master
+
+LABEL IDG.version.tag=${IDG_TAG}
+LABEL DP3.version.tag=${DP3_TAG}
diff --git a/Docker/Dockerfile_ci b/Docker/Dockerfile_ci
new file mode 100644
index 0000000000000000000000000000000000000000..63baad2426e2610d5975d353f07d15476114eb0f
--- /dev/null
+++ b/Docker/Dockerfile_ci
@@ -0,0 +1,8 @@
+FROM lofareosc/lofar-pipeline:latest
+SHELL ["/bin/bash", "-c"]
+
+ADD L570745_uv_first10.MS.tar.xz /data
+ADD skymodels.tar.gz /data/skymodels
+ADD test_data/A-Team_lowres.sourcedb /data/A-Team_lowres.sourcedb
+ADD test_data/example.h5 /data/example.h5
+ADD test_data/L570745_SB001_uv_MODEL.MS /data/L570745_SB001_uv_MODEL.MS
diff --git a/Docker/L570745_uv_first10.MS.tar.xz.REMOVED.git-id b/Docker/L570745_uv_first10.MS.tar.xz.REMOVED.git-id
new file mode 100644
index 0000000000000000000000000000000000000000..2266c40d2de8785cb1e771f88823e6fb2f02be07
--- /dev/null
+++ b/Docker/L570745_uv_first10.MS.tar.xz.REMOVED.git-id
@@ -0,0 +1 @@
+fac322047acad1a9cf1425d64a08d7bc3548f462
\ No newline at end of file
diff --git a/Docker/build_docker.sh b/Docker/build_docker.sh
new file mode 100755
index 0000000000000000000000000000000000000000..b908b90fc708c4726fb1010c202c5945eafda4e2
--- /dev/null
+++ b/Docker/build_docker.sh
@@ -0,0 +1,52 @@
+#! /bin/bash
+set -e
+
+git_clone_or_pull () {
+ REPO=$1
+ BRANCH=$2
+ DIR=$3
+ if [ -d $DIR ]
+ then
+ git -C "${DIR}" pull
+ else
+ git clone --depth 1 -b "${BRANCH}" "${REPO}" "${DIR}"
+ fi
+}
+
+# SOFTWARE VERSIONS
+DP3_TAG=v4.1
+IDG_TAG=0.6
+LSMTool_TAG=v1.4.2
+LoSoTo_TAG=f0398b
+TOIL_VERSION=3.20.0
+STMAN_TAG=master
+Dysco_TAG=v1.2
+Prefactor_TAG=master
+
+# FETCHES THE IDG
+git_clone_or_pull https://gitlab.com/astron-idg/idg/ ${IDG_TAG} idg
+
+# FETCHES DP3
+git_clone_or_pull https://github.com/lofar-astron/DP3 ${DP3_TAG} DP3
+
+# FETCHES STMAN
+git_clone_or_pull https://github.com/lofar-astron/LofarStMan ${STMAN_TAG} STMAN
+
+# FETCHES Dysco
+git_clone_or_pull https://github.com/aroffringa/dysco.git ${Dysco_TAG} Dysco
+
+
+SCRIPT_PATH=$(realpath ${BASH_SOURCE[0]})
+
+DOCKER_PATH=$(dirname ${SCRIPT_PATH})
+
+docker build ${DOCKER_PATH} --build-arg=IDG_TAG=${IDG_TAG}\
+ --build-arg=DP3_TAG=${DP3_TAG}\
+ --build-arg=LSMTool_TAG=${LSMTool_TAG}\
+ --build-arg=LoSoTo_TAG=${LoSoTo_TAG}\
+ --build-arg=STMAN_TAG=${STMAN_TAG}\
+ --build-arg=Dysco_TAG=${Dysco_TAG}\
+ --build-arg=TOIL_VERSION=${TOIL_VERSION}\
+ -t lofareosc/lofar-pipeline
+
+docker build ${DOCKER_PATH} -f ${DOCKER_PATH}/Dockerfile_ci -t lofareosc/lofar-pipeline-ci
diff --git a/Docker/losoto.patch b/Docker/losoto.patch
new file mode 100644
index 0000000000000000000000000000000000000000..fbdfb47754296d2b358e162afe777cd1e41fea75
--- /dev/null
+++ b/Docker/losoto.patch
@@ -0,0 +1,13 @@
+--- /src/losoto/losoto/operations/__init__.py 2019-11-07 12:43:46.220577945 +0000
++++ /src/losoto/losoto/operations/__init__.py 2019-11-07 12:43:34.401765786 +0000
+@@ -2,8 +2,7 @@
+
+ __all__ = [ os.path.basename(f)[:-3] for f in glob.glob(os.path.dirname(__file__)+"/*.py") if f[0] != '_']
+
+-for x in __all__:
+- __import__(x, locals(), globals())
++from . import *
+
+ class Timer(object):
+ """
+
diff --git a/Docker/skymodels.tar.gz.REMOVED.git-id b/Docker/skymodels.tar.gz.REMOVED.git-id
new file mode 100644
index 0000000000000000000000000000000000000000..e2362e5aa6bd4e3fe266bd71966dee96de2449ae
--- /dev/null
+++ b/Docker/skymodels.tar.gz.REMOVED.git-id
@@ -0,0 +1 @@
+2377705b2f6824c2a1bd59ddabfe5647a981f83e
\ No newline at end of file
diff --git a/Docker/test_data/A-Team_lowres.sourcedb b/Docker/test_data/A-Team_lowres.sourcedb
new file mode 100644
index 0000000000000000000000000000000000000000..b4e89906093cc01438b6a7b81f3ea9bfb06ba520
Binary files /dev/null and b/Docker/test_data/A-Team_lowres.sourcedb differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/ANTENNA/table.dat b/Docker/test_data/L570745_SB001_uv_MODEL.MS/ANTENNA/table.dat
new file mode 100644
index 0000000000000000000000000000000000000000..3670600f2692d3b4b77cc07eddbdb44261d3c214
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/ANTENNA/table.dat differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/ANTENNA/table.f0 b/Docker/test_data/L570745_SB001_uv_MODEL.MS/ANTENNA/table.f0
new file mode 100644
index 0000000000000000000000000000000000000000..811482a52b56513bf3d8dc5c6083549f8092b2c0
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/ANTENNA/table.f0 differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/ANTENNA/table.info b/Docker/test_data/L570745_SB001_uv_MODEL.MS/ANTENNA/table.info
new file mode 100644
index 0000000000000000000000000000000000000000..a1a976ed49439abda34d2c64cfd0360cfcdef651
--- /dev/null
+++ b/Docker/test_data/L570745_SB001_uv_MODEL.MS/ANTENNA/table.info
@@ -0,0 +1,3 @@
+Type =
+SubType =
+
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/ANTENNA/table.lock b/Docker/test_data/L570745_SB001_uv_MODEL.MS/ANTENNA/table.lock
new file mode 100644
index 0000000000000000000000000000000000000000..0472b61a97b3474a8587d7db73c4914e1104166e
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/ANTENNA/table.lock differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/DATA_DESCRIPTION/table.dat b/Docker/test_data/L570745_SB001_uv_MODEL.MS/DATA_DESCRIPTION/table.dat
new file mode 100644
index 0000000000000000000000000000000000000000..20ab416e6b2089d0bb7ed946963b87aa3a535c04
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/DATA_DESCRIPTION/table.dat differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/DATA_DESCRIPTION/table.f0 b/Docker/test_data/L570745_SB001_uv_MODEL.MS/DATA_DESCRIPTION/table.f0
new file mode 100644
index 0000000000000000000000000000000000000000..bc76518f7b4e25a64bb5a1e2570ddf4df13bfffa
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/DATA_DESCRIPTION/table.f0 differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/DATA_DESCRIPTION/table.info b/Docker/test_data/L570745_SB001_uv_MODEL.MS/DATA_DESCRIPTION/table.info
new file mode 100644
index 0000000000000000000000000000000000000000..a1a976ed49439abda34d2c64cfd0360cfcdef651
--- /dev/null
+++ b/Docker/test_data/L570745_SB001_uv_MODEL.MS/DATA_DESCRIPTION/table.info
@@ -0,0 +1,3 @@
+Type =
+SubType =
+
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/DATA_DESCRIPTION/table.lock b/Docker/test_data/L570745_SB001_uv_MODEL.MS/DATA_DESCRIPTION/table.lock
new file mode 100644
index 0000000000000000000000000000000000000000..0472b61a97b3474a8587d7db73c4914e1104166e
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/DATA_DESCRIPTION/table.lock differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/FEED/table.dat b/Docker/test_data/L570745_SB001_uv_MODEL.MS/FEED/table.dat
new file mode 100644
index 0000000000000000000000000000000000000000..08041c586fc1cdae160f90e7234b95baa35b18f8
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/FEED/table.dat differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/FEED/table.f0 b/Docker/test_data/L570745_SB001_uv_MODEL.MS/FEED/table.f0
new file mode 100644
index 0000000000000000000000000000000000000000..2a2ba196d7a2a608270dd2ce05896abbd1bc3fc6
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/FEED/table.f0 differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/FEED/table.f0i b/Docker/test_data/L570745_SB001_uv_MODEL.MS/FEED/table.f0i
new file mode 100644
index 0000000000000000000000000000000000000000..59ef50ecf03311aabb897346169b2ca04e874254
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/FEED/table.f0i differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/FEED/table.info b/Docker/test_data/L570745_SB001_uv_MODEL.MS/FEED/table.info
new file mode 100644
index 0000000000000000000000000000000000000000..a1a976ed49439abda34d2c64cfd0360cfcdef651
--- /dev/null
+++ b/Docker/test_data/L570745_SB001_uv_MODEL.MS/FEED/table.info
@@ -0,0 +1,3 @@
+Type =
+SubType =
+
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/FEED/table.lock b/Docker/test_data/L570745_SB001_uv_MODEL.MS/FEED/table.lock
new file mode 100644
index 0000000000000000000000000000000000000000..0472b61a97b3474a8587d7db73c4914e1104166e
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/FEED/table.lock differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/FIELD/table.dat b/Docker/test_data/L570745_SB001_uv_MODEL.MS/FIELD/table.dat
new file mode 100644
index 0000000000000000000000000000000000000000..51bf40ea2f1d0e463019ed6e3f3a111f05309ce4
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/FIELD/table.dat differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/FIELD/table.f0 b/Docker/test_data/L570745_SB001_uv_MODEL.MS/FIELD/table.f0
new file mode 100644
index 0000000000000000000000000000000000000000..85defe9931c79cf71617a32c6cb8d712339260aa
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/FIELD/table.f0 differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/FIELD/table.f0i b/Docker/test_data/L570745_SB001_uv_MODEL.MS/FIELD/table.f0i
new file mode 100644
index 0000000000000000000000000000000000000000..bb9a3f1cc36a284642d65c5f733c2f333f975e99
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/FIELD/table.f0i differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/FIELD/table.info b/Docker/test_data/L570745_SB001_uv_MODEL.MS/FIELD/table.info
new file mode 100644
index 0000000000000000000000000000000000000000..a1a976ed49439abda34d2c64cfd0360cfcdef651
--- /dev/null
+++ b/Docker/test_data/L570745_SB001_uv_MODEL.MS/FIELD/table.info
@@ -0,0 +1,3 @@
+Type =
+SubType =
+
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/FIELD/table.lock b/Docker/test_data/L570745_SB001_uv_MODEL.MS/FIELD/table.lock
new file mode 100644
index 0000000000000000000000000000000000000000..0472b61a97b3474a8587d7db73c4914e1104166e
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/FIELD/table.lock differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/FLAG_CMD/table.dat b/Docker/test_data/L570745_SB001_uv_MODEL.MS/FLAG_CMD/table.dat
new file mode 100644
index 0000000000000000000000000000000000000000..360359641eafdad08c4c32e80d43b7af36c1328a
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/FLAG_CMD/table.dat differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/FLAG_CMD/table.f0 b/Docker/test_data/L570745_SB001_uv_MODEL.MS/FLAG_CMD/table.f0
new file mode 100644
index 0000000000000000000000000000000000000000..acf64ff0cee834d15e3054ae8c3bab48645789f0
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/FLAG_CMD/table.f0 differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/FLAG_CMD/table.info b/Docker/test_data/L570745_SB001_uv_MODEL.MS/FLAG_CMD/table.info
new file mode 100644
index 0000000000000000000000000000000000000000..a1a976ed49439abda34d2c64cfd0360cfcdef651
--- /dev/null
+++ b/Docker/test_data/L570745_SB001_uv_MODEL.MS/FLAG_CMD/table.info
@@ -0,0 +1,3 @@
+Type =
+SubType =
+
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/FLAG_CMD/table.lock b/Docker/test_data/L570745_SB001_uv_MODEL.MS/FLAG_CMD/table.lock
new file mode 100644
index 0000000000000000000000000000000000000000..0472b61a97b3474a8587d7db73c4914e1104166e
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/FLAG_CMD/table.lock differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/HISTORY/table.dat b/Docker/test_data/L570745_SB001_uv_MODEL.MS/HISTORY/table.dat
new file mode 100644
index 0000000000000000000000000000000000000000..fce2d0ed9c73849ca8b1fe90f8fd4bd118a6671e
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/HISTORY/table.dat differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/HISTORY/table.f0 b/Docker/test_data/L570745_SB001_uv_MODEL.MS/HISTORY/table.f0
new file mode 100644
index 0000000000000000000000000000000000000000..2156633a62feafe8d17c140f2a1f4636670511d1
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/HISTORY/table.f0 differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/HISTORY/table.info b/Docker/test_data/L570745_SB001_uv_MODEL.MS/HISTORY/table.info
new file mode 100644
index 0000000000000000000000000000000000000000..a1a976ed49439abda34d2c64cfd0360cfcdef651
--- /dev/null
+++ b/Docker/test_data/L570745_SB001_uv_MODEL.MS/HISTORY/table.info
@@ -0,0 +1,3 @@
+Type =
+SubType =
+
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/HISTORY/table.lock b/Docker/test_data/L570745_SB001_uv_MODEL.MS/HISTORY/table.lock
new file mode 100644
index 0000000000000000000000000000000000000000..6e709e7a4fef28bff98f0eac7806aafbaafb3a85
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/HISTORY/table.lock differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_ANTENNA_FIELD/table.dat b/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_ANTENNA_FIELD/table.dat
new file mode 100644
index 0000000000000000000000000000000000000000..bc9a355ad27c4dbc51e94ed910ab7f572e16a578
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_ANTENNA_FIELD/table.dat differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_ANTENNA_FIELD/table.f0 b/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_ANTENNA_FIELD/table.f0
new file mode 100644
index 0000000000000000000000000000000000000000..fad4cc47eaab9825cca50ae6fc5fdcbaedc828f8
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_ANTENNA_FIELD/table.f0 differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_ANTENNA_FIELD/table.f0i b/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_ANTENNA_FIELD/table.f0i
new file mode 100644
index 0000000000000000000000000000000000000000..9bcba53e53d7e6d618d837ebe3a8c1bb77b48ea2
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_ANTENNA_FIELD/table.f0i differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_ANTENNA_FIELD/table.info b/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_ANTENNA_FIELD/table.info
new file mode 100644
index 0000000000000000000000000000000000000000..a1a976ed49439abda34d2c64cfd0360cfcdef651
--- /dev/null
+++ b/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_ANTENNA_FIELD/table.info
@@ -0,0 +1,3 @@
+Type =
+SubType =
+
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_ANTENNA_FIELD/table.lock b/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_ANTENNA_FIELD/table.lock
new file mode 100644
index 0000000000000000000000000000000000000000..0472b61a97b3474a8587d7db73c4914e1104166e
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_ANTENNA_FIELD/table.lock differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_ELEMENT_FAILURE/table.dat b/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_ELEMENT_FAILURE/table.dat
new file mode 100644
index 0000000000000000000000000000000000000000..9ba0c7141eed0faa82b9afc1ea616ffea109bdf9
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_ELEMENT_FAILURE/table.dat differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_ELEMENT_FAILURE/table.f0 b/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_ELEMENT_FAILURE/table.f0
new file mode 100644
index 0000000000000000000000000000000000000000..a890c6a767ea4d04d226092c34457cccd9f04000
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_ELEMENT_FAILURE/table.f0 differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_ELEMENT_FAILURE/table.info b/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_ELEMENT_FAILURE/table.info
new file mode 100644
index 0000000000000000000000000000000000000000..a1a976ed49439abda34d2c64cfd0360cfcdef651
--- /dev/null
+++ b/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_ELEMENT_FAILURE/table.info
@@ -0,0 +1,3 @@
+Type =
+SubType =
+
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_ELEMENT_FAILURE/table.lock b/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_ELEMENT_FAILURE/table.lock
new file mode 100644
index 0000000000000000000000000000000000000000..0472b61a97b3474a8587d7db73c4914e1104166e
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_ELEMENT_FAILURE/table.lock differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_STATION/table.dat b/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_STATION/table.dat
new file mode 100644
index 0000000000000000000000000000000000000000..07e024d0a4f85176f62a37b2ed66cd42667f10f7
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_STATION/table.dat differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_STATION/table.f0 b/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_STATION/table.f0
new file mode 100644
index 0000000000000000000000000000000000000000..0a4478675199281251b78a13a770e1a7c1383edc
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_STATION/table.f0 differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_STATION/table.info b/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_STATION/table.info
new file mode 100644
index 0000000000000000000000000000000000000000..a1a976ed49439abda34d2c64cfd0360cfcdef651
--- /dev/null
+++ b/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_STATION/table.info
@@ -0,0 +1,3 @@
+Type =
+SubType =
+
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_STATION/table.lock b/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_STATION/table.lock
new file mode 100644
index 0000000000000000000000000000000000000000..0472b61a97b3474a8587d7db73c4914e1104166e
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/LOFAR_STATION/table.lock differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/OBSERVATION/table.dat b/Docker/test_data/L570745_SB001_uv_MODEL.MS/OBSERVATION/table.dat
new file mode 100644
index 0000000000000000000000000000000000000000..e51b8f283d20f6d0e00166c2689297f87865e40d
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/OBSERVATION/table.dat differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/OBSERVATION/table.f0 b/Docker/test_data/L570745_SB001_uv_MODEL.MS/OBSERVATION/table.f0
new file mode 100644
index 0000000000000000000000000000000000000000..a6ea794cfbb9b734fe866f6dd998f771c8b702e9
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/OBSERVATION/table.f0 differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/OBSERVATION/table.info b/Docker/test_data/L570745_SB001_uv_MODEL.MS/OBSERVATION/table.info
new file mode 100644
index 0000000000000000000000000000000000000000..a1a976ed49439abda34d2c64cfd0360cfcdef651
--- /dev/null
+++ b/Docker/test_data/L570745_SB001_uv_MODEL.MS/OBSERVATION/table.info
@@ -0,0 +1,3 @@
+Type =
+SubType =
+
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/OBSERVATION/table.lock b/Docker/test_data/L570745_SB001_uv_MODEL.MS/OBSERVATION/table.lock
new file mode 100644
index 0000000000000000000000000000000000000000..0472b61a97b3474a8587d7db73c4914e1104166e
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/OBSERVATION/table.lock differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/POINTING/table.dat b/Docker/test_data/L570745_SB001_uv_MODEL.MS/POINTING/table.dat
new file mode 100644
index 0000000000000000000000000000000000000000..514b7b9136fab8d2b4bffcd379588330974dab89
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/POINTING/table.dat differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/POINTING/table.f0 b/Docker/test_data/L570745_SB001_uv_MODEL.MS/POINTING/table.f0
new file mode 100644
index 0000000000000000000000000000000000000000..7c4e0b3ba494f51fb1fcc5f43305c92697a9d062
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/POINTING/table.f0 differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/POINTING/table.f0i b/Docker/test_data/L570745_SB001_uv_MODEL.MS/POINTING/table.f0i
new file mode 100644
index 0000000000000000000000000000000000000000..58715d9bac6e3dd1229c3a3e78c5aede36c34b9d
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/POINTING/table.f0i differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/POINTING/table.info b/Docker/test_data/L570745_SB001_uv_MODEL.MS/POINTING/table.info
new file mode 100644
index 0000000000000000000000000000000000000000..a1a976ed49439abda34d2c64cfd0360cfcdef651
--- /dev/null
+++ b/Docker/test_data/L570745_SB001_uv_MODEL.MS/POINTING/table.info
@@ -0,0 +1,3 @@
+Type =
+SubType =
+
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/POINTING/table.lock b/Docker/test_data/L570745_SB001_uv_MODEL.MS/POINTING/table.lock
new file mode 100644
index 0000000000000000000000000000000000000000..0472b61a97b3474a8587d7db73c4914e1104166e
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/POINTING/table.lock differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/POLARIZATION/table.dat b/Docker/test_data/L570745_SB001_uv_MODEL.MS/POLARIZATION/table.dat
new file mode 100644
index 0000000000000000000000000000000000000000..15d239d1be0e401e157d39b9a204533cbd948ad9
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/POLARIZATION/table.dat differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/POLARIZATION/table.f0 b/Docker/test_data/L570745_SB001_uv_MODEL.MS/POLARIZATION/table.f0
new file mode 100644
index 0000000000000000000000000000000000000000..d487c40cb0089a9062007bdf010a13075950f19d
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/POLARIZATION/table.f0 differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/POLARIZATION/table.f0i b/Docker/test_data/L570745_SB001_uv_MODEL.MS/POLARIZATION/table.f0i
new file mode 100644
index 0000000000000000000000000000000000000000..247a4aff500069af7c07dbcd0073a7299984dba3
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/POLARIZATION/table.f0i differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/POLARIZATION/table.info b/Docker/test_data/L570745_SB001_uv_MODEL.MS/POLARIZATION/table.info
new file mode 100644
index 0000000000000000000000000000000000000000..a1a976ed49439abda34d2c64cfd0360cfcdef651
--- /dev/null
+++ b/Docker/test_data/L570745_SB001_uv_MODEL.MS/POLARIZATION/table.info
@@ -0,0 +1,3 @@
+Type =
+SubType =
+
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/POLARIZATION/table.lock b/Docker/test_data/L570745_SB001_uv_MODEL.MS/POLARIZATION/table.lock
new file mode 100644
index 0000000000000000000000000000000000000000..0472b61a97b3474a8587d7db73c4914e1104166e
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/POLARIZATION/table.lock differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/PROCESSOR/table.dat b/Docker/test_data/L570745_SB001_uv_MODEL.MS/PROCESSOR/table.dat
new file mode 100644
index 0000000000000000000000000000000000000000..7a2b0a8e519f8082715a83d168243a717ad3a544
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/PROCESSOR/table.dat differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/PROCESSOR/table.f0 b/Docker/test_data/L570745_SB001_uv_MODEL.MS/PROCESSOR/table.f0
new file mode 100644
index 0000000000000000000000000000000000000000..b78be96341e9baa7b8afea51e0d526f7c365eb06
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/PROCESSOR/table.f0 differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/PROCESSOR/table.info b/Docker/test_data/L570745_SB001_uv_MODEL.MS/PROCESSOR/table.info
new file mode 100644
index 0000000000000000000000000000000000000000..a1a976ed49439abda34d2c64cfd0360cfcdef651
--- /dev/null
+++ b/Docker/test_data/L570745_SB001_uv_MODEL.MS/PROCESSOR/table.info
@@ -0,0 +1,3 @@
+Type =
+SubType =
+
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/PROCESSOR/table.lock b/Docker/test_data/L570745_SB001_uv_MODEL.MS/PROCESSOR/table.lock
new file mode 100644
index 0000000000000000000000000000000000000000..0472b61a97b3474a8587d7db73c4914e1104166e
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/PROCESSOR/table.lock differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_BASELINE_STATISTIC/table.dat b/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_BASELINE_STATISTIC/table.dat
new file mode 100644
index 0000000000000000000000000000000000000000..66891e1254c8625436f0a79978d094e88bb1f9d0
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_BASELINE_STATISTIC/table.dat differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_BASELINE_STATISTIC/table.f0 b/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_BASELINE_STATISTIC/table.f0
new file mode 100644
index 0000000000000000000000000000000000000000..9d0dd62f6e87d1e6ae22f8fb7dd9e363df7361b8
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_BASELINE_STATISTIC/table.f0 differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_BASELINE_STATISTIC/table.info b/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_BASELINE_STATISTIC/table.info
new file mode 100644
index 0000000000000000000000000000000000000000..a1a976ed49439abda34d2c64cfd0360cfcdef651
--- /dev/null
+++ b/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_BASELINE_STATISTIC/table.info
@@ -0,0 +1,3 @@
+Type =
+SubType =
+
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_BASELINE_STATISTIC/table.lock b/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_BASELINE_STATISTIC/table.lock
new file mode 100644
index 0000000000000000000000000000000000000000..0472b61a97b3474a8587d7db73c4914e1104166e
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_BASELINE_STATISTIC/table.lock differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_FREQUENCY_STATISTIC/table.dat b/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_FREQUENCY_STATISTIC/table.dat
new file mode 100644
index 0000000000000000000000000000000000000000..534ab977cdbfd6c21726bdb52e78ca8f765cfabc
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_FREQUENCY_STATISTIC/table.dat differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_FREQUENCY_STATISTIC/table.f0 b/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_FREQUENCY_STATISTIC/table.f0
new file mode 100644
index 0000000000000000000000000000000000000000..6b4995c6cebf3a47ad41f96907d10c4d817696f6
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_FREQUENCY_STATISTIC/table.f0 differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_FREQUENCY_STATISTIC/table.info b/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_FREQUENCY_STATISTIC/table.info
new file mode 100644
index 0000000000000000000000000000000000000000..a1a976ed49439abda34d2c64cfd0360cfcdef651
--- /dev/null
+++ b/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_FREQUENCY_STATISTIC/table.info
@@ -0,0 +1,3 @@
+Type =
+SubType =
+
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_FREQUENCY_STATISTIC/table.lock b/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_FREQUENCY_STATISTIC/table.lock
new file mode 100644
index 0000000000000000000000000000000000000000..0472b61a97b3474a8587d7db73c4914e1104166e
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_FREQUENCY_STATISTIC/table.lock differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_KIND_NAME/table.dat b/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_KIND_NAME/table.dat
new file mode 100644
index 0000000000000000000000000000000000000000..5ab547b065b59ee35f5f75b4cb971c7850560324
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_KIND_NAME/table.dat differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_KIND_NAME/table.f0 b/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_KIND_NAME/table.f0
new file mode 100644
index 0000000000000000000000000000000000000000..b970716f71e6adc2ccd8079da078b19fa229b810
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_KIND_NAME/table.f0 differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_KIND_NAME/table.info b/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_KIND_NAME/table.info
new file mode 100644
index 0000000000000000000000000000000000000000..a1a976ed49439abda34d2c64cfd0360cfcdef651
--- /dev/null
+++ b/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_KIND_NAME/table.info
@@ -0,0 +1,3 @@
+Type =
+SubType =
+
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_KIND_NAME/table.lock b/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_KIND_NAME/table.lock
new file mode 100644
index 0000000000000000000000000000000000000000..0472b61a97b3474a8587d7db73c4914e1104166e
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_KIND_NAME/table.lock differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_TIME_STATISTIC/table.dat b/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_TIME_STATISTIC/table.dat
new file mode 100644
index 0000000000000000000000000000000000000000..638f3dd45f2c2e45f24c7638de10b581c8b2129f
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_TIME_STATISTIC/table.dat differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_TIME_STATISTIC/table.f0 b/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_TIME_STATISTIC/table.f0
new file mode 100644
index 0000000000000000000000000000000000000000..40f3c17a4924760e9be56d6b5bbdb3809aea5cf6
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_TIME_STATISTIC/table.f0 differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_TIME_STATISTIC/table.info b/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_TIME_STATISTIC/table.info
new file mode 100644
index 0000000000000000000000000000000000000000..a1a976ed49439abda34d2c64cfd0360cfcdef651
--- /dev/null
+++ b/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_TIME_STATISTIC/table.info
@@ -0,0 +1,3 @@
+Type =
+SubType =
+
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_TIME_STATISTIC/table.lock b/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_TIME_STATISTIC/table.lock
new file mode 100644
index 0000000000000000000000000000000000000000..0472b61a97b3474a8587d7db73c4914e1104166e
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/QUALITY_TIME_STATISTIC/table.lock differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/SPECTRAL_WINDOW/table.dat b/Docker/test_data/L570745_SB001_uv_MODEL.MS/SPECTRAL_WINDOW/table.dat
new file mode 100644
index 0000000000000000000000000000000000000000..f4cd2bc15998b2583474b25a88c4e253a2fdd249
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/SPECTRAL_WINDOW/table.dat differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/SPECTRAL_WINDOW/table.f0 b/Docker/test_data/L570745_SB001_uv_MODEL.MS/SPECTRAL_WINDOW/table.f0
new file mode 100644
index 0000000000000000000000000000000000000000..6489aad3aa55d3c4a9799f7351635f5076b107d7
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/SPECTRAL_WINDOW/table.f0 differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/SPECTRAL_WINDOW/table.f0i b/Docker/test_data/L570745_SB001_uv_MODEL.MS/SPECTRAL_WINDOW/table.f0i
new file mode 100644
index 0000000000000000000000000000000000000000..7f3944944fa4e5e33c75c0959274bcf3c95a979f
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/SPECTRAL_WINDOW/table.f0i differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/SPECTRAL_WINDOW/table.info b/Docker/test_data/L570745_SB001_uv_MODEL.MS/SPECTRAL_WINDOW/table.info
new file mode 100644
index 0000000000000000000000000000000000000000..a1a976ed49439abda34d2c64cfd0360cfcdef651
--- /dev/null
+++ b/Docker/test_data/L570745_SB001_uv_MODEL.MS/SPECTRAL_WINDOW/table.info
@@ -0,0 +1,3 @@
+Type =
+SubType =
+
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/SPECTRAL_WINDOW/table.lock b/Docker/test_data/L570745_SB001_uv_MODEL.MS/SPECTRAL_WINDOW/table.lock
new file mode 100644
index 0000000000000000000000000000000000000000..0472b61a97b3474a8587d7db73c4914e1104166e
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/SPECTRAL_WINDOW/table.lock differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/STATE/table.dat b/Docker/test_data/L570745_SB001_uv_MODEL.MS/STATE/table.dat
new file mode 100644
index 0000000000000000000000000000000000000000..e83d60922df567c9b906d353e590114d0f06ed9e
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/STATE/table.dat differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/STATE/table.f0 b/Docker/test_data/L570745_SB001_uv_MODEL.MS/STATE/table.f0
new file mode 100644
index 0000000000000000000000000000000000000000..a3e1ee3d9b6c1c0d1ca4c4089a5c1105d3060cd4
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/STATE/table.f0 differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/STATE/table.info b/Docker/test_data/L570745_SB001_uv_MODEL.MS/STATE/table.info
new file mode 100644
index 0000000000000000000000000000000000000000..a1a976ed49439abda34d2c64cfd0360cfcdef651
--- /dev/null
+++ b/Docker/test_data/L570745_SB001_uv_MODEL.MS/STATE/table.info
@@ -0,0 +1,3 @@
+Type =
+SubType =
+
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/STATE/table.lock b/Docker/test_data/L570745_SB001_uv_MODEL.MS/STATE/table.lock
new file mode 100644
index 0000000000000000000000000000000000000000..0472b61a97b3474a8587d7db73c4914e1104166e
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/STATE/table.lock differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.dat b/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.dat
new file mode 100644
index 0000000000000000000000000000000000000000..fdb2a941e91645871bb6b836fc2952f499e2c509
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.dat differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f0 b/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f0
new file mode 100644
index 0000000000000000000000000000000000000000..fbc772f920efcd59df4b726f48ec68c621c73f85
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f0 differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f1 b/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f1
new file mode 100644
index 0000000000000000000000000000000000000000..0b21992bc5062953e70c5b9bd5b08c292b161441
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f1 differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f1i b/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f1i
new file mode 100644
index 0000000000000000000000000000000000000000..2308328019d48cc87144699973b89461338d1a0b
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f1i differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f2 b/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f2
new file mode 100644
index 0000000000000000000000000000000000000000..15c5ebbdf772ebe0bcc4c233905345afea749e51
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f2 differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f2_TSM0 b/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f2_TSM0
new file mode 100644
index 0000000000000000000000000000000000000000..d0be7c43dc806e8e9c2cacab6062eb9f8c01c241
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f2_TSM0 differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f3 b/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f3
new file mode 100644
index 0000000000000000000000000000000000000000..a72b78ccd766304c88ebbdaa83d50a6a1012c3d8
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f3 differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f3_TSM0.REMOVED.git-id b/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f3_TSM0.REMOVED.git-id
new file mode 100644
index 0000000000000000000000000000000000000000..e5c61f434f829fb218f5c674e4e1d142717c1ede
--- /dev/null
+++ b/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f3_TSM0.REMOVED.git-id
@@ -0,0 +1 @@
+7637ff184c49f778fa3641fbc6548d043393cd86
\ No newline at end of file
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f4 b/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f4
new file mode 100644
index 0000000000000000000000000000000000000000..e598c71dd69b1f060fae01b0642b8c24a3f5d9c0
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f4 differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f4_TSM0 b/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f4_TSM0
new file mode 100644
index 0000000000000000000000000000000000000000..614ee4b023cd4f57066a1b8d6264ac3cf88aaaad
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f4_TSM0 differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f5 b/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f5
new file mode 100644
index 0000000000000000000000000000000000000000..3b4b570d2e2ab2a20fdaa61eb1d4b41725d30fd9
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f5 differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f5_TSM0 b/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f5_TSM0
new file mode 100644
index 0000000000000000000000000000000000000000..5a8f3fadec9e2a3cc53886a0dbf8b7f966f4d20f
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f5_TSM0 differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f6 b/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f6
new file mode 100644
index 0000000000000000000000000000000000000000..d7d657412e746495a916cd47adf4b13ca2373497
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f6 differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f6_TSM0.REMOVED.git-id b/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f6_TSM0.REMOVED.git-id
new file mode 100644
index 0000000000000000000000000000000000000000..3ea3217072e12722bffbde57d4ba11baf5bf27d2
--- /dev/null
+++ b/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f6_TSM0.REMOVED.git-id
@@ -0,0 +1 @@
+10891d14e7571d21ec73dd29362401082041c711
\ No newline at end of file
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f7 b/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f7
new file mode 100644
index 0000000000000000000000000000000000000000..92e945567782b8c27f4076795be9b77022ce2fc7
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f7 differ
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f7_TSM0.REMOVED.git-id b/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f7_TSM0.REMOVED.git-id
new file mode 100644
index 0000000000000000000000000000000000000000..38eb0c84b1e63548143a97f65b3a3f4176db1ea0
--- /dev/null
+++ b/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.f7_TSM0.REMOVED.git-id
@@ -0,0 +1 @@
+2dedb806a06539078faf5e62c82a98b21dc512b8
\ No newline at end of file
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.info b/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.info
new file mode 100644
index 0000000000000000000000000000000000000000..105af65c13868a224843eb0cb3596828614e8fd0
--- /dev/null
+++ b/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.info
@@ -0,0 +1,5 @@
+Type = Measurement Set
+SubType =
+
+This is a MeasurementSet Table holding measurements from a Telescope
+This is a LOFAR MeasurementSet Table
diff --git a/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.lock b/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.lock
new file mode 100644
index 0000000000000000000000000000000000000000..c0c07ecaa3d0c0ff6d59a423b8d0e752b7070563
Binary files /dev/null and b/Docker/test_data/L570745_SB001_uv_MODEL.MS/table.lock differ
diff --git a/Docker/test_data/example.h5.REMOVED.git-id b/Docker/test_data/example.h5.REMOVED.git-id
new file mode 100644
index 0000000000000000000000000000000000000000..ee55446f16fed96380113889ea423eb37e2bdbd0
--- /dev/null
+++ b/Docker/test_data/example.h5.REMOVED.git-id
@@ -0,0 +1 @@
+554933b577a0dcf1b2aa3fa91c7de5b2477fd724
\ No newline at end of file
diff --git a/steps/AOFlagger.cwl b/steps/AOFlagger.cwl
new file mode 100644
index 0000000000000000000000000000000000000000..e3421e14bc87cb1c9615f4e21f6a981a2849ebc8
--- /dev/null
+++ b/steps/AOFlagger.cwl
@@ -0,0 +1,121 @@
+#!/usr/bin/env cwl-runner
+
+class: CommandLineTool
+cwlVersion: v1.0
+id: aoflagger
+
+requirements:
+ - class: InlineJavascriptRequirement
+ - class: InitialWorkDirRequirement
+ listing:
+ - entry: $(inputs.msin)
+ writable: true
+
+hints:
+ DockerRequirement:
+ dockerPull: lofareosc/prefactor-ci:master
+
+baseCommand:
+ - aoflagger
+inputs:
+ - id: msin
+ type: Directory[]
+ doc: Input measurement set
+ inputBinding:
+ position: 0
+ - id: verbose
+ type: boolean?
+
+ doc: will produce verbose output
+ inputBinding:
+ valueFrom: -v
+ - id: n_threads
+ type: int?
+ doc: |
+ overrides the number of threads specified in the strategy
+ (default: one thread for each CPU core)
+ inputBinding:
+ prefix: -j
+
+ - id: strategy
+ type: File?
+ doc: |
+ specifies a customized strategy
+ inputBinding:
+ prefix: -strategy
+
+#### READ MODES
+ - id: direct_read
+ type: boolean?
+ doc: |
+ Will perform the slowest IO but will always work.
+ inputBinding:
+ prefix: -direct-read
+ - id: indirect_read
+ type: boolean?
+ doc: |
+ Will reorder the measurement set before starting, which is normally
+ faster but requires free disk space to reorder the data to.
+ inputBinding:
+ prefix: -indirect-read
+ - id: memory_read
+ type: boolean?
+ doc: |
+ Will read the entire measurement set in memory. This is the fastest,
+ but requires much memory.
+ inputBinding:
+ prefix: -memory-read
+ - id: auto_read
+ type: boolean?
+ doc: |
+ Will select either memory or direct mode based on available memory (default).
+ inputBinding:
+ prefix: -auto-read
+##############
+ - id: skip_flagged
+ type: boolean?
+ doc: |
+ Will skip an ms if it has already been processed by AOFlagger according
+ to its HISTORY table.
+ inputBinding:
+ prefix: -skip-flagged
+ - id: uvw
+ type: File?
+ doc: Reads uvw values (some exotic strategies require these)
+ inputBinding:
+ prefix: -uvw
+ - id: column
+ type: string?
+ doc: Specify column to flag
+ inputBinding:
+ prefix: -column
+ - id: bands
+ type: int[]?
+ doc: Comma separated list of (zero-indexed) band ids to process
+ inputBinding:
+ prefix: -bands
+ itemSeparator: ','
+ - id: fields
+ type: int[]?
+ doc: Comma separated list of (zero-indexed) field ids to process
+ inputBinding:
+ prefix: -fields
+ itemSeparator: ','
+ - id: combine-spws
+ type: boolean?
+ doc: Join all SPWs together in frequency direction before flagging
+ inputBinding:
+ prefix: -combine-spws
+ - id: bandpass
+ type: File?
+ doc: Set bandpass correction file for any 'Apply passband' action
+ inputBinding:
+ prefix: bandpass
+
+outputs:
+ - id: output_ms
+ doc: Output image
+ type: Directory[]
+ outputBinding:
+ outputEval: $(inputs.msin)
+label: AOFlagger
diff --git a/steps/DP3.AOFlaggerStepGenerator.cwl b/steps/DP3.AOFlaggerStepGenerator.cwl
new file mode 100755
index 0000000000000000000000000000000000000000..2a70f602771d7e5e669ba46ee395550807bb379d
--- /dev/null
+++ b/steps/DP3.AOFlaggerStepGenerator.cwl
@@ -0,0 +1,132 @@
+#!/usr/bin/env cwl-runner
+
+class: Workflow
+cwlVersion: v1.0
+id: aoflagger_step_generator
+
+inputs:
+ - id: steps
+ type: Any[]?
+ default: []
+ - id: step_name
+ type: string
+ doc: unique name for the step
+ default: aoflag
+ - id: count.save
+ type: boolean
+ default: false
+ doc: >-
+ If true, the flag percentages per frequency are saved to a table with
+ extension .flagfreq and percentages per station to a table with extension
+ .flagstat. The basename of the table is the MS name (without extension)
+ followed by the stepname and extension.
+ - id: count.path
+ type: string
+ default: ""
+ doc: >-
+ The directory where to create the flag percentages table. If empty, the
+ path of the input MS is used.
+ - id: strategy
+ type: File?
+ doc: >-
+ The name of the strategy file to use. If no name is given, the default
+ strategy is used which is fine for HBA. For LBA data the strategy
+ LBAdefault should be used.
+ - id: memoryperc
+ type: int
+ default: 0
+ doc: >-
+ If >0, percentage of the machines memory to use. If memorymax nor
+ memoryperc is given, all memory will be used (minus 2 GB (at most 50%) for
+ other purposes). Accepts only integer values (LOFAR v2.16). Limiting the
+ available memory too much affects flagging accuracy; in general try to use
+ at least 10 GB of memory.
+ - id: memorymax
+ type: double
+ default: 0
+ doc: >-
+ Maximum amount of memory (in GB) to use. â‡0 means no maximum. As stated
+ above, this affects flagging accuracy.
+ - id: timewindow
+ type: int
+ default: 0
+ doc: >-
+ Number of time slots to be flagged jointly. The larger the time window,
+ the better the flagging performs. 0 means that it will be deduced from the
+ memory to use. Note that the time window can be extended with an overlap
+ on the left and right side to minimize possible boundary effects.
+ - id: overlapperc
+ type: double
+ default: 0
+ doc: >-
+ If >0, percentage of time window to be added to the left and right side
+ for overlap purposes (to minimize boundary effects). If overlapmax is not
+ given, it defaults to 1%.
+ - id: overlapmax
+ type: int
+ default: 0
+ doc: Maximum overlap value (0 is no maximum).
+ - id: autocorr
+ type: boolean
+ default: true
+ doc: Flag autocorrelations?
+ - id: pulsar
+ type: boolean
+ default: false
+ doc: Use flagging strategy optimized for pulsar observations?
+ - id: pedantic
+ type: boolean
+ default: false
+ doc: Be more pedantic when flagging?
+ - id: keepstatistics
+ type: boolean
+ default: true
+ doc: Write the quality statistics?
+outputs:
+ - id: augmented_steps
+ outputSource:
+ - DP3_GenericStep/augmented_steps
+ type: Any[]
+
+steps:
+ - id: DP3_GenericStep
+ in:
+ - id: step_type
+ default: 'aoflagger'
+ - id: step_id
+ source: step_name
+ - id: steps
+ source: steps
+ - id: parameters
+ valueFrom: $(inputs)
+ - id: count.save
+ source: count.save
+ - id: count.path
+ source: count.save
+ - id: strategy
+ source: strategy
+ - id: memoryperc
+ source: memoryperc
+ - id: memorymax
+ source: memorymax
+ - id: timewindow
+ source: timewindow
+ - id: overlapperc
+ source: overlapperc
+ - id: overlapmax
+ source: overlapmax
+ - id: autocorr
+ source: autocorr
+ - id: pulsar
+ source: pulsar
+ - id: pedantic
+ source: pedantic
+ - id: keepstatistics
+ source: keepstatistics
+ out:
+ - augmented_steps
+ run: ../steps/DP3.GenericStep.cwl
+requirements:
+ - class: StepInputExpressionRequirement
+ - class: InlineJavascriptRequirement
+ - class: MultipleInputFeatureRequirement
diff --git a/steps/DP3.AveragerStepGenerator.cwl b/steps/DP3.AveragerStepGenerator.cwl
new file mode 100755
index 0000000000000000000000000000000000000000..d5424919111a6c38f65053bfec5f2e299a98c076
--- /dev/null
+++ b/steps/DP3.AveragerStepGenerator.cwl
@@ -0,0 +1,85 @@
+#!/usr/bin/env cwl-runner
+
+class: Workflow
+cwlVersion: v1.0
+id: average_step_generator
+
+inputs:
+ - id: steps
+ type: Any[]?
+ default: []
+ - id: step_name
+ type: string
+ doc: unique name for the step
+ default: average
+ - id: timestep
+ type: int
+ doc: >-
+ Number of time slots to average. It is truncated if exceeding the actual
+ number of times.
+ default: 1
+ - id: freqstep
+ type: int
+ doc: >-
+ Number of channels to average. It is truncated if exceeding the actual
+ number of channels.
+ default: 1
+ - id: minpoints
+ type: int
+ doc: >-
+ If number of averaged unflagged input points < minpoints, the averaged
+ point is flagged.
+ default: 0
+ - id: minperc
+ type: float
+ doc: 'Like minpoints, but expressed as a percentage of timestep*freqstep.'
+ default: 0
+ - id: timeresolution
+ type: float
+ doc: >-
+ Target time resolution, in seconds. If this is given, and bigger than
+ zero, it overrides <step>.timestep
+ default: 0
+ - id: freqresolution
+ type: float
+ doc: >-
+ Target frequency resolution, in Hz (or append "MHz" or "kHz" to specify it
+ in those units). If this is given, and bigger than zero, it overrides
+ <step>.freqstep
+ default: 0
+outputs:
+ - id: augmented_steps
+ outputSource:
+ - DP3_GenericStep/augmented_steps
+ type: Any[]
+
+steps:
+ - id: DP3_GenericStep
+ in:
+ - id: step_type
+ default: 'averager'
+ - id: step_id
+ source: step_name
+ - id: steps
+ source: steps
+ - id: parameters
+ valueFrom: $(inputs)
+ - id: timestep
+ source: timestep
+ - id: freqstep
+ source: freqstep
+ - id: minpoints
+ source: minpoints
+ - id: minperc
+ source: minperc
+ - id: timeresolution
+ source: timeresolution
+ - id: freqresolution
+ source: freqresolution
+ out:
+ - augmented_steps
+ run: ../steps/DP3.GenericStep.cwl
+requirements:
+ - class: StepInputExpressionRequirement
+ - class: InlineJavascriptRequirement
+ - class: MultipleInputFeatureRequirement
diff --git a/steps/DP3.DemixerStepGenerator.cwl b/steps/DP3.DemixerStepGenerator.cwl
new file mode 100755
index 0000000000000000000000000000000000000000..c88bdd1721d9a544f872ef89f814c361eb56dee5
--- /dev/null
+++ b/steps/DP3.DemixerStepGenerator.cwl
@@ -0,0 +1,190 @@
+#!/usr/bin/env cwl-runner
+
+class: Workflow
+cwlVersion: v1.0
+$namespaces:
+ sbg: 'https://www.sevenbridges.com/'
+id: demix_step_generator
+
+inputs:
+ - id: steps
+ type: Any[]?
+ default: []
+ - id: step_name
+ type: string
+ doc: unique name for the step
+ default: demix
+ - id: baseline
+ default: ""
+ type: string
+ doc: Baselines to demix. See Description of baseline selection parameters.
+ - id: blrange
+ default: []
+ type: double[]
+ doc: Baselines to demix. See Description of baseline selection parameters.
+ - id: corrtype
+ default: cross
+ type: string
+ doc: >-
+ Baselines to demix. Correlation type to match? Must be auto, cross, or an
+ empty string.
+ - id: timestep
+ default: 1
+ type: int
+ doc: >-
+ Number of time slots to average when subtracting. It is truncated if
+ exceeding the actual number of times. Note that the data itself will also
+ be averaged by this amount.
+ - id: freqstep
+ default: 1
+ type: int
+ doc: >-
+ Number of channels to average when subtracting. It is truncated if
+ exceeding the actual number of channels. Note that the data itself will
+ also be averaged by this amount.
+ - id: demixtimestep
+ default: 0
+ type: int
+ doc: >-
+ Number of time slots to average when demixing. It is truncated if
+ exceeding the actual number of times. It defaults to the averaging used
+ for the subtract.
+ - id: demixfreqstep
+ default: 0
+ type: int
+ doc: >-
+ Number of channels to average when demixing. It is truncated if exceeding
+ the actual number of channels. It defaults to the averaging used for the
+ subtract.
+ - id: ntimechunk
+ default: null
+ type: int
+ doc: >-
+ Number of demix time slots (after averaging) that are processed jointly in
+ as much a parallel way as possible. If subtract uses different time
+ averaging, it has to fit integrally.
+ - id: skymodel
+ default: sky
+ type: [File, Directory, string]
+ doc: The name of the SourceDB to use (i.e., the output of makesourcedb).
+ - id: instrumentmodel
+ default: instrument
+ type: string
+ doc: >-
+ The name of the ParmDB to use. The ParmDB does not need to exist. If it
+ does not exist it will be created.
+ - id: subtractsources
+ default: null
+ type: string[]?
+ doc: >-
+ Names of the sources to subtract. If none are given, demixing comes down
+ to averaging. The sources must exist as patches in the SourceDB.
+ - id: modelsources
+ default: []
+ type: string[]
+ doc: >-
+ Names of sources with models to take into account when solving. the
+ sources must exist as patches in the SourceDB. Note that the target should
+ NOT be part of this parameter. If a model of the target has to be used, it
+ has to be given in parameter targetsource.
+ - id: targetsource
+ default: ''
+ type: string
+ doc: >-
+ It can be used to specify the name of the source model of the target. If
+ given, the target source model (its patch in the SourceDB) is taken into
+ account when solving; in this case parameter othersources cannot be given.
+ It cannot be given if ignoretarget=true. If not given, the target is
+ projected away or ignored (depending on parameter ignoretarget).
+ - id: ignoretarget
+ default: false
+ type: boolean
+ doc: false = project the target source away; true = ignore the target
+ - id: othersources
+ default: []
+ type: string[]
+ doc: >-
+ Names of sources of which the direction is taken into account when
+ demixing by projecting the directions away. The direction needs to be
+ specified if the source is unknown (which is usually the case). It can be
+ done using parameters <step>.<sourcename>.phasecenter.
+ - id: propagatesolutions
+ default: true
+ type: boolean
+ doc: >-
+ If set to true, solutions of a time slot are used as initial values for
+ the next time slot. If set to false, the diagonal elements of the Jones
+ matrix are initialized to one and the off-diagonal elements to zero.
+ - id: defaultgain
+ default: 1
+ type: double
+ doc: >-
+ The default and initial gain for the directional gains that are computed
+ internally.
+ - id: maxiter
+ default: 50
+ type: int
+ doc: Maximum number of iterations used in the LM solve
+outputs:
+- id: augmented_steps
+ outputSource:
+ - DP3_GenericStep/augmented_steps
+ type: Any[]
+
+steps:
+ - id: DP3_GenericStep
+ in:
+ - id: steps
+ source: steps
+ - id: output_files
+ default: ["instrumentmodel"]
+
+ - id: parameters
+ valueFrom: $(inputs)
+ - id: step_id
+ source: step_name
+ - id: step_type
+ default: 'demixer'
+ - id: baseline
+ source: baseline
+ - id: blrange
+ source: blrange
+ - id: corrtype
+ source: corrtype
+ - id: timestep
+ source: timestep
+ - id: freqstep
+ source: freqstep
+ - id: demixtimestep
+ source: demixtimestep
+ - id: demixfreqstep
+ source: demixfreqstep
+ - id: ntimechunk
+ source: ntimechunk
+ - id: skymodel
+ source: skymodel
+ - id: instrumentmodel
+ source: instrumentmodel
+ - id: subtractsources
+ source: subtractsources
+ - id: modelsources
+ source: modelsources
+ - id: targetsource
+ source: targetsource
+ - id: ignoretarget
+ source: ignoretarget
+ - id: othersources
+ source: othersources
+ - id: propagatesolutions
+ source: propagatesolutions
+ - id: defaultgain
+ source: defaultgain
+ - id: maxiter
+ source: maxiter
+ out:
+ - augmented_steps
+ run: ../steps/DP3.GenericStep.cwl
+requirements:
+ - class: StepInputExpressionRequirement
+ - class: InlineJavascriptRequirement
+ - class: MultipleInputFeatureRequirement
diff --git a/steps/DP3.EmptyParsetGenerator.cwl b/steps/DP3.EmptyParsetGenerator.cwl
new file mode 100755
index 0000000000000000000000000000000000000000..f3e2c175ced26ad327e9d56e473bbaf0039996a5
--- /dev/null
+++ b/steps/DP3.EmptyParsetGenerator.cwl
@@ -0,0 +1,25 @@
+#!/usr/bin/env cwl-runner
+
+cwlVersion: v1.0
+class: CommandLineTool
+id: empty_parset_generator
+baseCommand: [echo, 'steps=[]']
+
+requirements:
+ InlineJavascriptRequirement: {}
+
+inputs: []
+
+stdout: output_parset
+outputs:
+ - id: output_parset
+ doc: Empty parset file
+ streamable: True
+ type: File
+ outputBinding:
+ glob: output_parset
+ - id: output_secondary_files
+ doc: files needed to execute the step
+ type: File[]
+ outputBinding:
+ outputEval: $([])
diff --git a/steps/DP3.Execute.cwl b/steps/DP3.Execute.cwl
new file mode 100644
index 0000000000000000000000000000000000000000..bf15ea45d9ac61ab6d23521605ea0955324e7e45
--- /dev/null
+++ b/steps/DP3.Execute.cwl
@@ -0,0 +1,79 @@
+class: Workflow
+cwlVersion: v1.0
+id: dp3_execute
+label: DP3.Execute
+$namespaces:
+ sbg: 'https://www.sevenbridges.com/'
+inputs:
+ - id: msout_name
+ type: string
+ 'sbg:x': -425.39886474609375
+ 'sbg:y': 127
+ - id: msin
+ type: Directory
+ 'sbg:x': -254.39886474609375
+ 'sbg:y': 242
+ - id: steps
+ type: 'Any[]?'
+ 'sbg:x': -518.3988647460938
+ 'sbg:y': -88
+outputs:
+ - id: secondary_output_files
+ outputSource:
+ - dppp/secondary_output_files
+ type: Any
+ 'sbg:x': 213.60113525390625
+ 'sbg:y': -184
+ - id: secondary_output_directories
+ outputSource:
+ - dppp/secondary_output_directories
+ type: Any
+ 'sbg:x': 248.60113525390625
+ 'sbg:y': -47
+ - id: msout
+ outputSource:
+ - dppp/msout
+ type: Directory
+ 'sbg:x': 224.60113525390625
+ 'sbg:y': 130
+steps:
+ - id: generic_step
+ in:
+ - id: steps
+ source:
+ - steps
+ out:
+ - id: parset
+ - id: input_files
+ - id: input_directories
+ - id: output_file_names
+ - id: output_directory_names
+ run: ./DP3.ParsetGenerator.cwl
+ 'sbg:x': -294
+ 'sbg:y': -84
+ - id: dppp
+ in:
+ - id: parset
+ source: generic_step/parset
+ - id: msin
+ source: msin
+ - id: msout_name
+ source: msout_name
+ - id: secondary_files
+ source:
+ - generic_step/input_files
+ - id: secondary_directories
+ source:
+ - generic_step/input_directories
+ - id: output_file_names
+ source: generic_step/output_file_names
+ - id: output_directory_names
+ source: generic_step/output_directory_names
+ out:
+ - id: msout
+ - id: secondary_output_files
+ - id: secondary_output_directories
+ run: ./DPPP.cwl
+ 'sbg:x': 26
+ 'sbg:y': -46
+requirements: []
diff --git a/steps/DP3.GenericStep.cwl b/steps/DP3.GenericStep.cwl
new file mode 100755
index 0000000000000000000000000000000000000000..38829da42ed0deaac51bcf92c7c105e9139d4108
--- /dev/null
+++ b/steps/DP3.GenericStep.cwl
@@ -0,0 +1,63 @@
+#!/usr/bin/env cwl-runner
+
+cwlVersion: v1.0
+class: ExpressionTool
+id: generic_step
+
+requirements:
+ - class: InlineJavascriptRequirement
+
+inputs:
+ - id: steps
+ type: Any[]
+
+ - id: parameters
+ type: Any?
+
+outputs:
+ - id: augmented_steps
+ type: Any[]
+expression: |-
+ ${
+ var step_type = inputs.parameters.step_type;
+ var step_id = inputs.parameters.step_id;
+
+ var output_file_type = inputs.parameters.output_files;
+ var output_files = {}
+
+ for(var index in output_file_type){
+ var file_type = output_file_type[index]
+ if(file_type != undefined && file_type != null){
+ output_files[file_type] = inputs.parameters[file_type]
+ }
+ }
+
+ var input_secondary_files = inputs.parameters.input_secondary_files;
+
+ delete inputs.parameters.step_type;
+ delete inputs.parameters.step_id;
+ delete inputs.parameters.steps;
+ delete inputs.parameters.parameters;
+ delete inputs.parameters.steps;
+
+
+ var new_step = {'step_type': step_type,
+ 'step_id': step_id,
+ 'parameters': inputs.parameters,
+ 'output_files': output_files};
+
+
+ var in_parameters = inputs.steps;
+
+ // Checks if the step_id is duplicate if so raised an exception
+ var step_ids = []
+ for(var step in in_parameters){
+ step = in_parameters[step]
+ step_ids.push(step.step_id)
+ }
+ if(step_ids.indexOf(step_id) >= 0) throw 'step_id has to be unique'
+
+ var out_parameters = in_parameters == null ? [new_step] : in_parameters.concat(new_step)
+
+ return {'augmented_steps': out_parameters};
+ }
diff --git a/steps/DP3.ParsetGenerator.cwl b/steps/DP3.ParsetGenerator.cwl
new file mode 100644
index 0000000000000000000000000000000000000000..f5837e9a3528135180143bee7fb406ef736b1efb
--- /dev/null
+++ b/steps/DP3.ParsetGenerator.cwl
@@ -0,0 +1,148 @@
+#!/usr/bin/env cwl-runner
+
+cwlVersion: v1.0
+class: CommandLineTool
+id: generic_step
+
+requirements:
+ - class: InlineJavascriptRequirement
+
+baseCommand:
+ - python3
+
+arguments:
+ - prefix: -c
+ valueFrom: |
+ import sys
+
+ true = True
+ false = False
+ null = None
+ steps_list = $(inputs.steps)
+ steps_id_list = list(map(lambda item: item['step_id'], steps_list))
+
+ for step in steps_list:
+ try:
+ step_id = step.get('step_id')
+ step_type = step.get('step_type')
+ parameters = step.get('parameters')
+
+
+ line = '.'.join([step_id, 'type'])
+ line += '=%s' % step_type
+ print(line)
+
+ for parameter_name, parameter in parameters.items():
+ if isinstance(parameter, dict):
+ if parameter['class'] in ['File', 'Directory']:
+ parameter = parameter['basename']
+ else:
+ raise NotImplemented()
+ if parameter is None:
+ continue
+ line = '.'.join([step_id, parameter_name])
+ line = '%s=%s' % (line, parameter)
+
+ print(line)
+ except KeyError as e:
+ print('invalid input for step ', step, ':', e)
+ sys.exit(1)
+ last_step_line = 'steps=[%s]' % ', '.join(steps_id_list)
+ print(last_step_line)
+inputs:
+ - id: steps
+ type: Any[]?
+
+outputs:
+ - id: parset
+ doc: Parset output file
+ type: File
+ outputBinding:
+ glob: parset
+ - id: input_files
+ doc: additional input files
+ type: File[]
+ outputBinding:
+ outputEval: |-
+ ${
+ var outfiles = []
+ for(var step in inputs.steps){
+ step = inputs.steps[step]
+ for(var parameter in step['parameters']){
+ parameter = step['parameters'][parameter]
+
+ if (parameter != null && parameter.class =='File'){
+ outfiles.push(parameter)
+ }
+ }
+ }
+ return outfiles
+ }
+ - id: input_directories
+ doc: additional input directory
+ type: Directory[]
+ outputBinding:
+ outputEval: |
+ ${
+ var outdirs = []
+ for(var step in inputs.steps){
+ step = inputs.steps[step]
+ for(var parameter in step['parameters']){
+ parameter = step['parameters'][parameter]
+ if (parameter != null && parameter.class=='Directory'){
+ outdirs.push(parameter)
+ }
+ }
+ }
+ return outdirs
+ }
+
+ - id: output_file_names
+ doc: expected output files
+ type: Any
+ outputBinding:
+ outputEval: |
+ ${
+ var out_names = {}
+ for(var step_idx in inputs.steps){
+ var step = inputs.steps[step_idx]
+ var step_id = step['step_id']
+ var step_dirs = {}
+ if(!step.hasOwnProperty("output_files")) continue;
+
+ var step_files = {}
+
+ for(var index in step['output_files']){
+ var file_id = index
+ var file_name = step['output_files'][file_id]
+ step_files[file_id] = file_name
+ }
+
+ out_names[step_id] = step_files
+ }
+ return out_names
+ }
+ - id: output_directory_names
+ doc: expected output directories
+ type: Any
+ outputBinding:
+ outputEval: |-
+ ${
+ var out_names = {}
+ for(var step_idx in inputs.steps){
+ var step = inputs.steps[step_idx]
+ var step_id = step['step_id']
+ var step_dirs = {}
+ if(!step.hasOwnProperty("output_directories")) continue;
+
+ for(var index in step['output_directories']){
+ var file_id = index
+ var file_name = step['output_directories'][file_id]
+ step_dirs[file_id] = file_name
+ }
+ out_names[step_id] = step_dirs
+ }
+ return out_names
+ }
+stdout:
+ parset
diff --git a/steps/DP3.PreflaggerStepGenerator.cwl b/steps/DP3.PreflaggerStepGenerator.cwl
new file mode 100755
index 0000000000000000000000000000000000000000..03f6c11d6f16db4d72a38666e4a294a2f568fb28
--- /dev/null
+++ b/steps/DP3.PreflaggerStepGenerator.cwl
@@ -0,0 +1,246 @@
+#!/usr/bin/env cwl-runner
+
+class: Workflow
+cwlVersion: v1.0
+id: preflag_step_generator
+
+inputs:
+ - id: steps
+ type: Any[]?
+ default: []
+ - id: step_name
+ type: string
+ doc: unique name for the step
+ default: preflag
+ - id: count.save
+ default: false
+ type: boolean
+ doc: >-
+ If true, the flag percentages per frequency are saved to a table with
+ extension .flagfreq and percentages per station to a table with extension
+ .flagstat. The basename of the table is the MS name (without extension)
+ followed by the stepname and extension.
+ - id: count.path
+ type: string?
+ doc: >-
+ The directory where to create the flag percentages table. If empty, the
+ path of the input MS is used.
+ - id: mode
+ default: set
+ type: string
+ doc: >-
+ Case-insensitive string telling what to do with the flags of the data
+ matching (or not matching) the selection criteria given in the other
+ parameters.
+ - id: expr
+ default: ''
+ type: string?
+ doc: >-
+ Expression of preflagger keyword sets (see above). Operators AND, OR, and
+ NOT are possible (or their equivalents &&,&, ||, |, and !). Parentheses
+ can be used to change precedence order. For example:: c1 and (c2 or c3)
+ - default:
+ id: timeofday
+ type: string?
+ doc: >-
+ Ranges of UTC time-of-day given as st..end or val+-delta. Each value must
+ be given as 12:34:56.789, 12h34m56.789, or as a value followed by a unit
+ like h, min, or s.
+ - id: abstime
+ default: []
+ type: string[]
+ doc: >-
+ Ranges of absolute UTC date/time given as st..end or val+-delta. Each
+ value (except delta) must be given as a date/time in casacore MVTime
+ format, for instance 12-Mar-2010/11:31:00.000. A delta value must be given
+ as a time (for instance 1:30:0 or 20s).
+ - id: reltime
+ default: []
+ type: string[]
+ doc: >-
+ Ranges of times (using .. or +-) since the start of the observation. A
+ time can be given like 1:30:0 or 20s.
+ - id: timeslot
+ default: []
+ type: string[]
+ doc: >-
+ Time slot sequence numbers. First time slot is 0. st..end means end
+ inclusive.
+ - id: lst
+ default: []
+ type: string[]
+ doc: >-
+ Ranges of Local Apparent Sidereal Times like 1:30:0 +- 20min. The LST of a
+ time slot is calculated for the array position, thus not per antenna.
+ - id: azimuth
+ default: []
+ type: float[]
+ doc: >-
+ Ranges of azimuth angles given as st..end or val+-delta. Each value has to
+ be given as a casacore direction like 12:34:56.789 or 12h34m56.789,
+ 12.34.56.789 or 12d34m56.789, or a value followed by a unit like rad or
+ deg.
+ - id: elevation
+ type: string?
+ doc: >-
+ Ranges of elevation angles (similar to azimuth). For example::
+ 0deg..10deg
+ - id: baseline
+ default: []
+ type: string[]
+ doc: See Description of baseline selection parameters.
+ - id: corrtype
+ default: ''
+ type: string
+ doc: Correlation type to match? Must be auto, cross, or an empty string.
+ - id: blmin
+ default: -1
+ type: float
+ doc: If blmin > 0, baselines with length < blmin meter will match.
+ - id: blmax
+ default: -1
+ type: float
+ doc: If blmax > 0, baselines with length > blmax meter will match.
+ - id: uvmmin
+ default: -1
+ type: float
+ doc: >-
+ If uvmmin > 0, baselines with UV-distance < uvmmin meter will match. Note
+ that the UV-distance is the projected baseline length.
+ - id: uvmmax
+ default: -1
+ type: float
+ doc: If uvmmax > 0, baselines with UV-distance > uvmmax meter will match.
+ - id: freqrange
+ default: ''
+ type: string
+ doc: >-
+ Channels in the given frequency ranges will match. Each value in the
+ vector is a range which can be given as start..end or start+-delta. A
+ value can be followed by a unit like KHz. If only one value in a range has
+ a unit, the unit is also applied to the other value. If a range has no
+ unit, it defaults to MHz. For example: freqrange=[1.2 .. 1.4 MHz,
+ 1.8MHz+-50KHz] flags channels between 1.2MHz and 1.4MHz and between
+ 1.75MHz and 1.85MHz. The example shows that blanks can be used at will.
+ - id: chan
+ default: ''
+ type: string
+ doc: >-
+ The given channels will match (start counting at 0). Channels exceeding
+ the number of channels are ignored. Similar to msin, it is possible to
+ specify the channels as an expression of nchan. Furthermore, .. can be
+ used to specify ranges. For example: chan=[0..nchan/32-1,
+ 31*nchan/32..nchan-1] to flag the first and last 2 or 8 channels
+ (depending on 64 or 256 channels in the observation).
+ - id: amplmin
+ default: -1e+30
+ type: float
+ doc: >-
+ Correlation data with amplitude < amplmin will match. It can be given per
+ correlation. For example, amplmin=[100,,,100] matches data points with XX
+ or YY amplitude < 100. The non-specified amplitudes get the default value.
+ - id: amplmax
+ default: 1e+30
+ type: float
+ doc: Correlation data with amplitude > amplmax will match.
+ - id: phasemin
+ default: -1e+30
+ type: float
+ doc: Correlation data with phase < phasemin (in radians) will match.
+ - id: phasemax
+ default: 1e+30
+ type: float
+ doc: Correlation data with phase > phasemax (in radians) will match.
+ - id: realmin
+ default: -1e+30
+ type: float
+ doc: Correlation data with real complex part < realmin will match.
+ - id: realmax
+ default: 1e+30
+ type: float
+ doc: Correlation data with real complex part > realmax will match.
+ - id: imagmin
+ default: -1e+30
+ type: float
+ doc: Correlation data with imaginary complex part < imagmin will match.
+ - id: imagmax
+ default: 1e+30
+ type: float
+ doc: Correlation data with imaginary complex part > imagmax will match.
+outputs:
+- id: augmented_steps
+ outputSource:
+ - DP3_GenericStep/augmented_steps
+ type: Any[]
+
+steps:
+ - id: DP3_GenericStep
+ in:
+ - id: step_type
+ default: 'preflagger'
+ - id: step_id
+ source: step_name
+ - id: steps
+ source: steps
+ - id: parameters
+ valueFrom: $(inputs)
+ - id: count.save
+ source: count.save
+ - id: count.path
+ source: count.path
+ - id: mode
+ source: mode
+ - id: expr
+ source: expr
+ - id: abstime
+ source: abstime
+ - id: reltime
+ source: reltime
+ - id: timeslot
+ source: timeslot
+ - id: lst
+ source: lst
+ - id: azimuth
+ source: azimuth
+ - id: elevation
+ source: elevation
+ - id: baseline
+ source: baseline
+ - id: corrtype
+ source: corrtype
+ - id: blmin
+ source: blmin
+ - id: blmax
+ source: blmax
+ - id: uvmmin
+ source: uvmmin
+ - id: uvmmax
+ source: uvmmax
+ - id: freqrange
+ source: freqrange
+ - id: chan
+ source: chan
+ - id: amplmin
+ source: amplmin
+ - id: amplmax
+ source: amplmax
+ - id: phasemin
+ source: phasemin
+ - id: phasemax
+ source: phasemax
+ - id: realmin
+ source: realmin
+ - id: realmax
+ source: realmax
+ - id: imagmin
+ source: imagmin
+ - id: imagmax
+ source: imagmax
+ out:
+ - augmented_steps
+ run: ../steps/DP3.GenericStep.cwl
+
+requirements:
+ - class: StepInputExpressionRequirement
+ - class: InlineJavascriptRequirement
+ - class: MultipleInputFeatureRequirement
diff --git a/steps/DP3.Selector.cwl b/steps/DP3.Selector.cwl
new file mode 100755
index 0000000000000000000000000000000000000000..b50754a85ab040136df19917a0e01a5464e8c7c9
--- /dev/null
+++ b/steps/DP3.Selector.cwl
@@ -0,0 +1,24 @@
+class: ExpressionTool
+cwlVersion: v1.0
+id: selector
+inputs:
+ - id: select_a
+ type: boolean
+ doc: 'if true selects a_steps if false selects b_steps'
+ - id: a_steps
+ type: Any[]
+ default: []
+ - id: b_steps
+ type: Any[]
+ default: []
+outputs:
+ - id: selected_steps
+ type: Any
+expression: |
+ $(
+ {'selected_steps': inputs.select_a ? inputs.a_steps : inputs.b_steps}
+ )
+label: Selector
+
+requirements:
+ - class: InlineJavascriptRequirement
diff --git a/steps/DPPP.cwl b/steps/DPPP.cwl
new file mode 100755
index 0000000000000000000000000000000000000000..70d824f02518f594ecf5d9b8975041b1febee5bc
--- /dev/null
+++ b/steps/DPPP.cwl
@@ -0,0 +1,89 @@
+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+ sbg: 'https://www.sevenbridges.com/'
+id: dppp
+baseCommand:
+ - DPPP
+inputs:
+ - id: parset
+ type: File?
+ inputBinding:
+ position: -1
+ - id: msin
+ type: Directory?
+ inputBinding:
+ position: 0
+ prefix: msin=
+ separate: false
+ doc: Input Measurement Set
+ - default: out.MS
+ id: msout_name
+ type: string
+ inputBinding:
+ position: 0
+ prefix: msout=
+ separate: false
+ doc: Output Measurement Set
+ - id: secondary_files
+ type: 'File[]?'
+ doc: Secondary files needed for the step
+ - id: secondary_directories
+ type: 'Directory[]?'
+ doc: Secondary directories needed for the step
+ - id: output_file_names
+ type: Any
+ doc: Expected output file names
+ - id: output_directory_names
+ type: Any
+ doc: Expected output file names
+outputs:
+ - id: msout
+ doc: Output Measurement Set
+ type: Directory
+ outputBinding:
+ glob: '$(inputs.msout_name=="."?inputs.msin:inputs.msout_name)'
+ - id: secondary_output_files
+ doc: Secondary output files
+ type: Any
+ outputBinding:
+ outputEval: |-
+ ${
+ var output={}
+ for(var step_name in inputs.output_file_names){
+ var file_per_idx = inputs.output_file_names[step_name];
+ for(var file_idx in file_per_idx){
+ var file_name = file_per_idx[file_idx];
+
+ output[file_idx] = {'class':'File', 'path': file_name};
+ }
+ }
+ return output
+ }
+ - id: secondary_output_directories
+ doc: Secondary output directories
+ type: Any
+ outputBinding:
+ outputEval: |-
+ ${
+ var output={}
+ for(var step_name in inputs.output_directory_names){
+ var file_per_idx = inputs.output_directory_names[step_name];
+ for(var file_idx in file_per_idx){
+ var file_name = file_per_idx[file_idx];
+
+ output[file_idx] = {'class':'Directory', 'path': file_name};
+ }
+ }
+ return output
+ }
+ - id: logfile
+ type: File?
+ outputBinding:
+ glob: DPPP.log
+hints:
+ - class: DockerRequirement
+ dockerPull: lofareosc/prefactor
+stdout: DPPP.log
+requirements:
+ - class: InlineJavascriptRequirement
diff --git a/steps/DirSelector.cwl b/steps/DirSelector.cwl
new file mode 100644
index 0000000000000000000000000000000000000000..d5bd04038a703c3ecfbc5f07aa917892b87bd85d
--- /dev/null
+++ b/steps/DirSelector.cwl
@@ -0,0 +1,39 @@
+class: ExpressionTool
+cwlVersion: v1.0
+id: dirselector
+inputs:
+ - id: inputs
+ type: Any
+ doc: input files
+ - id: step_name
+ type: string
+ doc: >-
+ id of the step that generates the output file or directory
+ (e.g. 'demixstepgenerator')
+ - id: key_name
+ type: string
+ doc: >-
+ key of the parset that contains the name of the outputfile
+ (e.g. 'instrumentmodel')
+outputs:
+ - id: output
+ type: [Directory, 'Directory[]']
+
+expression: |
+ ${
+ var step_name = inputs.step_name
+ var key_name = inputs.key_name
+
+ if(inputs.inputs.hasOwnProperty(step_name)){
+ var inputs_per_step = inputs.inputs[step_name]
+ if(inputs_per_step.hasOwnProperty(key_name)){
+ return {'output': inputs_per_step[key_name]}
+ }
+ }
+ return {'output': []}
+
+ }
+label: DirSelector
+
+requirements:
+ - class: InlineJavascriptRequirement
diff --git a/steps/FileSelector.cwl b/steps/FileSelector.cwl
new file mode 100644
index 0000000000000000000000000000000000000000..f935adca0125043812f663d913ef039819f6a52e
--- /dev/null
+++ b/steps/FileSelector.cwl
@@ -0,0 +1,31 @@
+class: ExpressionTool
+cwlVersion: v1.0
+id: fileselector
+inputs:
+ - id: inputs
+ type: Any
+ doc: input files
+ - id: step_name
+ type: string
+ doc: >-
+ id of the step that generates the output file or directory
+ (e.g. 'demixstepgenerator')
+ - id: key_name
+ type: string
+ doc: >-
+ key of the parset that contains the name of the outputfile
+ (e.g. 'instrumentmodel')
+outputs:
+ - id: output
+ type: File
+
+expression: |
+ ${
+ var step_name = inputs.step_name
+ var key_name = inputs.key_name
+ return {'output': inputs.inputs[step_name][key_name]}
+ }
+label: FileSelector
+
+requirements:
+ - class: InlineJavascriptRequirement
diff --git a/steps/H5ParmCollector.cwl b/steps/H5ParmCollector.cwl
new file mode 100755
index 0000000000000000000000000000000000000000..e2e33e20acab94f1e092866d37a2086185e93530
--- /dev/null
+++ b/steps/H5ParmCollector.cwl
@@ -0,0 +1,70 @@
+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+ sbg: 'https://www.sevenbridges.com/'
+ lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
+$schema:
+ - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
+
+id: h5parm_collector
+
+baseCommand:
+ - H5parm_collector.py
+inputs:
+ - id: h5parmFiles
+ type: 'File[]'
+ format: lofar:#H5Parm
+ inputBinding:
+ position: 0
+ doc: List of h5parm files
+ - default: sol000
+ id: insolset
+ type: string?
+ inputBinding:
+ position: 0
+ prefix: '--insolset'
+ doc: Input solset name
+ - default: output.h5
+ id: outh5parmname
+ type: string
+ doc: Output h5parm name
+ - id: squeeze
+ type: boolean
+ default: false
+ inputBinding:
+ position: 0
+ prefix: '-q'
+ doc: removes all axes with the length of 1
+ - default: true
+ id: verbose
+ type: boolean
+ inputBinding:
+ position: 0
+ prefix: '-v'
+ doc: verbose output
+ - default: true
+ id: clobber
+ type: boolean
+ inputBinding:
+ position: 0
+ prefix: '-c'
+ doc: overwrite output
+outputs:
+ - id: outh5parm
+ doc: Output h5parm
+ type: File
+ format: lofar:#H5Parm
+ outputBinding:
+ glob: $(inputs.outh5parmname)
+ - id: log
+ type: File
+ outputBinding:
+ glob: $(inputs.outh5parmname)-parm_collector_output.log
+label: H5parm_collector
+hints:
+ - class: DockerRequirement
+ dockerPull: 'lofareosc/prefactor-ci:master'
+stdout: $(inputs.outh5parmname)-parm_collector_output.log
+requirements:
+ - class: InlineJavascriptRequirement
+
diff --git a/steps/LoSoTo.Abs.cwl b/steps/LoSoTo.Abs.cwl
new file mode 100755
index 0000000000000000000000000000000000000000..43f5c28c5a5fd18157707d9ba273bd85c6620532
--- /dev/null
+++ b/steps/LoSoTo.Abs.cwl
@@ -0,0 +1,52 @@
+#!/usr/bin/env cwl-runner
+
+class: CommandLineTool
+cwlVersion: v1.0
+id: losoto_abs
+
+$namespaces:
+ lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
+doc: |
+ Take absolute value. Needed before smooth if amplitudes are negative!
+ WEIGHT: no need to be weight compliant
+
+requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entryname: 'parset.config'
+ entry: |
+ [abs]
+ soltab = $(inputs.soltab)
+ operation = ABS
+ - entryname: $(inputs.input_h5parm.basename)
+ entry: $(inputs.input_h5parm)
+ writable: true
+
+baseCommand: "losoto"
+
+arguments:
+ - $(inputs.input_h5parm.basename)
+ - parset.config
+
+hints:
+ DockerRequirement:
+ dockerPull: lofareosc/prefactor-ci:master
+
+inputs:
+ - id: input_h5parm
+ type: File
+ format: lofar:#H5Parm
+ - id: soltab
+ type: string
+ doc: "Solution table"
+
+outputs:
+ - id: output_h5parm
+ type: File
+ format: lofar:#H5Parm
+ outputBinding:
+ glob: $(inputs.input_h5parm.basename)
+
+
+$schema:
+ - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
diff --git a/steps/LoSoTo.Duplicate.cwl b/steps/LoSoTo.Duplicate.cwl
new file mode 100644
index 0000000000000000000000000000000000000000..9111602fa6ebde8aaca0b425afeef3c3d1e39d75
--- /dev/null
+++ b/steps/LoSoTo.Duplicate.cwl
@@ -0,0 +1,56 @@
+#!/usr/bin/env cwl-runner
+
+class: CommandLineTool
+cwlVersion: v1.0
+id: losoto_duplicate
+
+doc: |
+ Duplicate a table
+
+requirements:
+ InlineJavascriptRequirement: {}
+ InitialWorkDirRequirement:
+ listing:
+ - entryname: 'parset.config'
+ entry: |
+ [duplicate]
+ soltab = $(inputs.soltab)
+ operation = DUPLICATE
+ $(inputs.soltabOut !== null? 'soltabOut=' + inputs.soltabOut: '')
+
+ - entryname: $(inputs.input_h5parm.basename)
+ entry: $(inputs.input_h5parm)
+ writable: true
+
+baseCommand: "losoto"
+
+arguments:
+ - $(inputs.input_h5parm.basename)
+ - parset.config
+
+hints:
+ DockerRequirement:
+ dockerPull: lofareosc/prefactor-ci:master
+
+inputs:
+ - id: input_h5parm
+ type: File
+ format: lofar:#H5Parm
+ - id: soltab
+ type: string
+ doc: "Solution table"
+ - id: soltabOut
+ type: string?
+ doc: Output table name. By default choose next available from table type.
+outputs:
+ - id: output_h5parm
+ type: File
+ format: lofar:#H5Parm
+ outputBinding:
+ glob: $(inputs.input_h5parm.basename)
+
+
+$namespaces:
+ lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
+$schema:
+ - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
diff --git a/steps/LoSoTo.Norm.cwl b/steps/LoSoTo.Norm.cwl
new file mode 100644
index 0000000000000000000000000000000000000000..f60ee4c469c938e01c52b5d226e275bba4fe2a1f
--- /dev/null
+++ b/steps/LoSoTo.Norm.cwl
@@ -0,0 +1,61 @@
+#!/usr/bin/env cwl-runner
+
+class: CommandLineTool
+cwlVersion: v1.0
+id: losoto_abs
+
+doc: |
+ Normalize the solutions to a given value WEIGHT: Weights compliant
+
+requirements:
+ InlineJavascriptRequirement: {}
+ InitialWorkDirRequirement:
+ listing:
+ - entryname: 'parset.config'
+ entry: |
+ [norm]
+ soltab = $(inputs.soltab)
+ operation = NORM
+ axesToNorm = $(inputs.axesToNorm)
+ $(inputs.normVal !== null? 'normVal=' + inputs.normVal: '')
+
+ - entryname: $(inputs.input_h5parm.basename)
+ entry: $(inputs.input_h5parm)
+ writable: true
+
+baseCommand: "losoto"
+
+arguments:
+ - $(inputs.input_h5parm.basename)
+ - parset.config
+
+hints:
+ DockerRequirement:
+ dockerPull: lofareosc/prefactor-ci:master
+
+inputs:
+ - id: input_h5parm
+ type: File
+ format: lofar:#H5Parm
+ - id: soltab
+ type: string
+ doc: "Solution table"
+ - id: axesToNorm
+ type: string[]
+ doc: Axes along which compute the normalization
+ - id: normVal
+ type: float?
+ doc: Number to normalize to vals = vals * (normVal/valsMean), by default 1.
+
+outputs:
+ - id: output_h5parm
+ type: File
+ format: lofar:#H5Parm
+ outputBinding:
+ glob: $(inputs.input_h5parm.basename)
+
+
+$namespaces:
+ lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
+$schema:
+ - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
diff --git a/steps/LoSoTo.Plot.cwl b/steps/LoSoTo.Plot.cwl
new file mode 100644
index 0000000000000000000000000000000000000000..e5d5cfb054bbda082cf00a311d9df0904ea22f1f
--- /dev/null
+++ b/steps/LoSoTo.Plot.cwl
@@ -0,0 +1,148 @@
+#!/usr/bin/env cwl-runner
+
+class: CommandLineTool
+cwlVersion: v1.0
+id: losoto_plot
+
+doc: |
+ This operation for LoSoTo implements basic plotting WEIGHT:
+ flag-only compliant, no need for weight
+
+requirements:
+ InlineJavascriptRequirement:
+ expressionLib:
+ - |
+ function get_config() {
+ var par = ['soltab = ' + inputs.soltab]
+ if (inputs.ncpu !== null) par.push('ncpu='+inputs.ncpu);
+ par.push('[plot]')
+ par.push('operation = PLOT')
+ for(var field_name in inputs){
+ if(field_name === 'input_h5parm' ||
+ field_name === 'soltab' ||
+ field_name === 'ncpu') continue;
+ if(inputs[field_name] === null ||
+ inputs[field_name] === 'null') continue;
+ par.push(field_name+'='+inputs[field_name])
+ }
+ return par
+ }
+ InitialWorkDirRequirement:
+ listing:
+ - entryname: 'parset.config'
+ entry: $(get_config().join('\n'))
+ - entryname: $(inputs.input_h5parm.basename)
+ entry: $(inputs.input_h5parm)
+ writable: true
+
+
+baseCommand: "losoto"
+
+arguments:
+ - $(inputs.input_h5parm.basename)
+ - parset.config
+
+hints:
+ DockerRequirement:
+ dockerPull: lofareosc/prefactor-ci:master
+
+inputs:
+ - id: input_h5parm
+ type: File
+ format: lofar:#H5Parm
+ - id: soltab
+ type: string[]
+ doc: "Tabs to plot"
+
+ - id: axesInPlot
+ type: string[]?
+ default: []
+ doc: |
+ 1- or 2-element array which says the coordinates to plot (2 for 3D plots).
+ - id: axisInTable
+ type: string?
+ doc: |
+ the axis to plot on a page - e.g. ant to get all antenna’s on one file.
+ - id: axisInCol
+ type: string?
+ doc: |
+ The axis to plot in different colours - e.g. pol to get correlations with
+ different colors.
+ - id: axisDiff
+ type: string?
+ doc: |
+ This must be a len=2 axis and the plot will have the differential value
+ - e.g. ‘pol’ to plot XX-YY.
+ - id: NColFig
+ type: int?
+ doc: |
+ Number of columns in a multi-table image. By default is automatically
+ chosen.
+ - id: figSize
+ type: int[]
+ default: [0,0]
+ doc: |
+ Size of the image [x,y], if one of the values is 0, then it is
+ automatically chosen. By default automatic set.
+ - id: markerSize
+ type: int?
+ default: 2
+ doc: |
+ Size of the markers in the 2D plot. By default 2.
+ - id: minmax
+ type: float[]?
+ doc: |
+ Min max value for the independent variable (0 means automatic).
+ - id: log
+ type: string?
+ doc: |
+ Use Log=’XYZ’ to set which axes to put in Log.
+ - id: plotFlag
+ type: boolean?
+ default: false
+ doc: Whether to plot also flags as red points in 2D plots.
+ - id: doUnwrap
+ type: boolean?
+ default: false
+ doc: Unwrap phases.
+ - id: refAnt
+ type: string?
+ default: ''
+ doc: |
+ Reference antenna for phases. By default None.
+ - id: soltabsToAdd
+ type: string?
+ doc: |
+ Tables to “add†(e.g. ‘sol000/tec000’), it works only for tec and clock
+ to be added to phases.
+ - id: makeAntPlot
+ default: false
+ type: boolean?
+ doc: |
+ Make a plot containing antenna coordinates in x,y and in color the value
+ to plot, axesInPlot must be [ant].
+ - id: makeMovie
+ default: false
+ type: boolean?
+ doc: |
+ Make a movie summing up all the produced plots.
+ - id: prefix
+ type: string?
+ default: 'losoto.plot.'
+ doc: |
+ Prefix to add before the self-generated filename.
+ - id: ncpu
+ type: int?
+ doc: Number of cpus, by default all available.
+
+outputs:
+ - id: output_plots
+ type: File[]
+ outputBinding:
+ glob: "$(inputs.prefix)*.png"
+
+
+$namespaces:
+ lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
+$schema:
+ - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
diff --git a/steps/LoSoTo.Replicateonaxis.cwl b/steps/LoSoTo.Replicateonaxis.cwl
new file mode 100644
index 0000000000000000000000000000000000000000..c676467d19611e57252c17bdfd83b99d8a5fc821
--- /dev/null
+++ b/steps/LoSoTo.Replicateonaxis.cwl
@@ -0,0 +1,77 @@
+#!/usr/bin/env cwl-runner
+
+class: CommandLineTool
+cwlVersion: v1.0
+id: losoto_replicate_on_axis
+
+doc: |
+ Replace the values along a certain axis taking them from one specific axis
+ cell
+
+requirements:
+ InlineJavascriptRequirement: {}
+ InitialWorkDirRequirement:
+ listing:
+ - entryname: "replicate_axis.py"
+ entry: |
+ import sys
+ from losoto.h5parm import openSoltab
+ from losoto.operations import replicateonaxis
+
+ soltab_address = "$(inputs.soltab)"
+ soltab = openSoltab(sys.argv[1], address=soltab_address)
+
+ axisReplicate = "$(inputs.axisReplicate)"
+ fromCell = "$(inputs.fromCell !== null ? inputs.fromCell : 0)"
+ updateWeights = "$(inputs.updateWeights? inputs.updateWeights : 'True')"
+
+ replicateonaxis.run(soltab, axisReplicate=axisReplicate,
+ fromCell=fromCell,
+ updateWeights=updateWeights)
+
+ - entryname: $(inputs.input_h5parm.basename)
+ entry: $(inputs.input_h5parm)
+ writable: true
+
+baseCommand: "python3"
+
+arguments:
+ - replicate_axis.py
+ - $(inputs.input_h5parm.basename)
+
+
+hints:
+ DockerRequirement:
+ dockerPull: lofareosc/prefactor-ci:master
+
+inputs:
+ - id: input_h5parm
+ type: File
+ format: lofar:#H5Parm
+ - id: soltab
+ type: string
+ doc: "Solution table"
+ - id: axisReplicate
+ type: string
+ doc: Axis along which replicate the values.
+ - id: fromCell
+ type: string?
+ doc: A cell value in axisReplicate from which to copy the data values.
+ - id: updateWeights
+ type: boolean?
+ doc: |
+ If False then weights are untouched, if True they are replicated
+ like data. Default: True.
+
+outputs:
+ - id: output_h5parm
+ type: File
+ format: lofar:#H5Parm
+ outputBinding:
+ glob: $(inputs.input_h5parm.basename)
+
+
+$namespaces:
+ lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
+$schema:
+ - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
diff --git a/steps/LoSoTo.Reset.cwl b/steps/LoSoTo.Reset.cwl
new file mode 100644
index 0000000000000000000000000000000000000000..99594d12046ee2d48622dda0e08de72e25c8efcd
--- /dev/null
+++ b/steps/LoSoTo.Reset.cwl
@@ -0,0 +1,57 @@
+#!/usr/bin/env cwl-runner
+
+class: CommandLineTool
+cwlVersion: v1.0
+id: losoto_reset
+
+doc: Subtract/divide two tables or a clock/tec/tec3rd/rm from a phase.
+
+requirements:
+ InlineJavascriptRequirement: {}
+ InitialWorkDirRequirement:
+ listing:
+ - entryname: 'parset.config'
+ entry: |
+ [reset]
+ soltab = $(inputs.soltab)
+ operation = RESET
+ $(inputs.dataVal !== null? 'dataVal=' + inputs.dataVal: '')
+
+ - entryname: $(inputs.input_h5parm.basename)
+ entry: $(inputs.input_h5parm)
+ writable: true
+
+baseCommand: "losoto"
+
+arguments:
+ - $(inputs.input_h5parm.basename)
+ - parset.config
+
+hints:
+ DockerRequirement:
+ dockerPull: lofareosc/prefactor-ci:master
+
+inputs:
+ - id: input_h5parm
+ type: File
+ format: lofar:#H5Parm
+ - id: soltab
+ type: string
+ doc: "Solution table"
+ - id: dataVal
+ type: float?
+ doc: |
+ If given set values to this number, otherwise uses 1 for amplitude and 0
+ for all other soltab types.
+outputs:
+ - id: output_h5parm
+ type: File
+ format: lofar:#H5Parm
+ outputBinding:
+ glob: $(inputs.input_h5parm.basename)
+
+
+$namespaces:
+ lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
+$schema:
+ - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
diff --git a/steps/LoSoTo.Residual.cwl b/steps/LoSoTo.Residual.cwl
new file mode 100644
index 0000000000000000000000000000000000000000..4bc566cd7d878f8532d69cd74aad7a6d4a72bf8b
--- /dev/null
+++ b/steps/LoSoTo.Residual.cwl
@@ -0,0 +1,59 @@
+#!/usr/bin/env cwl-runner
+
+class: CommandLineTool
+cwlVersion: v1.0
+id: losoto_residual
+
+doc: Subtract/divide two tables or a clock/tec/tec3rd/rm from a phase.
+
+requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entryname: 'parset.config'
+ entry: |
+ [residuals]
+ soltab = $(inputs.soltab)
+ operation=RESIDUALS
+ soltabsToSub=$(inputs.soltabsToSub)
+ ratio=$(inputs.ratio)
+
+ - entryname: $(inputs.input_h5parm.basename)
+ entry: $(inputs.input_h5parm)
+ writable: true
+
+baseCommand: "losoto"
+
+arguments:
+ - $(inputs.input_h5parm.basename)
+ - parset.config
+
+hints:
+ DockerRequirement:
+ dockerPull: lofareosc/prefactor-ci:master
+
+inputs:
+ - id: input_h5parm
+ type: File
+ format: lofar:#H5Parm
+ - id: soltab
+ type: string
+ doc: "Solution table"
+ - id: soltabsToSub
+ type: string[]
+ doc: List of soltabs to subtract
+ - id: ratio
+ type: boolean?
+ default: false
+ doc: Return the ratio instead of subtracting.
+
+outputs:
+ - id: output_h5parm
+ type: File
+ format: lofar#H5Parm
+ outputBinding:
+ glob: $(inputs.input_h5parm.basename)
+
+$namespaces:
+ lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
+$schema:
+ - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
diff --git a/steps/RadioDatatypes.yaml b/steps/RadioDatatypes.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..0a1679541e2bf66a9bc447669dfd27a6d94c2ce1
--- /dev/null
+++ b/steps/RadioDatatypes.yaml
@@ -0,0 +1,8 @@
+type: array
+name: MeasurementSet
+label: The measurement set data location
+items:
+ - name: main_directory
+ type: Directory
+ inputBinding:
+ position: 0
diff --git a/steps/blsmooth.cwl b/steps/blsmooth.cwl
new file mode 100644
index 0000000000000000000000000000000000000000..d139c594c6f1738109a4048f0abe7691c998d079
--- /dev/null
+++ b/steps/blsmooth.cwl
@@ -0,0 +1,91 @@
+class: CommandLineTool
+cwlVersion: v1.0
+id: blsmooth
+label: BLsmooth
+baseCommand:
+ - BLsmooth.py
+inputs:
+ - id: msin
+ type: Directory
+ inputBinding:
+ position: 1
+ doc: Input measurement set
+ - default: 0.2
+ id: ionfactor
+ type: float
+ inputBinding:
+ position: 0
+ prefix: '-f'
+ doc: Gives an indication on how strong is the ionosphere
+ - id: do_smooth
+ type: boolean
+ doc: 'If true performs smoothing'
+ inputBinding:
+ prefix: '--smooth=SMOOTH'
+
+ - default: 0.5
+ id: bscalefactor
+ type: float
+ inputBinding:
+ position: 0
+ prefix: '-s'
+ doc: Gives an indication on how the smoothing varies with
+ - default: DATA
+ id: in_column_name
+ type: string
+ inputBinding:
+ position: 0
+ prefix: '-i'
+ doc: Column name to smooth
+ - default: SMOOTHED_DATA
+ id: out_column
+ type: string
+ inputBinding:
+ position: 0
+ prefix: '-o'
+ doc: Output column
+ - default: false
+ id: weight
+ type: boolean
+ inputBinding:
+ position: 0
+ prefix: '-w'
+ doc: >-
+ Save the newly computed WEIGHT_SPECTRUM, this action permanently modify
+ the MS!
+ - default: false
+ id: restore
+ type: boolean
+ inputBinding:
+ position: 0
+ prefix: '-r'
+ doc: If WEIGHT_SPECTRUM_ORIG exists then restore it before smoothing
+ - default: false
+ id: nobackup
+ type: boolean
+ inputBinding:
+ position: 0
+ prefix: '-b'
+ doc: Do not backup the old WEIGHT_SPECTRUM in WEIGHT_SPECTRUM_ORIG
+ - default: false
+ id: onlyamp
+ type: boolean
+ inputBinding:
+ position: 0
+ prefix: '-a'
+ doc: Smooth only amplitudes
+outputs:
+ - id: msout
+ doc: MS set output
+ type: Directory
+ outputBinding:
+ glob: $(inputs.msin.basename)
+hints:
+ - class: DockerRequirement
+ dockerPull: lofareosc/prefactor-ci:master
+requirements:
+ - class: InitialWorkDirRequirement
+ listing:
+ - entry: $(inputs.msin)
+ writable: true
+ - class: InlineJavascriptRequirement
diff --git a/steps/check_ateam_separation.cwl b/steps/check_ateam_separation.cwl
new file mode 100755
index 0000000000000000000000000000000000000000..977d9f86e63f3b003833aee677d330f1f188253d
--- /dev/null
+++ b/steps/check_ateam_separation.cwl
@@ -0,0 +1,44 @@
+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+ sbg: 'https://www.sevenbridges.com/'
+id: check_ateam_separation
+baseCommand:
+ - python3
+ - /usr/local/bin/check_Ateam_separation.py
+inputs:
+ - id: ms
+ type:
+ - Directory
+ - type: array
+ items: Directory
+ inputBinding:
+ position: 0
+ doc: Input measurement set
+ - default: Ateam_separation.png
+ id: output_image_name
+ type: string?
+ inputBinding:
+ position: 2
+ prefix: '--outputimage'
+ - id: min_separation
+ type: int
+ inputBinding:
+ position: 1
+ prefix: '--min_separation'
+outputs:
+ - id: output_imag
+ doc: Output image
+ type: File?
+ outputBinding:
+ glob: $(inputs.output_image_name)
+ - id: logfile
+ type: File?
+ outputBinding:
+ glob: Ateam_separation.log
+label: check_Ateam_separation
+hints:
+ - class: DockerRequirement
+ dockerPull: lofareosc/prefactor
+ - class: InlineJavascriptRequirement
+stdout: Ateam_separation.log
diff --git a/steps/ddecal.cwl b/steps/ddecal.cwl
new file mode 100644
index 0000000000000000000000000000000000000000..c534d5362f563941d9503b980ae0dd6a4b766acf
--- /dev/null
+++ b/steps/ddecal.cwl
@@ -0,0 +1,111 @@
+#!/usr/bin/env cwl-runner
+
+class: CommandLineTool
+cwlVersion: v1.0
+id: ddecal
+baseCommand: [DPPP]
+
+requirements:
+ InlineJavascriptRequirement: {}
+
+hints:
+ DockerRequirement:
+ dockerPull: lofareosc/prefactor-ci:master
+
+arguments:
+ - steps=[ddecal]
+ - ddecal.mode=rotation+diagonal
+ - ddecal.uvlambdamin=300
+ - ddecal.maxiter=50
+ - ddecal.nchan=1
+ - ddecal.solint=1
+ - ddecal.propagateconvergedonly=True
+ - ddecal.flagdivergedonly=True
+ - ddecal.tolerance=1.e-3
+ - ddecal.usemodelcolumn=True
+
+
+inputs:
+ - id: msin
+ type: Directory?
+ doc: Input Measurement Set
+ inputBinding:
+ prefix: msin=
+ separate: false
+
+ - id: msin_datacolumn
+ type: string
+ default: DATA
+ doc: Input data Column
+ inputBinding:
+ prefix: msin.datacolumn=
+ separate: false
+
+ - id: msin_modelcolum
+ type: string
+ default: MODEL_DATA
+ doc: Model data Column
+ inputBinding:
+ prefix: msin.modelcolumn=
+ separate: false
+
+ - id: output_name_h5parm
+ type: string
+ default: instrument.h5
+ inputBinding:
+ prefix: ddecal.h5parm=
+ separate: false
+ - id: msout_name
+ type: string
+ doc: Output Measurement Set
+ default: out.MS
+ inputBinding:
+ prefix: msout=
+ separate: false
+
+#--------------------
+ - id: propagate_solutions
+ type: boolean
+ default: true
+ inputBinding:
+ prefix: propagatesolutions=True
+ - id: flagunconverged
+ type: boolean
+ default: false
+ doc: |
+ Flag unconverged solutions (i.e., those from solves that did not converge
+ within maxiter iterations).
+ inputBinding:
+ prefix: flagdivergedonly=True
+ - id: flagdivergedonly
+ default: false
+ type: boolean
+ doc: |
+ Flag only the unconverged solutions for which divergence was detected.
+ At the moment, this option is effective only for rotation+diagonal
+ solves, where divergence is detected when the amplitudes of any station
+ are found to be more than a factor of 5 from the mean amplitude over all
+ stations.
+ If divergence for any one station is detected, all stations are flagged
+ for that solution interval. Only effective when flagunconverged=true
+ and mode=rotation+diagonal.
+ inputBinding:
+ prefix: flagdivergedonly=True
+outputs:
+ - id: msout
+ doc: Output Measurement Set
+ type: Directory
+ outputBinding:
+ glob: $(inputs.msout_name=="."?inputs.msin:inputs.msout_name)
+
+ - id: h5parm
+ doc: Filename of output H5Parm (to be read by e.g. losoto)
+ type: File
+ format: lofar:#H5Parm
+ outputBinding:
+ glob: $(inputs.output_name_h5parm)
+
+$namespaces:
+ lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
+$schema:
+ - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
diff --git a/steps/find_skymodel_cal.cwl b/steps/find_skymodel_cal.cwl
new file mode 100644
index 0000000000000000000000000000000000000000..51a1dc4a8f7500fccac5cef763cd15d50ecaa505
--- /dev/null
+++ b/steps/find_skymodel_cal.cwl
@@ -0,0 +1,59 @@
+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+ sbg: 'https://www.sevenbridges.com/'
+id: find_skymodel_cal_py
+baseCommand:
+ - python3
+inputs:
+ - id: msin
+ type: Directory
+ doc: MS containing the calibrator
+ inputBinding:
+ position: 0
+ - id: skymodels
+ type: Directory
+ doc: Directory containing the sky models
+ - id: skymodels_extension
+ type: string?
+ doc: path extension of the sky models
+
+label: find_skymodel_cal.py
+arguments:
+ - '-c'
+ - |
+ import sys
+
+ from unittest.mock import MagicMock
+ sys.modules['lofarpipe.support.data_map'] = MagicMock()
+
+
+ from find_skymodel_cal import main as find_skymodel
+ import json
+ mss = sys.argv[1:]
+ skymodels = "$(inputs.skymodels.path)"
+
+ extension = "$(inputs.skymodels_extension)"
+ output = {}
+ if extension != 'null':
+ output = find_skymodel(mss, skymodels, './', extension)
+ else:
+ output = find_skymodel(mss, skymodels, './')
+
+ cwl_output = {'class': 'File', 'path': output['SkymodelCal']}
+ with open('./out.json', 'w') as fp:
+ json.dump(cwl_output, fp)
+outputs:
+ - id: output_models
+ type: File
+ outputBinding:
+ loadContents: true
+ glob: 'out.json'
+ outputEval: $(JSON.parse(self[0].contents))
+
+requirements:
+ - class: InlineJavascriptRequirement
+
+hints:
+ DockerRequirement:
+ dockerPull: lofareosc/prefactor-ci:master
diff --git a/steps/interpolate_visibilities.cwl b/steps/interpolate_visibilities.cwl
new file mode 100644
index 0000000000000000000000000000000000000000..90772b4f072cdecd9f03fd5f1d6258b66b995d11
--- /dev/null
+++ b/steps/interpolate_visibilities.cwl
@@ -0,0 +1,52 @@
+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+ sbg: 'https://www.sevenbridges.com/'
+id: interpolate_visibilities
+
+baseCommand:
+ - DPPP
+arguments:
+ - msout.datacolumn=INTERP_DATA
+ - msout=.
+ - msin.datacolumn=DATA
+ - steps=[interpolate]
+inputs:
+ - id: msin
+ type: Directory
+ inputBinding:
+ prefix: 'msin='
+ separate: false
+ - id: storage_manager
+ type: string
+ default: Dysco
+ inputBinding:
+ separate: false
+ prefix: 'msout.storagemanager='
+ - id: storage_manager_databitrate
+ type: int
+ default: 0
+ inputBinding:
+ prefix: 'msout.storagemanager.databitrate='
+ separate: false
+ - id: window_size
+ type: int
+ inputBinding:
+ prefix: 'interpolate.windowsize='
+ separate: false
+
+outputs:
+ - id: msout
+ type: Directory
+ outputBinding:
+ glob: $(inputs.msin.basename)
+label: interpolate_visibilities
+hints:
+ - class: DockerRequirement
+ dockerPull: lofareosc/prefactor-ci:master
+
+requirements:
+ - class: InitialWorkDirRequirement
+ listing:
+ - entry: $(inputs.msin)
+ writable: true
diff --git a/steps/makesourcedb.cwl b/steps/makesourcedb.cwl
new file mode 100755
index 0000000000000000000000000000000000000000..eb5c2e7c7b8f09c0a2c97ea4512bb241cff17fb5
--- /dev/null
+++ b/steps/makesourcedb.cwl
@@ -0,0 +1,56 @@
+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+ sbg: 'https://www.sevenbridges.com/'
+id: make_sourcedb
+baseCommand:
+ - makesourcedb
+inputs:
+ - id: sky_model
+ type:
+ - File
+ - string
+ inputBinding:
+ position: 0
+ prefix: in=
+ separate: false
+ - default: Ateam.sourcedb
+ id: output_file_name
+ type: string?
+ inputBinding:
+ position: 1
+ prefix: out=
+ separate: false
+ valueFrom: Ateam.sourcedb
+ - default: blob
+ id: outtype
+ type: string?
+ inputBinding:
+ position: 2
+ prefix: outtype=
+ separate: false
+ - default: <
+ id: format
+ type: string?
+ inputBinding:
+ position: 3
+ prefix: format=
+ separate: false
+outputs:
+ - id: sourcedb
+ type:
+ - Directory
+ - File
+ outputBinding:
+ glob: $(inputs.output_file_name)
+ - id: log
+ type: File?
+ outputBinding:
+ glob: make_sourcedb_ateam.log
+label: make_sourcedb_ateam
+hints:
+ - class: DockerRequirement
+ dockerPull: 'lofareosc/prefactor:latest'
+stdout: make_sourcedb_ateam.log
+requirements:
+ - class: InlineJavascriptRequirement
diff --git a/steps/ms_concat.cwl b/steps/ms_concat.cwl
new file mode 100644
index 0000000000000000000000000000000000000000..d9038e8df9506c24c6093b7ed9c4b27fb2a88196
--- /dev/null
+++ b/steps/ms_concat.cwl
@@ -0,0 +1,59 @@
+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+ sbg: 'https://www.sevenbridges.com/'
+id: ms_concat
+
+
+baseCommand:
+ - concat_MS.py
+inputs:
+ - id: min_length
+ type: int?
+ default: 50
+ inputBinding:
+ position: 0
+ prefix: '--min_length'
+ doc: Minimum amount of subbands to concatenate in frequency.
+ - id: overhead
+ type: float?
+ inputBinding:
+ position: 0
+ prefix: '--overhead'
+ doc: |
+ Only use this fraction of the available memory for deriving the amount
+ of data to be concatenated.
+ - id: msin
+ type: 'Directory[]'
+ inputBinding:
+ position: 1
+ shellQuote: false
+ - default: out.MS
+ id: msout
+ type: string
+ inputBinding:
+ position: 2
+outputs:
+ - id: concat_meta_ms
+ type: Directory[]
+ outputBinding:
+ glob: >
+ $(inputs.msout)_[0-9*]
+ - id: concat_additional_ms
+ type: Directory[]
+ outputBinding:
+ glob: $(inputs.msout)_*_CONCAT
+ - id: ms_outs
+ type: Directory[]
+ outputBinding:
+ outputEval: $(inputs.msin)
+label: ms_concat
+requirements:
+ - class: ShellCommandRequirement
+ - class: DockerRequirement
+ dockerPull: lofareosc/prefactor-ci:master
+ - class: InlineJavascriptRequirement
+ - class: InitialWorkDirRequirement
+ listing:
+ - entry: $(inputs.msin)
+ writable: true
diff --git a/steps/msinfo.cwl b/steps/msinfo.cwl
new file mode 100644
index 0000000000000000000000000000000000000000..0c7e2f5b779360baef059c0fe61f753a428d8707
--- /dev/null
+++ b/steps/msinfo.cwl
@@ -0,0 +1,40 @@
+#!/usr/bin/env cwl-runner
+
+class: CommandLineTool
+cwlVersion: v1.1
+id: msinfo
+
+requirements:
+ - class: InlineJavascriptRequirement
+ - class: InitialWorkDirRequirement
+ listing:
+ - entryname: msinfo.py
+ entry: |
+ import sys
+ print(sys.argv)
+ - class: SchemaDefRequirement
+ types:
+ - $import: RadioDatatypes.yaml
+hints:
+ DockerRequirement:
+ dockerPull: lofareosc/prefactor-ci:master
+
+baseCommand:
+ - python3
+ - msinfo.py
+
+inputs:
+ data:
+ type: RadioDatatypes.yaml#MeasurementSet
+ inputBinding:
+ position: 0
+
+stdout: std.output
+
+outputs:
+ info:
+ type: File
+ outputBinding:
+ glob: std.output
+label: msinfo
+
diff --git a/steps/predict.cwl b/steps/predict.cwl
new file mode 100644
index 0000000000000000000000000000000000000000..eeb14aef083ee2e7ec43056e68fd018c47023e50
--- /dev/null
+++ b/steps/predict.cwl
@@ -0,0 +1,71 @@
+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+ sbg: 'https://www.sevenbridges.com/'
+id: predict
+baseCommand:
+ - DPPP
+inputs:
+ - id: msin
+ type: Directory
+ inputBinding:
+ position: 0
+ prefix: msin=
+ separate: false
+ doc: Input Measurement Set
+ - default: DATA
+ id: msin_datacolumn
+ type: string
+ inputBinding:
+ position: 0
+ prefix: msin.datacolumn=
+ separate: false
+ doc: Input data Column
+ - default: MODEL_DATA
+ id: msout_datacolumn
+ type: string
+ inputBinding:
+ position: 0
+ prefix: msout.datacolumn=
+ separate: false
+ - id: sources_db
+ type: File
+ inputBinding:
+ position: 0
+ prefix: predict.sourcedb=
+ separate: false
+ - default: null
+ id: sources
+ type: 'string[]'
+ inputBinding:
+ position: 0
+ prefix: predict.sources=
+ separate: false
+ itemSeparator: ','
+ valueFrom: '[$(self)]'
+ - default: false
+ id: usebeammodel
+ type: boolean
+ inputBinding:
+ position: 0
+ prefix: predict.usebeammodel=True
+outputs:
+ - id: msout
+ doc: Output Measurement Set
+ type: Directory
+ outputBinding:
+ glob: $(inputs.msin.basename)
+arguments:
+ - 'steps=[predict]'
+ - predict.beammode=array_factor
+ - predict.usechannelfreq=False
+ - msout=.
+requirements:
+ - class: InitialWorkDirRequirement
+ listing:
+ - entry: $(inputs.msin)
+ writable: true
+ - class: InlineJavascriptRequirement
+hints:
+ - class: DockerRequirement
+ dockerPull: lofareosc/prefactor-ci:master
diff --git a/steps/wsclean.cwl b/steps/wsclean.cwl
new file mode 100644
index 0000000000000000000000000000000000000000..2bb63f2066cf8237103211ced00a92b2c5c33170
--- /dev/null
+++ b/steps/wsclean.cwl
@@ -0,0 +1,42 @@
+class: CommandLineTool
+cwlVersion: v1.0
+id: wsclean
+baseCommand:
+ - wsclean
+inputs:
+ - id: image_size
+ type: int[]
+ default: [1000, 1000]
+ inputBinding:
+ position: 1
+ prefix: -size
+ - id: image_scale
+ type: float
+ default: 0.01
+ inputBinding:
+ position: 1
+ prefix: -scale
+ valueFrom: $(self)deg
+ - id: image_name
+ default: image
+ type: string
+ inputBinding:
+ position: 1
+ prefix: -name
+ - id: msin
+ type: Directory
+ inputBinding:
+ position: 2
+outputs:
+ dirty_image:
+ type: File
+ outputBinding:
+ glob: $(inputs.image_name)-dirty.fits
+ image:
+ type: File
+ outputBinding:
+ glob: $(inputs.image_name)-image.fits
+label: WSClean
+requirements:
+ - class: DockerRequirement
+ dockerPull: lofareosc/prefactor-ci:master
diff --git a/test_data/A-Team_lowres.sourcedb b/test_data/A-Team_lowres.sourcedb
new file mode 100644
index 0000000000000000000000000000000000000000..b4e89906093cc01438b6a7b81f3ea9bfb06ba520
Binary files /dev/null and b/test_data/A-Team_lowres.sourcedb differ
diff --git a/test_data/L570745_uv_first10.MS.tar.xz.REMOVED.git-id b/test_data/L570745_uv_first10.MS.tar.xz.REMOVED.git-id
new file mode 100644
index 0000000000000000000000000000000000000000..2266c40d2de8785cb1e771f88823e6fb2f02be07
--- /dev/null
+++ b/test_data/L570745_uv_first10.MS.tar.xz.REMOVED.git-id
@@ -0,0 +1 @@
+fac322047acad1a9cf1425d64a08d7bc3548f462
\ No newline at end of file
diff --git a/test_data/example.h5.REMOVED.git-id b/test_data/example.h5.REMOVED.git-id
new file mode 100644
index 0000000000000000000000000000000000000000..ee55446f16fed96380113889ea423eb37e2bdbd0
--- /dev/null
+++ b/test_data/example.h5.REMOVED.git-id
@@ -0,0 +1 @@
+554933b577a0dcf1b2aa3fa91c7de5b2477fd724
\ No newline at end of file
diff --git a/test_data/skymodels/3C196-pandey.skymodel b/test_data/skymodels/3C196-pandey.skymodel
new file mode 100644
index 0000000000000000000000000000000000000000..47424b9f86b2b6e06d789f5a54d15a787dffca7c
--- /dev/null
+++ b/test_data/skymodels/3C196-pandey.skymodel
@@ -0,0 +1,11 @@
+# (Name, Type, Patch, Ra, Dec, I, Q, U, V, ReferenceFrequency='150e6', SpectralIndex='[-0.76, -0.00]', MajorAxis, MinorAxis, Orientation) = format
+
+, , 3c196_4c, 08:13:36.0, 48.13.0
+3C196A1, GAUSSIAN, 3c196_4c, 08:13:35.925, +48.13.00.061, 33.5514, 0.0, 0.0, 0.0, 150e6, [-0.572, -0.042], 1.476, 1.091, 135.27
+3C196A2, GAUSSIAN, 3c196_4c, 08:13:35.772, +48.13.02.507, 12.3129, 0.0, 0.0, 0.0, 150e6, [-0.973, -0.319], 2.738, 1.754, 93.30
+3C196B1, GAUSSIAN, 3c196_4c, 08:13:36.182, +48.13.04.725, 14.6030, 0.0, 0.0, 0.0, 150e6, [-0.557, -0.270], 1.034, 0.248, 124.03
+3C196B2, GAUSSIAN, 3c196_4c, 08:13:36.389, +48.13.02.735, 22.6653, 0.0, 0.0, 0.0, 150e6, [-0.840, -0.288], 1.456, 2.159, 324.58
+# Pandey: 31July2014 converted for the normal bbs updated on April 17th from pandey_March26_2014. DO NOT DELETE THIS LINE
+# Tag:fitted version 2: 17-Apr-2014: pandey@astron.nl using L192832_1CH12S
+# Modified to include a patch name on 24.11.2015 by A. Horneffer
+
diff --git a/test_data/skymodels/3c147-SH.skymodel b/test_data/skymodels/3c147-SH.skymodel
new file mode 100644
index 0000000000000000000000000000000000000000..d8e7cd7cab357223c68320e5662228f2e331af40
--- /dev/null
+++ b/test_data/skymodels/3c147-SH.skymodel
@@ -0,0 +1,8 @@
+# (Name, Type, Patch, Ra, Dec, I, Q, U, V, ReferenceFrequency='74e6', SpectralIndex='[]', MajorAxis, MinorAxis, Orientation) = format
+
+# The above line defines the field order and is required.
+
+# Scaife and Heald Calibrator model.
+, , 3c147, 05:42:36.1, 49.51.07
+3c147_1, POINT, 3c147, 05:42:36.1, 49.51.07, 66.738, 0.0, 0.0, 0.0, 150.e6, [-0.022, -1.012, 0.549]
+
diff --git a/test_data/skymodels/3c286-SH.skymodel b/test_data/skymodels/3c286-SH.skymodel
new file mode 100644
index 0000000000000000000000000000000000000000..817c01a72a7e141579d04a7c87d794abca143091
--- /dev/null
+++ b/test_data/skymodels/3c286-SH.skymodel
@@ -0,0 +1,8 @@
+# (Name, Type, Patch, Ra, Dec, I, Q, U, V, ReferenceFrequency='74e6', SpectralIndex='[]', MajorAxis, MinorAxis, Orientation) = format
+
+# The above line defines the field order and is required.
+
+# Scaife and Heald Calibrator model.
+, , 3c286, 13:31:08.3, 30.30.33
+3c286_1, POINT, 3c286, 13:31:08.3, 30.30.33, 27.477, 0.0, 0.0, 0.0, 150.e6, [-0.158, 0.032, -0.180]
+
diff --git a/test_data/skymodels/3c287-SH.skymodel b/test_data/skymodels/3c287-SH.skymodel
new file mode 100644
index 0000000000000000000000000000000000000000..692009b074139d50389774f206b435eed5bb8307
--- /dev/null
+++ b/test_data/skymodels/3c287-SH.skymodel
@@ -0,0 +1,8 @@
+# (Name, Type, Patch, Ra, Dec, I, Q, U, V, ReferenceFrequency='74e6', SpectralIndex='[]', MajorAxis, MinorAxis, Orientation) = format
+
+# The above line defines the field order and is required.
+
+# Scaife and Heald Calibrator model.
+, , 3c287, 13:30:37.7, 25.09.11
+3c287_1, POINT, 3c287, 13:30:37.7, 25.09.11, 16.367, 0.0, 0.0, 0.0, 150.e6, [-0.364]
+
diff --git a/test_data/skymodels/3c295-twocomp.skymodel b/test_data/skymodels/3c295-twocomp.skymodel
new file mode 100644
index 0000000000000000000000000000000000000000..957cadd564d9a2e8ea1010d5e2a35c5d8d33cb4b
--- /dev/null
+++ b/test_data/skymodels/3c295-twocomp.skymodel
@@ -0,0 +1,5 @@
+# (Name, Type, Patch, Ra, Dec, I, Q, U, V, ReferenceFrequency='150.e6', SpectralIndex='[]', MajorAxis, MinorAxis, Orientation) = format
+
+ , , 3c295_2c, 14:11:20.64, +52.12.09.30
+3c295A, POINT, 3c295_2c, 14:11:20.49, +52.12.10.70, 48.8815, 0.0, 0.0, 0.0, 150.e6, [-0.582, -0.298, 0.583, -0.363]
+3c295B, POINT, 3c295_2c, 14:11:20.79, +52.12.07.90, 48.8815, 0.0, 0.0, 0.0, 150.e6, [-0.582, -0.298, 0.583, -0.363]
diff --git a/test_data/skymodels/3c380-SH.skymodel b/test_data/skymodels/3c380-SH.skymodel
new file mode 100644
index 0000000000000000000000000000000000000000..5a911080bbac0d42bf8a66afc670085a910cdd57
--- /dev/null
+++ b/test_data/skymodels/3c380-SH.skymodel
@@ -0,0 +1,8 @@
+# (Name, Type, Patch, Ra, Dec, I, Q, U, V, ReferenceFrequency='74e6', SpectralIndex='[]', MajorAxis, MinorAxis, Orientation) = format
+
+# The above line defines the field order and is required.
+
+# Scaife and Heald Calibrator model.
+, , 3c380, 18:29:31.8, 48.44.46
+3c380_1, POINT, 3c380, 18:29:31.8, 48.44.46, 77.352, 0.0, 0.0, 0.0, 150.e6, [-0.767]
+
diff --git a/test_data/skymodels/3c48-SH.skymodel b/test_data/skymodels/3c48-SH.skymodel
new file mode 100644
index 0000000000000000000000000000000000000000..e2c3f50da26c04a646fedde67ce861ead3e68133
--- /dev/null
+++ b/test_data/skymodels/3c48-SH.skymodel
@@ -0,0 +1,7 @@
+# (Name, Type, Patch, Ra, Dec, I, Q, U, V, ReferenceFrequency='74e6', SpectralIndex='[]', MajorAxis, MinorAxis, Orientation) = format
+
+# The above line defines the field order and is required.
+
+# Scaife and Heald Calibrator models.
+, , 3c48, 01:37:41.3, 33.09.35
+3c48_1, POINT, 3c48, 01:37:41.3, 33.09.35, 64.768, 0.0, 0.0, 0.0, 150.e6, [-0.387, -0.420, 0.181]
diff --git a/test_data/skymodels/A-Team_4_Highres.skymodel.REMOVED.git-id b/test_data/skymodels/A-Team_4_Highres.skymodel.REMOVED.git-id
new file mode 100644
index 0000000000000000000000000000000000000000..3a77caad0a210449d18cd8cf9e7f9ff5094f0bd9
--- /dev/null
+++ b/test_data/skymodels/A-Team_4_Highres.skymodel.REMOVED.git-id
@@ -0,0 +1 @@
+92830adec7ead19c19340071ea59a3131bd82212
\ No newline at end of file
diff --git a/test_data/skymodels/A-Team_lowres.skymodel b/test_data/skymodels/A-Team_lowres.skymodel
new file mode 100644
index 0000000000000000000000000000000000000000..cbb1296b7520bbbfede15403b33011cdfbad5e80
--- /dev/null
+++ b/test_data/skymodels/A-Team_lowres.skymodel
@@ -0,0 +1,78 @@
+# (Name, Type, Patch, Ra, Dec, I, Q, U, V, ReferenceFrequency='74e6', SpectralIndex='[]', MajorAxis, MinorAxis, Orientation) = format
+
+# The above line defines the field order and is required.
+
+# models for the A-Team members
+
+, , CygA, 19:59:26, +40.44.00
+CygA_4_2, POINT, CygA, 19:59:30.433, +40.43.56.221, 4.827e+03, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 7.63889e-03, 6.94444e-03, 1.00637e+02
+CygA_4_1, GAUSSIAN, CygA, 19:59:28.476, +40.44.01.804, 4.165e+03, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.20516e-02, 5.34011e-03, 1.03230e+02
+CygA_4_3, POINT, CygA, 19:59:22.168, +40.44.28.591, 3.896e+03, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 7.63889e-03, 6.94444e-03, 9.21061e+01
+CygA_4_4, POINT, CygA, 19:59:24.413, +40.44.18.617, 2.798e+03, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 7.63889e-03, 6.94444e-03, 6.74922e+01
+CygA_4_5, POINT, CygA, 19:59:25.555, +40.43.52.813, 1.560e+02, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 7.63889e-03, 6.94444e-03, 0.00000e+00
+
+, , CasA, 23:23:28, +58.48.42
+CasA_4_3, GAUSSIAN, CasA,23:23:21.044, +58.49.42.000, 4.193e+03, 0.0, 0.0, 0.0, 7.37817e+07, [-0.8], 4.32664e-02, 1.79281e-02, 2.73207e+01
+CasA_4_2, GAUSSIAN, CasA,23:23:33.918, +58.50.13.374, 4.131e+03, 0.0, 0.0, 0.0, 7.37817e+07, [-0.8], 4.25963e-02, 1.87784e-02, 1.12516e+02
+CasA_4_8, GAUSSIAN, CasA,23:23:34.032, +58.47.29.883, 2.536e+03, 0.0, 0.0, 0.0, 7.37817e+07, [-0.8], 2.77739e-02, 1.42664e-02, 6.26062e+01
+CasA_4_5, GAUSSIAN, CasA,23:23:15.292, +58.50.00.221, 2.316e+03, 0.0, 0.0, 0.0, 7.37817e+07, [-0.8], 3.15448e-02, 9.02869e-03, 4.11857e+01
+CasA_4_1, GAUSSIAN, CasA,23:23:39.632, +58.48.32.671, 2.245e+03, 0.0, 0.0, 0.0, 7.37817e+07, [-0.8], 2.25927e-02, 1.70874e-02, 8.48056e+01
+CasA_4_7, GAUSSIAN, CasA,23:23:24.110, +58.47.43.607, 2.201e+03, 0.0, 0.0, 0.0, 7.37817e+07, [-0.8], 4.64283e-02, 1.49372e-02, 2.12611e+01
+CasA_4_4, GAUSSIAN, CasA,23:23:14.939, +58.48.09.511, 2.099e+03, 0.0, 0.0, 0.0, 7.37817e+07, [-0.8], 3.09146e-02, 1.42990e-02, 1.61549e+02
+CasA_4_9, GAUSSIAN, CasA,23:23:10.817, +58.48.46.409, 9.790e+02, 0.0, 0.0, 0.0, 7.37817e+07, [-0.8], 9.62058e-03, 7.79284e-03, 1.77482e+02
+CasA_4_6, GAUSSIAN, CasA,23:23:26.327, +58.51.14.147, 2.444e+02, 0.0, 0.0, 0.0, 7.37817e+07, [-0.8], 8.75852e-03, 6.94444e-03, 8.12790e+01
+
+, , TauA, 05:34:32, +21.59.55
+TauA_4_1, GAUSSIAN, TauA, 05:34:32.269, +21.59.47.597, 1.911e+02, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 3.94949e-02, 1.31559e-02, 1.65623e+02
+TauA_4_11, GAUSSIAN, TauA, 05:34:28.639, +22.00.53.809, 1.344e+02, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.82500e-02, 1.22959e-02, 2.49193e+01
+TauA_4_3, GAUSSIAN, TauA, 05:34:25.671, +22.01.26.361, 1.247e+02, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.76133e-02, 1.13962e-02, 5.60626e+01
+TauA_4_9, GAUSSIAN, TauA, 05:34:23.420, +22.00.12.563, 1.096e+02, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.82128e-02, 1.71142e-02, 3.38306e+01
+TauA_4_15, GAUSSIAN, TauA, 05:34:22.884, +22.01.56.603, 9.243e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.99931e-02, 1.65627e-02, 1.76338e+02
+TauA_4_20, GAUSSIAN, TauA, 05:34:29.086, +21.59.21.722, 8.792e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 3.23261e-02, 9.03648e-03, 1.09196e+02
+TauA_4_10, GAUSSIAN, TauA, 05:34:37.530, +21.59.06.719, 8.641e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.13390e-02, 1.33726e-02, 1.27644e+02
+TauA_4_2, POINT, TauA, 05:34:30.550, +21.59.58.106, 8.276e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 7.63889e-03, 6.94444e-03, 0.00000e+00
+TauA_4_5, GAUSSIAN, TauA, 05:34:40.748, +21.59.31.921, 7.008e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.59971e-02, 1.25206e-02, 1.51745e+02
+TauA_4_14, GAUSSIAN, TauA, 05:34:30.570, +21.58.33.304, 6.187e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.87859e-02, 1.14519e-02, 8.42720e+01
+TauA_4_7, GAUSSIAN, TauA, 05:34:36.888, +22.01.01.174, 5.537e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.93364e-02, 1.21462e-02, 1.39794e+02
+TauA_4_8, GAUSSIAN, TauA, 05:34:35.324, +22.00.10.883, 5.525e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.81832e-02, 1.35652e-02, 1.06533e+02
+TauA_4_4, GAUSSIAN, TauA, 05:34:23.931, +21.58.55.252, 5.076e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.77800e-02, 1.52796e-02, 7.50003e+01
+TauA_4_13, GAUSSIAN, TauA, 05:34:36.235, +21.58.03.089, 4.862e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 3.59649e-02, 9.18833e-03, 8.58303e+01
+TauA_4_6, GAUSSIAN, TauA, 05:34:27.470, +21.58.50.002, 4.223e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.54141e-02, 7.87103e-03, 5.52923e+01
+TauA_4_21, GAUSSIAN, TauA, 05:34:34.585, +22.00.45.580, 4.175e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 3.19587e-02, 6.70123e-03, 1.72839e+02
+TauA_4_19, GAUSSIAN, TauA, 05:34:19.534, +22.01.07.470, 3.906e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.20399e-02, 1.14898e-02, 1.76500e+02
+TauA_4_17, GAUSSIAN, TauA, 05:34:29.426, +21.59.51.003, 2.848e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.10631e-02, 5.13228e-03, 1.27622e+02
+TauA_4_23, GAUSSIAN, TauA, 05:34:26.246, +22.02.05.085, 2.699e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.28687e-02, 1.05568e-02, 1.44302e+02
+TauA_4_16, GAUSSIAN, TauA, 05:34:32.269, +22.01.21.318, 2.557e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.16784e-02, 7.66067e-03, 1.43877e+02
+TauA_4_12, GAUSSIAN, TauA, 05:34:22.991, +22.00.33.000, 2.127e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.49607e-02, 8.16407e-03, 1.16642e+02
+TauA_4_18, GAUSSIAN, TauA, 05:34:26.292, +22.02.39.501, 1.727e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.19757e-02, 6.32082e-03, 1.08185e+02
+TauA_4_22, GAUSSIAN, TauA, 05:34:37.023, +21.58.27.987, 1.357e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 8.54110e-03, 5.01393e-03, 7.39302e+01
+TauA_4_24, GAUSSIAN, TauA, 05:34:31.108, +22.00.32.726, 1.106e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.59017e-02, 6.94444e-03, 5.84987e+01
+TauA_4_26, GAUSSIAN, TauA, 05:32:23.875, +21.50.58.488, 1.780e+00, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.05524e-02, 3.94647e-03, 6.84955e+01
+TauA_4_25, POINT, TauA, 05:34:30.647, +21.59.16.928, 9.776e-01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 7.63889e-03, 6.94444e-03, 2.05933e+01
+
+, , VirA, 12:30:35, +12.19.45
+VirA_4_17, GAUSSIAN, VirA, 12:30:34.383, +12.20.04.012, 3.219e+02, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 8.42972e-02, 6.11535e-02, 6.50364e+01
+VirA_4_2, GAUSSIAN, VirA, 12:30:47.034, +12.23.43.975, 2.591e+02, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 8.54400e-03, 4.07169e-03, 1.06884e+02
+VirA_4_1, POINT, VirA, 12:30:48.961, +12.23.33.770, 1.516e+02, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 7.63889e-03, 6.94444e-03, 1.70815e+02
+VirA_4_19, GAUSSIAN, VirA, 12:30:41.047, +12.17.36.983, 1.094e+02, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 4.63137e-02, 2.79725e-02, 8.64763e+01
+VirA_4_20, GAUSSIAN, VirA, 12:30:49.250, +12.24.02.102, 1.069e+02, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 9.92793e-02, 2.33402e-02, 1.08930e+01
+VirA_4_7, GAUSSIAN, VirA, 12:31:00.320, +12.26.38.867, 8.851e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 3.83541e-02, 3.47954e-02, 1.53069e+02
+VirA_4_6, GAUSSIAN, VirA, 12:30:55.490, +12.23.49.301, 7.499e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 6.64979e-02, 1.79950e-02, 1.72468e+01
+VirA_4_4, GAUSSIAN, VirA, 12:30:59.933, +12.22.42.187, 7.388e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.52817e-01, 1.69367e-02, 3.28830e+01
+VirA_4_21, GAUSSIAN, VirA, 12:30:42.588, +12.22.06.778, 6.153e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 3.81610e-02, 1.87921e-02, 2.03833e+01
+VirA_4_9, GAUSSIAN, VirA, 12:30:50.661, +12.23.49.618, 5.456e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 7.36474e-02, 8.25082e-03, 8.58262e+01
+VirA_4_3, POINT, VirA, 12:30:48.239, +12.23.18.068, 5.398e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 7.63889e-03, 6.94444e-03, 1.11922e+02
+VirA_4_14, GAUSSIAN, VirA, 12:30:39.222, +12.23.13.204, 5.142e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.40448e-02, 1.75673e-02, 1.70476e+02
+VirA_4_5, POINT, VirA, 12:30:45.851, +12.23.49.562, 4.189e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 7.63889e-03, 6.94444e-03, 8.15603e+01
+VirA_4_8, GAUSSIAN, VirA, 12:30:52.150, +12.18.29.472, 3.936e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 3.38379e-02, 2.30198e-02, 1.60497e+02
+VirA_4_15, GAUSSIAN, VirA, 12:31:01.387, +12.24.37.942, 3.313e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.14509e-02, 1.18403e-02, 7.44971e+01
+VirA_4_12, GAUSSIAN, VirA, 12:31:03.637, +12.23.31.633, 2.835e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 3.23407e-02, 9.02810e-03, 3.03022e+00
+VirA_4_11, GAUSSIAN, VirA, 12:30:59.512, +12.22.44.364, 2.701e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.66163e-02, 1.08450e-02, 1.34667e+02
+VirA_4_13, GAUSSIAN, VirA, 12:31:00.272, +12.23.41.572, 2.676e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.52105e-02, 1.11420e-02, 1.04009e+02
+VirA_4_10, POINT, VirA, 12:30:48.452, +12.23.32.663, 2.217e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 7.63889e-03, 6.94444e-03, 0.00000e+00
+VirA_4_16, GAUSSIAN, VirA, 12:30:43.137, +12.23.59.978, 1.788e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.68245e-02, 9.05564e-03, 9.84731e+01
+VirA_4_24, GAUSSIAN, VirA, 12:30:36.867, +12.18.49.897, 1.130e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.28602e-02, 1.66793e-02, 1.45577e+02
+VirA_4_18, POINT, VirA, 12:30:47.114, +12.23.26.692, 8.179e+00, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 7.63889e-03, 6.94444e-03, 0.00000e+00
+VirA_4_23, GAUSSIAN, VirA, 12:30:44.464, +12.19.17.889, 4.819e+00, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.37460e-02, 1.36631e-02, 1.48507e+02
+VirA_4_22, GAUSSIAN, VirA, 12:30:44.127, +12.23.08.175, 4.207e+00, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 8.85467e-03, 5.44859e-03, 1.33216e+02
+
diff --git a/test_data/skymodels/Ateam_LBA_CC.skymodel.REMOVED.git-id b/test_data/skymodels/Ateam_LBA_CC.skymodel.REMOVED.git-id
new file mode 100644
index 0000000000000000000000000000000000000000..c143443c567168c864095201d920d70a5351f4bc
--- /dev/null
+++ b/test_data/skymodels/Ateam_LBA_CC.skymodel.REMOVED.git-id
@@ -0,0 +1 @@
+ce968f5b4fb7491d2a63c642cc51d01da2bce829
\ No newline at end of file
diff --git a/test_data/skymodels/Calibrators_SH_lowres.skymodel b/test_data/skymodels/Calibrators_SH_lowres.skymodel
new file mode 100644
index 0000000000000000000000000000000000000000..cbb1296b7520bbbfede15403b33011cdfbad5e80
--- /dev/null
+++ b/test_data/skymodels/Calibrators_SH_lowres.skymodel
@@ -0,0 +1,78 @@
+# (Name, Type, Patch, Ra, Dec, I, Q, U, V, ReferenceFrequency='74e6', SpectralIndex='[]', MajorAxis, MinorAxis, Orientation) = format
+
+# The above line defines the field order and is required.
+
+# models for the A-Team members
+
+, , CygA, 19:59:26, +40.44.00
+CygA_4_2, POINT, CygA, 19:59:30.433, +40.43.56.221, 4.827e+03, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 7.63889e-03, 6.94444e-03, 1.00637e+02
+CygA_4_1, GAUSSIAN, CygA, 19:59:28.476, +40.44.01.804, 4.165e+03, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.20516e-02, 5.34011e-03, 1.03230e+02
+CygA_4_3, POINT, CygA, 19:59:22.168, +40.44.28.591, 3.896e+03, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 7.63889e-03, 6.94444e-03, 9.21061e+01
+CygA_4_4, POINT, CygA, 19:59:24.413, +40.44.18.617, 2.798e+03, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 7.63889e-03, 6.94444e-03, 6.74922e+01
+CygA_4_5, POINT, CygA, 19:59:25.555, +40.43.52.813, 1.560e+02, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 7.63889e-03, 6.94444e-03, 0.00000e+00
+
+, , CasA, 23:23:28, +58.48.42
+CasA_4_3, GAUSSIAN, CasA,23:23:21.044, +58.49.42.000, 4.193e+03, 0.0, 0.0, 0.0, 7.37817e+07, [-0.8], 4.32664e-02, 1.79281e-02, 2.73207e+01
+CasA_4_2, GAUSSIAN, CasA,23:23:33.918, +58.50.13.374, 4.131e+03, 0.0, 0.0, 0.0, 7.37817e+07, [-0.8], 4.25963e-02, 1.87784e-02, 1.12516e+02
+CasA_4_8, GAUSSIAN, CasA,23:23:34.032, +58.47.29.883, 2.536e+03, 0.0, 0.0, 0.0, 7.37817e+07, [-0.8], 2.77739e-02, 1.42664e-02, 6.26062e+01
+CasA_4_5, GAUSSIAN, CasA,23:23:15.292, +58.50.00.221, 2.316e+03, 0.0, 0.0, 0.0, 7.37817e+07, [-0.8], 3.15448e-02, 9.02869e-03, 4.11857e+01
+CasA_4_1, GAUSSIAN, CasA,23:23:39.632, +58.48.32.671, 2.245e+03, 0.0, 0.0, 0.0, 7.37817e+07, [-0.8], 2.25927e-02, 1.70874e-02, 8.48056e+01
+CasA_4_7, GAUSSIAN, CasA,23:23:24.110, +58.47.43.607, 2.201e+03, 0.0, 0.0, 0.0, 7.37817e+07, [-0.8], 4.64283e-02, 1.49372e-02, 2.12611e+01
+CasA_4_4, GAUSSIAN, CasA,23:23:14.939, +58.48.09.511, 2.099e+03, 0.0, 0.0, 0.0, 7.37817e+07, [-0.8], 3.09146e-02, 1.42990e-02, 1.61549e+02
+CasA_4_9, GAUSSIAN, CasA,23:23:10.817, +58.48.46.409, 9.790e+02, 0.0, 0.0, 0.0, 7.37817e+07, [-0.8], 9.62058e-03, 7.79284e-03, 1.77482e+02
+CasA_4_6, GAUSSIAN, CasA,23:23:26.327, +58.51.14.147, 2.444e+02, 0.0, 0.0, 0.0, 7.37817e+07, [-0.8], 8.75852e-03, 6.94444e-03, 8.12790e+01
+
+, , TauA, 05:34:32, +21.59.55
+TauA_4_1, GAUSSIAN, TauA, 05:34:32.269, +21.59.47.597, 1.911e+02, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 3.94949e-02, 1.31559e-02, 1.65623e+02
+TauA_4_11, GAUSSIAN, TauA, 05:34:28.639, +22.00.53.809, 1.344e+02, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.82500e-02, 1.22959e-02, 2.49193e+01
+TauA_4_3, GAUSSIAN, TauA, 05:34:25.671, +22.01.26.361, 1.247e+02, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.76133e-02, 1.13962e-02, 5.60626e+01
+TauA_4_9, GAUSSIAN, TauA, 05:34:23.420, +22.00.12.563, 1.096e+02, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.82128e-02, 1.71142e-02, 3.38306e+01
+TauA_4_15, GAUSSIAN, TauA, 05:34:22.884, +22.01.56.603, 9.243e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.99931e-02, 1.65627e-02, 1.76338e+02
+TauA_4_20, GAUSSIAN, TauA, 05:34:29.086, +21.59.21.722, 8.792e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 3.23261e-02, 9.03648e-03, 1.09196e+02
+TauA_4_10, GAUSSIAN, TauA, 05:34:37.530, +21.59.06.719, 8.641e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.13390e-02, 1.33726e-02, 1.27644e+02
+TauA_4_2, POINT, TauA, 05:34:30.550, +21.59.58.106, 8.276e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 7.63889e-03, 6.94444e-03, 0.00000e+00
+TauA_4_5, GAUSSIAN, TauA, 05:34:40.748, +21.59.31.921, 7.008e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.59971e-02, 1.25206e-02, 1.51745e+02
+TauA_4_14, GAUSSIAN, TauA, 05:34:30.570, +21.58.33.304, 6.187e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.87859e-02, 1.14519e-02, 8.42720e+01
+TauA_4_7, GAUSSIAN, TauA, 05:34:36.888, +22.01.01.174, 5.537e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.93364e-02, 1.21462e-02, 1.39794e+02
+TauA_4_8, GAUSSIAN, TauA, 05:34:35.324, +22.00.10.883, 5.525e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.81832e-02, 1.35652e-02, 1.06533e+02
+TauA_4_4, GAUSSIAN, TauA, 05:34:23.931, +21.58.55.252, 5.076e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.77800e-02, 1.52796e-02, 7.50003e+01
+TauA_4_13, GAUSSIAN, TauA, 05:34:36.235, +21.58.03.089, 4.862e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 3.59649e-02, 9.18833e-03, 8.58303e+01
+TauA_4_6, GAUSSIAN, TauA, 05:34:27.470, +21.58.50.002, 4.223e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.54141e-02, 7.87103e-03, 5.52923e+01
+TauA_4_21, GAUSSIAN, TauA, 05:34:34.585, +22.00.45.580, 4.175e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 3.19587e-02, 6.70123e-03, 1.72839e+02
+TauA_4_19, GAUSSIAN, TauA, 05:34:19.534, +22.01.07.470, 3.906e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.20399e-02, 1.14898e-02, 1.76500e+02
+TauA_4_17, GAUSSIAN, TauA, 05:34:29.426, +21.59.51.003, 2.848e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.10631e-02, 5.13228e-03, 1.27622e+02
+TauA_4_23, GAUSSIAN, TauA, 05:34:26.246, +22.02.05.085, 2.699e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.28687e-02, 1.05568e-02, 1.44302e+02
+TauA_4_16, GAUSSIAN, TauA, 05:34:32.269, +22.01.21.318, 2.557e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.16784e-02, 7.66067e-03, 1.43877e+02
+TauA_4_12, GAUSSIAN, TauA, 05:34:22.991, +22.00.33.000, 2.127e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.49607e-02, 8.16407e-03, 1.16642e+02
+TauA_4_18, GAUSSIAN, TauA, 05:34:26.292, +22.02.39.501, 1.727e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.19757e-02, 6.32082e-03, 1.08185e+02
+TauA_4_22, GAUSSIAN, TauA, 05:34:37.023, +21.58.27.987, 1.357e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 8.54110e-03, 5.01393e-03, 7.39302e+01
+TauA_4_24, GAUSSIAN, TauA, 05:34:31.108, +22.00.32.726, 1.106e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.59017e-02, 6.94444e-03, 5.84987e+01
+TauA_4_26, GAUSSIAN, TauA, 05:32:23.875, +21.50.58.488, 1.780e+00, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.05524e-02, 3.94647e-03, 6.84955e+01
+TauA_4_25, POINT, TauA, 05:34:30.647, +21.59.16.928, 9.776e-01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 7.63889e-03, 6.94444e-03, 2.05933e+01
+
+, , VirA, 12:30:35, +12.19.45
+VirA_4_17, GAUSSIAN, VirA, 12:30:34.383, +12.20.04.012, 3.219e+02, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 8.42972e-02, 6.11535e-02, 6.50364e+01
+VirA_4_2, GAUSSIAN, VirA, 12:30:47.034, +12.23.43.975, 2.591e+02, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 8.54400e-03, 4.07169e-03, 1.06884e+02
+VirA_4_1, POINT, VirA, 12:30:48.961, +12.23.33.770, 1.516e+02, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 7.63889e-03, 6.94444e-03, 1.70815e+02
+VirA_4_19, GAUSSIAN, VirA, 12:30:41.047, +12.17.36.983, 1.094e+02, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 4.63137e-02, 2.79725e-02, 8.64763e+01
+VirA_4_20, GAUSSIAN, VirA, 12:30:49.250, +12.24.02.102, 1.069e+02, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 9.92793e-02, 2.33402e-02, 1.08930e+01
+VirA_4_7, GAUSSIAN, VirA, 12:31:00.320, +12.26.38.867, 8.851e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 3.83541e-02, 3.47954e-02, 1.53069e+02
+VirA_4_6, GAUSSIAN, VirA, 12:30:55.490, +12.23.49.301, 7.499e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 6.64979e-02, 1.79950e-02, 1.72468e+01
+VirA_4_4, GAUSSIAN, VirA, 12:30:59.933, +12.22.42.187, 7.388e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.52817e-01, 1.69367e-02, 3.28830e+01
+VirA_4_21, GAUSSIAN, VirA, 12:30:42.588, +12.22.06.778, 6.153e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 3.81610e-02, 1.87921e-02, 2.03833e+01
+VirA_4_9, GAUSSIAN, VirA, 12:30:50.661, +12.23.49.618, 5.456e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 7.36474e-02, 8.25082e-03, 8.58262e+01
+VirA_4_3, POINT, VirA, 12:30:48.239, +12.23.18.068, 5.398e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 7.63889e-03, 6.94444e-03, 1.11922e+02
+VirA_4_14, GAUSSIAN, VirA, 12:30:39.222, +12.23.13.204, 5.142e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.40448e-02, 1.75673e-02, 1.70476e+02
+VirA_4_5, POINT, VirA, 12:30:45.851, +12.23.49.562, 4.189e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 7.63889e-03, 6.94444e-03, 8.15603e+01
+VirA_4_8, GAUSSIAN, VirA, 12:30:52.150, +12.18.29.472, 3.936e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 3.38379e-02, 2.30198e-02, 1.60497e+02
+VirA_4_15, GAUSSIAN, VirA, 12:31:01.387, +12.24.37.942, 3.313e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.14509e-02, 1.18403e-02, 7.44971e+01
+VirA_4_12, GAUSSIAN, VirA, 12:31:03.637, +12.23.31.633, 2.835e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 3.23407e-02, 9.02810e-03, 3.03022e+00
+VirA_4_11, GAUSSIAN, VirA, 12:30:59.512, +12.22.44.364, 2.701e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.66163e-02, 1.08450e-02, 1.34667e+02
+VirA_4_13, GAUSSIAN, VirA, 12:31:00.272, +12.23.41.572, 2.676e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.52105e-02, 1.11420e-02, 1.04009e+02
+VirA_4_10, POINT, VirA, 12:30:48.452, +12.23.32.663, 2.217e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 7.63889e-03, 6.94444e-03, 0.00000e+00
+VirA_4_16, GAUSSIAN, VirA, 12:30:43.137, +12.23.59.978, 1.788e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.68245e-02, 9.05564e-03, 9.84731e+01
+VirA_4_24, GAUSSIAN, VirA, 12:30:36.867, +12.18.49.897, 1.130e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.28602e-02, 1.66793e-02, 1.45577e+02
+VirA_4_18, POINT, VirA, 12:30:47.114, +12.23.26.692, 8.179e+00, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 7.63889e-03, 6.94444e-03, 0.00000e+00
+VirA_4_23, GAUSSIAN, VirA, 12:30:44.464, +12.19.17.889, 4.819e+00, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.37460e-02, 1.36631e-02, 1.48507e+02
+VirA_4_22, GAUSSIAN, VirA, 12:30:44.127, +12.23.08.175, 4.207e+00, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 8.85467e-03, 5.44859e-03, 1.33216e+02
+
diff --git a/test_jobs/NDPPP.json b/test_jobs/NDPPP.json
new file mode 100644
index 0000000000000000000000000000000000000000..d010d681ec66a6e25ae050193c8a40ac871d40d6
--- /dev/null
+++ b/test_jobs/NDPPP.json
@@ -0,0 +1,21 @@
+{
+ "parset": {
+ "class": "File",
+ "path": "ndppp_step.parset"
+ },
+ "msin": {
+ "class": "Directory",
+ "path": "/data/L570745_SB000_uv_first10.MS"
+ },
+ "output_file_names": {
+ "gaincal": {
+ "name": "test.h5"
+ },
+ },
+ "output_directory_names": {
+ "count": {
+ "count_flagfreq": "L570745_SB000_uv_first10_count.flagfreq",
+ "count_flagstat": "L570745_SB000_uv_first10_count.flagstat"
+ }
+ }
+}
diff --git a/test_jobs/aoflagger.json b/test_jobs/aoflagger.json
new file mode 100644
index 0000000000000000000000000000000000000000..138c5022e2e071f4f84a79fdadbdd063b51bfe3f
--- /dev/null
+++ b/test_jobs/aoflagger.json
@@ -0,0 +1,12 @@
+{
+ "msin": [
+ {
+ "class": "Directory",
+ "path": "/data/L570745_SB000_uv_first10.MS"
+ },
+ {
+ "class": "Directory",
+ "path": "/data/L570745_SB001_uv_first10.MS"
+ }
+ ]
+}
diff --git a/test_jobs/aoflagger_step_generator.json b/test_jobs/aoflagger_step_generator.json
new file mode 100644
index 0000000000000000000000000000000000000000..4c6b5085d4492d2a008d0c0cdd0bbd4139c97132
--- /dev/null
+++ b/test_jobs/aoflagger_step_generator.json
@@ -0,0 +1,8 @@
+{
+ "input_parset": [{
+ "step_type": "name",
+ "step_id": "id",
+ "parameters": []
+}],
+ "strategy": {"class":"File", "path":"/dev/null"}
+}
diff --git a/test_jobs/averager_step_generator.json b/test_jobs/averager_step_generator.json
new file mode 100644
index 0000000000000000000000000000000000000000..4bfe8ddc068c92271e8bb29fad6219214d24a199
--- /dev/null
+++ b/test_jobs/averager_step_generator.json
@@ -0,0 +1,7 @@
+{
+ "input_parset": [{
+ "step_type": "name",
+ "step_id": "id",
+ "parameters": []
+}]
+}
diff --git a/test_jobs/averager_step_generator_error.json b/test_jobs/averager_step_generator_error.json
new file mode 100644
index 0000000000000000000000000000000000000000..c0a124a2a47027e9261e360d5557c31a5de16b4e
--- /dev/null
+++ b/test_jobs/averager_step_generator_error.json
@@ -0,0 +1,9 @@
+{
+ "input_parset": [{
+ "step_type": "name",
+ "step_id": "id",
+ "parameters": []
+}],
+ "step_id": "id",
+ "strategy": {"class":"File", "path":"/dev/null"}
+}
diff --git a/test_jobs/blsmooth.json b/test_jobs/blsmooth.json
new file mode 100644
index 0000000000000000000000000000000000000000..5b300249de098dafe2ad49b7b68b8212ce6ba563
--- /dev/null
+++ b/test_jobs/blsmooth.json
@@ -0,0 +1,7 @@
+{
+ "msin": {
+ "class": "Directory",
+ "path": "/data/L570745_SB003_uv_first10.MS"
+ },
+ "do_smooth": true
+}
diff --git a/test_jobs/ddecal.json b/test_jobs/ddecal.json
new file mode 100644
index 0000000000000000000000000000000000000000..13a89c1f35db2da2d1be6966d5f213619360e514
--- /dev/null
+++ b/test_jobs/ddecal.json
@@ -0,0 +1,8 @@
+{
+ "msin": {
+ "class": "Directory",
+ "path": "/data/L570745_SB001_uv_MODEL.MS"
+ },
+ "msin_datacolumn": "DATA",
+ "usemodelcolumn": true
+}
diff --git a/test_jobs/demo_prefactor.json b/test_jobs/demo_prefactor.json
new file mode 100644
index 0000000000000000000000000000000000000000..ef2e0db960d13f382cdd2272969ce975e4c7d77f
--- /dev/null
+++ b/test_jobs/demo_prefactor.json
@@ -0,0 +1,17 @@
+{
+ "msin": [
+ {
+ "class": "Directory",
+ "path": "/data/L570745_SB003_uv_first10.MS"
+ },
+ {
+ "class": "Directory",
+ "path": "/data/L570745_SB005_uv_first10.MS"
+ },
+ {
+ "class": "Directory",
+ "path": "/data/L570745_SB008_uv_first10.MS"
+ }
+ ]
+}
+
diff --git a/test_jobs/dp3_execute.json b/test_jobs/dp3_execute.json
new file mode 100644
index 0000000000000000000000000000000000000000..f21dff5b5c3fc6f6c7bd58edc1d6aee45ce5446b
--- /dev/null
+++ b/test_jobs/dp3_execute.json
@@ -0,0 +1,19 @@
+{
+ "msin": {"class": "Directory", "path":"/data/L570745_SB003_uv_first10.MS"},
+ "steps": [
+ {
+ "step_type": "averager",
+ "step_id": "average",
+ "parameters": {
+ "timestep": 1,
+ "freqstep": 1,
+ "minpoints": 0,
+ "minperc": 0,
+ "timeresolution": 0,
+ "freqresolution": 0
+ },
+ "output_files": {}
+ }
+ ],
+ "msout_name": "TEST"
+}
diff --git a/test_jobs/file_selector.json b/test_jobs/file_selector.json
new file mode 100644
index 0000000000000000000000000000000000000000..77e4a30217226c2250882fd1ca55810695577c13
--- /dev/null
+++ b/test_jobs/file_selector.json
@@ -0,0 +1,12 @@
+{
+ "inputs": {
+ "average": {
+ "parmdb": {
+ "class": "File",
+ "location": "ndppp_step.parset"
+ }
+ }
+ },
+ "step_name": "average",
+ "key_name": "parmdb"
+}
diff --git a/test_jobs/find_skymodel_cal.json b/test_jobs/find_skymodel_cal.json
new file mode 100644
index 0000000000000000000000000000000000000000..1a0067244b08ded29eccd45b1ec5b3394f72b164
--- /dev/null
+++ b/test_jobs/find_skymodel_cal.json
@@ -0,0 +1,10 @@
+{
+ "msin": {
+ "class": "Directory",
+ "path": "/data/L570745_SB000_uv_first10.MS"
+ },
+ "skymodels": {
+ "class": "Directory",
+ "path": "/data/skymodels"
+ }
+}
diff --git a/test_jobs/flag_average.json b/test_jobs/flag_average.json
new file mode 100644
index 0000000000000000000000000000000000000000..36154f55330703582bc2a83c16cf666d27560148
--- /dev/null
+++ b/test_jobs/flag_average.json
@@ -0,0 +1,6 @@
+{
+ "msin": {
+ "class": "Directory",
+ "path": "/data/L570745_SB001_uv_first10.MS"
+ }
+}
diff --git a/test_jobs/h5parm_collector.json b/test_jobs/h5parm_collector.json
new file mode 100644
index 0000000000000000000000000000000000000000..42c60bb8d15f9f8e49673c916c7d19cec07ae35a
--- /dev/null
+++ b/test_jobs/h5parm_collector.json
@@ -0,0 +1,9 @@
+{
+ "h5parmFiles": [
+ {
+ "class": "File",
+ "path": "/data/example.h5",
+ "format": "lofar:#H5Parm"
+ }
+ ]
+}
diff --git a/test_jobs/integration_test_prefactor_calibrator.json b/test_jobs/integration_test_prefactor_calibrator.json
new file mode 100644
index 0000000000000000000000000000000000000000..672f979893679218fb67c1cc97ecb8721962aa7c
--- /dev/null
+++ b/test_jobs/integration_test_prefactor_calibrator.json
@@ -0,0 +1,14 @@
+{
+ "msin" : [
+ {"class": "Directory", "path": "/data/L570745_SB000_uv_first10.MS"},
+ {"class": "Directory", "path": "/data/L570745_SB001_uv_first10.MS"},
+ {"class": "Directory", "path": "/data/L570745_SB002_uv_first10.MS"}
+ ],
+ "raw_data": false,
+ "demix": false,
+ "skymodels": {
+ "class": "Directory",
+ "path": "/data/skymodels"
+ },
+ "A-Team_sky_model": "/data/skymodels/Ateam_LBA_CC.skymodel"
+}
diff --git a/test_jobs/interpolate_visibilities.json b/test_jobs/interpolate_visibilities.json
new file mode 100644
index 0000000000000000000000000000000000000000..a9ce3dea491423d5a858e8a500f78b3df24a63d3
--- /dev/null
+++ b/test_jobs/interpolate_visibilities.json
@@ -0,0 +1,4 @@
+{
+ "msin": { "class":"Directory", "path":"/data/L570745_SB000_uv_first10.MS"},
+ "window_size": 5
+}
diff --git a/test_jobs/losoto_abs.json b/test_jobs/losoto_abs.json
new file mode 100644
index 0000000000000000000000000000000000000000..a283362c08eefe03c690a9a3ad3791dbf4ed74bc
--- /dev/null
+++ b/test_jobs/losoto_abs.json
@@ -0,0 +1,8 @@
+{
+ "input_h5parm": {
+ "class": "File",
+ "path": "/data/example.h5",
+ "format": "lofar:#H5Parm"
+ },
+ "soltab": "sol000/amplitude000"
+}
diff --git a/test_jobs/losoto_duplicate.json b/test_jobs/losoto_duplicate.json
new file mode 100644
index 0000000000000000000000000000000000000000..a283362c08eefe03c690a9a3ad3791dbf4ed74bc
--- /dev/null
+++ b/test_jobs/losoto_duplicate.json
@@ -0,0 +1,8 @@
+{
+ "input_h5parm": {
+ "class": "File",
+ "path": "/data/example.h5",
+ "format": "lofar:#H5Parm"
+ },
+ "soltab": "sol000/amplitude000"
+}
diff --git a/test_jobs/losoto_norm.json b/test_jobs/losoto_norm.json
new file mode 100644
index 0000000000000000000000000000000000000000..d6d689949f55ce2c769956e474508c78c7c28f32
--- /dev/null
+++ b/test_jobs/losoto_norm.json
@@ -0,0 +1,9 @@
+{
+ "input_h5parm": {
+ "class": "File",
+ "path": "/data/example.h5",
+ "format": "lofar:#H5Parm"
+ },
+ "soltab": "sol000/amplitude000",
+ "axesToNorm": ["time"]
+}
diff --git a/test_jobs/losoto_plot.json b/test_jobs/losoto_plot.json
new file mode 100644
index 0000000000000000000000000000000000000000..1316e959d7b7af2e331e320f412a16357d69643d
--- /dev/null
+++ b/test_jobs/losoto_plot.json
@@ -0,0 +1,6 @@
+{
+ "input_h5parm": {"class": "File", "path": "/data/example.h5", "format": "lofar:#H5Parm"},
+ "soltab": ["sol000/phase000"],
+ "axesInPlot": ["time"],
+ "ncpu": 1
+}
diff --git a/test_jobs/losoto_replicateonaxis.json b/test_jobs/losoto_replicateonaxis.json
new file mode 100644
index 0000000000000000000000000000000000000000..0abfc35d9280bdc84605590f00479a9b5d36d7c6
--- /dev/null
+++ b/test_jobs/losoto_replicateonaxis.json
@@ -0,0 +1,10 @@
+{
+ "input_h5parm": {
+ "class": "File",
+ "path": "/data/example.h5",
+ "format": "lofar:#H5Parm"
+ },
+ "soltab": "sol000/amplitude000",
+ "axisReplicate": "time",
+ "updateWeight": true
+}
diff --git a/test_jobs/losoto_reset.json b/test_jobs/losoto_reset.json
new file mode 100644
index 0000000000000000000000000000000000000000..2d773afb6baa7bb4d83c8fe0d85becf472b54f69
--- /dev/null
+++ b/test_jobs/losoto_reset.json
@@ -0,0 +1,8 @@
+{
+ "input_h5parm": {
+ "class": "File",
+ "path": "/data/example.h5",
+ "format": "lofar:#H5Parm"
+ },
+ "soltab": "sol000/amplitude000"
+}
diff --git a/test_jobs/losoto_residual.json b/test_jobs/losoto_residual.json
new file mode 100644
index 0000000000000000000000000000000000000000..4a56f1c6e77dcb29aac66414ba1cb19253597bd6
--- /dev/null
+++ b/test_jobs/losoto_residual.json
@@ -0,0 +1,9 @@
+{
+ "input_h5parm": {
+ "class": "File",
+ "path": "/data/example.h5",
+ "format": "lofar:#H5Parm"
+ },
+ "soltab": "sol000/amplitude000",
+ "soltabsToSub": ["amplitude000"]
+}
diff --git a/test_jobs/ms_concat.json b/test_jobs/ms_concat.json
new file mode 100644
index 0000000000000000000000000000000000000000..935a917a5ee9ed30ebbb7782305899cecf88bd70
--- /dev/null
+++ b/test_jobs/ms_concat.json
@@ -0,0 +1,14 @@
+{
+ "min_length": 1,
+ "msin": [
+ {
+ "class": "Directory",
+ "path": "../test_data/L570745_SB000_uv_first10.MS"
+ },
+ {
+ "class": "Directory",
+ "path": "../test_data/L570745_SB001_uv_first10.MS"
+ }
+ ]
+}
+
diff --git a/test_jobs/ndppp_step.parset b/test_jobs/ndppp_step.parset
new file mode 100644
index 0000000000000000000000000000000000000000..450d95c45cfad84ae9af23bd3d51de49035a55ea
--- /dev/null
+++ b/test_jobs/ndppp_step.parset
@@ -0,0 +1,7 @@
+count.save=True
+count.path="."
+gaincal.parmdb=test.h5
+gaincal.usemodelcolumn=True
+gaincal.caltype=diagonal
+msin.modelcolumn=DATA
+steps=[count, gaincal]
diff --git a/test_jobs/parset_generator.json b/test_jobs/parset_generator.json
new file mode 100644
index 0000000000000000000000000000000000000000..5b3c748396c5cc9928b04ce0d57ae41842137835
--- /dev/null
+++ b/test_jobs/parset_generator.json
@@ -0,0 +1,7 @@
+{
+ "parameters_input": [{
+ "step_type": "name",
+ "step_id": "id",
+ "parameters": {"timestep": 1}
+}]
+}
diff --git a/test_jobs/parset_generator_file_input.json b/test_jobs/parset_generator_file_input.json
new file mode 100644
index 0000000000000000000000000000000000000000..c29a5a08f0230adfe440117df3bac1a4ca0023fd
--- /dev/null
+++ b/test_jobs/parset_generator_file_input.json
@@ -0,0 +1,22 @@
+{
+ "parameters_input": [{
+ "step_type": "averager",
+ "step_id": "average",
+ "output_files": {
+ "parmdb": "fily"
+ },
+ "output_directories": {
+ "parmdb": "diry"
+ },
+ "parameters": {
+ "timestep": 1,
+ "freqstep": 1,
+ "strategy": {
+ "class": "File",
+ "location": "file:///etc/hostname",
+ "size": 0,
+ "nameext": ""
+ }
+ }
+}]
+}
diff --git a/test_jobs/predict.json b/test_jobs/predict.json
new file mode 100644
index 0000000000000000000000000000000000000000..2a493f8755d1350243a7d0f04783dbddc2dda7ea
--- /dev/null
+++ b/test_jobs/predict.json
@@ -0,0 +1,10 @@
+{
+ "msin": {
+ "class": "Directory",
+ "path": "/data/L570745_SB001_uv_first10.MS"
+ },
+ "sources_db": {
+ "class": "File",
+ "path": "/data/A-Team_lowres.sourcedb"
+ }
+}
diff --git a/test_jobs/preflagger_step_generator.json b/test_jobs/preflagger_step_generator.json
new file mode 100644
index 0000000000000000000000000000000000000000..4c6b5085d4492d2a008d0c0cdd0bbd4139c97132
--- /dev/null
+++ b/test_jobs/preflagger_step_generator.json
@@ -0,0 +1,8 @@
+{
+ "input_parset": [{
+ "step_type": "name",
+ "step_id": "id",
+ "parameters": []
+}],
+ "strategy": {"class":"File", "path":"/dev/null"}
+}
diff --git a/test_jobs/selector.json b/test_jobs/selector.json
new file mode 100644
index 0000000000000000000000000000000000000000..3025390d76a1ae4ba3d6aab8482129ece27319cb
--- /dev/null
+++ b/test_jobs/selector.json
@@ -0,0 +1,4 @@
+{
+ "select_a": true,
+ "a_arguments": [{"name": "a"}]
+}