Squashed 'lofar-cwl/' content from commit c5ab49c

git-subtree-dir: lofar-cwl
git-subtree-split: c5ab49c3af528e46503ab714e2052285f091afc5

.gitignore

+test_data/*.MS
+Docker/scripts
+Docker/DP3
+Docker/Dysco
+Docker/STMAN
+Docker/idg
+Docker/LOFARBeam
+*.sif
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# Not idea
+.idea

.gitlab-ci.yml

+image: lofareosc/lofar-pipeline-ci:latest
+
+variables:
+  TEST_DATASET_NAME: "L570745_uv_first10.MS.tar.xz"
+  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+
+stages:
+  - build_docker
+  - push_docker
+  - validate_cwl_scripts
+  - test_steps
+  - integration_test
+
+before_script:
+   - mkdir workdir
+   - mkdir logs
+
+after_script:
+  - echo "All done"
+
+build_docker:
+  image: docker:latest
+  stage: build_docker
+  only:
+    - tags
+  script:
+    - apk add git subversion git-lfs bash
+    - cd Docker
+    - bash build_docker.sh
+
+push_docker_tags:
+  image: docker:latest
+  allow_failure: true
+  stage: push_docker
+  only:
+    - tags
+  script:
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - docker tag lofareosc/prefactor-ci:latest $CI_REGISTRY/prefactor-ci:$CI_COMMIT_REF_NAME
+    - docker push $CI_REGISTRY/prefactor-ci:$CI_COMMIT_REF_NAME
+
+validate_scripts:
+  stage: validate_cwl_scripts
+  script:
+    - for file in `find . -name 'steps/*.cwl'`; do cwltool --validate $file; done
+
+average_step_generator:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwl-runner steps/DP3.AveragerStepGenerator.cwl test_jobs/averager_step_generator.json
+    - ! cwl-runner steps/DP3.AveragerStepGenerator.cwl test_jobs/averager_step_generator_error.json
+
+applycal_step_generator:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwl-runner steps/DP3.ApplyCalStepGenerator.cwl test_jobs/applycal_step_generator.json
+
+aoflagger_step_generator:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwl-runner steps/DP3.AOFlaggerStepGenerator.cwl test_jobs/aoflagger_step_generator.json
+
+preflagger_step_generator:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwl-runner steps/DP3.PreflaggerStepGenerator.cwl test_jobs/preflagger_step_generator.json
+
+losoto_abs:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwl-runner --no-container steps/LoSoTo.Abs.cwl test_jobs/losoto_abs.json
+
+losoto_duplicate:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwl-runner --no-container steps/LoSoTo.Duplicate.cwl test_jobs/losoto_duplicate.json
+
+losoto_plot:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwl-runner --no-container steps/LoSoTo.Plot.cwl test_jobs/losoto_plot.json
+losoto_flag:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwl-runner --no-container steps/LoSoTo.Flag.cwl test_jobs/losoto_flag.json
+losoto_norm:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwl-runner --no-container steps/LoSoTo.Norm.cwl test_jobs/losoto_norm.json
+
+losoto_replicateonaxis:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwl-runner --no-container steps/LoSoTo.Replicateonaxis.cwl test_jobs/losoto_replicateonaxis.json
+
+losoto_reset:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwl-runner --no-container steps/LoSoTo.Reset.cwl test_jobs/losoto_reset.json
+
+losoto_structure:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwl-runner --no-container steps/LoSoTo.Structure.cwl test_jobs/losoto_structure.json
+
+losoto_smooth:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwl-runner --no-container steps/LoSoTo.Smooth.cwl test_jobs/losoto_smooth.json
+
+losoto_prefactor_bandpass:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwl-runner --no-container steps/LoSoTo.PrefactorBandpass.cwl test_jobs/losoto_prefactorbandpass.json
+
+losoto_clip:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwl-runner --no-container steps/LoSoTo.Clip.cwl test_jobs/losoto_clip.json
+
+losoto_clocktec:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwl-runner --no-container steps/LoSoTo.ClockTec.cwl test_jobs/losoto_clocktec.json
+
+losoto_flagextend:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwl-runner --no-container steps/LoSoTo.Flagextend.cwl test_jobs/losoto_flagextend.json
+
+losoto_prefactor_xyoffset:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwl-runner --no-container steps/LoSoTo.PrefactorXYoffset.cwl test_jobs/losoto_prefactorxyoffset.json
+
+losoto_flagstation:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwl-runner --no-container steps/LoSoTo.FlagStation.cwl test_jobs/losoto_flagstation.json
+losoto_faraday:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwl-runner --no-container steps/LoSoTo.Faraday.cwl test_jobs/losoto_faraday.json
+
+losoto_interpolate:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwl-runner --no-container steps/LoSoTo.Interpolate.cwl test_jobs/losoto_interpolate.json
+
+losoto_reweight:
+  stage: test_steps
+  allow_failure: true
+  script:
+   - cwl-runner --no-container steps/LoSoTo.Reweight.cwl test_jobs/losoto_reweight.json
+
+losoto_lofarbeam:
+  stage: test_steps
+  allow_failure: true
+  script:
+   - cwl-runner --no-container --preserve-environment PYTHONPATH steps/LoSoTo.Lofarbeam.cwl test_jobs/losoto_lofarbeam.json
+
+losoto_residual:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwl-runner --no-container steps/LoSoTo.Residual.cwl test_jobs/losoto_residual.json
+
+losoto_splitleak:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwl-runner --no-container steps/LoSoTo.Splitleak.cwl test_jobs/losoto_splitleak.json
+
+losoto_polalign:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwl-runner --no-container steps/LoSoTo.Polalign.cwl test_jobs/losoto_polalign.json
+
+
+parset_selector:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwl-runner steps/DP3.Selector.cwl test_jobs/selector.json
+
+file_selector:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwl-runner steps/FileSelector.cwl test_jobs/file_selector.json
+
+
+NDPPP:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH steps/DPPP.cwl test_jobs/NDPPP.json
+
+DP3.Execute:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH steps/DP3.Execute.cwl test_jobs/dp3_execute.json
+
+h5parm_collector:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/H5ParmCollector.cwl test_jobs/h5parm_collector.json
+
+aoflagger:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH steps/AOFlagger.cwl test_jobs/aoflagger.json
+
+interpolate_visibilities:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/interpolate_visibilities.cwl test_jobs/interpolate_visibilities.json
+
+h5parm_collector:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/H5ParmCollector.cwl test_jobs/h5parm_collector.json
+
+ddecal:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/ddecal.cwl test_jobs/ddecal.json
+
+gaincal:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/gaincal.cwl test_jobs/gaincal.json
+
+predict:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/predict.cwl test_jobs/predict.json

Docker/.entrypoint

+#!/bin/bash
+set -e
+
+exec "$@"
+

Docker/Dockerfile

+FROM kernsuite/base:5 as builder
+SHELL ["/bin/bash", "-c"]
+
+RUN export DEBIAN_FRONTEND=noninteractive && \ 
+    apt-get update && \
+    apt-get install -y \
+        casacore-data \
+        casacore-dev \
+        libboost-python-dev \
+        libcfitsio-dev \
+        python-dev \
+        python3-numpy \
+        cmake \
+        build-essential \
+        libhdf5-serial-dev \
+        libarmadillo-dev \
+        libboost-filesystem-dev \
+        libboost-system-dev \
+        libboost-date-time-dev \
+        libboost-numpy-dev \
+        libboost-signals-dev \ 
+        libboost-program-options-dev \
+        libboost-test-dev \
+        libxml2-dev \
+        libpng-dev \
+        pkg-config \
+        aoflagger-dev \
+        libgtkmm-3.0-dev \
+        git \
+        wget \
+        libfftw3-dev \
+        libgsl-dev
+        
+RUN mkdir -p /src/
+WORKDIR /src/
+
+
+# Build the IDG version from source
+
+
+ADD ./idg /src/idg/src
+RUN mkdir idg/build && cd idg/build && cmake -DCMAKE_INSTALL_PREFIX=/usr/local/ ../src && make install -j2
+
+ADD ./DP3 /src/dp3/src
+RUN cd /src/dp3/  && \
+    mkdir build && \
+    cd build && \
+    cmake ../src -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPORTABLE=True &&\
+    make -j2 && \
+    make install -j2
+
+ADD ./STMAN /src/STMAN/src
+RUN cd /src/STMAN/ && \
+    mkdir build && \
+    cd build && \
+    cmake ../src -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPORTABLE=True && \
+    make -j2 && \
+    make install -j2
+    
+ADD ./Dysco /src/Dysco/src
+RUN cd /src/Dysco/ && \
+    mkdir build && \
+    cd build && \
+    cmake ../src -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPORTABLE=True && \
+    make -j2 && \
+    make install -j2
+
+ADD ./LOFARBeam /src/LOFARBeam/src
+RUN cd /src/LOFARBeam/ && \
+    mkdir build && \
+    cd build && \
+    cmake ../src -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPYTHON_EXECUTABLE=/usr/bin/python3 && \
+    make -j2 && \
+    make install -j2
+
+#----------------------------------------------
+FROM kernsuite/base:5 as runner
+
+COPY --from=builder /usr/local/ /usr/local
+
+SHELL ["/bin/bash", "-c"]
+
+RUN export DEBIAN_FRONTEND=noninteractive && \
+    apt-get update && \
+    apt-get install -y \
+        aoflagger \
+        casacore-data \
+        libarmadillo8 \
+        libboost-date-time1.65.1 \
+        libboost-filesystem1.65.1 \
+        libboost-numpy1.65.1 \
+        libboost-python1.65.1 \
+        libboost-program-options1.65.1 \
+        libboost-system1.65.1 \
+        libboost-signals1.65.1 \
+        libboost-test1.65.1 \
+        libboost-python1.65-dev \
+        libstationresponse2 \
+        libcasa-* \
+        libcfitsio5 \
+        libgtkmm-3.0 \
+        libfftw3-3 \
+        libhdf5-cpp-100 \
+        libpng16-16 \
+        libxml2 \
+        python3.7 \
+        python3-casacore \
+        python3-numpy \
+        python3-scipy && \
+    rm -rf /var/lib/apt/lists/*
+
+
+RUN mkdir /home/lofaruser/ && \ 
+    groupadd -r lofaruser && \
+    useradd --no-log-init -r -g lofaruser lofaruser && \
+    chown lofaruser:lofaruser /home/lofaruser
+
+ARG TOIL_VERSION=3.20.0
+LABEL toil.version=${TOIL_VERSION}
+
+ARG LSMTool_TAG=master
+LABEL LSMTool.version.tag=${LSMTool_TAG}
+
+ARG LoSoTo_TAG=master
+LABEL LoSoTo.version.tag=${LoSoTo_TAG}
+
+ARG STMAN_TAG=master
+LABEL STMAN.version.tag=${STMAN_TAG}
+
+ARG Dysco_TAG=master
+LABEL Dysco.version.tag=${Dysco_TAG}
+
+ARG LofarBeam_TAG=master
+LABEL LOFARBeam.version.tag=${LofarBeam_TAG}
+    
+RUN apt-get --allow-releaseinfo-change update -y  && apt install -y nodejs python3-pip git wsclean
+RUN python3 -m pip install cwltool cwl-runner -e "git://github.com/darafferty/LSMTool.git@${LSMTool_TAG}#egg=LSMTool" \
+                                   "toil[cwl]"==${TOIL_VERSION} \
+                                   matplotlib
+
+RUN python3 -m pip install -e "git+https://github.com/revoltek/losoto.git@${LoSoTo_TAG}#egg=LoSoTo"
+
+ADD losoto.patch /home/lofaruser/losoto.patch
+RUN patch /src/losoto/losoto/operations/__init__.py /home/lofaruser/losoto.patch 
+ADD .entrypoint /home/lofaruser/.entrypoint
+RUN chown lofaruser:lofaruser /home/lofaruser/.entrypoint && \ 
+    chmod +rx /home/lofaruser/.entrypoint
+
+WORKDIR /home/lofaruser
+
+
+RUN chmod +rx /usr/local/bin/*
+ENTRYPOINT ["/home/lofaruser/.entrypoint"]
+USER lofaruser
+ENV PYTHONPATH=/usr/local/lib/python3.6/site-packages/
+ARG DP3_TAG=master
+ARG IDG_TAG=master
+
+LABEL IDG.version.tag=${IDG_TAG}
+LABEL DP3.version.tag=${DP3_TAG}

Docker/Dockerfile_ci

+FROM lofareosc/lofar-pipeline:latest
+SHELL ["/bin/bash", "-c"]
+
+ADD L570745_uv_first10.MS.tar.xz /data
+ADD skymodels.tar.gz /data/skymodels
+ADD test_data/A-Team_lowres.sourcedb /data/A-Team_lowres.sourcedb
+ADD test_data/example.h5 /data/example.h5
+ADD test_data/L570745_SB001_uv_MODEL.MS /data/L570745_SB001_uv_MODEL.MS

Docker/build_docker.sh

+#! /bin/bash
+set -e
+
+git_clone_or_pull () {
+    REPO=$1
+    BRANCH=$2
+    DIR=$3
+    if [ -d $DIR ] 
+    then
+     git -C "${DIR}" pull
+    else
+     git clone --depth 1 -b "${BRANCH}" "${REPO}" "${DIR}"
+    fi
+}
+
+# SOFTWARE VERSIONS
+DP3_TAG=v4.1
+IDG_TAG=0.6
+LSMTool_TAG=v1.4.2
+LoSoTo_TAG=f0398b
+TOIL_VERSION=3.20.0
+STMAN_TAG=master
+Dysco_TAG=v1.2
+Prefactor_TAG=master
+LofarBeam_TAG=master
+
+# FETCHES THE IDG
+git_clone_or_pull https://gitlab.com/astron-idg/idg/ ${IDG_TAG} idg
+
+# FETCHES DP3
+git_clone_or_pull https://github.com/lofar-astron/DP3 ${DP3_TAG} DP3
+
+# FETCHES STMAN
+git_clone_or_pull https://github.com/lofar-astron/LofarStMan ${STMAN_TAG} STMAN
+
+# FETCHES Dysco
+git_clone_or_pull https://github.com/aroffringa/dysco.git ${Dysco_TAG} Dysco
+
+# FETCHES LOFARBeam
+git_clone_or_pull https://github.com/lofar-astron/LOFARBeam ${LofarBeam_TAG} LOFARBeam
+
+SCRIPT_PATH=$(realpath ${BASH_SOURCE[0]})
+
+DOCKER_PATH=$(dirname ${SCRIPT_PATH})
+
+docker build ${DOCKER_PATH} --build-arg=IDG_TAG=${IDG_TAG}\
+                            --build-arg=DP3_TAG=${DP3_TAG}\
+                            --build-arg=LSMTool_TAG=${LSMTool_TAG}\
+                            --build-arg=LoSoTo_TAG=${LoSoTo_TAG}\
+                            --build-arg=STMAN_TAG=${STMAN_TAG}\
+                            --build-arg=Dysco_TAG=${Dysco_TAG}\
+                            --build-arg=TOIL_VERSION=${TOIL_VERSION}\
+			    --build-arg=LofarBeam_TAG=${LofarBeam_TAG}\
+                            -t lofareosc/lofar-pipeline
+
+docker build ${DOCKER_PATH} -f ${DOCKER_PATH}/Dockerfile_ci -t lofareosc/lofar-pipeline-ci

Docker/losoto.patch

+---  /src/losoto/losoto/operations/__init__.py      2019-11-07 12:43:46.220577945 +0000
++++  /src/losoto/losoto/operations/__init__.py      2019-11-07 12:43:34.401765786 +0000
+@@ -2,8 +2,7 @@
+ 
+ __all__ = [ os.path.basename(f)[:-3] for f in glob.glob(os.path.dirname(__file__)+"/*.py") if f[0] != '_']
+ 
+-for x in __all__:
+-    __import__(x, locals(), globals())
++from . import *
+ 
+ class Timer(object):
+     """
+

Docker/test_data/L570745_SB001_uv_MODEL.MS/ANTENNA/table.info

+Type = 
+SubType = 
+

Docker/test_data/L570745_SB001_uv_MODEL.MS/DATA_DESCRIPTION/table.info

+Type = 
+SubType = 
+