.dockerignore

+build/
+dist/
+Docker/
+docs/build/
+*.egg-info/
+venv/
+.git/

.gitlab-ci.yml

-image: lofareosc/prefactor3-cwl:latest
+workflow:
+  rules:
+    # Don't create a pipeline if commit is on a branch with open merge requests
+    - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
+      when: never
+    - when: always
 
 variables:
   TEST_HBA_DATASET_NAME: "test_data.tar.gz"
+  TEST_LBA_DATASET_NAME: "test_data_lba.tar.gz"
   CALIBRATOR_HBA_RESULTS_NAME: "results_calibrator.tar.gz"
+  CALIBRATOR_LBA_RESULTS_NAME: 'results_calibrator_lba.tar.gz'
   TARGET_HBA_RESULTS_NAME: "results_target.tar.gz"
+  TARGET_LBA_RESULTS_NAME: "results_target_lba.tar.gz"
+  TARGET_HBA_SELFCAL_RESULTS_NAME: "results_target_selfcal.tar.gz"
+  TARGET_HBA_VLBI_RESULTS_NAME: "results_target_VLBI.tar.gz"
   PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+  BUILD_DOCKER_IMAGE: "0"
+  GIT_STRATEGY: clone  # ensure every job starts with a pristine working copy
+  GIT_DEPTH: 0  # do not do shallow clones, needed for `git describe --tags`
 
 stages:
-  - download_data
-  - validate_cwl_scripts
-  - test_steps
-  - test_full
+  - initialize
+  - versioning
+  - build
+  - install
+  - prepare_tests
+  - run_tests
+  - docs
+  - deploy
+  - finalize
 
-before_script:
-  - mkdir workdir
-  - mkdir logs
+.release:
+  before_script:
+    # Make sure release branch is checked out when building a release
+    - export RELEASE=$(echo -n $CI_COMMIT_BRANCH | sed -n 's,^releases/,,p')
+    - |
+      if test -n "$RELEASE"
+      then
+        echo "Checking out branch '$CI_COMMIT_BRANCH'"
+        git checkout $CI_COMMIT_BRANCH
+      fi
 
-after_script:
-  - echo "All done"
+.versioning:
+  image: python
+  before_script:
+    - !reference [.release, before_script]
+    - pip install setuptools_scm
+  script:
+    # Determine the LINC version, and the docker image tag. If RELEASE is set,
+    # check if it adheres to the PEP-440 naming conventions, and if so, set the LINC
+    # version string.  If RELEASE is not set, then determine the LINC version string
+    # using `setuptools_scm`. For the docker image tag, replace all the characters
+    # in the LINC version string that are not allowed in a tag string with a dash.
+    - |
+      if test -n "$RELEASE"
+      then
+        LINC_VERSION=$(python -c 'import os; from packaging.version import Version; print(Version(os.getenv("RELEASE")))')
+      else
+        LINC_VERSION=$(python -m setuptools_scm)
+      fi
+    - LINC_TAG=${LINC_VERSION//[^[:alnum:]_.-]/-}
+    - echo "LINC_VERSION=$LINC_VERSION" >> versions.env
+    - echo "LINC_TAG=$LINC_TAG" >> versions.env
+  artifacts:
+    reports:
+      dotenv: versions.env
+
+.setup_git:
+  image: bitnami/git
+  before_script:
+    # Use a token that has git push permissions. Token must be created in the
+    # GitLab GUI; a project variable must be created containing the token.
+    # See file `DEVELOPER.md` for details.
+    - !reference [.release, before_script]
+    - git config user.email $GITLAB_USER_EMAIL
+    - git config user.name $GITLAB_USER_NAME
+    - git remote set-url origin https://oauth2:$GIT_PUSH_TOKEN@$CI_SERVER_HOST/$CI_PROJECT_PATH.git
+
+.setup_docker:
+  stage: build
+  image: docker
+  tags:
+    - dind
+  before_script:
+    - !reference [.release, before_script]
+    - echo "Logging in as $CI_REGISTRY_USER @ $CI_REGISTRY"
+    - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
+
+.run_tests:
+  stage: run_tests
+  image: $INTEGRATION_IMAGE
+  before_script:
+    - !reference [.release, before_script]
+    - mkdir -p workdir
+    - mkdir -p logs
+
+.deploy:
+  stage: deploy
+  extends: .setup_docker
+  tags:
+    - dind
+  before_script:
+    - !reference [.setup_docker, before_script]
+    - echo "Logging in as $DH_REGISTRY_USER @ DockerHub"
+    - echo $DH_REGISTRY_PASSWORD | docker login -u $DH_REGISTRY_USER --password-stdin
+
+.docs:
+  stage: docs
+  image: $INTEGRATION_IMAGE
+
+.finalize:
+  stage: finalize
+  extends: .setup_git
+
+
+### Stage: initialize
+
+prepare_release:
+  stage: initialize
+  image: python
+  extends: .setup_git
+  rules:
+    # Run this job when current branch is a release branch
+    - if: '$CI_COMMIT_BRANCH =~ /^releases//'
+      when: always
+  artifacts:
+    # Create a marker file if release tag already exists. The presence of this
+    # file is used in the `rollback_release` job, to decide that the existing
+    # tag should not be removed.
+    paths:
+      - .tag.exists
+    untracked: true
+    when: on_failure
+  before_script:
+    - !reference [.versioning, before_script]
+    # When building a release, bail out if release tag already exists
+    - |
+      if git ls-remote --tags --exit-code origin $RELEASE > /dev/null
+      then
+        echo "*** Release '$RELEASE' already exists. Bailing out! ***"
+        touch .tag.exists
+        exit 1
+      fi
+    - !reference [.setup_git, before_script]
+  script:
+    # Use .versioning:script to get LINC_TAG
+    - !reference [.versioning, script]
+    - echo "Updating dockerPull URI to use image tag '$LINC_TAG'"
+    - sed -ri "/dockerPull/s,['\"]?(astronrd/linc).*,\1:$LINC_TAG," steps/*.cwl
+    - git add -u steps/*.cwl
+    # Only commit if there are changes
+    - |
+      if test -n "$(git status -uno --porcelain)"
+      then
+        git commit -m "Tag $RELEASE added to dockerPull URI in CWL steps (by GitLab CI)"
+      fi
+    # Tag current revision
+    - git tag -a $RELEASE -m "Git tag $RELEASE created (by GitLab CI)"
+    # Skip CI on this push
+    - git push --follow-tags -o ci.skip
+
+
+### Stage: versioning
+
+versioning:
+  stage: versioning
+  extends: .versioning
+  script:
+    - !reference [.versioning, script]
+    - ./Docker/fetch_commit_hashes.sh | tee commits.txt >> versions.env
+    # Use hash of commits to determine version of base image (and rebuild if necessary)
+    - echo INTEGRATION_BASE_IMAGE=$CI_REGISTRY_IMAGE/integration_base:$(sha256sum commits.txt | cut -d " " -f 1) >> versions.env
+    - echo INTEGRATION_IMAGE=$CI_REGISTRY_IMAGE/integration_full:$(git log -n 1 --pretty=format:%H) >> versions.env
+    - cat versions.env
+
+
+### Stage: build
+
+build_base:
+  stage: build
+  extends: .setup_docker
+  script:
+    - |
+      if ! docker manifest inspect $INTEGRATION_BASE_IMAGE > /dev/null || [ "$BUILD_DOCKER_IMAGE" = "1" ]; then
+        if [ "$BUILD_DOCKER_IMAGE" = "1" ] ; then
+          DOCKER_CACHE_PARAMETERS="--no-cache"
+        else
+          DOCKER_CACHE_PARAMETERS=""
+        fi
+        docker build \
+          $DOCKER_CACHE_PARAMETERS \
+          --build-arg CASACORE_COMMIT \
+          --build-arg LOFARSTMAN_COMMIT \
+          --build-arg IDG_COMMIT \
+          --build-arg AOFLAGGER_COMMIT \
+          --build-arg EVERYBEAM_COMMIT \
+          --build-arg SAGECAL_COMMIT \
+          --build-arg DP3_COMMIT \
+          --build-arg WSCLEAN_COMMIT \
+          --build-arg PYTHONCASACORE_COMMIT \
+          --build-arg LINC_VERSION \
+          --file Docker/Dockerfile-base \
+          --tag $INTEGRATION_BASE_IMAGE \
+          .
+        docker push $INTEGRATION_BASE_IMAGE
+      fi
+
+
+### Stage: install
+
+install_linc:
+  stage: install
+  extends: .setup_docker
+  script:
+    - |
+      if [ "$BUILD_DOCKER_IMAGE" = "1" ] ; then
+        DOCKER_CACHE_PARAMETERS="--no-cache"
+      else
+        DOCKER_CACHE_PARAMETERS=""
+      fi
+      docker build \
+        $DOCKER_CACHE_PARAMETERS \
+        --build-arg BASE_IMAGE=$INTEGRATION_BASE_IMAGE \
+        --build-arg LINC_VERSION=$LINC_VERSION \
+        --file Docker/Dockerfile-linc \
+        --tag $INTEGRATION_IMAGE \
+        .
+    - docker push $INTEGRATION_IMAGE
+
+
+### Stage: prepare_tests
 
 download_data:
-  stage: download_data
+  image: $INTEGRATION_IMAGE
+  stage: prepare_tests
   script:
     - mkdir data && cd data
-    - wget -q www.astron.nl/citt/ci_data/prefactor/$TEST_HBA_DATASET_NAME -O $TEST_HBA_DATASET_NAME && tar xfz $TEST_HBA_DATASET_NAME && rm -f $TEST_HBA_DATASET_NAME
-    - wget -q www.astron.nl/citt/ci_data/prefactor/$CALIBRATOR_HBA_RESULTS_NAME -O $CALIBRATOR_HBA_RESULTS_NAME && tar xfz $CALIBRATOR_HBA_RESULTS_NAME && rm -f $CALIBRATOR_HBA_RESULTS_NAME
-    - wget -q www.astron.nl/citt/ci_data/prefactor/$TARGET_HBA_RESULTS_NAME -O $TARGET_HBA_RESULTS_NAME && tar xfz $TARGET_HBA_RESULTS_NAME && rm -f $TARGET_HBA_RESULTS_NAME
-    - wget -q https://git.astron.nl/eosc/prefactor3-cwl/-/raw/master/test_data/example.h5 -O example.h5
-    - wget -q https://git.astron.nl/eosc/prefactor3-cwl/-/raw/master/test_data/A-Team_lowres.sourcedb -O A-Team_lowres.sourcedb
+    - wget -nv https://support.astron.nl/software/ci_data/linc/$TEST_HBA_DATASET_NAME -O $TEST_HBA_DATASET_NAME && tar xfz $TEST_HBA_DATASET_NAME && rm -f $TEST_HBA_DATASET_NAME
+    - wget -nv https://support.astron.nl/software/ci_data/linc/$TEST_LBA_DATASET_NAME -O $TEST_LBA_DATASET_NAME && tar xfz $TEST_LBA_DATASET_NAME && rm -f $TEST_LBA_DATASET_NAME
+    - wget -nv https://support.astron.nl/software/ci_data/linc/$CALIBRATOR_HBA_RESULTS_NAME -O $CALIBRATOR_HBA_RESULTS_NAME && tar xfz $CALIBRATOR_HBA_RESULTS_NAME && rm -f $CALIBRATOR_HBA_RESULTS_NAME
+    - wget -nv https://support.astron.nl/software/ci_data/linc/$CALIBRATOR_LBA_RESULTS_NAME -O $CALIBRATOR_LBA_RESULTS_NAME && tar xfz $CALIBRATOR_LBA_RESULTS_NAME && rm -f $CALIBRATOR_LBA_RESULTS_NAME
+    - wget -nv https://support.astron.nl/software/ci_data/linc/$TARGET_HBA_RESULTS_NAME -O $TARGET_HBA_RESULTS_NAME && tar xfz $TARGET_HBA_RESULTS_NAME && rm -f $TARGET_HBA_RESULTS_NAME
+    - wget -nv https://support.astron.nl/software/ci_data/linc/$TARGET_LBA_RESULTS_NAME -O $TARGET_LBA_RESULTS_NAME && tar xfz $TARGET_LBA_RESULTS_NAME && rm -f $TARGET_LBA_RESULTS_NAME
+    - wget -nv https://support.astron.nl/software/ci_data/linc/$TARGET_HBA_SELFCAL_RESULTS_NAME -O $TARGET_HBA_SELFCAL_RESULTS_NAME && tar xfz $TARGET_HBA_SELFCAL_RESULTS_NAME && rm -f $TARGET_HBA_SELFCAL_RESULTS_NAME
+    - wget -nv https://support.astron.nl/software/ci_data/linc/$TARGET_HBA_VLBI_RESULTS_NAME -O $TARGET_HBA_VLBI_RESULTS_NAME && tar xfz $TARGET_HBA_VLBI_RESULTS_NAME && rm -f $TARGET_HBA_VLBI_RESULTS_NAME
   artifacts:
     paths:
-    - data
+      - data
+
+
+### Stage: run_tests
 
 validate_scripts:
-  stage: validate_cwl_scripts
+  extends: .run_tests
   script:
-    - for file in `find . -name 'steps/*.cwl'`; do cwltool --validate $file; done
+    - |
+      errors=0
+      for file in $(find steps -name "*.cwl")
+      do
+        cwltool --validate $file || ((errors++))
+      done
+      ((errors == 0))
 
 blsmooth:
-  stage: test_steps
-  allow_failure: true
-  needs: ["download_data"]
+  extends: .run_tests
   script:
     - cwltool --no-container steps/blsmooth.cwl test_jobs/blsmooth.json
 
 find_skymodel_cal:
-  stage: test_steps
-  allow_failure: true
-  needs: ["download_data"]
+  extends: .run_tests
   script:
     - cwltool --no-container --preserve-environment PYTHONPATH steps/find_skymodel_cal.cwl test_jobs/find_skymodel_cal.json
 
 check_ateam_separation:
-  stage: test_steps
-  allow_failure: true
-  needs: ["download_data"]
+  extends: .run_tests
   script:
-    - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/check_ateam_separation.cwl test_jobs/check_ateam_separation.json
+    - cwltool --no-container --preserve-environment PATH --preserve-environment PYTHONPATH steps/check_ateam_separation.cwl test_jobs/check_ateam_separation.json
 
 run_hba_calibrator:
-  stage: test_full
-  allow_failure: true
-  needs: ["download_data"]
+  extends: .run_tests
   script:
-    - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH workflows/HBA_calibrator.cwl test_jobs/HBA_calibrator.json
-    - test_jobs/check_workflow_results.py results /builds/eosc/prefactor3-cwl/data/results_calibrator
+    - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_calibrator/ workflows/HBA_calibrator.cwl test_jobs/HBA_calibrator.json
+    - test_jobs/check_workflow_results.py results $CI_PROJECT_DIR/data/results_calibrator
+  after_script:
+    - find /tmp/run_hba_calibrator -name "*.log" -print0 | tar czf hba_calibrator_logs.tar.gz --null -T -
+  artifacts:
+    paths:
+      - hba_calibrator_logs.tar.gz
+    when: on_failure
 
 run_hba_target:
-  stage: test_full
-  allow_failure: true
-  needs: ["download_data"]
+  extends: .run_tests
+  script:
+    - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_target/ workflows/HBA_target.cwl test_jobs/HBA_target.json
+    - test_jobs/check_workflow_results.py results $CI_PROJECT_DIR/data/results_target
+  after_script:
+    - find /tmp/run_hba_target -name "*.log" -print0 | tar czf hba_target_logs.tar.gz --null -T -
+    - find /tmp/run_hba_target -name "*.png" -print0 | tar czf inspection.tar.gz --null -T -
+    - find $CI_PROJECT_DIR/results/ -name "cal_solutions.h5" -print0 | tar czf cal_solutions.tar.gz --null -T -
+  artifacts:
+    paths:
+      - hba_target_logs.tar.gz
+      - inspection.tar.gz
+      - cal_solutions.tar.gz
+    when: on_failure
+
+run_hba_target_selfcal:
+  extends: .run_tests
+  script:
+    - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_target/ workflows/HBA_target.cwl test_jobs/HBA_target_selfcal.json
+    - test_jobs/check_workflow_results.py --skip_soltabs TGSSphase_final results $CI_PROJECT_DIR/data/results_target_selfcal
+  after_script:
+    - find /tmp/run_hba_target -name "*.log" -print0 | tar czf hba_target_selfcal_logs.tar.gz --null -T -
+    - find /tmp/run_hba_target -name "*.png" -print0 | tar czf inspection.tar.gz --null -T -
+    - find $CI_PROJECT_DIR/results/ -name "cal_solutions.h5" -print0 | tar czf cal_solutions.tar.gz --null -T -
+  artifacts:
+    paths:
+      - hba_target_selfcal_logs.tar.gz
+      - inspection.tar.gz
+      - cal_solutions.tar.gz
+    when: on_failure
+
+run_hba_target_vlbi:
+  extends: .run_tests
+  script:
+    - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_target/ workflows/HBA_target_VLBI.cwl test_jobs/HBA_target.json
+    - test_jobs/check_workflow_results.py results $CI_PROJECT_DIR/data/results_target_VLBI
+  after_script:
+    - find /tmp/run_hba_target -name "*.log" -print0 | tar czf hba_target_vlbi_logs.tar.gz --null -T -
+    - find /tmp/run_hba_target -name "*.png" -print0 | tar czf inspection.tar.gz --null -T -
+    - find $CI_PROJECT_DIR/results/ -name "cal_solutions.h5" -print0 | tar czf cal_solutions.tar.gz --null -T -
+  artifacts:
+    paths:
+      - hba_target_vlbi_logs.tar.gz
+      - inspection.tar.gz
+      - cal_solutions.tar.gz
+    when: on_failure
+
+run_lba_calibrator:
+  extends: .run_tests
+  script:
+    - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_lba_calibrator workflows/LBA_calibrator.cwl test_jobs/LBA_calibrator.json
+    - test_jobs/check_workflow_results.py results $CI_PROJECT_DIR/data/results_calibrator_lba
+  after_script:
+    - find /tmp/run_lba_calibrator -name "*.log" -print0 | tar czf lba_calibrator_logs.tar.gz --null -T -
+  artifacts:
+    paths:
+      - lba_calibrator_logs.tar.gz
+    when: on_failure
+
+run_lba_target:
+  extends: .run_tests
+  script:
+    - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_lba_target/ workflows/LBA_target.cwl test_jobs/LBA_target.json
+    - test_jobs/check_workflow_results.py --skip_soltabs GSMtec_final results $CI_PROJECT_DIR/data/results_target_lba
+  after_script:
+    - find /tmp/run_lba_target -name "*.log" -print0 | tar czf lba_target_logs.tar.gz --null -T -
+    - find /tmp/run_lba_target -name "*.png" -print0 | tar czf inspection.tar.gz --null -T -
+    - find $CI_PROJECT_DIR/results/ -name "cal_solutions.h5" -print0 | tar czf cal_solutions.tar.gz --null -T -
+  artifacts:
+    paths:
+      - lba_target_logs.tar.gz
+      - inspection.tar.gz
+      - cal_solutions.tar.gz
+    when: on_failure
+
+
+### Stage: docs
+
+build_doc:
+  extends: .docs
+  before_script:
+    - apt-get update
+    - apt-get install -y make
+    - pip install sphinx
+  script:
+    - cd docs
+    - make html
+  artifacts:
+    paths:
+      - docs/build/html
+  rules:
+    # Only add job for commits to the default branch or on merge request events
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE != "schedule"'
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+    - changes:
+        - docs/**/*
+
+
+### Stage: deploy
+
+deploy_docker:
+  stage: deploy
+  extends: .deploy
+  script:
+    - echo "Deploying to DockerHub, using tag '$LINC_TAG'"
+    - docker pull $INTEGRATION_IMAGE
+    - docker tag $INTEGRATION_IMAGE astronrd/linc:$LINC_TAG
+    - docker tag $INTEGRATION_IMAGE astronrd/linc:latest
+    - docker push astronrd/linc:$LINC_TAG
+    - docker push astronrd/linc:latest
+  rules:
+    # Run on the default branch or on a release branch
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+    - if: '$CI_COMMIT_BRANCH =~ /^releases//'
+      when: on_success
+
+deploy_docker_tag_stable:
+  stage: deploy
+  extends: .deploy
+  when: manual
+  script:
+    - docker pull $INTEGRATION_IMAGE
+    - docker tag $INTEGRATION_IMAGE astronrd/linc:stable
+    - docker push astronrd/linc:stable
+
+
+### Stage: finalize
+
+rollback_release:
+  stage: finalize
+  extends: .finalize
+  rules:
+    # Run this job if the pipeline fails, to undo changes made in prepare_release.
+    # We only care about removing the tag; other changes can remain.
+    - if: '$CI_COMMIT_BRANCH =~ /^releases//'
+      when: on_failure
+  script:
+    - echo "Rolling back release $RELEASE"
+    - |
+      if test -f .tag.exists
+      then
+        echo "*** Not removing existing tag '$RELEASE'! ***"
+      else
+        git push origin -d $RELEASE
+      fi
+
+finalize_release:
+  stage: finalize
+  extends: .finalize
+  rules:
+    # Run this job if the pipeline succeeds, to create a versioned release.
+    # A versioned release is a release whose branch/tag name matches a string
+    # that looks like "v<major-version>[.<minor_version>][...]".
+    # We want to ensure that the release tag also exists on the default branch,
+    # so that `git describe --tags` yields the desired output. Hence, we must
+    # undo the changes made in the `prepare_release` job, before merging to
+    # the default branch.
+    - if: '$CI_COMMIT_BRANCH =~ /^releases\/v[0-9]+(\.[0-9]+)*/'
+      when: on_success
   script:
-    - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH workflows/HBA_target.cwl test_jobs/HBA_target.json
-    - test_jobs/check_workflow_results.py results /builds/eosc/prefactor3-cwl/data/results_target
+    - echo "Finalizing release $RELEASE"
+    # Update dockerPull image URI in CWL steps by removing version tag
+    - sed -ri "/dockerPull/s,(astronrd/linc).*,\1," steps/*.cwl
+    - git add -u steps/*.cwl
+    # Only commit if there are changes
+    - |
+      if test -n "$(git status -uno --porcelain)"
+      then
+        git commit -m "Tag removed from dockerPull URI in CWL steps (by GitLab CI)"
+      fi
+    # Skip CI on this push
+    - git push -o ci.skip
+    # Next switch to the default branch and make sure it's up to date
+    - git checkout $CI_DEFAULT_BRANCH
+    - git pull
+    - echo "Merging changes from $CI_COMMIT_BRANCH into $CI_DEFAULT_BRANCH"
+    # Merge release branch into the default branch
+    - git merge $CI_COMMIT_BRANCH -m"Merged release branch into default branch (by Gitlab CI)"
+    # Skip CI on this push
+    - git push -o ci.skip

.readthedocs.yaml

+# Read the Docs configuration file for Sphinx projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the OS, Python version and other tools you might need
+build:
+  os: ubuntu-22.04
+  jobs:
+    # See https://github.com/readthedocs/readthedocs.org/issues/5989
+    post_checkout:
+      - git fetch --unshallow
+  tools:
+    python: "3.12"
+    # You can also specify other tool versions:
+    # nodejs: "20"
+    # rust: "1.70"
+    # golang: "1.20"
+
+# Build documentation in the "docs/" directory with Sphinx
+sphinx:
+  configuration: docs/source/conf.py
+  # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs
+  # builder: "dirhtml"
+  # Fail on all warnings to avoid broken references
+  # fail_on_warning: true
+
+# Optionally build your docs in additional formats such as PDF and ePub
+# formats:
+#   - pdf
+#   - epub
+
+# Optional but recommended, declare the Python requirements required
+# to build your documentation
+# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+  install:
+    - requirements: docs/requirements.txt
+    # LINC needs to be installed to generate the documentation.
+    - method: pip
+      path: .

DEVELOPER.md

+# Developer notes
+
+## Creating a release
+
+Creating a LINC release is a bit more involved than simply pressing a button in the GitLab GUI. The main reason is that all the CWL files that contain a `dockerPull` line need to be updated on-the-fly to contain the proper reference to the release. This is taken care of by the CI/CD pipeline, but it is good to understand how this pipeline works under the hood. First, let's properly configure the GitLab project.
+
+### Configure the project in the GitLab GUI
+
+The CI/CD pipelines needs to have push rights to the Git repository. We use an access token to grant these rights. The following preparations need to be done only once (though they have to be repeated when the access token expires):
+
+- Create a project access token (`Settings > Access tokens`), and grant it the `Maintainer` role and the `write_repository` scope. Copy the token directly after you've created it, because there is no way to retrieve it later on.
+- Create a project variable (`Settings > CI/CD > Variables`) named `GIT_PUSH_TOKEN`, and set its value to the access token that you just created (and copied!). Mark the variable as masked and protected. This variable will be used when a release is created (see `.gitlab-ci.yml` file).
+
+**NOTE**: The name of the access token will be used in the `Created by` column in the `Pipelines` view for pipelines that were triggered as a result of an automatic `git push` action.
+
+### What does the CI/CD pipeline consider a release?
+
+The CI/CD pipeline will consider every branch whose name starts with `releases/` a release branch. To create a release named `<my-release>`, you need to manually create a branch named `releases/<my-release>`.
+
+**NOTE**:`<my-release>` must be a valid name according to the [PEP-440 versioning scheme](
+https://peps.python.org/pep-0440/#version-scheme). For example, `v5.0rc1` is a valid name, whereas `ldv_404` is not. However, you can add a local identifier to the name, e.g. `v5.0rc1+ldv_404`.
+
+### What does the CI/CD pipeline do with a release branch?
+
+When the CI/CD pipeline is run on a release branch, it will, in addition to a normal CI/CD pipeline run, do the following:
+
+- update all the CWL files containing a `dockerPull` line with the correct image tag, and commit the changes
+- tag the release using `<my-release>` as tag
+- deploy the docker image to [Docker Hub](https://hub.docker.com/repository/docker/astronrd/linc)
+- if `<my-release>` is a versioned release:[^1]
+  - undo the changes to the CWL files, and commit
+  - merge the changes into the default branch
+
+[^1]: A versioned release is a release whose name starts with a `V` or `v`.
+
+The rationale behind the two extra steps for a versioned release is that we want the git tag to be visible on the default branch, so that `git describe --tags` will return the desired result.
+
+**NOTE**: If a tag `<my-release>` already exists, the pipeline will bail out with an error. In order to let it run successfully, the tag `<my-release>` needs to be removed manually in the GitLab GUI. This is to avoid that an existing release will be clobbered!
+
+### Creating the release in the GitLab GUI
+
+Once the CI/CD pipeline has run, and the new release has been tagged, a new release can be created in the GitLab GUI. Navigate to "Deploy | Releases", click on the "Create new release" button, and proceed as you would normally do when creating a new release.

Docker/Dockerfile

-FROM lofareosc/lofar-pipeline:latest
-USER root
-ENV PREFACTOR_VERSION='master' \
-    PREFACTOR_NAME=prefactor \
-    LC_ALL=C.UTF-8 \
-    PREFACTOR_DATA_ROOT=/usr/local/share/prefactor \
-    PYTHONPATH="/usr/local/bin:${PYTHONPATH}"
-
-RUN \
-  apt-get update && \
-  apt-get install -y vim wget casacore-tools && \
-  rm -rf /var/cache/apt 
-RUN \
-  pip3 install --upgrade pip && \
-  git clone https://github.com/lofar-astron/prefactor.git ${PREFACTOR_NAME} && \
-  cd ${PREFACTOR_NAME} && \
-  git checkout ${PREFACTOR_VERSION} && \
-  git pull && \
-  pip3 install --upgrade $PWD && \
-  cd .. && \
-  rm -rfv ${PREFACTOR_NAME}
-
-USER lofaruser
-

Docker/Dockerfile-base

-FROM kernsuite/base:6 as builder
-SHELL ["/bin/bash", "-c"]
+# This Dockerfile builds the base image for the LINC pipeline,
+# containing all its dependencies.
+
+FROM ubuntu:22.04 AS builder
+
+# Set default versions. Can be overridden from the command-line
+ARG AOFLAGGER_COMMIT=master
+ARG CASACORE_COMMIT=master
+ARG DP3_COMMIT=master
+ARG EVERYBEAM_COMMIT=master
+ARG IDG_COMMIT=master
+ARG LOFARSTMAN_COMMIT=main
+ARG PYTHONCASACORE_COMMIT=master
+ARG SAGECAL_COMMIT=master
+ARG WSCLEAN_COMMIT=master
+
+# Build binaries targeted at the oldest CPU that supports AVX2
+ARG TARGET_CPU=haswell
 
-RUN export DEBIAN_FRONTEND=noninteractive && \ 
+# Suppress warning from pip when installing packages as root
+ENV PIP_ROOT_USER_ACTION=ignore
+
+# Set our working directory
+WORKDIR /src
+
+# Install all build-time dependencies (+pip for debugging)
+RUN export DEBIAN_FRONTEND=noninteractive && \
     apt-get update && \
     apt-get install -y \
-        gfortran \ 
-        flex \
         bison \
-        wcslib-dev \        
-        libncurses5-dev \
-        casacore-data \
-        casacore-dev \
-        libboost-python-dev \
-        libcfitsio-dev \
-        python-dev \
-        python3-numpy \
-        libcasa* \
-        cmake \
         build-essential \
-        liblua5.3-dev \
-        libhdf5-serial-dev \
-        libarmadillo-dev \
-        libboost-filesystem-dev \
-        libboost-system-dev \
+        flex \
+        gfortran \
+        git \
+        libblas-dev \
         libboost-date-time-dev \
+        libboost-filesystem-dev \
         libboost-numpy-dev \
-        libboost-signals-dev \ 
         libboost-program-options-dev \
+        libboost-python-dev \
+        libboost-system-dev \
         libboost-test-dev \
-        pybind11-dev   \
-        libxml2-dev \
+        libcfitsio-dev \
+        libfftw3-dev \
+        libgsl-dev \
+        libgtkmm-3.0-dev \
+        libhdf5-serial-dev \
+        liblapack-dev \
+        liblua5.3-dev \
+        libncurses5-dev \
         libpng-dev \
+        libpython3-dev \
         pkg-config \
-        libgtkmm-3.0-dev \
-        git \
-        wget \
-        libfftw3-dev \
-        libgsl-dev
-        
-RUN mkdir -p /src/
-WORKDIR /src/
-
-RUN \
-  git clone https://github.com/lofar-astron/LofarStMan && \
-  mkdir LofarStMan/build && \
-  cd LofarStMan/build && \
-  cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPORTABLE=True && \
-  make install -j`nproc --all`
-
-RUN \
-  git clone https://github.com/aroffringa/dysco.git && \
-  mkdir dysco/build && \
-  cd dysco/build && \
-  cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPORTABLE=True && \
-  make install -j`nproc --all`
-    
-RUN \
-  git clone https://git.astron.nl/RD/idg.git && \
-  mkdir idg/build && \
-  cd idg/build && \
-  cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ && \
-  make install -j`nproc --all`
-
-RUN \
-  git clone https://gitlab.com/aroffringa/aoflagger.git && \
-  mkdir aoflagger/build && \
-  cd aoflagger/build && \
-  cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPYTHON_EXECUTABLE=/usr/bin/python3.6 -DPORTABLE=True && \
-  make install -j`nproc --all`
-
-RUN \
-  git clone https://github.com/lofar-astron/LOFARBeam.git && \
-  mkdir LOFARBeam/build && \
-  cd LOFARBeam/build && \
-  cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPYTHON_EXECUTABLE=/usr/bin/python3.6 && \
-  make install -j`nproc --all`
-
-RUN \
-  git clone https://git.astron.nl/RD/EveryBeam.git && \
-  mkdir EveryBeam/build && \
-  cd EveryBeam/build && \
-  cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ && \
-  make install -j`nproc --all`
-    
-RUN \
-  git clone https://git.astron.nl/RD/DP3.git && \
-  mkdir DP3/build && \
-  cd DP3/build && \
-  cmake ../ \
-    -DCMAKE_INSTALL_PREFIX=/usr/local/ \
-    -DPYTHON_EXECUTABLE=/usr/bin/python3.6 \
-    -DPORTABLE=True \
-    -DAOFlagger_DIR=/usr/local/ && \
-  make install -j`nproc --all`
-
-RUN \
-  git clone https://gitlab.com/aroffringa/wsclean.git && \
-  mkdir wsclean/build && \
-  cd wsclean/build && \
-  cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPORTABLE=True && \
-  make install -j`nproc --all`
+        python3 \
+        python3-casacore \
+        python3-numpy \
+        wcslib-dev \
+        wget
+
+# Do not use `pip` from the Debian repository, but fetch it from PyPA.
+# This way, we are sure that the latest versions of `pip`, `setuptools`, and
+# `wheel` are installed in /usr/local, the only directory we're going to copy
+# over to the next build stage.
+RUN wget -q https://bootstrap.pypa.io/get-pip.py && \
+    python3 get-pip.py
+
+# Install required python packages
+RUN python3 -m pip install --no-cache-dir --upgrade \
+    cmake \
+    nodejs-wheel \
+    toil[cwl]
+
+# Install the casacore measures data. We purposely do not install these from
+# the Ubuntu repository, but download the latest version directly from the
+# ASTRON ftp site.
+# Note: The file on the ftp site is updated daily. When warnings regarding
+# leap seconds appear, ignore them or regenerate the docker image.
+RUN mkdir -p /usr/local/share/casacore/data && \
+    wget -qO - ftp://ftp.astron.nl/outgoing/Measures/WSRT_Measures.ztar | \
+        tar -C /usr/local/share/casacore/data -xzf -
+
+RUN git clone --no-checkout \
+        https://github.com/casacore/casacore.git && \
+    mkdir casacore/build && \
+    cd casacore && git checkout ${CASACORE_COMMIT} && \
+    cd build && \
+    cmake .. && \
+    make install -j`nproc`
+
+RUN git clone --no-checkout \
+        https://github.com/lofar-astron/LofarStMan && \
+    mkdir LofarStMan/build && \
+    cd LofarStMan && git checkout ${LOFARSTMAN_COMMIT} && \
+    cd build && \
+    cmake .. -DPORTABLE=TRUE && \
+    make install -j`nproc`
+
+RUN git clone --no-checkout \
+        https://git.astron.nl/RD/idg.git && \
+    mkdir idg/build && \
+    cd idg && git checkout ${IDG_COMMIT} && \
+    cd build && \
+    cmake .. -DTARGET_CPU=${TARGET_CPU} && \
+    make install -j`nproc`
+
+RUN git clone --no-checkout \
+        https://gitlab.com/aroffringa/aoflagger.git && \
+    mkdir aoflagger/build && \
+    cd aoflagger && git checkout ${AOFLAGGER_COMMIT} && \
+    cd build && \
+    cmake .. -DTARGET_CPU=${TARGET_CPU} && \
+    make install -j`nproc`
+
+RUN git clone --no-checkout \
+        https://git.astron.nl/RD/EveryBeam.git && \
+    mkdir EveryBeam/build && \
+    cd EveryBeam && git checkout ${EVERYBEAM_COMMIT} && \
+    cd build && \
+    cmake .. -DTARGET_CPU=${TARGET_CPU} && \
+    make install -j`nproc`
+
+RUN git clone --no-checkout \
+        https://github.com/nlesc-dirac/sagecal && \
+    mkdir sagecal/build && \
+    cd sagecal && git checkout ${SAGECAL_COMMIT} && \
+    cd build && \
+    cmake .. -DLIB_ONLY=1 && \
+    make install -j`nproc`
+
+RUN git clone --no-checkout \
+        https://git.astron.nl/RD/DP3.git && \
+    mkdir DP3/build && \
+    cd DP3 && git checkout ${DP3_COMMIT} && \
+    cd build && \
+    cmake .. -DTARGET_CPU=${TARGET_CPU} -DLIBDIRAC_PREFIX=/usr/local/ && \
+    make install -j`nproc`
+
+RUN git clone --no-checkout \
+        https://gitlab.com/aroffringa/wsclean.git && \
+    mkdir wsclean/build && \
+    cd wsclean && git checkout ${WSCLEAN_COMMIT} && \
+    cd build && \
+    cmake .. -DTARGET_CPU=${TARGET_CPU} && \
+    make install -j`nproc`
+
+RUN git clone --no-checkout \
+    https://github.com/casacore/python-casacore.git && \
+    cd python-casacore && git checkout ${PYTHONCASACORE_COMMIT} && \
+    CASACORE_DATA=/usr/local/share/casacore/data python3 -m pip install -v .
+
+# We need to `pip install` EveryBeam separately, because the C++ build
+# does not produce a proper Python package.
+RUN cd EveryBeam && pip install -v .
+
+# Install current version of LINC. By adding this to the base image
+# we can speed up the build of the final image, because all of LINC's
+# dependencies will have been installed already and (probably) don't need to
+# be updated.
+COPY . linc
+
+# Note: LINC_VERSION should be provided as build argument
+ARG LINC_VERSION=0.0.0
+RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_LINC=${LINC_VERSION} \
+    python3 -m pip install --upgrade ./linc
+
 
 #---------------------------------------------------------------------------
-# The image is now rebuild without adding the source, to get a smaller image
-# Further binary and/or Python dependencies are also added here.
+# The image will now be rebuilt without adding the sources, in order to
+# reduce the size of the image.
 #---------------------------------------------------------------------------
-FROM kernsuite/base:6 as runner
+FROM ubuntu:22.04 AS runner
 
-COPY --from=builder /usr/local/ /usr/local
+COPY --from=builder /usr/local /usr/local
 RUN chmod +rx /usr/local/bin/*
 
 SHELL ["/bin/bash", "-c"]
 
+# Set default versions. Can be overridden from the command-line
+ARG AOFLAGGER_COMMIT=master
+ARG CASACORE_COMMIT=master
+ARG DP3_COMMIT=master
+ARG EVERYBEAM_COMMIT=master
+ARG IDG_COMMIT=master
+ARG LOFARSTMAN_COMMIT=main
+ARG PYTHONCASACORE_COMMIT=master
+ARG SAGECAL_COMMIT=master
+ARG WSCLEAN_COMMIT=master
+
+# Add version information to the metadata of the image
+LABEL \
+    nl.astron.linc.aoflagger.version=${AOFLAGGER_COMMIT} \
+    nl.astron.linc.casacore.version=${CASACORE_COMMIT} \
+    nl.astron.linc.dp3.version=${DP3_COMMIT} \
+    nl.astron.linc.everybeam.version=${EVERYBEAM_COMMIT} \
+    nl.astron.linc.idg.version=${IDG_COMMIT} \
+    nl.astron.linc.lofarstman.version=${LOFARSTMAN_COMMIT} \
+    nl.astron.linc.python-casacore.version=${PYTHONCASACORE_COMMIT} \
+    nl.astron.linc.sagecal.version=${SAGECAL_COMMIT} \
+    nl.astron.linc.wsclean.version=${WSCLEAN_COMMIT}
+
+# Only install run-time required packages
 RUN export DEBIAN_FRONTEND=noninteractive && \
     apt-get update && \
     apt-get install -y \
-        casacore-data \
-        wcslib-dev \        
-        libarmadillo8 \
-        bison \
-        libncurses5 \ 
-        flex \
-        libboost-date-time1.65.1 \
-        libboost-filesystem1.65.1 \
-        libboost-numpy1.65.1 \
-        libboost-python1.65.1 \
-        libboost-program-options1.65.1 \
-        libboost-system1.65.1 \
-        libboost-signals1.65.1 \
-        libboost-test1.65.1 \
-        libboost-python1.65-dev \
-        libstationresponse3 \
-        liblua5.3-dev \
-        libcasa-* \
-        pybind11-dev   \
-        libcfitsio5 \
-        libcfitsio-dev \
-        libgtkmm-3.0 \
-        libfftw3-3 \
-        libhdf5-cpp-100 \
+        gdb \
+        git \
+        libatkmm-1.6-1v5 \
+        libblas3 \
+        libboost-filesystem1.74.0 \
+        libboost-program-options1.74.0 \
+        libboost-python1.74.0 \
+        libcairomm-1.0-1v5 \
+        libcfitsio9 \
+        libfftw3-double3 \
+        libfftw3-single3 \
+        libgfortran5 \
+        libglib2.0-0 \
+        libglibmm-2.4-1v5 \
+        libgomp1 \
+        libgsl27 \
+        libgtkmm-3.0-1v5 \
+        libhdf5-103-1 \
+        libhdf5-cpp-103-1 \
+        liblapack3 \
+        liblua5.3-0 \
+        libpangomm-1.4-1v5 \
         libpng16-16 \
-        libxml2 \
-        python3.7 \
-        python3-casacore \
-	dysco \
-        python3-numpy \
-        python3-scipy && \
+        libpython3.10 \
+        libsigc++-2.0-0v5 \
+        libstdc++6 \
+        python3 \
+        python3-distutils \
+        rsync \
+        wget && \
     rm -rf /var/lib/apt/lists/*
 
-ARG CASACORE_TAG=master
-LABEL casacore.version.tag=${CASACORE_TAG}
-    
-ARG TOIL_VERSION=3.20.0
-LABEL toil.version=${TOIL_VERSION}
-
-ARG LSMTool_TAG=master
-LABEL LSMTool.version.tag=${LSMTool_TAG}
-
-ARG LoSoTo_TAG=master
-LABEL LoSoTo.version.tag=${LoSoTo_TAG}
-
-ARG STMAN_TAG=master
-LABEL STMAN.version.tag=${STMAN_TAG}
-
-ARG Dysco_TAG=master
-LABEL Dysco.version.tag=${Dysco_TAG}
-
-ARG LofarBeam_TAG=master
-LABEL LOFARBeam.version.tag=${LofarBeam_TAG}
-
-ARG DP3_TAG=master
-LABEL DP3.version.tag=${DP3_TAG}
-
-ARG IDG_TAG=master
-LABEL IDG.version.tag=${IDG_TAG}
-
-ARG Everybeam_TAG
-LABEL Everybeam.version.tag=${Everybeam_TAG}
-
-ARG Aoflagger_TAG
-LABEL Aoflagger.version.tag=${Aoflagger_TAG}
-
-ARG RMextract_TAG
-LABEL RMextract.version.tag=${RMExtract_TAG}
-
-ARG wsclean_TAG
-LABEL wsclean.version.tag=${wsclean_TAG}
-    
-RUN \
-  apt-get --allow-releaseinfo-change update -y && \
-  apt-get install -y nodejs python3-pip git gfortran && \
-  python3 \
-    -m pip install cwltool cwl-runner \
-    -e "git://github.com/darafferty/LSMTool.git@${LSMTool_TAG}#egg=LSMTool" \
-       "toil[cwl]"==${TOIL_VERSION} \
-       matplotlib && \
-  python3 \
-    -m pip install \
-    -e "git+https://github.com/revoltek/losoto.git@${LoSoTo_TAG}#egg=LoSoTo" \
-    ipython "git+https://github.com/lofar-astron/RMextract.git@${RMextract_TAG}#egg=RMextract"
-
-RUN \
-  mkdir /home/lofaruser/ && \ 
-  groupadd -r lofaruser && \
-  useradd --no-log-init -r -g lofaruser lofaruser && \
-  chown lofaruser:lofaruser /home/lofaruser
-
-WORKDIR /home/lofaruser
-
-USER lofaruser
-ENV PYTHONPATH=/usr/local/lib/python3.6/site-packages/
-
+# Try to run the compiled tools to make sure they run without
+# a problem (e.g. no missing libraries).
+RUN aoflagger --version && \
+    DP3 --version && \
+    wsclean --version

Docker/Dockerfile-linc

+ARG BASE_IMAGE
+FROM $BASE_IMAGE
+
+ENV EVERYBEAM_DATADIR=/usr/local/share/everybeam \
+    LINC_DATA_ROOT=/usr/local/share/linc \
+    PYTHONPATH=/usr/local/bin
+
+# Install LINC
+COPY . /tmp/linc
+
+# Note: LINC_VERSION should be provided as build argument
+ARG LINC_VERSION=0.0.0
+RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_LINC=${LINC_VERSION} \
+    python3 -m pip install --no-cache-dir --upgrade /tmp/linc && \
+    rm -rf /tmp/linc

Docker/build_docker.sh

-#! /bin/bash
-set -e
+#!/bin/bash
+#
+# This script build the base docker image and the full docker image for LINC.
+# It is inspired by the way the docker images are built in the CI/CD pipeline
+# (described in the `.gitlab-ci.yml` file).
 
-DOCKER_TAG=latest
+set -euo pipefail
 
-docker build ${PWD} -f Dockerfile-base -t lofareosc/lofar-pipeline:${DOCKER_TAG}
-docker push lofareosc/lofar-pipeline:${DOCKER_TAG}
+SCRIPT_DIR=$(cd "$(dirname "${0}")" && pwd)
+REPO_ROOT="$(git rev-parse --show-toplevel)"
 
-docker build --no-cache ${PWD} -t lofareosc/prefactor3-cwl:${DOCKER_TAG}
-docker push lofareosc/prefactor3-cwl:${DOCKER_TAG}
+echo "Determining version information ..."
+eval $(${SCRIPT_DIR}/fetch_commit_hashes.sh | tee commits.txt)
+
+BASE_NAME=linc-base
+BASE_TAG=$(sha256sum commits.txt | cut -d" " -f1)
+BASE_IMAGE=${BASE_NAME}:${BASE_TAG}
+FULL_NAME=linc-full
+FULL_TAG=$(git log -n 1 --pretty=format:%H)
+FULL_IMAGE=${FULL_NAME}:${FULL_TAG}
+LINC_VERSION=$(${SCRIPT_DIR}/get_scm_version.sh)
+LINC_NAME=astronrd/linc
+LINC_TAG=${LINC_VERSION//[^[:alnum:]_.-]/-}
+LINC_IMAGE=${LINC_NAME}:${LINC_TAG}
+
+# Build the base image, using the previously determined git commit hashes for
+# the packages we need to build
+docker build \
+  --build-arg CASACORE_COMMIT=${CASACORE_COMMIT} \
+  --build-arg LOFARSTMAN_COMMIT=${LOFARSTMAN_COMMIT} \
+  --build-arg IDG_COMMIT=${IDG_COMMIT} \
+  --build-arg AOFLAGGER_COMMIT=${AOFLAGGER_COMMIT} \
+  --build-arg EVERYBEAM_COMMIT=${EVERYBEAM_COMMIT} \
+  --build-arg SAGECAL_COMMIT=${SAGECAL_COMMIT} \
+  --build-arg DP3_COMMIT=${DP3_COMMIT} \
+  --build-arg WSCLEAN_COMMIT=${WSCLEAN_COMMIT} \
+  --build-arg PYTHONCASACORE_COMMIT=${PYTHONCASACORE_COMMIT} \
+  --build-arg LINC_VERSION=${LINC_VERSION} \
+  --progress plain \
+  --file ${SCRIPT_DIR}/Dockerfile-base \
+  --tag ${BASE_IMAGE} \
+  ${REPO_ROOT}
+
+# Build the full image, taking the base image as starting point
+docker build \
+  --build-arg BASE_IMAGE=${BASE_IMAGE} \
+  --build-arg LINC_VERSION=${LINC_VERSION} \
+  --progress plain \
+  --file ${SCRIPT_DIR}/Dockerfile-linc \
+  --tag ${FULL_IMAGE} \
+  ${REPO_ROOT}
+
+# Tag the full image with LINC version number
+docker tag \
+  ${FULL_IMAGE} \
+  ${LINC_IMAGE}
+
+# Tag the full image as latest
+docker tag \
+  ${FULL_IMAGE} \
+  latest

Docker/fetch_commit_hashes.sh

+#!/bin/sh
+git ls-remote https://gitlab.com/aroffringa/aoflagger.git     HEAD   | awk '{ print "AOFLAGGER_COMMIT="$1 }'
+git ls-remote https://github.com/casacore/casacore.git        v3.6.1 | awk '{ print "CASACORE_COMMIT="$1 }'
+git ls-remote https://git.astron.nl/RD/DP3.git                HEAD   | awk '{ print "DP3_COMMIT="$1 }'
+git ls-remote https://git.astron.nl/RD/EveryBeam.git          HEAD   | awk '{ print "EVERYBEAM_COMMIT="$1 }'
+git ls-remote https://git.astron.nl/RD/idg.git                HEAD   | awk '{ print "IDG_COMMIT="$1 }'
+git ls-remote https://github.com/lofar-astron/LofarStMan.git  HEAD   | awk '{ print "LOFARSTMAN_COMMIT="$1 }'
+git ls-remote https://github.com/casacore/python-casacore.git v3.6.1 | awk '{ print "PYTHONCASACORE_COMMIT="$1 }'
+git ls-remote https://github.com/nlesc-dirac/sagecal.git      HEAD   | awk '{ print "SAGECAL_COMMIT="$1 }'
+git ls-remote https://gitlab.com/aroffringa/wsclean.git       HEAD   | awk '{ print "WSCLEAN_COMMIT="$1 }'

Docker/get_scm_version.sh

+#!/bin/bash -e
+#
+# shellcheck disable=SC1091
+#
+# Get version information from the Git repository, using the python package
+# `setuptools_scm`.
+
+# Determine top-level directory of the repository, and change directory to it.
+ROOT=$(git -C "$(dirname "${0}")" rev-parse --show-toplevel)
+cd "${ROOT}"
+
+# Check if module `setuptools_scm` is avaible.
+if ! python3 -c "import setuptools_scm" 2>/dev/null
+then
+  # Are we inside a virtual environment?
+  if [ -z "${VIRTUAL_ENV}" ]
+  then
+    # Activate virtual environment if it exists in `venv` directory,
+    # else create one first.
+    if ! [ -r "venv/bin/activate" ]
+    then
+      python3 -m venv venv
+    fi
+    . venv/bin/activate
+  fi
+  # Check again if module `setuptools_scm` is avaible, now in the virtualenv
+  # If not, install it
+  if ! python3 -c "import setuptools_scm" 2>/dev/null
+  then
+    pip install --disable-pip-version-check --quiet --require-virtualenv setuptools_scm
+  fi
+fi
+
+# Run the command to get the version information
+python3 -c "from setuptools_scm import get_version; print(get_version())"

Docker/losoto.patch

----  /src/losoto/losoto/operations/__init__.py      2019-11-07 12:43:46.220577945 +0000
-+++  /src/losoto/losoto/operations/__init__.py      2019-11-07 12:43:34.401765786 +0000
-@@ -2,8 +2,7 @@
- 
- __all__ = [ os.path.basename(f)[:-3] for f in glob.glob(os.path.dirname(__file__)+"/*.py") if f[0] != '_']
- 
--for x in __all__:
--    __import__(x, locals(), globals())
-+from . import *
- 
- class Timer(object):
-     """
-

Docker/test_data/L570745_SB001_uv_CORRECTED_DATA.MS/ANTENNA/table.info

-Type = 
-SubType = 
-