Sync with development version

.gitlab-ci.yml

@@ -36,7 +36,7 @@ stages:
     - |
       if test -n "$RELEASE"
       then
-        echo "Checking out branch $CI_COMMIT_BRANCH"
+        echo "Checking out branch '$CI_COMMIT_BRANCH'"
         git checkout $CI_COMMIT_BRANCH
       fi
 
@@ -66,7 +66,6 @@ stages:
 .run_tests:
   stage: run_tests
   image: $INTEGRATION_IMAGE
-  extends: .release
   before_script:
     - !reference [.release, before_script]
     - mkdir -p workdir
@@ -84,7 +83,6 @@ stages:
 
 .docs:
   stage: docs
-  extends: .release
   image: $INTEGRATION_IMAGE
 
 .finalize:
@@ -115,14 +113,21 @@ prepare_release:
     - |
       if git ls-remote --tags --exit-code origin $RELEASE > /dev/null
       then
-        echo "*** Release $RELEASE already exists. Bailing out! ***"
+        echo "*** Release '$RELEASE' already exists. Bailing out! ***"
         touch .tag.exists
         exit 1
       fi
     - !reference [.setup_git, before_script]
   script:
-    # Update dockerPull image URI in CWL steps with a tagged version
-    - sed -ri "/dockerPull/s,(astronrd/linc).*,\1:$RELEASE," steps/*.cwl
+    # Determine the name of the docker image tag. If this is a versioned release,
+    # strip off the leading "V" or "v", because that is what `setuptools_scm`
+    # does when setting LINC_VERSION in the `versioning` job.  LINC_VERSION
+    # is used later in the `deploy_docker` job to tag the docker image to be
+    # pushed to Docker Hub. Hence, the same tag must be used in the `dockerPull`
+    # URI in the CWL steps, so update the CWL files.
+    - LINC_TAG=$(echo $RELEASE | sed -r 's,^[Vv]([0-9]+),\1,')
+    - echo "Updating dockerPull URI to use image tag '$LINC_TAG'"
+    - sed -ri "/dockerPull/s,(astronrd/linc).*,\1:$LINC_TAG," steps/*.cwl
     - git add -u steps/*.cwl
     # Only commit if there are changes
     - |
@@ -140,11 +145,14 @@ prepare_release:
 
 versioning:
   stage: versioning
-  image: bitnami/git
-  extends: .release
+  image: python
+  before_script:
+    - !reference [.release, before_script]
+    - pip install setuptools_scm
   script:
     - ./Docker/fetch_latest_commits.sh | tee commits.txt > versions.env
-    - echo LINC_VERSION=$(git describe --tags --always) >> versions.env
+    # Use a sub-command, to catch the exit status from `setuptools_scm`
+    - (echo -n "LINC_VERSION="; python -m setuptools_scm) >> versions.env
     # Use hash of commits to determine version of base image (and rebuild if necessary)
     - echo INTEGRATION_BASE_IMAGE=$CI_REGISTRY_IMAGE/integration_base:$(sha256sum commits.txt | cut -d " " -f 1) >> versions.env
     - echo INTEGRATION_IMAGE=$CI_REGISTRY_IMAGE/integration_full:$(git log -n 1 --pretty=format:%H) >> versions.env
@@ -353,12 +361,14 @@ deploy_docker:
   stage: deploy
   extends: .deploy
   script:
-    - echo "Deploying to DockerHub, using $LINC_VERSION as image tag"
+    # Replace characters that are now allowed in a tag string with a dash
+    - LINC_TAG=${LINC_VERSION//[^[:alnum:]_.-]/-}
+    - echo "Deploying to DockerHub, using tag '$LINC_TAG'"
     - docker pull $INTEGRATION_IMAGE
-    - docker tag $INTEGRATION_IMAGE $CI_PROJECT_PATH:$LINC_VERSION
-    - docker tag $INTEGRATION_IMAGE $CI_PROJECT_PATH:latest
-    - docker push $CI_PROJECT_PATH:$LINC_VERSION
-    - docker push $CI_PROJECT_PATH:latest
+    - docker tag $INTEGRATION_IMAGE astronrd/linc:$LINC_TAG
+    - docker tag $INTEGRATION_IMAGE astronrd/linc:latest
+    - docker push astronrd/linc:$LINC_TAG
+    - docker push astronrd/linc:latest
   rules:
     # Run on the default branch or on a release branch
     - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
@@ -371,8 +381,8 @@ deploy_docker_tag_stable:
   when: manual
   script:
     - docker pull $INTEGRATION_IMAGE
-    - docker tag $INTEGRATION_IMAGE $CI_PROJECT_PATH:stable
-    - docker push $CI_PROJECT_PATH:stable
+    - docker tag $INTEGRATION_IMAGE astronrd/linc:stable
+    - docker push astronrd/linc:stable
 
 
 ### Stage: finalize
@@ -390,7 +400,7 @@ rollback_release:
     - |
       if test -f .tag.exists
       then
-        echo "*** Not removing existing tag $RELEASE! ***"
+        echo "*** Not removing existing tag '$RELEASE'! ***"
       else
         git push origin -d $RELEASE
       fi

Docker/Dockerfile-base

-FROM ubuntu:22.04 as builder
+FROM ubuntu:22.04 AS builder
 
 # This Docker image builds the dependencies for the Rapthor pipeline.
 # It lives on the head of its dependencies.
@@ -9,8 +9,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
     apt-get install -y \
         bison \
         build-essential \
-        casacore-data \
-        casacore-dev \
         cmake \
         flex \
         gfortran \
@@ -37,16 +35,38 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
         python3 \
         python3-casacore \
         python3-numpy \
-        python3-pip \
         wcslib-dev \
-        wget && \
-    mkdir -p /src
+        wget
 
 WORKDIR /src
 
-# Build portable binaries by default
+# Suppress warning from pip when installing packages as root
+ENV PIP_ROOT_USER_ACTION=ignore
+
+# Do not use `pip` from the Debian repository, but fetch it from PyPA.
+# This way, we are sure that the latest versions of `pip`, `setuptools`, and
+# `wheel` are installed in /usr/local, the only directory we're going to copy
+# over to the next build stage.
+RUN wget -q https://bootstrap.pypa.io/get-pip.py && \
+    python3 get-pip.py
+
+# Install required python packages
+RUN python3 -m pip install --no-cache-dir --upgrade \
+    nodejs-wheel \
+    toil[cwl]
+
+# Build binaries targeted at the oldest CPU that supports AVX2
 ARG TARGET_CPU=haswell
 
+ARG CASACORE_COMMIT=master
+RUN git clone --no-checkout \
+        https://github.com/casacore/casacore.git && \
+    mkdir casacore/build && \
+    cd casacore && git checkout ${CASACORE_COMMIT} && \
+    cd build && \
+    cmake .. -DTARGET_CPU=${TARGET_CPU} && \
+    make install -j`nproc`
+
 ARG LOFARSTMAN_COMMIT=main
 RUN git clone --no-checkout \
         https://github.com/lofar-astron/LofarStMan && \
@@ -132,17 +152,11 @@ RUN git clone --no-checkout \
     cmake .. -DTARGET_CPU=${TARGET_CPU} && \
     make install -j`nproc`
 
-# Do not use `pip` from the Debian repository, but fetch it from PyPA.
-# This way, we are sure that the latest versions of `pip`, `setuptools`, and
-# `wheel` are installed in /usr/local, the only directory we're going to copy
-# over to the next build stage.
-RUN wget -q https://bootstrap.pypa.io/get-pip.py && \
-    python3 get-pip.py
-
-# Install required python packages
-RUN python3 -m pip install --no-cache-dir --upgrade \
-    nodejs-wheel \
-    toil[cwl]
+ARG PYTHONCASACORE_COMMIT=master
+RUN git clone --no-checkout \
+    https://github.com/casacore/python-casacore.git && \
+    cd python-casacore && git checkout ${PYTHONCASACORE_COMMIT} && \
+    python3 -m pip install --verbose .
 
 # Install current version of LINC. By adding this to the base image
 # we can speed up the build of the final image, because all of LINC's
@@ -153,14 +167,14 @@ COPY . linc
 # Note: LINC_VERSION should be provided as build argument
 ARG LINC_VERSION=0.0.0
 RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_LINC=${LINC_VERSION} \
-    python3 -m pip install --no-cache-dir --upgrade ./linc
+    python3 -m pip install --upgrade ./linc
 
 
 #---------------------------------------------------------------------------
 # The image will now be rebuilt without adding the sources, in order to
 # reduce the size of the image.
 #---------------------------------------------------------------------------
-FROM ubuntu:22.04 as runner
+FROM ubuntu:22.04 AS runner
 
 COPY --from=builder /usr/local /usr/local
 RUN chmod +rx /usr/local/bin/*
@@ -168,18 +182,22 @@ RUN chmod +rx /usr/local/bin/*
 SHELL ["/bin/bash", "-c"]
 
 # Set default versions. Can be overridden from the command-line
+ARG CASACORE_COMMIT=master
 ARG LOFARSTMAN_COMMIT=main
 ARG DYSCO_COMMIT=master
-ARG SAGECAL_COMMIT=master
 ARG IDG_COMMIT=master
 ARG AOFLAGGER_COMMIT=master
 ARG LOFARBEAM_COMMIT=master
 ARG EVERYBEAM_COMMIT=master
+ARG SAGECAL_COMMIT=master
 ARG DP3_COMMIT=master
 ARG WSCLEAN_COMMIT=master
+ARG PYTHONCASACORE_COMMIT=master
 
 # Add version information to the metadata of the image
 LABEL \
+    nl.astron.nl.linc.casacore.version=${CASACORE_COMMIT} \
+    nl.astron.nl.linc.python-casacore.version=${PYTHONCASACORE_COMMIT} \
     nl.astron.linc.lofarstman.version=${LOFARSTMAN_COMMIT} \
     nl.astron.linc.dysco.version=${DYSCO_COMMIT} \
     nl.astron.linc.sagecal.version=${SAGECAL_COMMIT} \
@@ -194,7 +212,6 @@ LABEL \
 RUN export DEBIAN_FRONTEND=noninteractive && \
     apt-get update && \
     apt-get install -y \
-        casacore-tools \
         gdb \
         git \
         libatkmm-1.6-1v5 \
@@ -203,13 +220,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
         libboost-program-options1.74.0 \
         libboost-python1.74.0 \
         libcairomm-1.0-1v5 \
-        libcasa-casa6 \
-        libcasa-fits6 \
-        libcasa-measures6 \
-        libcasa-ms6 \
-        libcasa-python3-6 \
-        libcasa-scimath6 \
-        libcasa-tables6 \
         libcfitsio9 \
         libfftw3-double3 \
         libfftw3-single3 \
@@ -229,7 +239,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
         libsigc++-2.0-0v5 \
         libstdc++6 \
         python3 \
-        python3-casacore \
         python3-distutils \
         rsync \
         wget && \
@@ -238,11 +247,9 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
 # Install WSRT Measures (extra casacore data)
 # Note: The file on the ftp site is updated daily. When warnings regarding leap
 # seconds appear, ignore them or regenerate the docker image.
-RUN wget -q -O /WSRT_Measures.ztar \
-        ftp://ftp.astron.nl/outgoing/Measures/WSRT_Measures.ztar && \
-    cd /var/lib/casacore/data && \
-    tar xfz /WSRT_Measures.ztar && \
-    rm /WSRT_Measures.ztar
+RUN mkdir -p /usr/local/share/casacore/data && \
+    wget -qO - ftp://ftp.astron.nl/outgoing/Measures/WSRT_Measures.ztar | \
+        tar -C /usr/local/share/casacore/data -xzf -
 
 # Try to run the compiled tools to make sure they run without
 # a problem (e.g. no missing libraries).

Docker/fetch_latest_commits.sh

 #!/bin/sh
 
+git ls-remote https://github.com/casacore/casacore.git        v3.6.1 | awk '{ print "CASACORE_COMMIT="$1 }'
 git ls-remote https://github.com/lofar-astron/LofarStMan      HEAD   | awk '{ print "LOFARSTMAN_COMMIT="$1 }'
 git ls-remote https://github.com/aroffringa/dysco.git         HEAD   | awk '{ print "DYSCO_COMMIT="$1 }'
 git ls-remote https://git.astron.nl/RD/idg.git                HEAD   | awk '{ print "IDG_COMMIT="$1 }'
 git ls-remote https://gitlab.com/aroffringa/aoflagger.git     HEAD   | awk '{ print "AOFLAGGER_COMMIT="$1 }'
-git ls-remote https://github.com/nlesc-dirac/sagecal HEAD           | awk '{ print "SAGECAL_COMMIT="$1 }'
 git ls-remote https://github.com/lofar-astron/LOFARBeam.git   HEAD   | awk '{ print "LOFARBEAM_COMMIT="$1 }'
 git ls-remote https://git.astron.nl/RD/EveryBeam.git          HEAD   | awk '{ print "EVERYBEAM_COMMIT="$1 }'
+git ls-remote https://github.com/nlesc-dirac/sagecal          HEAD   | awk '{ print "SAGECAL_COMMIT="$1 }'
 git ls-remote https://git.astron.nl/RD/DP3.git                HEAD   | awk '{ print "DP3_COMMIT="$1 }'
 git ls-remote https://gitlab.com/aroffringa/wsclean.git       HEAD   | awk '{ print "WSCLEAN_COMMIT="$1 }'
+git ls-remote https://github.com/casacore/python-casacore.git v3.6.1 | awk '{ print "PYTHONCASACORE_COMMIT="$1 }'