diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 40df5a3ee364b5d9705f30385377b6a73f2d138a..319732ed5d98fc7ed2546aac6fb4f2d329b50c7a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -36,7 +36,7 @@ stages:
- |
if test -n "$RELEASE"
then
- echo "Checking out branch $CI_COMMIT_BRANCH"
+ echo "Checking out branch '$CI_COMMIT_BRANCH'"
git checkout $CI_COMMIT_BRANCH
fi
@@ -66,7 +66,6 @@ stages:
.run_tests:
stage: run_tests
image: $INTEGRATION_IMAGE
- extends: .release
before_script:
- !reference [.release, before_script]
- mkdir -p workdir
@@ -84,7 +83,6 @@ stages:
.docs:
stage: docs
- extends: .release
image: $INTEGRATION_IMAGE
.finalize:
@@ -115,14 +113,21 @@ prepare_release:
- |
if git ls-remote --tags --exit-code origin $RELEASE > /dev/null
then
- echo "*** Release $RELEASE already exists. Bailing out! ***"
+ echo "*** Release '$RELEASE' already exists. Bailing out! ***"
touch .tag.exists
exit 1
fi
- !reference [.setup_git, before_script]
script:
- # Update dockerPull image URI in CWL steps with a tagged version
- - sed -ri "/dockerPull/s,(astronrd/linc).*,\1:$RELEASE," steps/*.cwl
+ # Determine the name of the docker image tag. If this is a versioned release,
+ # strip off the leading "V" or "v", because that is what `setuptools_scm`
+ # does when setting LINC_VERSION in the `versioning` job. LINC_VERSION
+ # is used later in the `deploy_docker` job to tag the docker image to be
+ # pushed to Docker Hub. Hence, the same tag must be used in the `dockerPull`
+ # URI in the CWL steps, so update the CWL files.
+ - LINC_TAG=$(echo $RELEASE | sed -r 's,^[Vv]([0-9]+),\1,')
+ - echo "Updating dockerPull URI to use image tag '$LINC_TAG'"
+ - sed -ri "/dockerPull/s,(astronrd/linc).*,\1:$LINC_TAG," steps/*.cwl
- git add -u steps/*.cwl
# Only commit if there are changes
- |
@@ -140,11 +145,14 @@ prepare_release:
versioning:
stage: versioning
- image: bitnami/git
- extends: .release
+ image: python
+ before_script:
+ - !reference [.release, before_script]
+ - pip install setuptools_scm
script:
- ./Docker/fetch_latest_commits.sh | tee commits.txt > versions.env
- - echo LINC_VERSION=$(git describe --tags --always) >> versions.env
+ # Use a sub-command, to catch the exit status from `setuptools_scm`
+ - (echo -n "LINC_VERSION="; python -m setuptools_scm) >> versions.env
# Use hash of commits to determine version of base image (and rebuild if necessary)
- echo INTEGRATION_BASE_IMAGE=$CI_REGISTRY_IMAGE/integration_base:$(sha256sum commits.txt | cut -d " " -f 1) >> versions.env
- echo INTEGRATION_IMAGE=$CI_REGISTRY_IMAGE/integration_full:$(git log -n 1 --pretty=format:%H) >> versions.env
@@ -353,12 +361,14 @@ deploy_docker:
stage: deploy
extends: .deploy
script:
- - echo "Deploying to DockerHub, using $LINC_VERSION as image tag"
+ # Replace characters that are now allowed in a tag string with a dash
+ - LINC_TAG=${LINC_VERSION//[^[:alnum:]_.-]/-}
+ - echo "Deploying to DockerHub, using tag '$LINC_TAG'"
- docker pull $INTEGRATION_IMAGE
- - docker tag $INTEGRATION_IMAGE $CI_PROJECT_PATH:$LINC_VERSION
- - docker tag $INTEGRATION_IMAGE $CI_PROJECT_PATH:latest
- - docker push $CI_PROJECT_PATH:$LINC_VERSION
- - docker push $CI_PROJECT_PATH:latest
+ - docker tag $INTEGRATION_IMAGE astronrd/linc:$LINC_TAG
+ - docker tag $INTEGRATION_IMAGE astronrd/linc:latest
+ - docker push astronrd/linc:$LINC_TAG
+ - docker push astronrd/linc:latest
rules:
# Run on the default branch or on a release branch
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
@@ -371,8 +381,8 @@ deploy_docker_tag_stable:
when: manual
script:
- docker pull $INTEGRATION_IMAGE
- - docker tag $INTEGRATION_IMAGE $CI_PROJECT_PATH:stable
- - docker push $CI_PROJECT_PATH:stable
+ - docker tag $INTEGRATION_IMAGE astronrd/linc:stable
+ - docker push astronrd/linc:stable
### Stage: finalize
@@ -390,7 +400,7 @@ rollback_release:
- |
if test -f .tag.exists
then
- echo "*** Not removing existing tag $RELEASE! ***"
+ echo "*** Not removing existing tag '$RELEASE'! ***"
else
git push origin -d $RELEASE
fi
diff --git a/Docker/Dockerfile-base b/Docker/Dockerfile-base
index b24a5a54d60f107543b5c3f645eebe02f24ba0ae..509e7578a6a69d4a9c3adb353f2f705ad7d4b02c 100644
--- a/Docker/Dockerfile-base
+++ b/Docker/Dockerfile-base
@@ -1,4 +1,4 @@
-FROM ubuntu:22.04 as builder
+FROM ubuntu:22.04 AS builder
# This Docker image builds the dependencies for the Rapthor pipeline.
# It lives on the head of its dependencies.
@@ -9,8 +9,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
apt-get install -y \
bison \
build-essential \
- casacore-data \
- casacore-dev \
cmake \
flex \
gfortran \
@@ -37,16 +35,38 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
python3 \
python3-casacore \
python3-numpy \
- python3-pip \
wcslib-dev \
- wget && \
- mkdir -p /src
+ wget
WORKDIR /src
-# Build portable binaries by default
+# Suppress warning from pip when installing packages as root
+ENV PIP_ROOT_USER_ACTION=ignore
+
+# Do not use `pip` from the Debian repository, but fetch it from PyPA.
+# This way, we are sure that the latest versions of `pip`, `setuptools`, and
+# `wheel` are installed in /usr/local, the only directory we're going to copy
+# over to the next build stage.
+RUN wget -q https://bootstrap.pypa.io/get-pip.py && \
+ python3 get-pip.py
+
+# Install required python packages
+RUN python3 -m pip install --no-cache-dir --upgrade \
+ nodejs-wheel \
+ toil[cwl]
+
+# Build binaries targeted at the oldest CPU that supports AVX2
ARG TARGET_CPU=haswell
+ARG CASACORE_COMMIT=master
+RUN git clone --no-checkout \
+ https://github.com/casacore/casacore.git && \
+ mkdir casacore/build && \
+ cd casacore && git checkout ${CASACORE_COMMIT} && \
+ cd build && \
+ cmake .. -DTARGET_CPU=${TARGET_CPU} && \
+ make install -j`nproc`
+
ARG LOFARSTMAN_COMMIT=main
RUN git clone --no-checkout \
https://github.com/lofar-astron/LofarStMan && \
@@ -132,17 +152,11 @@ RUN git clone --no-checkout \
cmake .. -DTARGET_CPU=${TARGET_CPU} && \
make install -j`nproc`
-# Do not use `pip` from the Debian repository, but fetch it from PyPA.
-# This way, we are sure that the latest versions of `pip`, `setuptools`, and
-# `wheel` are installed in /usr/local, the only directory we're going to copy
-# over to the next build stage.
-RUN wget -q https://bootstrap.pypa.io/get-pip.py && \
- python3 get-pip.py
-
-# Install required python packages
-RUN python3 -m pip install --no-cache-dir --upgrade \
- nodejs-wheel \
- toil[cwl]
+ARG PYTHONCASACORE_COMMIT=master
+RUN git clone --no-checkout \
+ https://github.com/casacore/python-casacore.git && \
+ cd python-casacore && git checkout ${PYTHONCASACORE_COMMIT} && \
+ python3 -m pip install --verbose .
# Install current version of LINC. By adding this to the base image
# we can speed up the build of the final image, because all of LINC's
@@ -153,14 +167,14 @@ COPY . linc
# Note: LINC_VERSION should be provided as build argument
ARG LINC_VERSION=0.0.0
RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_LINC=${LINC_VERSION} \
- python3 -m pip install --no-cache-dir --upgrade ./linc
+ python3 -m pip install --upgrade ./linc
#---------------------------------------------------------------------------
# The image will now be rebuilt without adding the sources, in order to
# reduce the size of the image.
#---------------------------------------------------------------------------
-FROM ubuntu:22.04 as runner
+FROM ubuntu:22.04 AS runner
COPY --from=builder /usr/local /usr/local
RUN chmod +rx /usr/local/bin/*
@@ -168,18 +182,22 @@ RUN chmod +rx /usr/local/bin/*
SHELL ["/bin/bash", "-c"]
# Set default versions. Can be overridden from the command-line
+ARG CASACORE_COMMIT=master
ARG LOFARSTMAN_COMMIT=main
ARG DYSCO_COMMIT=master
-ARG SAGECAL_COMMIT=master
ARG IDG_COMMIT=master
ARG AOFLAGGER_COMMIT=master
ARG LOFARBEAM_COMMIT=master
ARG EVERYBEAM_COMMIT=master
+ARG SAGECAL_COMMIT=master
ARG DP3_COMMIT=master
ARG WSCLEAN_COMMIT=master
+ARG PYTHONCASACORE_COMMIT=master
# Add version information to the metadata of the image
LABEL \
+ nl.astron.nl.linc.casacore.version=${CASACORE_COMMIT} \
+ nl.astron.nl.linc.python-casacore.version=${PYTHONCASACORE_COMMIT} \
nl.astron.linc.lofarstman.version=${LOFARSTMAN_COMMIT} \
nl.astron.linc.dysco.version=${DYSCO_COMMIT} \
nl.astron.linc.sagecal.version=${SAGECAL_COMMIT} \
@@ -194,7 +212,6 @@ LABEL \
RUN export DEBIAN_FRONTEND=noninteractive && \
apt-get update && \
apt-get install -y \
- casacore-tools \
gdb \
git \
libatkmm-1.6-1v5 \
@@ -203,13 +220,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
libboost-program-options1.74.0 \
libboost-python1.74.0 \
libcairomm-1.0-1v5 \
- libcasa-casa6 \
- libcasa-fits6 \
- libcasa-measures6 \
- libcasa-ms6 \
- libcasa-python3-6 \
- libcasa-scimath6 \
- libcasa-tables6 \
libcfitsio9 \
libfftw3-double3 \
libfftw3-single3 \
@@ -229,7 +239,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
libsigc++-2.0-0v5 \
libstdc++6 \
python3 \
- python3-casacore \
python3-distutils \
rsync \
wget && \
@@ -238,11 +247,9 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
# Install WSRT Measures (extra casacore data)
# Note: The file on the ftp site is updated daily. When warnings regarding leap
# seconds appear, ignore them or regenerate the docker image.
-RUN wget -q -O /WSRT_Measures.ztar \
- ftp://ftp.astron.nl/outgoing/Measures/WSRT_Measures.ztar && \
- cd /var/lib/casacore/data && \
- tar xfz /WSRT_Measures.ztar && \
- rm /WSRT_Measures.ztar
+RUN mkdir -p /usr/local/share/casacore/data && \
+ wget -qO - ftp://ftp.astron.nl/outgoing/Measures/WSRT_Measures.ztar | \
+ tar -C /usr/local/share/casacore/data -xzf -
# Try to run the compiled tools to make sure they run without
# a problem (e.g. no missing libraries).
diff --git a/Docker/fetch_latest_commits.sh b/Docker/fetch_latest_commits.sh
index 101fc136281229f0c7f3dc69df069baa5e088365..b0af8ede3d632c9ab2f742459fd54991514900b8 100755
--- a/Docker/fetch_latest_commits.sh
+++ b/Docker/fetch_latest_commits.sh
@@ -1,11 +1,13 @@
#!/bin/sh
-git ls-remote https://github.com/lofar-astron/LofarStMan HEAD | awk '{ print "LOFARSTMAN_COMMIT="$1 }'
-git ls-remote https://github.com/aroffringa/dysco.git HEAD | awk '{ print "DYSCO_COMMIT="$1 }'
-git ls-remote https://git.astron.nl/RD/idg.git HEAD | awk '{ print "IDG_COMMIT="$1 }'
-git ls-remote https://gitlab.com/aroffringa/aoflagger.git HEAD | awk '{ print "AOFLAGGER_COMMIT="$1 }'
-git ls-remote https://github.com/nlesc-dirac/sagecal HEAD | awk '{ print "SAGECAL_COMMIT="$1 }'
-git ls-remote https://github.com/lofar-astron/LOFARBeam.git HEAD | awk '{ print "LOFARBEAM_COMMIT="$1 }'
-git ls-remote https://git.astron.nl/RD/EveryBeam.git HEAD | awk '{ print "EVERYBEAM_COMMIT="$1 }'
-git ls-remote https://git.astron.nl/RD/DP3.git HEAD | awk '{ print "DP3_COMMIT="$1 }'
-git ls-remote https://gitlab.com/aroffringa/wsclean.git HEAD | awk '{ print "WSCLEAN_COMMIT="$1 }'
+git ls-remote https://github.com/casacore/casacore.git v3.6.1 | awk '{ print "CASACORE_COMMIT="$1 }'
+git ls-remote https://github.com/lofar-astron/LofarStMan HEAD | awk '{ print "LOFARSTMAN_COMMIT="$1 }'
+git ls-remote https://github.com/aroffringa/dysco.git HEAD | awk '{ print "DYSCO_COMMIT="$1 }'
+git ls-remote https://git.astron.nl/RD/idg.git HEAD | awk '{ print "IDG_COMMIT="$1 }'
+git ls-remote https://gitlab.com/aroffringa/aoflagger.git HEAD | awk '{ print "AOFLAGGER_COMMIT="$1 }'
+git ls-remote https://github.com/lofar-astron/LOFARBeam.git HEAD | awk '{ print "LOFARBEAM_COMMIT="$1 }'
+git ls-remote https://git.astron.nl/RD/EveryBeam.git HEAD | awk '{ print "EVERYBEAM_COMMIT="$1 }'
+git ls-remote https://github.com/nlesc-dirac/sagecal HEAD | awk '{ print "SAGECAL_COMMIT="$1 }'
+git ls-remote https://git.astron.nl/RD/DP3.git HEAD | awk '{ print "DP3_COMMIT="$1 }'
+git ls-remote https://gitlab.com/aroffringa/wsclean.git HEAD | awk '{ print "WSCLEAN_COMMIT="$1 }'
+git ls-remote https://github.com/casacore/python-casacore.git v3.6.1 | awk '{ print "PYTHONCASACORE_COMMIT="$1 }'