Keep track of upstream version in the Docker image for reproducibility in production environments.

.gitlab-ci.yml

@@ -34,8 +34,10 @@ versioning:
   script:
     # Unshallowing ensures that 'git describe' works
     - git fetch --unshallow
-    - echo LINC_VERSION=$(git describe --tags --dirty --always) > versions.env
-    - echo INTEGRATION_BASE_IMAGE=${CI_REGISTRY_IMAGE}/integration_base:$(git log -n 1 --pretty=format:%H -- Docker/Dockerfile-base) >> versions.env
+    - ./Docker/fetch_latest_commits.sh | tee commits.txt > versions.env
+    - echo LINC_VERSION=$(git describe --tags --always) > versions.env
+    # Use hash of commits to determine version of base image (and rebuild if necessary)
+    - echo INTEGRATION_BASE_IMAGE=${CI_REGISTRY_IMAGE}/integration_base:$(sha256sum commits.txt | cut -d " " -f 1) >> versions.env
     - echo INTEGRATION_IMAGE=${CI_REGISTRY_IMAGE}/integration_full:$(git log -n 1 --pretty=format:%H) >> versions.env
     - cat versions.env
   artifacts:
@@ -60,7 +62,17 @@ build_base:
   script:
     - |
       if ! docker manifest inspect $INTEGRATION_BASE_IMAGE > /dev/null || [ "$BUILD_DOCKER_IMAGE" = "1" ]; then
-        docker build --tag $INTEGRATION_BASE_IMAGE -f Docker/Dockerfile-base .
+        docker build --tag $INTEGRATION_BASE_IMAGE \
+          --build-arg LOFARSTMAN_COMMIT \
+          --build-arg DYSCO_COMMIT \
+          --build-arg IDG_COMMIT \
+          --build-arg SAGECAL_COMMIT \
+          --build-arg AOFLAGGER_COMMIT \
+          --build-arg LOFARBEAM_COMMIT \
+          --build-arg EVERYBEAM_COMMIT \
+          --build-arg DP3_COMMIT \
+          --build-arg WSCLEAN_COMMIT \
+          -f Docker/Dockerfile-base .
         docker push $INTEGRATION_BASE_IMAGE
       fi
 
@@ -72,7 +84,7 @@ install_linc:
   script:
     - docker build --build-arg BASE_TAG=$INTEGRATION_BASE_IMAGE --tag $INTEGRATION_IMAGE -f Docker/Dockerfile-linc .
     - docker push $INTEGRATION_IMAGE
-    
+
 download_data:
   needs: ["versioning","install_linc"]
   image: $INTEGRATION_IMAGE
@@ -173,7 +185,7 @@ deploy_docker:
     - docker:20.10-dind
   before_script:
     - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
-    - echo $DH_REGISTRY_PASSWORD | docker login -u $DH_REGISTRY_USER --password-stdin 
+    - echo $DH_REGISTRY_PASSWORD | docker login -u $DH_REGISTRY_USER --password-stdin
   script:
     - docker tag $INTEGRATION_IMAGE astronrd/linc:$LINC_VERSION
     - docker tag $INTEGRATION_IMAGE astronrd/linc:latest
@@ -184,6 +196,37 @@ deploy_docker:
     - if: '$CI_COMMIT_BRANCH == "master"'
       when: always
 
+# Deploy a release version
+deploy_release:
+  stage: deploy
+  needs: ["versioning", "download_data"]
+  image: docker:20.10
+  services:
+    - docker:20.10-dind
+  before_script:
+    - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
+    - echo $DH_REGISTRY_PASSWORD | docker login -u $DH_REGISTRY_USER --password-stdin
+    - apk update && apk add git
+  script:
+    - echo -n "$CI_COMMIT_BRANCH" | sed -e "s/^releases\//export RELEASE=/" > version && source version
+    - docker tag $INTEGRATION_IMAGE astronrd/linc:$RELEASE
+    - docker push astronrd/linc:$RELEASE
+    # Update docker image in cwl steps by tagged version
+    - sed -i "s/astronrd\/linc/astronrd\/linc:$RELEASE/g" steps/*.cwl
+    # Create and push Git tag
+    - git add steps/*.cwl
+    - git config user.email $GITLAB_USER_EMAIL
+    - git config user.name $GITLAB_USER_NAME
+    # Skip CI on this commit and tag
+    - git commit -m "[SKIP CI] Replace latest with tag $RELEASE"
+    - git tag -a $RELEASE -m "[SKIP CI] Version $RELEASE created by gitlab-ci build"
+    - git push --all
+  rules:
+    # TODO: check if pushed tag triggers Pipeline
+    # Only run on release branches
+    - if: '$CI_COMMIT_BRANCH =~ /^releases//'
+      when: always
+
 deploy_doc:
   stage: deploy
   needs: ["versioning", "build_doc"]

Docker/Dockerfile-base

@@ -4,7 +4,7 @@ FROM ubuntu:20.04 as builder
 # It lives on the head of its dependencies.
 
 # Install all build-time dependencies
-RUN export DEBIAN_FRONTEND=noninteractive && \ 
+RUN export DEBIAN_FRONTEND=noninteractive && \
     apt-get update && \
     apt-get install -y \
         bison \
@@ -13,7 +13,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
         casacore-dev \
         cmake \
         flex \
-        gfortran \ 
+        gfortran \
         git \
         libblas-dev \
         libboost-date-time-dev \
@@ -46,77 +46,86 @@ WORKDIR /src
 # Build portable binaries by default
 ARG PORTABLE=TRUE
 
-ARG LOFARSTMAN_VERSION=master
-RUN git clone --depth 1 --branch ${LOFARSTMAN_VERSION} \
+ARG LOFARSTMAN_COMMIT=master
+RUN git clone --no-checkout \
         https://github.com/lofar-astron/LofarStMan && \
     mkdir LofarStMan/build && \
-    cd LofarStMan/build && \
+    cd LofarStMan && git checkout ${LOFARSTMAN_COMMIT} && \
+    cd build && \
     cmake .. -DPORTABLE=${PORTABLE} && \
     make install -j`nproc`
 
-ARG DYSCO_VERSION=master
-RUN git clone --depth 1 --branch ${DYSCO_VERSION} \
+ARG DYSCO_COMMIT=master
+RUN git clone --no-checkout \
         https://github.com/aroffringa/dysco.git && \
     mkdir dysco/build && \
-    cd dysco/build && \
+    cd dysco && git checkout ${DYSCO_COMMIT} && \
+    cd build && \
     cmake .. -DPORTABLE=${PORTABLE} && \
     make install -j`nproc`
-    
-ARG IDG_VERSION=master
-# IDG doesn't work with --depth 1, because it needs all branches to
+
+ARG IDG_COMMIT=master
+# IDG does not work with --depth 1, because it needs all branches to
 # determine its version :-(
-RUN git clone --branch ${IDG_VERSION} \
+RUN git clone  \
         https://git.astron.nl/RD/idg.git && \
     mkdir idg/build && \
-    cd idg/build && \
+    cd idg && git checkout ${IDG_COMMIT} && \
+    cd build && \
     cmake .. && \
     make install -j`nproc`
 
-ARG AOFLAGGER_VERSION=master
-RUN git clone --depth 1 --branch ${AOFLAGGER_VERSION} \
+ARG AOFLAGGER_COMMIT=master
+RUN git clone --no-checkout \
         https://gitlab.com/aroffringa/aoflagger.git && \
     mkdir aoflagger/build && \
-    cd aoflagger/build && \
+    cd aoflagger && git checkout ${AOFLAGGER_COMMIT} && \
+    cd build && \
     cmake .. -DPORTABLE=${PORTABLE} && \
     make install -j`nproc`
 
-ARG LOFARBEAM_VERSION=master
-RUN git clone  --depth 1 --branch ${LOFARBEAM_VERSION} \
+ARG LOFARBEAM_COMMIT=master
+RUN git clone --no-checkout \
         https://github.com/lofar-astron/LOFARBeam.git && \
     mkdir LOFARBeam/build && \
-    cd LOFARBeam/build && \
+    cd LOFARBeam && git checkout ${LOFARBEAM_COMMIT} && \
+    cd build && \
     cmake .. && \
     make install -j`nproc`
 
-ARG EVERYBEAM_VERSION=master
-RUN git clone --depth 1  --branch ${EVERYBEAM_VERSION} \
+ARG EVERYBEAM_COMMIT=master
+RUN git clone --no-checkout \
         https://git.astron.nl/RD/EveryBeam.git && \
     mkdir EveryBeam/build && \
-    cd EveryBeam/build && \
+    cd EveryBeam && git checkout ${EVERYBEAM_COMMIT} && \
+    cd build && \
     cmake .. && \
     make install -j`nproc`
 
-ARG SAGECAL_VERSION=master
-RUN git clone --depth 1 --branch ${SAGECAL_VERSION} \
+ARG SAGECAL_COMMIT=master
+RUN git clone --no-checkout \
         https://github.com/nlesc-dirac/sagecal && \
     mkdir sagecal/build && \
-    cd sagecal/build && \
+    cd sagecal && git checkout ${SAGECAL_COMMIT} && \
+    cd build && \
     cmake .. -DLIB_ONLY=1 && \
     make install -j`nproc`
 
-ARG DP3_VERSION=master
-RUN git clone --depth 1 --branch ${DP3_VERSION} \
+ARG DP3_COMMIT=master
+RUN git clone --no-checkout \
         https://git.astron.nl/RD/DP3.git && \
     mkdir DP3/build && \
-    cd DP3/build && \
-    cmake .. -DPORTABLE=${PORTABLE} -DLIBDIRAC_PREFIX=/usr/local/ && \
+    cd DP3 && git checkout ${DP3_COMMIT} && \
+    cd build && \
+    cmake .. -DPORTABLE=${PORTABLE} && \
     make install -j`nproc`
 
-ARG WSCLEAN_VERSION=master
-RUN git clone --depth 1 --branch ${WSCLEAN_VERSION} \
+ARG WSCLEAN_COMMIT=master
+RUN git clone --no-checkout \
         https://gitlab.com/aroffringa/wsclean.git && \
     mkdir wsclean/build && \
-    cd wsclean/build && \
+    cd wsclean && git checkout ${WSCLEAN_COMMIT} && \
+    cd build && \
     cmake .. -DPORTABLE=${PORTABLE} && \
     make install -j`nproc`
 
@@ -152,14 +161,15 @@ RUN chmod +rx /usr/local/bin/*
 SHELL ["/bin/bash", "-c"]
 
 # Set default versions. Can be overridden from the command-line
-ARG LOFARSTMAN_VERSION=master
-ARG DYSCO_VERSION=master
-ARG IDG_VERSION=master
-ARG AOFLAGGER_VERSION=master
-ARG LOFARBEAM_VERSION=master
-ARG EVERYBEAM_VERSION=master
-ARG DP3_VERSION=master
-ARG WSCLEAN_VERSION=master
+ARG LOFARSTMAN_COMMIT=master
+ARG DYSCO_COMMIT=master
+ARG SAGECAL_COMMIT=master
+ARG IDG_COMMIT=master
+ARG AOFLAGGER_COMMIT=master
+ARG LOFARBEAM_COMMIT=master
+ARG EVERYBEAM_COMMIT=master
+ARG DP3_COMMIT=master
+ARG WSCLEAN_COMMIT=master
 
 # Add version information to the metadata of the image
 LABEL \
@@ -172,7 +182,7 @@ LABEL \
     nl.astron.linc.dp3.version=${DP3_VERSION} \
     nl.astron.linc.wsclean.version=${WSCLEAN_VERSION}
 
-# Only install run-time required packages 
+# Only install run-time required packages
 RUN export DEBIAN_FRONTEND=noninteractive && \
     apt-get update && \
     apt-get install -y \

Docker/fetch_latest_commits.sh

+#!/bin/sh
+
+git ls-remote https://github.com/lofar-astron/LofarStMan HEAD       | awk '{ print "LOFARSTMAN_COMMIT="$1 }'
+git ls-remote https://github.com/aroffringa/dysco.git HEAD          | awk '{ print "DYSCO_COMMIT="$1 }'
+git ls-remote https://git.astron.nl/RD/idg.git HEAD                 | awk '{ print "IDG_COMMIT="$1 }'
+git ls-remote https://gitlab.com/aroffringa/aoflagger.git HEAD      | awk '{ print "AOFLAGGER_COMMIT="$1 }'
+git ls-remote https://github.com/nlesc-dirac/sagecal HEAD           | awk '{ print "SAGECAL_COMMIT="$1 }'
+git ls-remote https://github.com/lofar-astron/LOFARBeam.git HEAD    | awk '{ print "LOFARBEAM_COMMIT="$1 }'
+git ls-remote https://git.astron.nl/RD/EveryBeam.git HEAD           | awk '{ print "EVERYBEAM_COMMIT="$1 }'
+git ls-remote https://git.astron.nl/RD/DP3.git HEAD                 | awk '{ print "DP3_COMMIT="$1 }'
+git ls-remote https://gitlab.com/aroffringa/wsclean.git HEAD        | awk '{ print "WSCLEAN_COMMIT="$1 }'