SW-1009: Deploy LOFAR pipelines through gitlab CI/CD, migrating from Jenkins.

.gitlab-ci.yml

@@ -16,6 +16,13 @@ include:
 variables:
   VERSION: "0.0.0"
   USE_PYTHON: "true"
+  CEP_HEADNODE: "head.cep4.control.lofar"
+  MARK_PIPELINES_LATEST:
+    description: "Use deployed pipelines as default in production?"
+    value: "false"
+    options:
+      - "false"
+      - "true"
 
 #
 # This script snippet sets the VERSION argument to the correct version based on the LOFAR-Release tag.
@@ -248,18 +255,6 @@ build_ST_MAC:
     paths:
       - build/gnucxx11_opt
 
-build_lofar_pulp_docker_image:
-  stage: build
-  script:
-    - docker build -t ci_pulp:$CI_COMMIT_SHORT_SHA Docker/pulp-base
-    - docker build --build-arg BASE_VERSION=$CI_COMMIT_SHORT_SHA --build-arg LOFAR_TAG=$CI_COMMIT_REF_NAME -t ci_lofar_pulp:$CI_COMMIT_SHORT_SHA Docker/lofar-pulp
-    - docker login -u $CI_NEXUS_REGISTRY_USERNAME -p $CI_NEXUS_REGISTRY_PASSWORD $CI_NEXUS_REGISTRY
-    - docker tag ci_lofar_pulp:$CI_COMMIT_SHORT_SHA $CI_NEXUS_REGISTRY_LOCATION/ci_lofar_pulp:$CI_COMMIT_SHORT_SHA
-    - docker push $CI_NEXUS_REGISTRY_LOCATION/ci_lofar_pulp:$CI_COMMIT_SHORT_SHA
-    - docker logout $CI_NEXUS_REGISTRY
-  interruptible: true
-  when: manual # pulp docker images take a long time to build. It does not/hardly depend on currently developed lofar code. So an automatic build for each lofar commit is not needed.
-
 build_Docker:
   stage: build
   image: ci_mac:$CI_COMMIT_SHORT_SHA
@@ -279,14 +274,56 @@ build_Docker:
     paths:
       - build/gnucxx11_opt
 
-build_lofar_tbbwriter_control_docker_image:
+build_lofar_base:
+  stage: build
+  script:
+    - CPU_OPTIMISATION="haswell"   # also supported: "x86-64" and "skylake". Do we need to support switching build variants/optimizations in an interactive way?
+    - docker build --build-arg CPU_OPTIMISATION=$CPU_OPTIMISATION -t lofar-base:$CI_COMMIT_SHORT_SHA Docker/lofar-base
+  interruptible: true
+  when: manual # this takes a long time to build and the image is humongous. So let's not do that on every commit.
+
+build_lofar_pipeline:
+  stage: build
+  script:
+    - CPU_OPTIMISATION="haswell"   # also supported: "x86-64" and "skylake". Do we need to support switching build variants/optimizations in an interactive way?
+    - docker build --build-arg LOFAR_VERSION=$CI_COMMIT_REF_NAME --build-arg LOFAR_BASE_IMAGE_VERSION=$CI_COMMIT_SHORT_SHA -t ci_lofar_pipeline:$CI_COMMIT_SHORT_SHA Docker/lofar-pipeline
+    - docker login -u $CI_NEXUS_REGISTRY_USERNAME -p $CI_NEXUS_REGISTRY_PASSWORD $CI_NEXUS_REGISTRY
+    - docker tag ci_lofar_pipeline:$CI_COMMIT_SHORT_SHA $CI_NEXUS_REGISTRY_LOCATION/ci_lofar_pipeline:$CI_COMMIT_SHORT_SHA
+    - docker push $CI_NEXUS_REGISTRY_LOCATION/ci_lofar_pipeline:$CI_COMMIT_SHORT_SHA
+    - docker logout $CI_NEXUS_REGISTRY
+  interruptible: true
+  needs:
+    - build_lofar_base
+
+build_dynspec:
+  stage: build
+  script:
+    - CPU_OPTIMISATION="haswell"   # also supported: "x86-64" and "skylake". Do we need to support switching build variants/optimizations in an interactive way?
+    - docker build -t ci_dynspec:$CI_COMMIT_SHORT_SHA Docker/dynspec
+    - docker login -u $CI_NEXUS_REGISTRY_USERNAME -p $CI_NEXUS_REGISTRY_PASSWORD $CI_NEXUS_REGISTRY
+    - docker tag ci_dynspec:$CI_COMMIT_SHORT_SHA $CI_NEXUS_REGISTRY_LOCATION/ci_dynspec:$CI_COMMIT_SHORT_SHA
+    - docker push $CI_NEXUS_REGISTRY_LOCATION/ci_dynspec:$CI_COMMIT_SHORT_SHA
+    - docker logout $CI_NEXUS_REGISTRY
+  interruptible: true
+  when: manual # this takes a long time to build and the image is humongous. So let's not do that on every commit.
+
+build_lofar_pulp:
+  stage: build
+  script:
+    - docker build -t ci_pulp:$CI_COMMIT_SHORT_SHA Docker/pulp-base
+    - docker build --build-arg BASE_VERSION=$CI_COMMIT_SHORT_SHA --build-arg LOFAR_TAG=$CI_COMMIT_REF_NAME -t ci_lofar_pulp:$CI_COMMIT_SHORT_SHA Docker/lofar-pulp
+    - docker login -u $CI_NEXUS_REGISTRY_USERNAME -p $CI_NEXUS_REGISTRY_PASSWORD $CI_NEXUS_REGISTRY
+    - docker tag ci_lofar_pulp:$CI_COMMIT_SHORT_SHA $CI_NEXUS_REGISTRY_LOCATION/ci_lofar_pulp:$CI_COMMIT_SHORT_SHA
+    - docker push $CI_NEXUS_REGISTRY_LOCATION/ci_lofar_pulp:$CI_COMMIT_SHORT_SHA
+    - docker logout $CI_NEXUS_REGISTRY
+  interruptible: true
+  when: manual # pulp docker images take a long time to build. It does not/hardly depend on currently developed lofar code. So an automatic build for each lofar commit is not needed.
+
+build_lofar_tbbwriter_control:
   stage: build
   script:
-    - echo "Building Docker lofar-base..."
     - CPU_OPTIMISATION="x86-64"   # also supported: "haswell" and "skylake". Do we need to support switching build variants/optimizations in an interactive way?
     - cd build/gnucxx11_opt
-    - docker build --build-arg CPU_OPTIMISATION=$CPU_OPTIMISATION -t lofar-base:$CI_COMMIT_SHORT_SHA installed/share/docker/lofar-base
-    - echo "Building Docker lofar-tbbwriter-control..."
     - docker build --build-arg LOFAR_BASE_IMAGE_VERSION=$CI_COMMIT_SHORT_SHA -t ci_lofar_tbbwriter_control:$CI_COMMIT_SHORT_SHA installed/share/docker/lofar-tbbwriter-control
     - docker login -u $CI_NEXUS_REGISTRY_USERNAME -p $CI_NEXUS_REGISTRY_PASSWORD $CI_NEXUS_REGISTRY
     - docker tag ci_lofar_tbbwriter_control:$CI_COMMIT_SHORT_SHA $CI_NEXUS_REGISTRY_LOCATION/ci_lofar_tbbwriter_control:$CI_COMMIT_SHORT_SHA
@@ -295,6 +332,7 @@ build_lofar_tbbwriter_control_docker_image:
   interruptible: true
   needs:
     - build_Docker
+    - build_lofar_base
   when: manual # this takes a long time to build and the image is humongous. So let's not do that on every commit.
 
 #
@@ -667,8 +705,10 @@ deploy-CCU_MAC-prod:
   only:
     - tags
 
-deploy-PULP-prod:
+deploy-lofar-pipeline-prod:
   stage: deploy-prod
+  variables:
+    CEP_PARTITION: "cpu"
   before_script:
     # needed so that gitlab-runner can login at cep4
     - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )'
@@ -676,20 +716,81 @@ deploy-PULP-prod:
     - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
     - mkdir -p ~/.ssh
     - chmod 700 ~/.ssh
-    - ssh-keyscan head.cep4.control.lofar >> ~/.ssh/known_hosts
+    - ssh-keyscan ${CEP_HEADNODE} >> ~/.ssh/known_hosts
     - chmod 644 ~/.ssh/known_hosts
   script:
-    - ssh lofarsys@head.cep4.control.lofar "clush -S -P -w cpu[01-50] -w gpu[01-04] -w head[01-02] docker pull $CI_NEXUS_REGISTRY_LOCATION/ci_lofar_pulp:$CI_COMMIT_SHORT_SHA"
+    - NODES=$(ssh lofarsys@${CEP_HEADNODE} sinfo -p ${CEP_PARTITION} -h -o %N)
+    - echo "Deploying on ${NODES}"
+    - ssh lofarsys@${CEP_HEADNODE} "clush -S -P -w ${NODES} -w head[01-02] docker pull $CI_NEXUS_REGISTRY_LOCATION/ci_lofar_pipeline:$CI_COMMIT_SHORT_SHA"
+    # the pipeline is built to be run as lofar-pipeline:$CI_COMMIT_REF_NAME
+    - ssh lofarsys@${CEP_HEADNODE} "clush -S -P -w ${NODES} -w head[01-02] docker tag $CI_NEXUS_REGISTRY_LOCATION/ci_lofar_pipeline:$CI_COMMIT_SHORT_SHA lofar-pipeline:$CI_COMMIT_REF_NAME"
+    - if $MARK_PIPELINES_LATEST; then ssh lofarsys@${CEP_HEADNODE} "clush -S -P -w ${NODES} -w head[01-02] docker tag lofar-pipeline:$CI_COMMIT_REF_NAME lofar-pipeline:latest"; done
+  environment:
+    name: production
+  needs:
+    - build_lofar_pipeline
+  when: manual
+  only:
+    - tags
+
+deploy-dynspec-prod:
+  stage: deploy-prod
+  variables:
+    CEP_PARTITION: "cpu"
+  before_script:
+    # needed so that gitlab-runner can login at cep4
+    - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )'
+    - eval $(ssh-agent -s)
+    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
+    - mkdir -p ~/.ssh
+    - chmod 700 ~/.ssh
+    - ssh-keyscan ${CEP_HEADNODE} >> ~/.ssh/known_hosts
+    - chmod 644 ~/.ssh/known_hosts
+  script:
+    - NODES=$(ssh lofarsys@${CEP_HEADNODE} sinfo -p ${CEP_PARTITION} -h -o %N)
+    - echo "Deploying on ${NODES}"
+    - ssh lofarsys@${CEP_HEADNODE} "clush -S -P -w ${NODES} -w head[01-02] docker pull $CI_NEXUS_REGISTRY_LOCATION/ci_dynspec:$CI_COMMIT_SHORT_SHA"
+    # the pipeline is built to be run as dynspec:latest!!
+    - ssh lofarsys@${CEP_HEADNODE} "clush -S -P -w ${NODES} -w head[01-02] docker tag $CI_NEXUS_REGISTRY_LOCATION/ci_dynspec:$CI_COMMIT_SHORT_SHA dynspec:latest"
+  environment:
+    name: production
+  needs:
+    - build_dynspec
+  when: manual
+  only:
+    - tags
+
+deploy-lofar-pulp-prod:
+  stage: deploy-prod
+  variables:
+    CEP_PARTITION: "cpu"
+  before_script:
+    # needed so that gitlab-runner can login at cep4
+    - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )'
+    - eval $(ssh-agent -s)
+    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
+    - mkdir -p ~/.ssh
+    - chmod 700 ~/.ssh
+    - ssh-keyscan ${CEP_HEADNODE} >> ~/.ssh/known_hosts
+    - chmod 644 ~/.ssh/known_hosts
+  script:
+    - NODES=$(ssh lofarsys@${CEP_HEADNODE} sinfo -p ${CEP_PARTITION} -h -o %N)
+    - echo "Deploying on ${NODES}"
+    - ssh lofarsys@${CEP_HEADNODE} "clush -S -P -w ${NODES} -w head[01-02] docker pull $CI_NEXUS_REGISTRY_LOCATION/ci_lofar_pulp:$CI_COMMIT_SHORT_SHA"
     # the pulp pipeline is built to be run as lofar-pulp:$CI_COMMIT_REF_NAME through setting the LOFAR_TAG in build_lofar_pulp_docker_image
-    - ssh lofarsys@head.cep4.control.lofar "clush -S -P -w cpu[01-50] -w gpu[01-04] -w head[01-02] docker tag $CI_NEXUS_REGISTRY_LOCATION/ci_lofar_pulp:$CI_COMMIT_SHORT_SHA lofar-pulp:$CI_COMMIT_REF_NAME"
+    - ssh lofarsys@${CEP_HEADNODE} "clush -S -P -w ${NODES} -w head[01-02] docker tag $CI_NEXUS_REGISTRY_LOCATION/ci_lofar_pulp:$CI_COMMIT_SHORT_SHA lofar-pulp:$CI_COMMIT_REF_NAME"
+  environment:
+    name: production
   needs:
-    - build_lofar_pulp_docker_image
+    - build_lofar_pulp
   when: manual
   only:
     - tags
 
 deploy-tbbwriter-control-prod:
   stage: deploy-prod
+  variables:
+    CEP_PARTITION: "cpu"
   before_script:
     # needed so that gitlab-runner can login at cep4
     - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )'
@@ -697,14 +798,18 @@ deploy-tbbwriter-control-prod:
     - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
     - mkdir -p ~/.ssh
     - chmod 700 ~/.ssh
-    - ssh-keyscan head.cep4.control.lofar >> ~/.ssh/known_hosts
+    - ssh-keyscan ${CEP_HEADNODE} >> ~/.ssh/known_hosts
     - chmod 644 ~/.ssh/known_hosts
   script:
-    - ssh lofarsys@head.cep4.control.lofar "clush -S -P -w cpu[01-50] -w gpu[01-04] -w head[01-02] docker pull $CI_NEXUS_REGISTRY_LOCATION/ci_lofar_tbbwriter_control:$CI_COMMIT_SHORT_SHA"
+    - NODES=$(ssh lofarsys@${CEP_HEADNODE} sinfo -p ${CEP_PARTITION} -h -o %N)
+    - echo "Deploying on ${NODES}"
+    - ssh lofarsys@${CEP_HEADNODE} "clush -S -P -w ${NODES} -w head[01-02] docker pull $CI_NEXUS_REGISTRY_LOCATION/ci_lofar_tbbwriter_control:$CI_COMMIT_SHORT_SHA"
     # the tbbwriter pipeline is built to be run as lofar-tbbwriter-control:$CI_COMMIT_REF_NAME
-    - ssh lofarsys@head.cep4.control.lofar "clush -S -P -w cpu[01-50] -w gpu[01-04] -w head[01-02] docker tag $CI_NEXUS_REGISTRY_LOCATION/ci_lofar_tbbwriter_control:$CI_COMMIT_SHORT_SHA lofar-tbbwriter-control:$CI_COMMIT_REF_NAME"
+    - ssh lofarsys@${CEP_HEADNODE} "clush -S -P -w ${NODES} -w head[01-02] docker tag $CI_NEXUS_REGISTRY_LOCATION/ci_lofar_tbbwriter_control:$CI_COMMIT_SHORT_SHA lofar-tbbwriter-control:$CI_COMMIT_REF_NAME"
+  environment:
+    name: production
   needs:
-    - build_lofar_tbbwriter_control_docker_image
+    - build_lofar_tbbwriter_control
   when: manual
   only:
     - tags

CEP/CMakeLists.txt

 # $Id$
 
 lofar_add_package(Calibration)
-lofar_add_package(DP3)
 lofar_add_package(GSM)
 lofar_add_package(LMWCommon)
 lofar_add_package(MS)

CMake/FindWCSLIB.cmake

+# - Try to find WCSLIB: the FITS "World Coordinate System" library
+# Variables used by this module:
+#  WCSLIB_ROOT_DIR     - WCSLIB root directory
+# Variables defined by this module:
+#  WCSLIB_FOUND        - system has WCSLIB
+#  WCSLIB_INCLUDE_DIR  - the WCSLIB include directory (cached)
+#  WCSLIB_INCLUDE_DIRS - the WCSLIB include directories
+#                        (identical to WCSLIB_INCLUDE_DIR)
+#  WCSLIB_LIBRARY      - the WCSLIB library (cached)
+#  WCSLIB_LIBRARIES    - the WCSLIB libraries
+#                        (identical to WCSLIB_LIBRARY)
+
+# Copyright (C) 2009
+# ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+#
+# $Id$
+
+if(NOT WCSLIB_FOUND)
+
+  find_path(WCSLIB_INCLUDE_DIR wcslib/wcs.h
+    HINTS ${WCSLIB_ROOT_DIR} PATH_SUFFIXES include)
+  find_library(WCSLIB_LIBRARY wcs
+    HINTS ${WCSLIB_ROOT_DIR} PATH_SUFFIXES lib)
+  find_library(M_LIBRARY m)
+  mark_as_advanced(WCSLIB_INCLUDE_DIR WCSLIB_LIBRARY M_LIBRARY)
+
+  include(FindPackageHandleStandardArgs)
+  find_package_handle_standard_args(WCSLIB DEFAULT_MSG
+    WCSLIB_LIBRARY M_LIBRARY WCSLIB_INCLUDE_DIR)
+
+  set(WCSLIB_INCLUDE_DIRS ${WCSLIB_INCLUDE_DIR})
+  set(WCSLIB_LIBRARIES ${WCSLIB_LIBRARY} ${M_LIBRARY})
+
+endif(NOT WCSLIB_FOUND)

CMake/LofarPackageList.cmake

@@ -15,7 +15,6 @@
 if(NOT DEFINED LOFAR_PACKAGE_LIST_INCLUDED)
   set(LOFAR_PACKAGE_LIST_INCLUDED TRUE)
   set(Calibration_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/Calibration)
-  set(DP3_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/DP3)
   set(GSM_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/GSM)
   set(LMWCommon_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/LMWCommon)
   set(MS_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/MS)

Docker/CMakeLists.txt

@@ -23,8 +23,6 @@ lofar_add_bin_program(versiondocker versiondocker.cc)
 
 # Directories with Dockerfile.tmpl to parse
 set(DOCKER_TEMPLATE_DIRS
-  lofar-base
-  lofar-pipeline
   lofar-subbandtbbwriter
   lofar-tbbwriter-control)
 

Docker/lofar-base/Dockerfile.tmpl → Docker/lofar-base/Dockerfile

@@ -7,26 +7,6 @@ FROM ubuntu:18.04
 # the shell's string substitutions won't work.
 SHELL ["/bin/bash", "-c"]
 
-# Add build date to the environment
-ARG DOCKER_IMAGE_BUILD_DATE="The build date has to be set by a --build-arg parameter!"
-ENV DOCKER_IMAGE_BUILD_DATE=${DOCKER_IMAGE_BUILD_DATE}
-
-# Allow to specify the LOFAR branch at build time of the image.
-# Use master as the default if nothing is specified.
-ARG LOFAR_VERSION=latest
-ENV LOFAR_VERSION=${LOFAR_VERSION}
-
-# LOFAR_TAG is an environment variable that gets used
-# (at least) by the lofar-pipeline image config file
-# CEP/Pipeline/recipes/sip/pipeline.cfg.CEP4.
-# It determines which version of the lofar-pipeline
-# images gets executed on the nodes.
-ENV LOFAR_TAG=${LOFAR_VERSION}
-
-# Tell image build information.
-ARG LOFAR_BUILDVARIANT=gnucxx11_cep4_optarch
-ENV LOFAR_BUILDVARIANT=${LOFAR_BUILDVARIANT}
-
 # Allow to specify the wanted CXX
 # Default is the new one.
 ARG CXX_ABI=1
@@ -156,12 +136,12 @@ RUN aptitude install -y sasl2-bin libuuid1 libnss3 libnspr4 xqilla libssl1.1 lib
 RUN export BUILD_PACKAGES="git rsync swig ruby ruby-dev python-dev python-setuptools libsasl2-dev pkg-config cmake libtool uuid-dev libxerces-c-dev libnss3-dev libnspr4-dev help2man fakeroot build-essential g++ debhelper libssl-dev libxqilla-dev libboost-program-options${BOOST_VERSION}-dev libboost-filesystem${BOOST_VERSION}-dev" && \
     aptitude install -y ${BUILD_PACKAGES} && \
     mkdir ${INSTALLDIR}/qpid && \
-    git clone --depth 1 --shallow-submodules --branch ${LOFAR_VERSION//latest/master} https://git.astron.nl/ro/lofar.git /tmp/LOFAR && \
+    git clone https://git.astron.nl/ro/lofar.git /tmp/LOFAR && \
+    cd /tmp/LOFAR && \
     rsync --archive /tmp/LOFAR/LCS/MessageBus/qpid/ ${INSTALLDIR}/qpid/ && \
     rm -rf /tmp/LOFAR && \
     bash -c "HOME=/tmp ${INSTALLDIR}/qpid/local/sbin/build_qpid" && \
     bash -c "strip ${INSTALLDIR}/qpid/{bin,lib}/* || true" && \
-    bash -c "rm -rf /tmp/sources" && \
     aptitude purge -y ${BUILD_PACKAGES} && \
     aptitude clean && \
     aptitude autoclean
@@ -180,16 +160,6 @@ RUN aptitude install -y libqpid-proton8 libqpid-proton-cpp8 python3-qpid-proton
 # *******************
 RUN aptitude install -y python3-kombu
 
-# *******************
-#   Unconsumed build arguments
-# *******************
-
-# Apply a finger print to force a rebuild if the source code changes. Supply a unique ID here to force a rebuild.
-ARG LOFAR_FINGERPRINT=whatever
-
-# we do not use this, but will be configured to produce it
-ARG LOFAR_BASE_IMAGE_VERSION=latest
-
 #
 # entry
 #

Docker/lofar-pipeline/Dockerfile.tmpl → Docker/lofar-pipeline/Dockerfile

@@ -5,20 +5,11 @@
 # Allow to specify a specific base image version.
 ARG LOFAR_BASE_IMAGE_VERSION=latest
 FROM lofar-base:${LOFAR_BASE_IMAGE_VERSION}
+
 # Docker mandates that this has to be done again
 ARG LOFAR_BASE_IMAGE_VERSION=latest
 ENV LOFAR_BASE_IMAGE_VERSION=${LOFAR_BASE_IMAGE_VERSION}
 
-# Add build date to the environment
-ARG DOCKER_IMAGE_BUILD_DATE="The build date has to be set by a --build-arg parameter!"
-ENV DOCKER_IMAGE_BUILD_DATE=${DOCKER_IMAGE_BUILD_DATE}
-
-# Allow to specify the LOFAR branch at build time of the image.
-# Use master as the default if nothing is specified.
-ARG LOFAR_VERSION=latest
-ENV LOFAR_VERSION=${LOFAR_VERSION}
-
-
 ENV AOFLAGGER_VERSION=v3.1.0 \
     PYBDSF_VERSION=v1.9.0 \
     DYSCO_VERSION=v1.2 \
@@ -220,15 +211,28 @@ RUN aptitude install -y libncurses${NCURSES_VERSION} liblog4cplus-${LIBLOG4CPLUS
     aptitude clean && \
     aptitude autoclean
 
-# Apply a finger print to force a rebuild if the source code changes. Supply a unique ID here to force a rebuild.
-ARG LOFAR_FINGERPRINT=whatever
-ENV LOFAR_FINGERPRINT=${LOFAR_FINGERPRINT}
+# Tell image build information.
+ARG LOFAR_BUILDVARIANT=gnucxx11_cep4_optarch
+ENV LOFAR_BUILDVARIANT=${LOFAR_BUILDVARIANT}
+
+# Allow to specify the LOFAR branch at build time of the image.
+# Use master as the default if nothing is specified.
+ARG LOFAR_VERSION=latest
+ENV LOFAR_VERSION=${LOFAR_VERSION}
+
+# LOFAR_TAG is an environment variable that gets used
+# (at least) by the lofar-pipeline image config file
+# CEP/Pipeline/recipes/sip/pipeline.cfg.CEP4.
+# It determines which version of the lofar-pipeline
+# images gets executed on the nodes.
+ENV LOFAR_TAG=${LOFAR_VERSION}
 
 # Install
 RUN export BUILD_PACKAGES="git cmake g++ gfortran python3-setuptools bison flex libncurses-dev liblog4cplus-dev libboost${BOOST_VERSION}-all-dev libboost-python${BOOST_VERSION}-dev python3-dev libxml2-dev pkg-config libpng-dev liblapack-dev libfftw3-dev libunittest++-dev libxml++${LIBXMLPP_VERSION}-dev libgsl-dev libreadline-dev binutils-dev libcfitsio-dev wcslib-dev liblapack-dev libqpid-proton-dev libqpid-proton-cpp-dev libpqxx-dev libhdf5-dev libarmadillo-dev" && \
     aptitude install -y ${BUILD_PACKAGES} && \
     mkdir -p ${INSTALLDIR}/lofar/build/${LOFAR_BUILDVARIANT} && \
-    git clone --depth 1 --shallow-submodules --branch ${LOFAR_VERSION//latest/master} https://git.astron.nl/ro/lofar.git ${INSTALLDIR}/lofar/src && \
+    git clone https://git.astron.nl/ro/lofar.git ${INSTALLDIR}/lofar/src && \
+    cd ${INSTALLDIR}/lofar/src && git checkout ${LOFAR_VERSION} && \
     cd ${INSTALLDIR}/lofar/build/${LOFAR_BUILDVARIANT} && \
     sed -i "s/ABI=0/ABI=${CXX_ABI}/g" ../../src/CMake/variants/GNUCXX11.cmake && \
     cmake -DCMAKE_CXX_FLAGS="${CXX_FLAGS}" -DBUILD_PACKAGES=Offline -DBUILD_TESTING=OFF -DCMAKE_INSTALL_PREFIX=${INSTALLDIR}/lofar/ -DPYTHON_EXECUTABLE=/usr/bin/python3 -DCASACORE_ROOT_DIR=${INSTALLDIR}/casacore/ -DBLITZ_ROOT_DIR=${INSTALLDIR}/blitz -DAOFLAGGER_ROOT_DIR=${INSTALLDIR}/aoflagger/ -DBDSF_ROOT_DIR=${INSTALLDIR}/pybdsf/lib/python${PYTHON_VERSION}/site-packages/ -DQPID_ROOT_DIR=/opt/qpid/ -DUSE_OPENMP=True ${INSTALLDIR}/lofar/src/ && \