Split Dockerfile into static base and always-to-built Prefactor

.gitlab-ci.yml

-image: lofareosc/prefactor3-cwl:latest
+image: docker:stable
 
 workflow:
   rules:
@@ -15,12 +15,9 @@ variables:
 
 stages:
   - versioning
-  - prepare
-  - download_data
-  - validate_cwl_scripts
-  - test_steps
-  - test_full
   - build
+  - tests
+  - docs
   - deploy
 
 before_script:
@@ -37,7 +34,8 @@ versioning:
     # Unshallowing ensures that 'git describe' works
     - git fetch --unshallow
     - echo PREFACTOR3_CWL_VERSION=$(git describe --tags --dirty --always) > versions.env
-    - echo INTEGRATION_IMAGE=${CI_REGISTRY_IMAGE}/integration:$(git log -n 1 --pretty=format:%H) >> versions.env
+    - echo INTEGRATION_BASE_IMAGE=${CI_REGISTRY_IMAGE}/integration_base:$(git log -n 1 --pretty=format:%H -- Docker/Dockerfile-base) >> versions.env
+    - echo INTEGRATION_IMAGE=${CI_REGISTRY_IMAGE}/integration_full:$(git log -n 1 --pretty=format:%H) >> versions.env
     - cat versions.env
   artifacts:
     reports:
@@ -45,30 +43,39 @@ versioning:
 
 # Docker login
 .prepare:
-  stage: prepare
+  stage: build
   needs: ["versioning"]
-  image: docker:20.10
+  image: docker:stable
   services:
-    - docker:20.10-dind
+    - docker:dind
   before_script:
     - echo "Logging in as $CI_REGISTRY_USER @ $CI_REGISTRY"
     - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
 
 # Create and push the integration image to the gitlab registry, if it does not exist.
-prepare-integration:
+build_base:
+  stage: build
   extends: .prepare
   script:
     - |
-      if ! docker manifest inspect $INTEGRATION_IMAGE > /dev/null; then
-        docker build --tag $INTEGRATION_IMAGE -f Docker/Dockerfile .
-        docker push $INTEGRATION_IMAGE
+      if ! docker manifest inspect $INTEGRATION_BASE_IMAGE > /dev/null; then
+        docker build --tag $INTEGRATION_BASE_IMAGE -f Docker/Dockerfile-base .
+        docker push $INTEGRATION_BASE_IMAGE
       fi
 
+install_prefactor:
+  stage: build
+  needs: ["versioning","build_base"]
+  image: docker:stable
+  extends: .prepare
+  script:
+    - docker build --build-arg BASE_TAG=$INTEGRATION_BASE_IMAGE --tag $INTEGRATION_IMAGE -f Docker/Dockerfile-prefactor .
+    - docker push $INTEGRATION_IMAGE
+    
 download_data:
-  needs: ["versioning","prepare-integration"]
-  # This step does not really require the complete integration image, just an image with `wget`, could run in parrallel with the prepare stage
+  needs: ["versioning","install_prefactor"]
   image: $INTEGRATION_IMAGE
-  stage: download_data
+  stage: build
   script:
     - mkdir data && cd data
     - wget -nv www.astron.nl/citt/ci_data/prefactor/$TEST_HBA_DATASET_NAME -O $TEST_HBA_DATASET_NAME && tar xfz $TEST_HBA_DATASET_NAME && rm -f $TEST_HBA_DATASET_NAME
@@ -79,15 +86,15 @@ download_data:
     - data
 
 validate_scripts:
-  stage: validate_cwl_scripts
+  stage: tests
   # Versioning required for $INTEGRATION_IMAGE
-  needs: ["versioning", "prepare-integration"]
+  needs: ["versioning", "download_data"]
   image: $INTEGRATION_IMAGE
   script:
     - for file in `find . -name 'steps/*.cwl'`; do cwltool --validate $file; done
 
 blsmooth:
-  stage: test_steps
+  stage: tests
   # Versioning required for $INTEGRATION_IMAGE
   needs: ["versioning", "download_data"]
   image: $INTEGRATION_IMAGE
@@ -95,21 +102,21 @@ blsmooth:
     - cwltool --no-container steps/blsmooth.cwl test_jobs/blsmooth.json
 
 find_skymodel_cal:
-  stage: test_steps
+  stage: tests
   needs: ["versioning", "download_data"]
   image: $INTEGRATION_IMAGE
   script:
     - cwltool --no-container --preserve-environment PYTHONPATH steps/find_skymodel_cal.cwl test_jobs/find_skymodel_cal.json
 
 check_ateam_separation:
-  stage: test_steps
+  stage: tests
   needs: ["versioning", "download_data"]
   image: $INTEGRATION_IMAGE
   script:
     - cwltool --no-container --preserve-environment PATH --preserve-environment PYTHONPATH steps/check_ateam_separation.cwl test_jobs/check_ateam_separation.json
 
 run_hba_calibrator:
-  stage: test_full
+  stage: tests
   needs: ["versioning", "download_data"]
   image: $INTEGRATION_IMAGE
   script:
@@ -117,16 +124,16 @@ run_hba_calibrator:
     - test_jobs/check_workflow_results.py results /builds/eosc/prefactor3-cwl/data/results_calibrator
 
 run_hba_target:
-  stage: test_full
+  stage: tests
   needs: ["versioning", "download_data"]
   image: $INTEGRATION_IMAGE
   script:
     - cwltool --no-container --preserve-environment PATH --preserve-environment PYTHONPATH --outdir results workflows/HBA_target.cwl test_jobs/HBA_target.json
     - test_jobs/check_workflow_results.py results /builds/eosc/prefactor3-cwl/data/results_target
 
-build-doc:
-  stage: build
-  needs: ["versioning", "prepare-integration"]
+build_doc:
+  stage: docs
+  needs: ["versioning", "download_data"]
   image: $INTEGRATION_IMAGE
   script:
     - cd docs
@@ -142,9 +149,9 @@ build-doc:
         - docs/**/*
 
 # Deploy to docker hub
-deploy-docker:
+deploy_docker:
   stage: deploy
-  needs: ["versioning", "prepare-integration"]
+  needs: ["versioning", "download_data"]
   image: docker:20.10
   services:
     - docker:20.10-dind
@@ -161,9 +168,9 @@ deploy-docker:
     - if: '$CI_COMMIT_BRANCH == "master"'
       when: always
 
-deploy-doc:
+deploy_doc:
   stage: deploy
-  needs: ["versioning", "build-doc"]
+  needs: ["versioning", "build_doc"]
   image: $INTEGRATION_IMAGE
   before_script:
     - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'

Docker/Dockerfile → Docker/Dockerfile-base

@@ -183,10 +183,7 @@ LABEL \
   RMextract.version.tag=${RMextract_TAG} \
   WSClean.version.tag=${WSClean_TAG}
 
-ENV PREFACTOR_VERSION='master' \
-    PREFACTOR_NAME=prefactor \
-    LC_ALL=C.UTF-8 \
-    PREFACTOR_DATA_ROOT=/usr/local/share/prefactor \
+ENV LC_ALL=C.UTF-8 \
     PYTHONPATH="/usr/local/bin:/usr/local/lib/python3.8/site-packages"
 
 RUN \
@@ -200,29 +197,11 @@ RUN \
   \
 # Install cwtool & cwl-runner
   python3 -m pip install --upgrade pip && \
-  python3 -m pip install cwltool cwl-runner
-  
-
-# Install Prefactor CWL
-COPY . /${PREFACTOR_NAME}
-RUN cd /${PREFACTOR_NAME} && \
-  pip3 install --upgrade $PWD && \
-  cd .. && \
-  rm -rfv ${PREFACTOR_NAME} && \
-  \
-# A user 'lofaruser' is added (this might be legacy -- not sure this is
-# still relevant).
-  mkdir /home/lofaruser/ && \ 
-  groupadd -r lofaruser && \
-  useradd --no-log-init -r -g lofaruser lofaruser && \
-  chown lofaruser:lofaruser /home/lofaruser && \
+  python3 -m pip install cwltool cwl-runner && \
   \
 # Try to run the compiled tools to make sure they run without
 # a problem (e.g. no missing libraries).
   aoflagger --version && \
   DP3 --version && \
   wsclean --version
-
-WORKDIR /home/lofaruser
-
-USER lofaruser
+  
\ No newline at end of file

Docker/Dockerfile-prefactor

+ARG BASE_TAG=latest
+FROM $BASE_TAG
+
+ENV PREFACTOR_VERSION='master' \
+    PREFACTOR_NAME=prefactor \
+    PREFACTOR_DATA_ROOT=/usr/local/share/prefactor
+
+# Install Prefactor CWL
+COPY . /${PREFACTOR_NAME}
+RUN cd /${PREFACTOR_NAME} && \
+  pip3 install --upgrade $PWD && \
+  cd .. && \
+  rm -rfv ${PREFACTOR_NAME} && \
+  \
+# A user 'lofaruser' is added (this might be legacy -- not sure this is
+# still relevant).
+  mkdir /home/lofaruser/ && \ 
+  groupadd -r lofaruser && \
+  useradd --no-log-init -r -g lofaruser lofaruser && \
+  chown lofaruser:lofaruser /home/lofaruser
+
+WORKDIR /home/lofaruser
+
+USER lofaruser