Improve CI pipeline

.gitlab-ci.yml

@@ -2,7 +2,7 @@ image: docker:latest
 
 workflow:
   rules:
-    # don't create a pipeline if its a commit pipeline, on a branch and that branch has open merge requests.
+    # Don't create a pipeline if commit is on a branch with open merge requests
     - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
       when: never
     - when: always
@@ -21,7 +21,8 @@ variables:
 stages:
   - versioning
   - build
-  - tests
+  - prepare_tests
+  - run_tests
   - docs
   - deploy
 
@@ -51,15 +52,13 @@ versioning:
 # Docker login
 .prepare:
   stage: build
-  needs: ["versioning"]
-  image: docker:latest
   tags:
     - dind
   before_script:
     - echo "Logging in as $CI_REGISTRY_USER @ $CI_REGISTRY"
     - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
 
-# Create and push the integration image to the gitlab registry, if it does not exist.
+# Create and push the integration image to the gitlab registry, if it does not exist
 build_base:
   stage: build
   extends: .prepare
@@ -84,15 +83,9 @@ build_base:
           -f Docker/Dockerfile-base .
         docker push $INTEGRATION_BASE_IMAGE
       fi
-  tags:
-    - das6
 
 install_linc:
   stage: build
-  needs: ["versioning","build_base"]
-  image: docker:latest
-  tags:
-    - dind
   extends: .prepare
   script:
     - |
@@ -105,9 +98,8 @@ install_linc:
     - docker push $INTEGRATION_IMAGE
 
 download_data:
-  needs: ["versioning","install_linc"]
   image: $INTEGRATION_IMAGE
-  stage: build
+  stage: prepare_tests
   script:
     - mkdir data && cd data
     - wget -nv https://support.astron.nl/software/ci_data/linc/$TEST_HBA_DATASET_NAME -O $TEST_HBA_DATASET_NAME && tar xfz $TEST_HBA_DATASET_NAME && rm -f $TEST_HBA_DATASET_NAME
@@ -122,9 +114,7 @@ download_data:
     - data
 
 validate_scripts:
-  stage: tests
-  # Versioning required for $INTEGRATION_IMAGE
-  needs: ["versioning", "download_data"]
+  stage: run_tests
   image: $INTEGRATION_IMAGE
   script:
     - |
@@ -136,30 +126,25 @@ validate_scripts:
       ((errors == 0))
 
 blsmooth:
-  stage: tests
-  # Versioning required for $INTEGRATION_IMAGE
-  needs: ["versioning", "download_data"]
+  stage: run_tests
   image: $INTEGRATION_IMAGE
   script:
     - cwltool --no-container steps/blsmooth.cwl test_jobs/blsmooth.json
 
 find_skymodel_cal:
-  stage: tests
-  needs: ["versioning", "download_data"]
+  stage: run_tests
   image: $INTEGRATION_IMAGE
   script:
     - cwltool --no-container --preserve-environment PYTHONPATH steps/find_skymodel_cal.cwl test_jobs/find_skymodel_cal.json
 
 check_ateam_separation:
-  stage: tests
-  needs: ["versioning", "download_data"]
+  stage: run_tests
   image: $INTEGRATION_IMAGE
   script:
     - cwltool --no-container --preserve-environment PATH --preserve-environment PYTHONPATH steps/check_ateam_separation.cwl test_jobs/check_ateam_separation.json
 
 run_hba_calibrator:
-  stage: tests
-  needs: ["versioning", "download_data"]
+  stage: run_tests
   image: $INTEGRATION_IMAGE
   script:
     - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_calibrator/ workflows/HBA_calibrator.cwl test_jobs/HBA_calibrator.json
@@ -172,8 +157,7 @@ run_hba_calibrator:
     when: on_failure
 
 run_hba_target:
-  stage: tests
-  needs: ["versioning", "download_data"]
+  stage: run_tests
   image: $INTEGRATION_IMAGE
   script:
     - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_target/ workflows/HBA_target.cwl test_jobs/HBA_target.json
@@ -190,8 +174,7 @@ run_hba_target:
     when: on_failure
 
 run_hba_target_selfcal:
-  stage: tests
-  needs: ["versioning", "download_data"]
+  stage: run_tests
   image: $INTEGRATION_IMAGE
   script:
     - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_target/ workflows/HBA_target.cwl test_jobs/HBA_target_selfcal.json
@@ -208,8 +191,7 @@ run_hba_target_selfcal:
     when: on_failure
 
 run_lba_calibrator:
-  stage: tests
-  needs: ["versioning", "download_data"]
+  stage: run_tests
   image: $INTEGRATION_IMAGE
   script:
     - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_lba_calibrator workflows/LBA_calibrator.cwl test_jobs/LBA_calibrator.json
@@ -222,8 +204,7 @@ run_lba_calibrator:
     when: on_failure
 
 run_lba_target:
-  stage: tests
-  needs: ["versioning", "download_data"]
+  stage: run_tests
   image: $INTEGRATION_IMAGE
   script:
     - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_lba_target/ workflows/LBA_target.cwl test_jobs/LBA_target.json
@@ -241,7 +222,6 @@ run_lba_target:
 
 build_doc:
   stage: docs
-  needs: ["versioning", "download_data"]
   image: $INTEGRATION_IMAGE
   before_script:
     - apt-get update
@@ -263,10 +243,6 @@ build_doc:
 # Deploy to docker hub
 deploy_docker:
   stage: deploy
-  needs: ["versioning", "download_data"]
-  image: docker:latest
-  tags:
-    - dind
   before_script:
     - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
     - echo $DH_REGISTRY_PASSWORD | docker login -u $DH_REGISTRY_USER --password-stdin
@@ -284,8 +260,6 @@ deploy_docker:
 # Deploy a release version
 deploy_release:
   stage: deploy
-  needs: ["versioning", "download_data"]
-  image: docker:latest
   tags:
     - dind
   before_script:
@@ -315,9 +289,7 @@ deploy_release:
 
 deploy_docker_tag_stable:
   stage: deploy
-  needs: ["versioning", "download_data"]
   when: manual
-  image: docker:latest
   tags:
     - dind
   before_script: