Synced with development version of CI/CD pipeline

.gitlab-ci.yml

-image: docker:latest
-
 workflow:
   rules:
     # Don't create a pipeline if commit is on a branch with open merge requests
@@ -31,16 +29,21 @@ stages:
   - deploy
   - finalize
 
+.release:
   before_script:
-  - mkdir workdir
-  - mkdir logs
-
-after_script:
-  - echo "All done"
+    # Make sure release branch is checked out when building a release
+    - RELEASE=$(echo -n $CI_COMMIT_BRANCH | sed -n 's,^releases/,,p')
+    - |
+      if test -n "$RELEASE"
+      then
+        echo "Checking out branch $CI_COMMIT_BRANCH"
+        git checkout $CI_COMMIT_BRANCH
+      fi
 
 .setup_git:
   image: bitnami/git
-  script:
+  before_script:
+    - !reference [.release, before_script]
     - eval $(ssh-agent -s)
     - chmod 400 $SSH_PRIVATE_KEY
     - ssh-add $SSH_PRIVATE_KEY
@@ -52,12 +55,23 @@ after_script:
 
 .setup_docker:
   stage: build
+  image: docker
   tags:
     - dind
   before_script:
+    - !reference [.release, before_script]
     - echo "Logging in as $CI_REGISTRY_USER @ $CI_REGISTRY"
     - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
 
+.run_tests:
+  stage: run_tests
+  image: $INTEGRATION_IMAGE
+  extends: .release
+  before_script:
+    - !reference [.release, before_script]
+    - mkdir -p workdir
+    - mkdir -p logs
+
 .deploy:
   stage: deploy
   extends: .setup_docker
@@ -68,6 +82,16 @@ after_script:
     - echo "Logging in as $DH_REGISTRY_USER @ DockerHub"
     - echo $DH_REGISTRY_PASSWORD | docker login -u $DH_REGISTRY_USER --password-stdin
 
+.docs:
+  stage: docs
+  extends: .release
+  image: $INTEGRATION_IMAGE
+
+.finalize:
+  stage: finalize
+  extends: .setup_git
+
+
 ### Stage: initialize
 
 prepare_release:
@@ -86,8 +110,8 @@ prepare_release:
     untracked: true
     when: on_failure
   before_script:
+    - !reference [.release, before_script]
     # When building a release, bail out if release tag already exists
-    - RELEASE=$(echo -n $CI_COMMIT_BRANCH | sed 's,^releases/,,')
     - |
       if git ls-remote --tags --exit-code origin $RELEASE > /dev/null
       then
@@ -95,10 +119,8 @@ prepare_release:
         touch .tag.exists
         exit 1
       fi
-    - !reference [.setup_git, script]
+    - !reference [.setup_git, before_script]
   script:
-    # Make sure the current commit is checked out
-    - git checkout $CI_COMMIT_BRANCH
     # Update dockerPull image URI in CWL steps with a tagged version
     - sed -ri "/dockerPull/s,(astronrd/linc).*,\1:$RELEASE," steps/*.cwl
     - git add -u steps/*.cwl
@@ -113,20 +135,16 @@ prepare_release:
     # Skip CI on this push
     - git push --follow-tags -o ci.skip
 
+
 ### Stage: versioning
 
 versioning:
   stage: versioning
-  image: python
-  before_script:
-    - pip install setuptools_scm
+  image: bitnami/git
+  extends: .release
   script:
-    # Make sure the current branch is checked out
-    - git checkout $CI_COMMIT_BRANCH
     - ./Docker/fetch_latest_commits.sh | tee commits.txt > versions.env
-    # Use a sub-command, to catch the exit status from `setuptools_scm`
-    - (echo -n "LINC_VERSION="; python -m setuptools_scm) >> versions.env
-    - echo RELEASE=$(echo $CI_COMMIT_BRANCH | sed -n 's,^releases/,,p') >> versions.env
+    - echo LINC_VERSION=$(git describe --tags --always) >> versions.env
     # Use hash of commits to determine version of base image (and rebuild if necessary)
     - echo INTEGRATION_BASE_IMAGE=$CI_REGISTRY_IMAGE/integration_base:$(sha256sum commits.txt | cut -d " " -f 1) >> versions.env
     - echo INTEGRATION_IMAGE=$CI_REGISTRY_IMAGE/integration_full:$(git log -n 1 --pretty=format:%H) >> versions.env
@@ -135,6 +153,7 @@ versioning:
     reports:
       dotenv: versions.env
 
+
 ### Stage: build
 
 build_base:
@@ -163,6 +182,7 @@ build_base:
         docker push $INTEGRATION_BASE_IMAGE
       fi
 
+
 ### Stage: install
 
 install_linc:
@@ -184,7 +204,8 @@ install_linc:
         .
     - docker push $INTEGRATION_IMAGE
 
-### Stage: prepare_tests
+
+## Stage: prepare_tests
 
 download_data:
   image: $INTEGRATION_IMAGE
@@ -202,11 +223,11 @@ download_data:
     paths:
       - data
 
+
 ### Stage: run_tests
 
 validate_scripts:
-  stage: run_tests
-  image: $INTEGRATION_IMAGE
+  extends: .run_tests
   script:
     - |
       errors=0
@@ -217,26 +238,22 @@ validate_scripts:
       ((errors == 0))
 
 blsmooth:
-  stage: run_tests
-  image: $INTEGRATION_IMAGE
+  extends: .run_tests
   script:
     - cwltool --no-container steps/blsmooth.cwl test_jobs/blsmooth.json
 
 find_skymodel_cal:
-  stage: run_tests
-  image: $INTEGRATION_IMAGE
+  extends: .run_tests
   script:
     - cwltool --no-container --preserve-environment PYTHONPATH steps/find_skymodel_cal.cwl test_jobs/find_skymodel_cal.json
 
 check_ateam_separation:
-  stage: run_tests
-  image: $INTEGRATION_IMAGE
+  extends: .run_tests
   script:
     - cwltool --no-container --preserve-environment PATH --preserve-environment PYTHONPATH steps/check_ateam_separation.cwl test_jobs/check_ateam_separation.json
 
 run_hba_calibrator:
-  stage: run_tests
-  image: $INTEGRATION_IMAGE
+  extends: .run_tests
   script:
     - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_calibrator/ workflows/HBA_calibrator.cwl test_jobs/HBA_calibrator.json
     - test_jobs/check_workflow_results.py results $CI_PROJECT_DIR/data/results_calibrator
@@ -248,8 +265,7 @@ run_hba_calibrator:
     when: on_failure
 
 run_hba_target:
-  stage: run_tests
-  image: $INTEGRATION_IMAGE
+  extends: .run_tests
   script:
     - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_target/ workflows/HBA_target.cwl test_jobs/HBA_target.json
     - test_jobs/check_workflow_results.py results $CI_PROJECT_DIR/data/results_target
@@ -265,8 +281,7 @@ run_hba_target:
     when: on_failure
 
 run_hba_target_selfcal:
-  stage: run_tests
-  image: $INTEGRATION_IMAGE
+  extends: .run_tests
   script:
     - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_target/ workflows/HBA_target.cwl test_jobs/HBA_target_selfcal.json
     - test_jobs/check_workflow_results.py --skip_soltabs TGSSphase_final results $CI_PROJECT_DIR/data/results_target_selfcal
@@ -282,8 +297,7 @@ run_hba_target_selfcal:
     when: on_failure
 
 run_lba_calibrator:
-  stage: run_tests
-  image: $INTEGRATION_IMAGE
+  extends: .run_tests
   script:
     - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_lba_calibrator workflows/LBA_calibrator.cwl test_jobs/LBA_calibrator.json
     - test_jobs/check_workflow_results.py results $CI_PROJECT_DIR/data/results_calibrator_lba
@@ -295,8 +309,7 @@ run_lba_calibrator:
     when: on_failure
 
 run_lba_target:
-  stage: run_tests
-  image: $INTEGRATION_IMAGE
+  extends: .run_tests
   script:
     - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_lba_target/ workflows/LBA_target.cwl test_jobs/LBA_target.json
     - test_jobs/check_workflow_results.py --skip_soltabs GSMtec_final results $CI_PROJECT_DIR/data/results_target_lba
@@ -311,11 +324,11 @@ run_lba_target:
       - cal_solutions.tar.gz
     when: on_failure
 
+
 ### Stage: docs
 
 build_doc:
-  stage: docs
-  image: $INTEGRATION_IMAGE
+  extends: .docs
   before_script:
     - apt-get update
     - apt-get install -y make
@@ -333,18 +346,18 @@ build_doc:
     - changes:
         - docs/**/*
 
+
 ### Stage: deploy
 
 deploy_docker:
   stage: deploy
   extends: .deploy
   script:
-    # Replace characters that are now allowed in a tag string with a dash
-    - LINC_TAG=${LINC_VERSION//[^[:alnum:]_.-]/-}
+    - echo "Deploying to DockerHub, using $LINC_VERSION as image tag"
     - docker pull $INTEGRATION_IMAGE
-    - docker tag $INTEGRATION_IMAGE $CI_PROJECT_PATH:$LINC_TAG
+    - docker tag $INTEGRATION_IMAGE $CI_PROJECT_PATH:$LINC_VERSION
     - docker tag $INTEGRATION_IMAGE $CI_PROJECT_PATH:latest
-    - docker push $CI_PROJECT_PATH:$LINC_TAG
+    - docker push $CI_PROJECT_PATH:$LINC_VERSION
     - docker push $CI_PROJECT_PATH:latest
   rules:
     # Run on the default branch or on a release branch
@@ -361,19 +374,19 @@ deploy_docker_tag_stable:
     - docker tag $INTEGRATION_IMAGE $CI_PROJECT_PATH:stable
     - docker push $CI_PROJECT_PATH:stable
 
+
 ### Stage: finalize
 
 rollback_release:
   stage: finalize
+  extends: .finalize
   rules:
     # Run this job if the pipeline fails, to undo changes made in prepare_release.
     # We only care about removing the tag; other changes can remain.
     - if: '$CI_COMMIT_BRANCH =~ /^releases//'
       when: on_failure
-  before_script:
-    - !reference [.setup_git, script]
   script:
-    - RELEASE=$(echo -n $CI_COMMIT_BRANCH | sed 's,^releases/,,')
+    - echo "Rolling back release $RELEASE"
     - |
       if test -f .tag.exists
       then
@@ -384,7 +397,7 @@ rollback_release:
 
 finalize_release:
   stage: finalize
-  extends: .setup_git
+  extends: .finalize
   rules:
     # Run this job if the pipeline succeeds, to create a versioned release.
     # A versioned release is a release whose branch/tag name matches a string
@@ -395,11 +408,8 @@ finalize_release:
     # the default branch.
     - if: '$CI_COMMIT_BRANCH =~ /^releases\/v[0-9]+(\.[0-9]+)*/'
       when: on_success
-  before_script:
-    - !reference [.setup_git, script]
   script:
-    # Make sure the current branch is checked out
-    - git checkout $CI_COMMIT_BRANCH
+    - echo "Finalizing release $RELEASE"
     # Update dockerPull image URI in CWL steps by removing version tag
     - sed -ri "/dockerPull/s,(astronrd/linc).*,\1," steps/*.cwl
     - git add -u steps/*.cwl
@@ -414,6 +424,7 @@ finalize_release:
     # Next switch to the default branch and make sure it's up to date
     - git checkout $CI_DEFAULT_BRANCH
     - git pull
+    - echo "Merging changes from $CI_COMMIT_BRANCH into $CI_DEFAULT_BRANCH"
     # Merge release branch into the default branch
     - git merge $CI_COMMIT_BRANCH -m"Merged release branch into default branch (by Gitlab CI)"
     # Skip CI on this push