diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3e4455992c13b6139f9db300fcf987fc11c0becb..40df5a3ee364b5d9705f30385377b6a73f2d138a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,3 @@ -image: docker:latest - workflow: rules: # Don't create a pipeline if commit is on a branch with open merge requests @@ -31,16 +29,21 @@ stages: - deploy - finalize -before_script: - - mkdir workdir - - mkdir logs - -after_script: - - echo "All done" +.release: + before_script: + # Make sure release branch is checked out when building a release + - RELEASE=$(echo -n $CI_COMMIT_BRANCH | sed -n 's,^releases/,,p') + - | + if test -n "$RELEASE" + then + echo "Checking out branch $CI_COMMIT_BRANCH" + git checkout $CI_COMMIT_BRANCH + fi .setup_git: image: bitnami/git - script: + before_script: + - !reference [.release, before_script] - eval $(ssh-agent -s) - chmod 400 $SSH_PRIVATE_KEY - ssh-add $SSH_PRIVATE_KEY @@ -52,12 +55,23 @@ after_script: .setup_docker: stage: build + image: docker tags: - dind before_script: + - !reference [.release, before_script] - echo "Logging in as $CI_REGISTRY_USER @ $CI_REGISTRY" - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY +.run_tests: + stage: run_tests + image: $INTEGRATION_IMAGE + extends: .release + before_script: + - !reference [.release, before_script] + - mkdir -p workdir + - mkdir -p logs + .deploy: stage: deploy extends: .setup_docker @@ -68,6 +82,16 @@ after_script: - echo "Logging in as $DH_REGISTRY_USER @ DockerHub" - echo $DH_REGISTRY_PASSWORD | docker login -u $DH_REGISTRY_USER --password-stdin +.docs: + stage: docs + extends: .release + image: $INTEGRATION_IMAGE + +.finalize: + stage: finalize + extends: .setup_git + + ### Stage: initialize prepare_release: @@ -86,8 +110,8 @@ prepare_release: untracked: true when: on_failure before_script: + - !reference [.release, before_script] # When building a release, bail out if release tag already exists - - RELEASE=$(echo -n $CI_COMMIT_BRANCH | sed 's,^releases/,,') - | if git ls-remote --tags --exit-code origin $RELEASE > /dev/null then @@ -95,10 +119,8 @@ prepare_release: touch .tag.exists exit 1 fi - - !reference [.setup_git, script] + - !reference [.setup_git, before_script] script: - # Make sure the current commit is checked out - - git checkout $CI_COMMIT_BRANCH # Update dockerPull image URI in CWL steps with a tagged version - sed -ri "/dockerPull/s,(astronrd/linc).*,\1:$RELEASE," steps/*.cwl - git add -u steps/*.cwl @@ -113,20 +135,16 @@ prepare_release: # Skip CI on this push - git push --follow-tags -o ci.skip + ### Stage: versioning versioning: stage: versioning - image: python - before_script: - - pip install setuptools_scm + image: bitnami/git + extends: .release script: - # Make sure the current branch is checked out - - git checkout $CI_COMMIT_BRANCH - ./Docker/fetch_latest_commits.sh | tee commits.txt > versions.env - # Use a sub-command, to catch the exit status from `setuptools_scm` - - (echo -n "LINC_VERSION="; python -m setuptools_scm) >> versions.env - - echo RELEASE=$(echo $CI_COMMIT_BRANCH | sed -n 's,^releases/,,p') >> versions.env + - echo LINC_VERSION=$(git describe --tags --always) >> versions.env # Use hash of commits to determine version of base image (and rebuild if necessary) - echo INTEGRATION_BASE_IMAGE=$CI_REGISTRY_IMAGE/integration_base:$(sha256sum commits.txt | cut -d " " -f 1) >> versions.env - echo INTEGRATION_IMAGE=$CI_REGISTRY_IMAGE/integration_full:$(git log -n 1 --pretty=format:%H) >> versions.env @@ -135,6 +153,7 @@ versioning: reports: dotenv: versions.env + ### Stage: build build_base: @@ -163,6 +182,7 @@ build_base: docker push $INTEGRATION_BASE_IMAGE fi + ### Stage: install install_linc: @@ -184,7 +204,8 @@ install_linc: . - docker push $INTEGRATION_IMAGE -### Stage: prepare_tests + +## Stage: prepare_tests download_data: image: $INTEGRATION_IMAGE @@ -200,13 +221,13 @@ download_data: - wget -nv https://support.astron.nl/software/ci_data/linc/$TARGET_HBA_SELFCAL_RESULTS_NAME -O $TARGET_HBA_SELFCAL_RESULTS_NAME && tar xfz $TARGET_HBA_SELFCAL_RESULTS_NAME && rm -f $TARGET_HBA_SELFCAL_RESULTS_NAME artifacts: paths: - - data + - data + ### Stage: run_tests validate_scripts: - stage: run_tests - image: $INTEGRATION_IMAGE + extends: .run_tests script: - | errors=0 @@ -217,26 +238,22 @@ validate_scripts: ((errors == 0)) blsmooth: - stage: run_tests - image: $INTEGRATION_IMAGE + extends: .run_tests script: - cwltool --no-container steps/blsmooth.cwl test_jobs/blsmooth.json find_skymodel_cal: - stage: run_tests - image: $INTEGRATION_IMAGE + extends: .run_tests script: - cwltool --no-container --preserve-environment PYTHONPATH steps/find_skymodel_cal.cwl test_jobs/find_skymodel_cal.json check_ateam_separation: - stage: run_tests - image: $INTEGRATION_IMAGE + extends: .run_tests script: - cwltool --no-container --preserve-environment PATH --preserve-environment PYTHONPATH steps/check_ateam_separation.cwl test_jobs/check_ateam_separation.json run_hba_calibrator: - stage: run_tests - image: $INTEGRATION_IMAGE + extends: .run_tests script: - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_calibrator/ workflows/HBA_calibrator.cwl test_jobs/HBA_calibrator.json - test_jobs/check_workflow_results.py results $CI_PROJECT_DIR/data/results_calibrator @@ -248,8 +265,7 @@ run_hba_calibrator: when: on_failure run_hba_target: - stage: run_tests - image: $INTEGRATION_IMAGE + extends: .run_tests script: - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_target/ workflows/HBA_target.cwl test_jobs/HBA_target.json - test_jobs/check_workflow_results.py results $CI_PROJECT_DIR/data/results_target @@ -265,8 +281,7 @@ run_hba_target: when: on_failure run_hba_target_selfcal: - stage: run_tests - image: $INTEGRATION_IMAGE + extends: .run_tests script: - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_target/ workflows/HBA_target.cwl test_jobs/HBA_target_selfcal.json - test_jobs/check_workflow_results.py --skip_soltabs TGSSphase_final results $CI_PROJECT_DIR/data/results_target_selfcal @@ -282,8 +297,7 @@ run_hba_target_selfcal: when: on_failure run_lba_calibrator: - stage: run_tests - image: $INTEGRATION_IMAGE + extends: .run_tests script: - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_lba_calibrator workflows/LBA_calibrator.cwl test_jobs/LBA_calibrator.json - test_jobs/check_workflow_results.py results $CI_PROJECT_DIR/data/results_calibrator_lba @@ -295,8 +309,7 @@ run_lba_calibrator: when: on_failure run_lba_target: - stage: run_tests - image: $INTEGRATION_IMAGE + extends: .run_tests script: - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_lba_target/ workflows/LBA_target.cwl test_jobs/LBA_target.json - test_jobs/check_workflow_results.py --skip_soltabs GSMtec_final results $CI_PROJECT_DIR/data/results_target_lba @@ -311,11 +324,11 @@ run_lba_target: - cal_solutions.tar.gz when: on_failure + ### Stage: docs build_doc: - stage: docs - image: $INTEGRATION_IMAGE + extends: .docs before_script: - apt-get update - apt-get install -y make @@ -333,18 +346,18 @@ build_doc: - changes: - docs/**/* + ### Stage: deploy deploy_docker: stage: deploy extends: .deploy script: - # Replace characters that are now allowed in a tag string with a dash - - LINC_TAG=${LINC_VERSION//[^[:alnum:]_.-]/-} + - echo "Deploying to DockerHub, using $LINC_VERSION as image tag" - docker pull $INTEGRATION_IMAGE - - docker tag $INTEGRATION_IMAGE $CI_PROJECT_PATH:$LINC_TAG + - docker tag $INTEGRATION_IMAGE $CI_PROJECT_PATH:$LINC_VERSION - docker tag $INTEGRATION_IMAGE $CI_PROJECT_PATH:latest - - docker push $CI_PROJECT_PATH:$LINC_TAG + - docker push $CI_PROJECT_PATH:$LINC_VERSION - docker push $CI_PROJECT_PATH:latest rules: # Run on the default branch or on a release branch @@ -361,19 +374,19 @@ deploy_docker_tag_stable: - docker tag $INTEGRATION_IMAGE $CI_PROJECT_PATH:stable - docker push $CI_PROJECT_PATH:stable + ### Stage: finalize rollback_release: stage: finalize + extends: .finalize rules: # Run this job if the pipeline fails, to undo changes made in prepare_release. # We only care about removing the tag; other changes can remain. - if: '$CI_COMMIT_BRANCH =~ /^releases//' when: on_failure - before_script: - - !reference [.setup_git, script] script: - - RELEASE=$(echo -n $CI_COMMIT_BRANCH | sed 's,^releases/,,') + - echo "Rolling back release $RELEASE" - | if test -f .tag.exists then @@ -384,7 +397,7 @@ rollback_release: finalize_release: stage: finalize - extends: .setup_git + extends: .finalize rules: # Run this job if the pipeline succeeds, to create a versioned release. # A versioned release is a release whose branch/tag name matches a string @@ -395,11 +408,8 @@ finalize_release: # the default branch. - if: '$CI_COMMIT_BRANCH =~ /^releases\/v[0-9]+(\.[0-9]+)*/' when: on_success - before_script: - - !reference [.setup_git, script] script: - # Make sure the current branch is checked out - - git checkout $CI_COMMIT_BRANCH + - echo "Finalizing release $RELEASE" # Update dockerPull image URI in CWL steps by removing version tag - sed -ri "/dockerPull/s,(astronrd/linc).*,\1," steps/*.cwl - git add -u steps/*.cwl @@ -414,6 +424,7 @@ finalize_release: # Next switch to the default branch and make sure it's up to date - git checkout $CI_DEFAULT_BRANCH - git pull + - echo "Merging changes from $CI_COMMIT_BRANCH into $CI_DEFAULT_BRANCH" # Merge release branch into the default branch - git merge $CI_COMMIT_BRANCH -m"Merged release branch into default branch (by Gitlab CI)" # Skip CI on this push