Skip to content
Snippets Groups Projects
Commit 35d93bd6 authored by Marcel Loose's avatar Marcel Loose :sunglasses:
Browse files

Synced with development version of CI/CD pipeline

parent 497bb02d
Branches
Tags
No related merge requests found
Pipeline #103052 canceled
Stage: versioning
Stage: build
Stage: install
Stage: prepare_tests
Stage: run_tests
Stage: docs
Stage: deploy
Hide whitespace changes
Inline Side-by-side
Showing
with 66 additions and 55 deletions
.gitlab-ci.yml
+ 66
− 55
View file @ 35d93bd6
image: docker:latest
workflow: workflow:
rules: rules:
# Don't create a pipeline if commit is on a branch with open merge requests # Don't create a pipeline if commit is on a branch with open merge requests
...@@ -31,16 +29,21 @@ stages: ...@@ -31,16 +29,21 @@ stages:
- deploy - deploy
- finalize - finalize
before_script: .release:
- mkdir workdir before_script:
- mkdir logs # Make sure release branch is checked out when building a release
- RELEASE=$(echo -n $CI_COMMIT_BRANCH | sed -n 's,^releases/,,p')
after_script: - |
- echo "All done" if test -n "$RELEASE"
then
echo "Checking out branch $CI_COMMIT_BRANCH"
git checkout $CI_COMMIT_BRANCH
fi
.setup_git: .setup_git:
image: bitnami/git image: bitnami/git
script: before_script:
- !reference [.release, before_script]
- eval $(ssh-agent -s) - eval $(ssh-agent -s)
- chmod 400 $SSH_PRIVATE_KEY - chmod 400 $SSH_PRIVATE_KEY
- ssh-add $SSH_PRIVATE_KEY - ssh-add $SSH_PRIVATE_KEY
...@@ -52,12 +55,23 @@ after_script: ...@@ -52,12 +55,23 @@ after_script:
.setup_docker: .setup_docker:
stage: build stage: build
image: docker
tags: tags:
- dind - dind
before_script: before_script:
- !reference [.release, before_script]
- echo "Logging in as $CI_REGISTRY_USER @ $CI_REGISTRY" - echo "Logging in as $CI_REGISTRY_USER @ $CI_REGISTRY"
- echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
.run_tests:
stage: run_tests
image: $INTEGRATION_IMAGE
extends: .release
before_script:
- !reference [.release, before_script]
- mkdir -p workdir
- mkdir -p logs
.deploy: .deploy:
stage: deploy stage: deploy
extends: .setup_docker extends: .setup_docker
...@@ -68,6 +82,16 @@ after_script: ...@@ -68,6 +82,16 @@ after_script:
- echo "Logging in as $DH_REGISTRY_USER @ DockerHub" - echo "Logging in as $DH_REGISTRY_USER @ DockerHub"
- echo $DH_REGISTRY_PASSWORD | docker login -u $DH_REGISTRY_USER --password-stdin - echo $DH_REGISTRY_PASSWORD | docker login -u $DH_REGISTRY_USER --password-stdin
.docs:
stage: docs
extends: .release
image: $INTEGRATION_IMAGE
.finalize:
stage: finalize
extends: .setup_git
### Stage: initialize ### Stage: initialize
prepare_release: prepare_release:
...@@ -86,8 +110,8 @@ prepare_release: ...@@ -86,8 +110,8 @@ prepare_release:
untracked: true untracked: true
when: on_failure when: on_failure
before_script: before_script:
- !reference [.release, before_script]
# When building a release, bail out if release tag already exists # When building a release, bail out if release tag already exists
- RELEASE=$(echo -n $CI_COMMIT_BRANCH | sed 's,^releases/,,')
- | - |
if git ls-remote --tags --exit-code origin $RELEASE > /dev/null if git ls-remote --tags --exit-code origin $RELEASE > /dev/null
then then
...@@ -95,10 +119,8 @@ prepare_release: ...@@ -95,10 +119,8 @@ prepare_release:
touch .tag.exists touch .tag.exists
exit 1 exit 1
fi fi
- !reference [.setup_git, script] - !reference [.setup_git, before_script]
script: script:
# Make sure the current commit is checked out
- git checkout $CI_COMMIT_BRANCH
# Update dockerPull image URI in CWL steps with a tagged version # Update dockerPull image URI in CWL steps with a tagged version
- sed -ri "/dockerPull/s,(astronrd/linc).*,\1:$RELEASE," steps/*.cwl - sed -ri "/dockerPull/s,(astronrd/linc).*,\1:$RELEASE," steps/*.cwl
- git add -u steps/*.cwl - git add -u steps/*.cwl
...@@ -113,20 +135,16 @@ prepare_release: ...@@ -113,20 +135,16 @@ prepare_release:
# Skip CI on this push # Skip CI on this push
- git push --follow-tags -o ci.skip - git push --follow-tags -o ci.skip
### Stage: versioning ### Stage: versioning
versioning: versioning:
stage: versioning stage: versioning
image: python image: bitnami/git
before_script: extends: .release
- pip install setuptools_scm
script: script:
# Make sure the current branch is checked out
- git checkout $CI_COMMIT_BRANCH
- ./Docker/fetch_latest_commits.sh | tee commits.txt > versions.env - ./Docker/fetch_latest_commits.sh | tee commits.txt > versions.env
# Use a sub-command, to catch the exit status from `setuptools_scm` - echo LINC_VERSION=$(git describe --tags --always) >> versions.env
- (echo -n "LINC_VERSION="; python -m setuptools_scm) >> versions.env
- echo RELEASE=$(echo $CI_COMMIT_BRANCH | sed -n 's,^releases/,,p') >> versions.env
# Use hash of commits to determine version of base image (and rebuild if necessary) # Use hash of commits to determine version of base image (and rebuild if necessary)
- echo INTEGRATION_BASE_IMAGE=$CI_REGISTRY_IMAGE/integration_base:$(sha256sum commits.txt | cut -d " " -f 1) >> versions.env - echo INTEGRATION_BASE_IMAGE=$CI_REGISTRY_IMAGE/integration_base:$(sha256sum commits.txt | cut -d " " -f 1) >> versions.env
- echo INTEGRATION_IMAGE=$CI_REGISTRY_IMAGE/integration_full:$(git log -n 1 --pretty=format:%H) >> versions.env - echo INTEGRATION_IMAGE=$CI_REGISTRY_IMAGE/integration_full:$(git log -n 1 --pretty=format:%H) >> versions.env
...@@ -135,6 +153,7 @@ versioning: ...@@ -135,6 +153,7 @@ versioning:
reports: reports:
dotenv: versions.env dotenv: versions.env
### Stage: build ### Stage: build
build_base: build_base:
...@@ -163,6 +182,7 @@ build_base: ...@@ -163,6 +182,7 @@ build_base:
docker push $INTEGRATION_BASE_IMAGE docker push $INTEGRATION_BASE_IMAGE
fi fi
### Stage: install ### Stage: install
install_linc: install_linc:
...@@ -184,7 +204,8 @@ install_linc: ...@@ -184,7 +204,8 @@ install_linc:
. .
- docker push $INTEGRATION_IMAGE - docker push $INTEGRATION_IMAGE
### Stage: prepare_tests
## Stage: prepare_tests
download_data: download_data:
image: $INTEGRATION_IMAGE image: $INTEGRATION_IMAGE
...@@ -200,13 +221,13 @@ download_data: ...@@ -200,13 +221,13 @@ download_data:
- wget -nv https://support.astron.nl/software/ci_data/linc/$TARGET_HBA_SELFCAL_RESULTS_NAME -O $TARGET_HBA_SELFCAL_RESULTS_NAME && tar xfz $TARGET_HBA_SELFCAL_RESULTS_NAME && rm -f $TARGET_HBA_SELFCAL_RESULTS_NAME - wget -nv https://support.astron.nl/software/ci_data/linc/$TARGET_HBA_SELFCAL_RESULTS_NAME -O $TARGET_HBA_SELFCAL_RESULTS_NAME && tar xfz $TARGET_HBA_SELFCAL_RESULTS_NAME && rm -f $TARGET_HBA_SELFCAL_RESULTS_NAME
artifacts: artifacts:
paths: paths:
- data - data
### Stage: run_tests ### Stage: run_tests
validate_scripts: validate_scripts:
stage: run_tests extends: .run_tests
image: $INTEGRATION_IMAGE
script: script:
- | - |
errors=0 errors=0
...@@ -217,26 +238,22 @@ validate_scripts: ...@@ -217,26 +238,22 @@ validate_scripts:
((errors == 0)) ((errors == 0))
blsmooth: blsmooth:
stage: run_tests extends: .run_tests
image: $INTEGRATION_IMAGE
script: script:
- cwltool --no-container steps/blsmooth.cwl test_jobs/blsmooth.json - cwltool --no-container steps/blsmooth.cwl test_jobs/blsmooth.json
find_skymodel_cal: find_skymodel_cal:
stage: run_tests extends: .run_tests
image: $INTEGRATION_IMAGE
script: script:
- cwltool --no-container --preserve-environment PYTHONPATH steps/find_skymodel_cal.cwl test_jobs/find_skymodel_cal.json - cwltool --no-container --preserve-environment PYTHONPATH steps/find_skymodel_cal.cwl test_jobs/find_skymodel_cal.json
check_ateam_separation: check_ateam_separation:
stage: run_tests extends: .run_tests
image: $INTEGRATION_IMAGE
script: script:
- cwltool --no-container --preserve-environment PATH --preserve-environment PYTHONPATH steps/check_ateam_separation.cwl test_jobs/check_ateam_separation.json - cwltool --no-container --preserve-environment PATH --preserve-environment PYTHONPATH steps/check_ateam_separation.cwl test_jobs/check_ateam_separation.json
run_hba_calibrator: run_hba_calibrator:
stage: run_tests extends: .run_tests
image: $INTEGRATION_IMAGE
script: script:
- cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_calibrator/ workflows/HBA_calibrator.cwl test_jobs/HBA_calibrator.json - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_calibrator/ workflows/HBA_calibrator.cwl test_jobs/HBA_calibrator.json
- test_jobs/check_workflow_results.py results $CI_PROJECT_DIR/data/results_calibrator - test_jobs/check_workflow_results.py results $CI_PROJECT_DIR/data/results_calibrator
...@@ -248,8 +265,7 @@ run_hba_calibrator: ...@@ -248,8 +265,7 @@ run_hba_calibrator:
when: on_failure when: on_failure
run_hba_target: run_hba_target:
stage: run_tests extends: .run_tests
image: $INTEGRATION_IMAGE
script: script:
- cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_target/ workflows/HBA_target.cwl test_jobs/HBA_target.json - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_target/ workflows/HBA_target.cwl test_jobs/HBA_target.json
- test_jobs/check_workflow_results.py results $CI_PROJECT_DIR/data/results_target - test_jobs/check_workflow_results.py results $CI_PROJECT_DIR/data/results_target
...@@ -265,8 +281,7 @@ run_hba_target: ...@@ -265,8 +281,7 @@ run_hba_target:
when: on_failure when: on_failure
run_hba_target_selfcal: run_hba_target_selfcal:
stage: run_tests extends: .run_tests
image: $INTEGRATION_IMAGE
script: script:
- cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_target/ workflows/HBA_target.cwl test_jobs/HBA_target_selfcal.json - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_target/ workflows/HBA_target.cwl test_jobs/HBA_target_selfcal.json
- test_jobs/check_workflow_results.py --skip_soltabs TGSSphase_final results $CI_PROJECT_DIR/data/results_target_selfcal - test_jobs/check_workflow_results.py --skip_soltabs TGSSphase_final results $CI_PROJECT_DIR/data/results_target_selfcal
...@@ -282,8 +297,7 @@ run_hba_target_selfcal: ...@@ -282,8 +297,7 @@ run_hba_target_selfcal:
when: on_failure when: on_failure
run_lba_calibrator: run_lba_calibrator:
stage: run_tests extends: .run_tests
image: $INTEGRATION_IMAGE
script: script:
- cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_lba_calibrator workflows/LBA_calibrator.cwl test_jobs/LBA_calibrator.json - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_lba_calibrator workflows/LBA_calibrator.cwl test_jobs/LBA_calibrator.json
- test_jobs/check_workflow_results.py results $CI_PROJECT_DIR/data/results_calibrator_lba - test_jobs/check_workflow_results.py results $CI_PROJECT_DIR/data/results_calibrator_lba
...@@ -295,8 +309,7 @@ run_lba_calibrator: ...@@ -295,8 +309,7 @@ run_lba_calibrator:
when: on_failure when: on_failure
run_lba_target: run_lba_target:
stage: run_tests extends: .run_tests
image: $INTEGRATION_IMAGE
script: script:
- cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_lba_target/ workflows/LBA_target.cwl test_jobs/LBA_target.json - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_lba_target/ workflows/LBA_target.cwl test_jobs/LBA_target.json
- test_jobs/check_workflow_results.py --skip_soltabs GSMtec_final results $CI_PROJECT_DIR/data/results_target_lba - test_jobs/check_workflow_results.py --skip_soltabs GSMtec_final results $CI_PROJECT_DIR/data/results_target_lba
...@@ -311,11 +324,11 @@ run_lba_target: ...@@ -311,11 +324,11 @@ run_lba_target:
- cal_solutions.tar.gz - cal_solutions.tar.gz
when: on_failure when: on_failure
### Stage: docs ### Stage: docs
build_doc: build_doc:
stage: docs extends: .docs
image: $INTEGRATION_IMAGE
before_script: before_script:
- apt-get update - apt-get update
- apt-get install -y make - apt-get install -y make
...@@ -333,18 +346,18 @@ build_doc: ...@@ -333,18 +346,18 @@ build_doc:
- changes: - changes:
- docs/**/* - docs/**/*
### Stage: deploy ### Stage: deploy
deploy_docker: deploy_docker:
stage: deploy stage: deploy
extends: .deploy extends: .deploy
script: script:
# Replace characters that are now allowed in a tag string with a dash - echo "Deploying to DockerHub, using $LINC_VERSION as image tag"
- LINC_TAG=${LINC_VERSION//[^[:alnum:]_.-]/-}
- docker pull $INTEGRATION_IMAGE - docker pull $INTEGRATION_IMAGE
- docker tag $INTEGRATION_IMAGE $CI_PROJECT_PATH:$LINC_TAG - docker tag $INTEGRATION_IMAGE $CI_PROJECT_PATH:$LINC_VERSION
- docker tag $INTEGRATION_IMAGE $CI_PROJECT_PATH:latest - docker tag $INTEGRATION_IMAGE $CI_PROJECT_PATH:latest
- docker push $CI_PROJECT_PATH:$LINC_TAG - docker push $CI_PROJECT_PATH:$LINC_VERSION
- docker push $CI_PROJECT_PATH:latest - docker push $CI_PROJECT_PATH:latest
rules: rules:
# Run on the default branch or on a release branch # Run on the default branch or on a release branch
...@@ -361,19 +374,19 @@ deploy_docker_tag_stable: ...@@ -361,19 +374,19 @@ deploy_docker_tag_stable:
- docker tag $INTEGRATION_IMAGE $CI_PROJECT_PATH:stable - docker tag $INTEGRATION_IMAGE $CI_PROJECT_PATH:stable
- docker push $CI_PROJECT_PATH:stable - docker push $CI_PROJECT_PATH:stable
### Stage: finalize ### Stage: finalize
rollback_release: rollback_release:
stage: finalize stage: finalize
extends: .finalize
rules: rules:
# Run this job if the pipeline fails, to undo changes made in prepare_release. # Run this job if the pipeline fails, to undo changes made in prepare_release.
# We only care about removing the tag; other changes can remain. # We only care about removing the tag; other changes can remain.
- if: '$CI_COMMIT_BRANCH =~ /^releases//' - if: '$CI_COMMIT_BRANCH =~ /^releases//'
when: on_failure when: on_failure
before_script:
- !reference [.setup_git, script]
script: script:
- RELEASE=$(echo -n $CI_COMMIT_BRANCH | sed 's,^releases/,,') - echo "Rolling back release $RELEASE"
- | - |
if test -f .tag.exists if test -f .tag.exists
then then
...@@ -384,7 +397,7 @@ rollback_release: ...@@ -384,7 +397,7 @@ rollback_release:
finalize_release: finalize_release:
stage: finalize stage: finalize
extends: .setup_git extends: .finalize
rules: rules:
# Run this job if the pipeline succeeds, to create a versioned release. # Run this job if the pipeline succeeds, to create a versioned release.
# A versioned release is a release whose branch/tag name matches a string # A versioned release is a release whose branch/tag name matches a string
...@@ -395,11 +408,8 @@ finalize_release: ...@@ -395,11 +408,8 @@ finalize_release:
# the default branch. # the default branch.
- if: '$CI_COMMIT_BRANCH =~ /^releases\/v[0-9]+(\.[0-9]+)*/' - if: '$CI_COMMIT_BRANCH =~ /^releases\/v[0-9]+(\.[0-9]+)*/'
when: on_success when: on_success
before_script:
- !reference [.setup_git, script]
script: script:
# Make sure the current branch is checked out - echo "Finalizing release $RELEASE"
- git checkout $CI_COMMIT_BRANCH
# Update dockerPull image URI in CWL steps by removing version tag # Update dockerPull image URI in CWL steps by removing version tag
- sed -ri "/dockerPull/s,(astronrd/linc).*,\1," steps/*.cwl - sed -ri "/dockerPull/s,(astronrd/linc).*,\1," steps/*.cwl
- git add -u steps/*.cwl - git add -u steps/*.cwl
...@@ -414,6 +424,7 @@ finalize_release: ...@@ -414,6 +424,7 @@ finalize_release:
# Next switch to the default branch and make sure it's up to date # Next switch to the default branch and make sure it's up to date
- git checkout $CI_DEFAULT_BRANCH - git checkout $CI_DEFAULT_BRANCH
- git pull - git pull
- echo "Merging changes from $CI_COMMIT_BRANCH into $CI_DEFAULT_BRANCH"
# Merge release branch into the default branch # Merge release branch into the default branch
- git merge $CI_COMMIT_BRANCH -m"Merged release branch into default branch (by Gitlab CI)" - git merge $CI_COMMIT_BRANCH -m"Merged release branch into default branch (by Gitlab CI)"
# Skip CI on this push # Skip CI on this push
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment