Skip to content
Snippets Groups Projects
Commit d16d455c authored by Marcel Loose's avatar Marcel Loose :sunglasses:
Browse files

Rewrite of CI/CD pipeline (WIP)

parent 71c3e173
No related branches found
No related tags found
1 merge request!224Rewrite of CI/CD pipeline
Pipeline #101423 failed
......@@ -4,3 +4,4 @@ Docker/
docs/build/
*.egg-info/
venv/
.git/
......@@ -17,14 +17,19 @@ variables:
TARGET_HBA_SELFCAL_RESULTS_NAME: "results_target_selfcal.tar.gz"
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
BUILD_DOCKER_IMAGE: "0"
GIT_STRATEGY: clone # ensure every job starts with a pristine working copy
GIT_DEPTH: 0 # do not do shallow clones, needed for `git describe --tags`
stages:
- initialize
- versioning
- build
- install
- prepare_tests
- run_tests
- docs
- deploy
- finalize
before_script:
- mkdir workdir
......@@ -33,35 +38,108 @@ before_script:
after_script:
- echo "All done"
.setup_git:
image: bitnami/git
script:
- eval $(ssh-agent -s)
- chmod 400 $SSH_PRIVATE_KEY
- ssh-add $SSH_PRIVATE_KEY
- mkdir -p ~/.ssh
- ssh-keyscan $CI_SERVER_HOST > ~/.ssh/known_hosts
- git config user.email $GITLAB_USER_EMAIL
- git config user.name $GITLAB_USER_NAME
- git remote set-url origin git@$CI_SERVER_HOST:$CI_PROJECT_PATH.git
.setup_docker:
stage: build
tags:
- dind
before_script:
- echo "Logging in as $CI_REGISTRY_USER @ $CI_REGISTRY"
- echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
.deploy:
stage: deploy
extends: .setup_docker
tags:
- dind
before_script:
- !reference [.setup_docker, before_script]
- echo "Logging in as $DH_REGISTRY_USER @ DockerHub"
- echo $DH_REGISTRY_PASSWORD | docker login -u $DH_REGISTRY_USER --password-stdin
### Stage: initialize
prepare_release:
stage: initialize
extends: .setup_git
rules:
# Run this job when current branch is a release branch
- if: '$CI_COMMIT_BRANCH =~ /^releases//'
when: always
artifacts:
# Create a marker file if release tag already exists. The presence of this
# file is used in the `rollback_release` job, to decide that the existing
# tag should not be removed.
paths:
- .tag.exists
untracked: true
when: on_failure
before_script:
# When building a release, bail out if release tag already exists
- RELEASE=$(echo -n $CI_COMMIT_BRANCH | sed 's,^releases/,,')
- |
if git ls-remote --tags --exit-code origin $RELEASE > /dev/null
then
echo "*** Release $RELEASE already exists. Bailing out! ***"
touch .tag.exists
exit 1
fi
- !reference [.setup_git, script]
script:
# Make sure the current commit is checked out
- git checkout $CI_COMMIT_BRANCH
# Update dockerPull image URI in CWL steps with a tagged version
- sed -ri "/dockerPull/s,(astronrd/linc).*,\1:$RELEASE," steps/*.cwl
- git add -u steps/*.cwl
# Only commit if there are changes
- |
if test -n "$(git status -uno --porcelain)"
then
git commit -m "Tag $RELEASE added to dockerPull URI in CWL steps (by GitLab CI)"
fi
# Tag current revision
- git tag -a $RELEASE -m "Git tag $RELEASE created (by GitLab CI)"
# Skip CI on this push
- git push --follow-tags -o ci.skip
### Stage: versioning
versioning:
stage: versioning
image: bitnami/git
image: python
before_script:
- pip install setuptools_scm
script:
# Unshallowing ensures that 'git describe' works
- git fetch --unshallow
# Make sure the current branch is checked out
- git checkout $CI_COMMIT_BRANCH
- ./Docker/fetch_latest_commits.sh | tee commits.txt > versions.env
- echo LINC_VERSION=$(git describe --tags --always) >> versions.env
# Use a sub-command, to catch the exit status from `setuptools_scm`
- (echo -n "LINC_VERSION="; python -m setuptools_scm) >> versions.env
- echo RELEASE=$(echo $CI_COMMIT_BRANCH | sed -n 's,^releases/,,p') >> versions.env
# Use hash of commits to determine version of base image (and rebuild if necessary)
- echo INTEGRATION_BASE_IMAGE=${CI_REGISTRY_IMAGE}/integration_base:$(sha256sum commits.txt | cut -d " " -f 1) >> versions.env
- echo INTEGRATION_IMAGE=${CI_REGISTRY_IMAGE}/integration_full:$(git log -n 1 --pretty=format:%H) >> versions.env
- echo INTEGRATION_BASE_IMAGE=$CI_REGISTRY_IMAGE/integration_base:$(sha256sum commits.txt | cut -d " " -f 1) >> versions.env
- echo INTEGRATION_IMAGE=$CI_REGISTRY_IMAGE/integration_full:$(git log -n 1 --pretty=format:%H) >> versions.env
- cat versions.env
artifacts:
reports:
dotenv: versions.env
# Docker login
.prepare:
stage: build
tags:
- dind
before_script:
- echo "Logging in as $CI_REGISTRY_USER @ $CI_REGISTRY"
- echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
### Stage: build
# Create and push the integration image to the gitlab registry, if it does not exist
build_base:
stage: build
extends: .prepare
extends: .setup_docker
script:
- |
if ! docker manifest inspect $INTEGRATION_BASE_IMAGE > /dev/null || [ "$BUILD_DOCKER_IMAGE" = "1" ]; then
......@@ -70,7 +148,7 @@ build_base:
else
DOCKER_CACHE_PARAMETERS=""
fi
docker build --tag $INTEGRATION_BASE_IMAGE ${DOCKER_CACHE_PARAMETERS} \
docker build --tag $INTEGRATION_BASE_IMAGE $DOCKER_CACHE_PARAMETERS \
--build-arg LOFARSTMAN_COMMIT \
--build-arg DYSCO_COMMIT \
--build-arg IDG_COMMIT \
......@@ -80,29 +158,34 @@ build_base:
--build-arg EVERYBEAM_COMMIT \
--build-arg DP3_COMMIT \
--build-arg WSCLEAN_COMMIT \
--build-arg LINC_VERSION \
-f Docker/Dockerfile-base .
docker push $INTEGRATION_BASE_IMAGE
fi
### Stage: install
install_linc:
stage: build
extends: .prepare
needs: ["versioning", "build_base"]
stage: install
extends: .setup_docker
script:
- echo "BUILD_DOCKER_IMAGE=$BUILD_DOCKER_IMAGE"
- echo "INTEGRATION_BASE_IMAGE=$INTEGRATION_BASE_IMAGE"
- echo "INTEGRATION_IMAGE=$INTEGRATION_IMAGE"
- echo "DOCKER_CACHE_PARAMETERS=$DOCKER_CACHE_PARAMETERS"
- echo "PWD=$PWD"
- |
if [ "$BUILD_DOCKER_IMAGE" = "1" ] ; then
DOCKER_CACHE_PARAMETERS="--no-cache"
else
DOCKER_CACHE_PARAMETERS=""
fi
docker build --build-arg BASE_TAG=$INTEGRATION_BASE_IMAGE --tag $INTEGRATION_IMAGE ${DOCKER_CACHE_PARAMETERS} -f Docker/Dockerfile-linc .
docker build \
$DOCKER_CACHE_PARAMETERS \
--build-arg BASE_TAG=$INTEGRATION_BASE_IMAGE \
--build-arg LINC_VERSION=$LINC_VERSION \
--file Docker/Dockerfile-linc \
--tag $INTEGRATION_IMAGE \
.
- docker push $INTEGRATION_IMAGE
### Stage: prepare_tests
download_data:
image: $INTEGRATION_IMAGE
stage: prepare_tests
......@@ -119,6 +202,8 @@ download_data:
paths:
- data
### Stage: run_tests
validate_scripts:
stage: run_tests
image: $INTEGRATION_IMAGE
......@@ -154,7 +239,7 @@ run_hba_calibrator:
image: $INTEGRATION_IMAGE
script:
- cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_calibrator/ workflows/HBA_calibrator.cwl test_jobs/HBA_calibrator.json
- test_jobs/check_workflow_results.py results /builds/RD/LINC/data/results_calibrator
- test_jobs/check_workflow_results.py results $CI_PROJECT_DIR/data/results_calibrator
after_script:
- find /tmp/run_hba_calibrator -name "*.log" -print0 | tar czf hba_calibrator_logs.tar.gz --null -T -
artifacts:
......@@ -167,11 +252,11 @@ run_hba_target:
image: $INTEGRATION_IMAGE
script:
- cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_target/ workflows/HBA_target.cwl test_jobs/HBA_target.json
- test_jobs/check_workflow_results.py results /builds/RD/LINC/data/results_target
- test_jobs/check_workflow_results.py results $CI_PROJECT_DIR/data/results_target
after_script:
- find /tmp/run_hba_target -name "*.log" -print0 | tar czf hba_target_logs.tar.gz --null -T -
- find /tmp/run_hba_target -name "*.png" -print0 | tar czf inspection.tar.gz --null -T -
- find /builds/RD/LINC/results/ -name "cal_solutions.h5" -print0 | tar czf cal_solutions.tar.gz --null -T -
- find $CI_PROJECT_DIR/results/ -name "cal_solutions.h5" -print0 | tar czf cal_solutions.tar.gz --null -T -
artifacts:
paths:
- hba_target_logs.tar.gz
......@@ -184,11 +269,11 @@ run_hba_target_selfcal:
image: $INTEGRATION_IMAGE
script:
- cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_target/ workflows/HBA_target.cwl test_jobs/HBA_target_selfcal.json
- test_jobs/check_workflow_results.py --skip_soltabs TGSSphase_final results /builds/RD/LINC/data/results_target_selfcal
- test_jobs/check_workflow_results.py --skip_soltabs TGSSphase_final results $CI_PROJECT_DIR/data/results_target_selfcal
after_script:
- find /tmp/run_hba_target -name "*.log" -print0 | tar czf hba_target_selfcal_logs.tar.gz --null -T -
- find /tmp/run_hba_target -name "*.png" -print0 | tar czf inspection.tar.gz --null -T -
- find /builds/RD/LINC/results/ -name "cal_solutions.h5" -print0 | tar czf cal_solutions.tar.gz --null -T -
- find $CI_PROJECT_DIR/results/ -name "cal_solutions.h5" -print0 | tar czf cal_solutions.tar.gz --null -T -
artifacts:
paths:
- hba_target_selfcal_logs.tar.gz
......@@ -201,7 +286,7 @@ run_lba_calibrator:
image: $INTEGRATION_IMAGE
script:
- cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_lba_calibrator workflows/LBA_calibrator.cwl test_jobs/LBA_calibrator.json
- test_jobs/check_workflow_results.py results /builds/RD/LINC/data/results_calibrator_lba
- test_jobs/check_workflow_results.py results $CI_PROJECT_DIR/data/results_calibrator_lba
after_script:
- find /tmp/run_lba_calibrator -name "*.log" -print0 | tar czf lba_calibrator_logs.tar.gz --null -T -
artifacts:
......@@ -214,11 +299,11 @@ run_lba_target:
image: $INTEGRATION_IMAGE
script:
- cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_lba_target/ workflows/LBA_target.cwl test_jobs/LBA_target.json
- test_jobs/check_workflow_results.py --skip_soltabs GSMtec_final results /builds/RD/LINC/data/results_target_lba
- test_jobs/check_workflow_results.py --skip_soltabs GSMtec_final results $CI_PROJECT_DIR/data/results_target_lba
after_script:
- find /tmp/run_lba_target -name "*.log" -print0 | tar czf lba_target_logs.tar.gz --null -T -
- find /tmp/run_lba_target -name "*.png" -print0 | tar czf inspection.tar.gz --null -T -
- find /builds/RD/LINC/results/ -name "cal_solutions.h5" -print0 | tar czf cal_solutions.tar.gz --null -T -
- find $CI_PROJECT_DIR/results/ -name "cal_solutions.h5" -print0 | tar czf cal_solutions.tar.gz --null -T -
artifacts:
paths:
- lba_target_logs.tar.gz
......@@ -226,6 +311,8 @@ run_lba_target:
- cal_solutions.tar.gz
when: on_failure
### Stage: docs
build_doc:
stage: docs
image: $INTEGRATION_IMAGE
......@@ -240,70 +327,94 @@ build_doc:
paths:
- docs/build/html
rules:
# Only add job for commits to master or on merge request events
- if: '$CI_COMMIT_BRANCH == "master" && $CI_PIPELINE_SOURCE != "schedule"'
# Only add job for commits to the default branch or on merge request events
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE != "schedule"'
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
- changes:
- docs/**/*
# Deploy to docker hub
### Stage: deploy
deploy_docker:
stage: deploy
tags:
- dind
before_script:
- echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
- echo $DH_REGISTRY_PASSWORD | docker login -u $DH_REGISTRY_USER --password-stdin
extends: .deploy
script:
# Replace characters that are now allowed in a tag string with a dash
- LINC_TAG=${LINC_VERSION//[^[:alnum:]_.-]/-}
- docker pull $INTEGRATION_IMAGE
- docker tag $INTEGRATION_IMAGE astronrd/linc:$LINC_VERSION
- docker tag $INTEGRATION_IMAGE astronrd/linc:latest
- docker push astronrd/linc:$LINC_VERSION
- docker push astronrd/linc:latest
- docker tag $INTEGRATION_IMAGE $CI_PROJECT_PATH:$LINC_TAG
- docker tag $INTEGRATION_IMAGE $CI_PROJECT_PATH:latest
- docker push $CI_PROJECT_PATH:$LINC_TAG
- docker push $CI_PROJECT_PATH:latest
rules:
# Only run on master
- if: '$CI_COMMIT_BRANCH == "master"'
when: always
# Run on the default branch or on a release branch
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
- if: '$CI_COMMIT_BRANCH =~ /^releases//'
when: on_success
# Deploy a release version
deploy_release:
deploy_docker_tag_stable:
stage: deploy
tags:
- dind
before_script:
- echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
- echo $DH_REGISTRY_PASSWORD | docker login -u $DH_REGISTRY_USER --password-stdin
- apk update && apk add git
extends: .deploy
when: manual
script:
- echo -n "$CI_COMMIT_BRANCH" | sed -e "s/^releases\//export RELEASE=/" > version && source version
- docker pull $INTEGRATION_IMAGE
- docker tag $INTEGRATION_IMAGE astronrd/linc:$RELEASE
- docker push astronrd/linc:$RELEASE
# Update docker image in cwl steps by tagged version
- sed -i "s/astronrd\/linc/astronrd\/linc:$RELEASE/g" steps/*.cwl
# Create and push Git tag
- git add steps/*.cwl
- git config user.email $GITLAB_USER_EMAIL
- git config user.name $GITLAB_USER_NAME
# Skip CI on this commit and tag
- git commit -m "[SKIP CI] Replace latest with tag $RELEASE"
- git tag -a $RELEASE -m "[SKIP CI] Version $RELEASE created by gitlab-ci build"
- git push --all
- docker tag $INTEGRATION_IMAGE $CI_PROJECT_PATH:stable
- docker push $CI_PROJECT_PATH:stable
### Stage: finalize
rollback_release:
stage: finalize
rules:
# TODO: check if pushed tag triggers Pipeline
# Only run on release branches
# Run this job if the pipeline fails, to undo changes made in prepare_release.
# We only care about removing the tag; other changes can remain.
- if: '$CI_COMMIT_BRANCH =~ /^releases//'
when: always
when: on_failure
before_script:
- !reference [.setup_git, script]
script:
- RELEASE=$(echo -n $CI_COMMIT_BRANCH | sed 's,^releases/,,')
- |
if test -f .tag.exists
then
echo "*** Not removing existing tag $RELEASE! ***"
else
git push origin -d $RELEASE
fi
deploy_docker_tag_stable:
stage: deploy
when: manual
tags:
- dind
finalize_release:
stage: finalize
extends: .setup_git
rules:
# Run this job if the pipeline succeeds, to create a versioned release.
# A versioned release is a release whose branch/tag name matches a string
# that looks like "v<major-version>[.<minor_version>][...]".
# We want to ensure that the release tag also exists on the default branch,
# so that `git describe --tags` yields the desired output. Hence, we must
# undo the changes made in the `prepare_release` job, before merging to
# the default branch.
- if: '$CI_COMMIT_BRANCH =~ /^releases\/v[0-9]+(\.[0-9]+)*/'
when: on_success
before_script:
- echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
- echo $DH_REGISTRY_PASSWORD | docker login -u $DH_REGISTRY_USER --password-stdin
- !reference [.setup_git, script]
script:
- docker pull $INTEGRATION_IMAGE
- docker tag $INTEGRATION_IMAGE astronrd/linc:stable
- docker push astronrd/linc:stable
# Make sure the current branch is checked out
- git checkout $CI_COMMIT_BRANCH
# Update dockerPull image URI in CWL steps by removing version tag
- sed -ri "/dockerPull/s,(astronrd/linc).*,\1," steps/*.cwl
- git add -u steps/*.cwl
# Only commit if there are changes
- |
if test -n "$(git status -uno --porcelain)"
then
git commit -m "Tag removed from dockerPull URI in CWL steps (by GitLab CI)"
fi
# Skip CI on this push
- git push -o ci.skip
# Next switch to the default branch and make sure it's up to date
- git checkout $CI_DEFAULT_BRANCH
- git pull
# Merge release branch into the default branch
- git merge $CI_COMMIT_BRANCH -m"Merged release branch into default branch (by Gitlab CI)"
# Skip CI on this push
- git push -o ci.skip
......@@ -149,7 +149,11 @@ RUN python3 -m pip install --no-cache-dir --upgrade \
# dependencies will have been installed already and (probably) don't need to
# be updated.
COPY . linc
RUN python3 -m pip install --no-cache-dir --upgrade ./linc
# Note: LINC_VERSION should be provided as build argument
ARG LINC_VERSION=0.0.0
RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_LINC=${LINC_VERSION} \
python3 -m pip install --no-cache-dir --upgrade ./linc
#---------------------------------------------------------------------------
......
......@@ -7,7 +7,10 @@ ENV LINC_DATA_ROOT=/usr/local/share/linc \
# Install LINC
COPY . /tmp/linc
RUN python3 -m pip install --no-cache-dir --upgrade /tmp/linc && \
# Note: LINC_VERSION should be provided as build argument
ARG LINC_VERSION=0.0.0
RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_LINC=${LINC_VERSION} \
python3 -m pip install --no-cache-dir --upgrade /tmp/linc && \
rm -rf /tmp/linc
# # A user 'lofaruser' is added (this might be legacy -- not sure this is
......
......@@ -81,7 +81,7 @@ outputs:
label: H5parm_collector
hints:
- class: DockerRequirement
dockerPull: 'astronrd/linc'
dockerPull: astronrd/linc
stdout: $(inputs.outh5parmname)-parm_collector_output.log
stderr: $(inputs.outh5parmname)-parm_collector_output_err.log
requirements:
......
......@@ -76,7 +76,7 @@ stderr: add_missing_stations_err.log
label: add_missing_stations
hints:
- class: DockerRequirement
dockerPull: 'astronrd/linc'
dockerPull: astronrd/linc
requirements:
- class: InlineJavascriptRequirement
- class: InitialWorkDirRequirement
......
......@@ -109,4 +109,4 @@ requirements:
coresMin: $(inputs.max_dp3_threads)
hints:
- class: DockerRequirement
dockerPull: 'astronrd/linc'
dockerPull: astronrd/linc
......@@ -115,4 +115,4 @@ requirements:
coresMin: $(inputs.max_dp3_threads)
hints:
- class: DockerRequirement
dockerPull: 'astronrd/linc'
dockerPull: astronrd/linc
......@@ -116,4 +116,4 @@ requirements:
coresMin: 2
hints:
- class: DockerRequirement
dockerPull: 'astronrd/linc'
dockerPull: astronrd/linc
......@@ -92,6 +92,6 @@ requirements:
coresMin: 8
hints:
- class: DockerRequirement
dockerPull: 'astronrd/linc'
dockerPull: astronrd/linc
stdout: average.log
stderr: average_err.log
......@@ -163,7 +163,7 @@ requirements:
coresMin: 8
hints:
- class: DockerRequirement
dockerPull: 'astronrd/linc'
dockerPull: astronrd/linc
- class: InitialWorkDirRequirement
listing:
- entry: $(inputs.msin)
......
......@@ -58,6 +58,6 @@ outputs:
label: merge_skymodels
hints:
- class: DockerRequirement
dockerPull: 'astronrd/linc'
dockerPull: astronrd/linc
stdout: merge_skymodels.log
stderr: merge_skymodels_err.log
......@@ -41,7 +41,7 @@ outputs:
hints:
- class: DockerRequirement
dockerPull: 'astronrd/linc'
dockerPull: astronrd/linc
requirements:
- class: InlineJavascriptRequirement
stdout: plot_unflagged_fraction.log
......
......@@ -40,7 +40,7 @@ outputs:
label: TaQL
hints:
- class: DockerRequirement
dockerPull: 'astronrd/linc'
dockerPull: astronrd/linc
requirements:
- class: ShellCommandRequirement
- class: InplaceUpdateRequirement
......
......@@ -40,7 +40,7 @@ outputs:
label: TaQL
hints:
- class: DockerRequirement
dockerPull: 'astronrd/linc'
dockerPull: astronrd/linc
requirements:
- class: ShellCommandRequirement
- class: InitialWorkDirRequirement
......
......@@ -92,7 +92,7 @@ stderr: transfer_solutions_err.log
label: transfer_solutions
hints:
- class: DockerRequirement
dockerPull: 'astronrd/linc'
dockerPull: astronrd/linc
requirements:
- class: InlineJavascriptRequirement
- class: InitialWorkDirRequirement
......
......@@ -355,7 +355,7 @@ outputs:
label: WSClean
hints:
- class: DockerRequirement
dockerPull: 'astronrd/linc'
dockerPull: astronrd/linc
requirements:
- class: InplaceUpdateRequirement
inplaceUpdate: true
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment