From e924919753d4d481f932c9e3aebb3fb5a6fd54f4 Mon Sep 17 00:00:00 2001
From: Marcel Loose <loose@astron.nl>
Date: Thu, 5 Dec 2024 11:54:50 +0100
Subject: [PATCH] Sync with development
---
.gitlab-ci.yml | 68 ++++++++++++++++++++++++++------------------------
1 file changed, 36 insertions(+), 32 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 45085714..e7cdc0a1 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -5,6 +5,9 @@ workflow:
# Don't create a pipeline if commit is on a branch with open merge requests
- if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
when: never
+ # # Don't run if a tag is created, because this workflow itself may create a tag!
+ # - if: $CI_COMMIT_TAG
+ # when: never
- when: always
variables:
@@ -42,14 +45,15 @@ after_script:
image: bitnami/git
script:
- eval $(ssh-agent -s)
- - chmod 400 "${SSH_PRIVATE_KEY}"
- - ssh-add "${SSH_PRIVATE_KEY}"
+ - chmod 400 "$SSH_PRIVATE_KEY"
+ - ssh-add "$SSH_PRIVATE_KEY"
- mkdir -p ~/.ssh
- - ssh-keyscan ${CI_SERVER_HOST} > ~/.ssh/known_hosts
+ - ssh-keyscan $CI_SERVER_HOST > ~/.ssh/known_hosts
- git config user.email $GITLAB_USER_EMAIL
- git config user.name $GITLAB_USER_NAME
- git remote set-url origin git@$CI_SERVER_HOST:$CI_PROJECT_PATH.git
+# Docker login
.setup_docker:
stage: build
tags:
@@ -85,11 +89,11 @@ prepare_release:
when: on_failure
before_script:
# When building a release, bail out if release tag already exists
- - RELEASE="$(echo -n ${CI_COMMIT_BRANCH} | sed 's,^releases/,,')"
+ - RELEASE="$(echo -n $CI_COMMIT_BRANCH | sed 's,^releases/,,')"
- |
- if git ls-remote --tags --exit-code origin "${RELEASE}" > /dev/null
+ if git ls-remote --tags --exit-code origin "$RELEASE" > /dev/null
then
- echo "*** Release ${RELEASE} already exists. Bailing out! ***"
+ echo "*** Release $RELEASE already exists. Bailing out! ***"
touch .tag.exists
exit 1
fi
@@ -98,16 +102,16 @@ prepare_release:
# Make sure the current commit is checked out
- git checkout "$CI_COMMIT_BRANCH"
# Update dockerPull image URI in CWL steps with a tagged version
- - sed -ri "/dockerPull/s,(astronrd/linc).*,\1:${RELEASE}," steps/*.cwl
+ - sed -ri "/dockerPull/s,(astronrd/linc).*,\1:$RELEASE," steps/*.cwl
- git add -u steps/*.cwl
# Only commit if there are changes
- |
if test -n "$(git status -uno --porcelain)"
then
- git commit -m "Tag ${RELEASE} added to dockerPull URI in CWL steps (by GitLab CI)"
+ git commit -m "Tag $RELEASE added to dockerPull URI in CWL steps (by GitLab CI)"
fi
# Tag current revision
- - git tag -a ${RELEASE} -m "Git tag ${RELEASE} created (by GitLab CI)"
+ - git tag -a $RELEASE -m "Git tag ${RELEASE} created (by GitLab CI)"
# Skip CI on this push
- git push --follow-tags -o ci.skip
@@ -118,13 +122,13 @@ versioning:
- pip install setuptools_scm
script:
# Make sure the current branch is checked out
- - git checkout ${CI_COMMIT_BRANCH}
+ - git checkout $CI_COMMIT_BRANCH
- ./Docker/fetch_latest_commits.sh | tee commits.txt > versions.env
- echo LINC_VERSION=$(python -m setuptools_scm) >> versions.env
- - echo RELEASE=$(echo ${CI_COMMIT_BRANCH} | sed -n 's,^releases/,,p') >> versions.env
+ - echo RELEASE=$(echo $CI_COMMIT_BRANCH | sed -n 's,^releases/,,p') >> versions.env
# Use hash of commits to determine version of base image (and rebuild if necessary)
- - echo INTEGRATION_BASE_IMAGE=${CI_REGISTRY_IMAGE}/integration_base:$(sha256sum commits.txt | cut -d " " -f 1) >> versions.env
- - echo INTEGRATION_IMAGE=${CI_REGISTRY_IMAGE}/integration_full:$(git log -n 1 --pretty=format:%H) >> versions.env
+ - echo INTEGRATION_BASE_IMAGE=$CI_REGISTRY_IMAGE/integration_base:$(sha256sum commits.txt | cut -d " " -f 1) >> versions.env
+ - echo INTEGRATION_IMAGE=$CI_REGISTRY_IMAGE/integration_full:$(git log -n 1 --pretty=format:%H) >> versions.env
- cat versions.env
artifacts:
reports:
@@ -142,7 +146,7 @@ build_base:
else
DOCKER_CACHE_PARAMETERS=""
fi
- docker build --tag $INTEGRATION_BASE_IMAGE ${DOCKER_CACHE_PARAMETERS} \
+ docker build --tag $INTEGRATION_BASE_IMAGE $DOCKER_CACHE_PARAMETERS \
--build-arg LOFARSTMAN_COMMIT \
--build-arg DYSCO_COMMIT \
--build-arg IDG_COMMIT \
@@ -168,9 +172,9 @@ install_linc:
DOCKER_CACHE_PARAMETERS=""
fi
docker build \
- ${DOCKER_CACHE_PARAMETERS} \
+ $DOCKER_CACHE_PARAMETERS \
--build-arg BASE_TAG=$INTEGRATION_BASE_IMAGE \
- --build-arg LINC_VERSION=${LINC_VERSION} \
+ --build-arg LINC_VERSION=$LINC_VERSION \
--file Docker/Dockerfile-linc \
--tag $INTEGRATION_IMAGE \
.
@@ -227,7 +231,7 @@ run_hba_calibrator:
image: $INTEGRATION_IMAGE
script:
- cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_calibrator/ workflows/HBA_calibrator.cwl test_jobs/HBA_calibrator.json
- - test_jobs/check_workflow_results.py results ${CI_PROJECT_DIR}/data/results_calibrator
+ - test_jobs/check_workflow_results.py results $CI_PROJECT_DIR/data/results_calibrator
after_script:
- find /tmp/run_hba_calibrator -name "*.log" -print0 | tar czf hba_calibrator_logs.tar.gz --null -T -
artifacts:
@@ -240,11 +244,11 @@ run_hba_target:
image: $INTEGRATION_IMAGE
script:
- cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_target/ workflows/HBA_target.cwl test_jobs/HBA_target.json
- - test_jobs/check_workflow_results.py results ${CI_PROJECT_DIR}/data/results_target
+ - test_jobs/check_workflow_results.py results $CI_PROJECT_DIR/data/results_target
after_script:
- find /tmp/run_hba_target -name "*.log" -print0 | tar czf hba_target_logs.tar.gz --null -T -
- find /tmp/run_hba_target -name "*.png" -print0 | tar czf inspection.tar.gz --null -T -
- - find ${CI_PROJECT_DIR}/results/ -name "cal_solutions.h5" -print0 | tar czf cal_solutions.tar.gz --null -T -
+ - find $CI_PROJECT_DIR/results/ -name "cal_solutions.h5" -print0 | tar czf cal_solutions.tar.gz --null -T -
artifacts:
paths:
- hba_target_logs.tar.gz
@@ -257,11 +261,11 @@ run_hba_target_selfcal:
image: $INTEGRATION_IMAGE
script:
- cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_target/ workflows/HBA_target.cwl test_jobs/HBA_target_selfcal.json
- - test_jobs/check_workflow_results.py --skip_soltabs TGSSphase_final results ${CI_PROJECT_DIR}/data/results_target_selfcal
+ - test_jobs/check_workflow_results.py --skip_soltabs TGSSphase_final results $CI_PROJECT_DIR/data/results_target_selfcal
after_script:
- find /tmp/run_hba_target -name "*.log" -print0 | tar czf hba_target_selfcal_logs.tar.gz --null -T -
- find /tmp/run_hba_target -name "*.png" -print0 | tar czf inspection.tar.gz --null -T -
- - find ${CI_PROJECT_DIR}/results/ -name "cal_solutions.h5" -print0 | tar czf cal_solutions.tar.gz --null -T -
+ - find $CI_PROJECT_DIR/results/ -name "cal_solutions.h5" -print0 | tar czf cal_solutions.tar.gz --null -T -
artifacts:
paths:
- hba_target_selfcal_logs.tar.gz
@@ -274,7 +278,7 @@ run_lba_calibrator:
image: $INTEGRATION_IMAGE
script:
- cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_lba_calibrator workflows/LBA_calibrator.cwl test_jobs/LBA_calibrator.json
- - test_jobs/check_workflow_results.py results ${CI_PROJECT_DIR}/data/results_calibrator_lba
+ - test_jobs/check_workflow_results.py results $CI_PROJECT_DIR/data/results_calibrator_lba
after_script:
- find /tmp/run_lba_calibrator -name "*.log" -print0 | tar czf lba_calibrator_logs.tar.gz --null -T -
artifacts:
@@ -287,11 +291,11 @@ run_lba_target:
image: $INTEGRATION_IMAGE
script:
- cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_lba_target/ workflows/LBA_target.cwl test_jobs/LBA_target.json
- - test_jobs/check_workflow_results.py --skip_soltabs GSMtec_final results ${CI_PROJECT_DIR}/data/results_target_lba
+ - test_jobs/check_workflow_results.py --skip_soltabs GSMtec_final results $CI_PROJECT_DIR/data/results_target_lba
after_script:
- find /tmp/run_lba_target -name "*.log" -print0 | tar czf lba_target_logs.tar.gz --null -T -
- find /tmp/run_lba_target -name "*.png" -print0 | tar czf inspection.tar.gz --null -T -
- - find ${CI_PROJECT_DIR}/results/ -name "cal_solutions.h5" -print0 | tar czf cal_solutions.tar.gz --null -T -
+ - find $CI_PROJECT_DIR/results/ -name "cal_solutions.h5" -print0 | tar czf cal_solutions.tar.gz --null -T -
artifacts:
paths:
- lba_target_logs.tar.gz
@@ -314,7 +318,7 @@ build_doc:
- docs/build/html
rules:
# Only add job for commits to the default branch or on merge request events
- - if: '$CI_COMMIT_BRANCH == "${CI_DEFAULT_BRANCH}" && $CI_PIPELINE_SOURCE != "schedule"'
+ - if: '$CI_COMMIT_BRANCH == "$CI_DEFAULT_BRANCH" && $CI_PIPELINE_SOURCE != "schedule"'
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
- changes:
- docs/**/*
@@ -323,8 +327,8 @@ deploy_docker:
stage: deploy
extends: .deploy
script:
- - LINC_TAG=${LINC_VERSION//[^[:alnum:]_.-]/-}
- - echo "LINC_TAG=${LINC_TAG}"
+ - LINC_TAG=$LINC_VERSION//[^[:alnum:]_.-]/-
+ - echo "LINC_TAG=$LINC_TAG"
- docker pull $INTEGRATION_IMAGE
- docker tag $INTEGRATION_IMAGE $CI_PROJECT_PATH:$LINC_TAG
- docker tag $INTEGRATION_IMAGE $CI_PROJECT_PATH:latest
@@ -332,7 +336,7 @@ deploy_docker:
- docker push $CI_PROJECT_PATH:latest
rules:
# Run on the default branch or on a release branch
- - if: '$CI_COMMIT_BRANCH == "${CI_DEFAULT_BRANCH}"'
+ - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
- if: '$CI_COMMIT_BRANCH =~ /^releases//'
when: on_success
@@ -341,7 +345,7 @@ deploy_docker_tag_stable:
extends: .deploy
when: manual
script:
- - echo "INTEGRATION_IMAGE=${INTEGRATION_IMAGE}"
+ - echo "INTEGRATION_IMAGE=$INTEGRATION_IMAGE"
- docker pull $INTEGRATION_IMAGE
- docker tag $INTEGRATION_IMAGE $CI_PROJECT_PATH:stable
- docker push $CI_PROJECT_PATH:stable
@@ -356,14 +360,14 @@ rollback_release:
before_script:
- !reference [.setup_git, script]
script:
- - RELEASE="$(echo -n ${CI_COMMIT_BRANCH} | sed 's,^releases/,,')"
+ - RELEASE="$(echo -n $CI_COMMIT_BRANCH | sed 's,^releases/,,')"
- |
if test -f .tag.exists
then
- echo "*** Not removing existing tag ${RELEASE}! ***"
+ echo "*** Not removing existing tag $RELEASE! ***"
exit 1
else
- git push origin -d ${RELEASE}
+ git push origin -d $RELEASE
fi
finalize_release:
--
GitLab