Skip to content
Snippets Groups Projects
Commit 8f736fe6 authored by Jan David Mol's avatar Jan David Mol
Browse files

L2SS-2309: Allow triggering of station deployment by a different project

parent c21e68ee
No related branches found
No related tags found
1 merge request!1092Resolve L2SS-2309 "Trigger deployment"
variables:
STATION: ""
COMPONENTS: ""
stages:
- deploy
deploy_nomad:
stage: deploy
rules:
- if: $STATION != ""
needs:
- pipeline: $PARENT_PIPELINE_ID
job: render_levant
image:
name: hashicorp/nomad
entrypoint: [ "" ]
environment:
name: $STATION
script:
- |
if [ "${STATION}" == "dts-lab" ]; then
# dts-lab test station
HOSTNAME="dts-lab.lofar.net"
else
# core/remote station
HOSTNAME="${STATION}c.control.lofar"
fi
for COMPONENT in ${COMPONENTS}; do
echo "Running station ${STATION} component ${COMPONENT}"
nomad job run -address="http://${HOSTNAME}:4646" jobs/${STATION}/${COMPONENT}.nomad
done
...@@ -36,22 +36,46 @@ include: ...@@ -36,22 +36,46 @@ include:
- template: Security/Dependency-Scanning.gitlab-ci.yml - template: Security/Dependency-Scanning.gitlab-ci.yml
- template: Security/Secret-Detection.gitlab-ci.yml - template: Security/Secret-Detection.gitlab-ci.yml
# Local jobs are triggered by commits instead of
# external triggers.
.local_job:
needs:
- trigger_prepare
rules:
# The rules here do not accept jobs,
# only disregard them. As such, additional
# rules are always needed, either to filter
# further or to always accept.
- if: $CI_PIPELINE_SOURCE == "pipeline"
when: never
# Prepare image to run ci on # Prepare image to run ci on
trigger_prepare: trigger_prepare:
stage: prepare stage: prepare
extends: .local_job
needs: []
rules:
- !reference [.local_job, rules]
- if: $CI
trigger: trigger:
strategy: depend strategy: depend
include: .prepare.gitlab-ci.yml include: .prepare.gitlab-ci.yml
run_lint: run_lint:
stage: lint stage: lint
extends: .local_job
rules:
- !reference [.local_job, rules]
- if: $CI
script: script:
- tox -e lint - tox -e lint
run_shellcheck: run_shellcheck:
stage: lint stage: lint
needs: extends: .local_job
- trigger_prepare rules:
- !reference [.local_job, rules]
- if: $CI
script: script:
- shellcheck --version - shellcheck --version
- shellcheck **/*.sh - shellcheck **/*.sh
...@@ -262,10 +286,13 @@ secret_detection: ...@@ -262,10 +286,13 @@ secret_detection:
# Run all unit tests for Python versions except the base image # Run all unit tests for Python versions except the base image
run_unit_tests: run_unit_tests:
extends: .run_unit_test_version_base
needs:
- trigger_prepare
stage: test stage: test
extends:
- .run_unit_test_version_base
- .local_job
rules:
- !reference [.local_job, rules]
- if: $CI
allow_failure: true allow_failure: true
image: python:3.${PY_VERSION} image: python:3.${PY_VERSION}
script: script:
...@@ -275,10 +302,13 @@ run_unit_tests: ...@@ -275,10 +302,13 @@ run_unit_tests:
- PY_VERSION: [10, 11] - PY_VERSION: [10, 11]
run_unit_tests_coverage: run_unit_tests_coverage:
extends: .run_unit_test_version_base
needs:
- trigger_prepare
stage: test stage: test
extends:
- .run_unit_test_version_base
- .local_job
rules:
- !reference [.local_job, rules]
- if: $CI
script: script:
- tox -e coverage - tox -e coverage
coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/' coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/'
...@@ -295,8 +325,10 @@ run_unit_tests_coverage: ...@@ -295,8 +325,10 @@ run_unit_tests_coverage:
package_files: package_files:
stage: package stage: package
needs: extends: .local_job
- trigger_prepare rules:
- !reference [.local_job, rules]
- if: $CI
artifacts: artifacts:
expire_in: 1w expire_in: 1w
...@@ -308,6 +340,10 @@ package_files: ...@@ -308,6 +340,10 @@ package_files:
sphinx_documentation: sphinx_documentation:
stage: package stage: package
extends: .local_job
rules:
- !reference [.local_job, rules]
- if: $CI
artifacts: artifacts:
expire_in: 1w expire_in: 1w
paths: paths:
...@@ -318,11 +354,13 @@ sphinx_documentation: ...@@ -318,11 +354,13 @@ sphinx_documentation:
# See docker-compose/README.md for docker image behavior and explanation # See docker-compose/README.md for docker image behavior and explanation
.base_docker_images: .base_docker_images:
stage: images stage: images
extends: .local_job
rules:
- !reference [.local_job, rules]
- if: $CI
image: docker:latest image: docker:latest
tags: tags:
- dind - dind
needs:
- package_files
before_script: before_script:
- | - |
if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" && -z "$CI_COMMIT_TAG" ]]; then if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" && -z "$CI_COMMIT_TAG" ]]; then
...@@ -355,8 +393,11 @@ sphinx_documentation: ...@@ -355,8 +393,11 @@ sphinx_documentation:
# Download all remote images and store them on our image registry for tagged # Download all remote images and store them on our image registry for tagged
# master builds # master builds
docker_store_images_master_tag: docker_store_images_master_tag:
extends: .base_docker_store_images extends:
- .base_docker_store_images
- .local_job
rules: rules:
- !reference [.local_job, rules]
- if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG
# Download all remote images and store them on our image registry if .env changes # Download all remote images and store them on our image registry if .env changes
...@@ -375,7 +416,9 @@ docker_store_images_changes: ...@@ -375,7 +416,9 @@ docker_store_images_changes:
# Build and push custom images on merge request if relevant files changed # Build and push custom images on merge request if relevant files changed
docker_build: docker_build:
extends: .base_docker_images extends:
- .base_docker_store_images
- .local_job
parallel: parallel:
matrix: matrix:
- IMAGE: - IMAGE:
...@@ -386,6 +429,7 @@ docker_build: ...@@ -386,6 +429,7 @@ docker_build:
- snmp-exporter - snmp-exporter
- landing-page - landing-page
rules: rules:
- !reference [.local_job, rules]
- if: $CI_PIPELINE_SOURCE == "merge_request_event" - if: $CI_PIPELINE_SOURCE == "merge_request_event"
changes: changes:
- docker/$IMAGE/**/* - docker/$IMAGE/**/*
...@@ -412,12 +456,16 @@ docker_build_device_base: ...@@ -412,12 +456,16 @@ docker_build_device_base:
.run_integration_tests: .run_integration_tests:
allow_failure: true allow_failure: true
stage: integration stage: integration
image: docker:latest extends: .local_job
rules:
- !reference [.local_job, rules]
- if: $CI
needs: needs:
- run_unit_tests - run_unit_tests
- docker_build_device_base - docker_build_device_base
tags: tags:
- integration_tests - integration_tests
image: docker:latest
variables: variables:
JUMPPAD_HOME: $CI_PROJECT_DIR JUMPPAD_HOME: $CI_PROJECT_DIR
before_script: before_script:
...@@ -454,19 +502,34 @@ docker_build_device_base: ...@@ -454,19 +502,34 @@ docker_build_device_base:
- .jumppad/logs/ - .jumppad/logs/
run_integration_test_core: run_integration_test_core:
extends: .run_integration_tests extends:
- .run_integration_tests
- .local_job
rules:
- !reference [.local_job, rules]
- if: $CI
script: script:
# Do not remove 'bash' or statement will be ignored by primitive docker shell # Do not remove 'bash' or statement will be ignored by primitive docker shell
- bash -e $CI_PROJECT_DIR/sbin/run_integration_test.sh --no-build --save-logs --module="tango" --station=cs - bash -e $CI_PROJECT_DIR/sbin/run_integration_test.sh --no-build --save-logs --module="tango" --station=cs
run_integration_test_remote: run_integration_test_remote:
extends: .run_integration_tests extends:
- .run_integration_tests
- .local_job
rules:
- !reference [.local_job, rules]
- if: $CI
script: script:
# Do not remove 'bash' or statement will be ignored by primitive docker shell # Do not remove 'bash' or statement will be ignored by primitive docker shell
- bash -e $CI_PROJECT_DIR/sbin/run_integration_test.sh --no-build --save-logs --module="tango" --station=rs - bash -e $CI_PROJECT_DIR/sbin/run_integration_test.sh --no-build --save-logs --module="tango" --station=rs
run_service_test_docker: run_service_test_docker:
extends: .run_integration_tests extends:
- .run_integration_tests
- .local_job
rules:
- !reference [.local_job, rules]
- if: $CI
script: script:
# Do not remove 'bash' or statement will be ignored by primitive docker shell # Do not remove 'bash' or statement will be ignored by primitive docker shell
- bash -e $CI_PROJECT_DIR/sbin/run_integration_test.sh --no-build --skip-tests --module="services" --station=cs - bash -e $CI_PROJECT_DIR/sbin/run_integration_test.sh --no-build --skip-tests --module="services" --station=cs
...@@ -474,6 +537,10 @@ run_service_test_docker: ...@@ -474,6 +537,10 @@ run_service_test_docker:
run_multi_project_integration_test: run_multi_project_integration_test:
allow_failure: true allow_failure: true
stage: integration stage: integration
extends: .local_job
rules:
- !reference [.local_job, rules]
- if: $CI
needs: needs:
- docker_build_device_base - docker_build_device_base
variables: variables:
...@@ -491,12 +558,14 @@ run_multi_project_integration_test: ...@@ -491,12 +558,14 @@ run_multi_project_integration_test:
publish_on_gitlab: publish_on_gitlab:
stage: publish stage: publish
environment: gitlab environment: gitlab
needs: extends: .local_job
- package_files
rules: rules:
- !reference [.local_job, rules]
- if: '$CI_COMMIT_TAG && $CI_COMMIT_REF_PROTECTED == "true"' - if: '$CI_COMMIT_TAG && $CI_COMMIT_REF_PROTECTED == "true"'
- if: $CI_COMMIT_TAG - if: $CI_COMMIT_TAG
when: manual when: manual
needs:
- package_files
variables: variables:
TWINE_PASSWORD: ${CI_JOB_TOKEN} TWINE_PASSWORD: ${CI_JOB_TOKEN}
TWINE_USERNAME: gitlab-ci-token TWINE_USERNAME: gitlab-ci-token
...@@ -512,7 +581,9 @@ publish_on_gitlab: ...@@ -512,7 +581,9 @@ publish_on_gitlab:
release_job: release_job:
stage: publish stage: publish
image: registry.gitlab.com/gitlab-org/release-cli:latest image: registry.gitlab.com/gitlab-org/release-cli:latest
extends: .local_job
rules: rules:
- !reference [.local_job, rules]
- if: '$CI_COMMIT_TAG && $CI_COMMIT_REF_PROTECTED == "true"' - if: '$CI_COMMIT_TAG && $CI_COMMIT_REF_PROTECTED == "true"'
before_script: before_script:
- echo "running release_job before_script" - echo "running release_job before_script"
...@@ -522,43 +593,40 @@ release_job: ...@@ -522,43 +593,40 @@ release_job:
tag_name: '$CI_COMMIT_TAG' tag_name: '$CI_COMMIT_TAG'
description: '$CI_COMMIT_TAG - $CI_COMMIT_TAG_MESSAGE' description: '$CI_COMMIT_TAG - $CI_COMMIT_TAG_MESSAGE'
deploy_nomad: deploy_nomad_manual:
extends: .components
stage: deploy stage: deploy
extends:
- .components
- .local_job
rules: rules:
- !reference [.local_job, rules]
- if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG
when: manual when: manual
needs: needs:
- docker_build - docker_build
- docker_build_device_base - docker_build_device_base
- render_levant variables:
dependencies: PARENT_PIPELINE_ID: $CI_PIPELINE_ID
- render_levant trigger:
include: .deploy.gitlab-ci.yml
forward:
pipeline_variables: true
parallel: parallel:
matrix: matrix:
- STATION: - STATION:
- cs001 - cs001
- cs032 - cs032
- rs307 - rs307
image:
name: hashicorp/nomad
entrypoint: [ "" ]
environment:
name: $STATION
# override default before_script, job won't have Python available
before_script:
- uname
script:
- |
if [ "${STATION}" == "dts-lab" ]; then
# dts-lab test station
HOSTNAME="dts-lab.lofar.net"
else
# core/remote station
HOSTNAME="monitor.control.lofar"
fi
for COMPONENT in ${COMPONENTS}; do # Deploy on $STATION as set by a multi-project pipeline
echo "Running station ${STATION} component ${COMPONENT}" deploy_nomad_auto:
nomad job run -address="http://${HOSTNAME}:4646" jobs/${STATION}/${COMPONENT}.nomad stage: deploy
done extends: .components
rules:
- if: $CI_PIPELINE_SOURCE == "pipeline"
variables:
PARENT_PIPELINE_ID: $CI_PIPELINE_ID
trigger:
include: .deploy.gitlab-ci.yml
forward:
pipeline_variables: true
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment