diff --git a/.deploy.gitlab-ci.yml b/.deploy.gitlab-ci.yml
new file mode 100644
index 0000000000000000000000000000000000000000..2e4bd33131edb2c0838dfa600b85f8d1c5e560c9
--- /dev/null
+++ b/.deploy.gitlab-ci.yml
@@ -0,0 +1,33 @@
+variables:
+ STATION: ""
+ COMPONENTS: ""
+
+stages:
+ - deploy
+
+deploy_nomad:
+ stage: deploy
+ rules:
+ - if: $STATION != ""
+ needs:
+ - pipeline: $PARENT_PIPELINE_ID
+ job: render_levant
+ image:
+ name: hashicorp/nomad
+ entrypoint: [ "" ]
+ environment:
+ name: $STATION
+ script:
+ - |
+ if [ "${STATION}" == "dts-lab" ]; then
+ # dts-lab test station
+ HOSTNAME="dts-lab.lofar.net"
+ else
+ # core/remote station
+ HOSTNAME="${STATION}c.control.lofar"
+ fi
+
+ for COMPONENT in ${COMPONENTS}; do
+ echo "Running station ${STATION} component ${COMPONENT}"
+ nomad job run -address="http://${HOSTNAME}:4646" jobs/${STATION}/${COMPONENT}.nomad
+ done
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 662a73154347f0f37e3f8310222374e7e7d009bf..2a5cae16c566926e7f0ceb95377082631146e1bc 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -36,22 +36,46 @@ include:
- template: Security/Dependency-Scanning.gitlab-ci.yml
- template: Security/Secret-Detection.gitlab-ci.yml
+# Local jobs are triggered by commits instead of
+# external triggers.
+.local_job:
+ needs:
+ - trigger_prepare
+ rules:
+ # The rules here do not accept jobs,
+ # only disregard them. As such, additional
+ # rules are always needed, either to filter
+ # further or to always accept.
+ - if: $CI_PIPELINE_SOURCE == "pipeline"
+ when: never
+
# Prepare image to run ci on
trigger_prepare:
stage: prepare
+ extends: .local_job
+ needs: []
+ rules:
+ - !reference [.local_job, rules]
+ - if: $CI
trigger:
strategy: depend
include: .prepare.gitlab-ci.yml
run_lint:
stage: lint
+ extends: .local_job
+ rules:
+ - !reference [.local_job, rules]
+ - if: $CI
script:
- tox -e lint
run_shellcheck:
stage: lint
- needs:
- - trigger_prepare
+ extends: .local_job
+ rules:
+ - !reference [.local_job, rules]
+ - if: $CI
script:
- shellcheck --version
- shellcheck **/*.sh
@@ -262,10 +286,13 @@ secret_detection:
# Run all unit tests for Python versions except the base image
run_unit_tests:
- extends: .run_unit_test_version_base
- needs:
- - trigger_prepare
stage: test
+ extends:
+ - .run_unit_test_version_base
+ - .local_job
+ rules:
+ - !reference [.local_job, rules]
+ - if: $CI
allow_failure: true
image: python:3.${PY_VERSION}
script:
@@ -275,10 +302,13 @@ run_unit_tests:
- PY_VERSION: [10, 11]
run_unit_tests_coverage:
- extends: .run_unit_test_version_base
- needs:
- - trigger_prepare
stage: test
+ extends:
+ - .run_unit_test_version_base
+ - .local_job
+ rules:
+ - !reference [.local_job, rules]
+ - if: $CI
script:
- tox -e coverage
coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/'
@@ -295,8 +325,10 @@ run_unit_tests_coverage:
package_files:
stage: package
- needs:
- - trigger_prepare
+ extends: .local_job
+ rules:
+ - !reference [.local_job, rules]
+ - if: $CI
artifacts:
expire_in: 1w
@@ -308,6 +340,10 @@ package_files:
sphinx_documentation:
stage: package
+ extends: .local_job
+ rules:
+ - !reference [.local_job, rules]
+ - if: $CI
artifacts:
expire_in: 1w
paths:
@@ -318,11 +354,13 @@ sphinx_documentation:
# See docker-compose/README.md for docker image behavior and explanation
.base_docker_images:
stage: images
+ extends: .local_job
+ rules:
+ - !reference [.local_job, rules]
+ - if: $CI
image: docker:latest
tags:
- dind
- needs:
- - package_files
before_script:
- |
if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" && -z "$CI_COMMIT_TAG" ]]; then
@@ -355,8 +393,11 @@ sphinx_documentation:
# Download all remote images and store them on our image registry for tagged
# master builds
docker_store_images_master_tag:
- extends: .base_docker_store_images
+ extends:
+ - .base_docker_store_images
+ - .local_job
rules:
+ - !reference [.local_job, rules]
- if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG
# Download all remote images and store them on our image registry if .env changes
@@ -375,7 +416,9 @@ docker_store_images_changes:
# Build and push custom images on merge request if relevant files changed
docker_build:
- extends: .base_docker_images
+ extends:
+ - .base_docker_store_images
+ - .local_job
parallel:
matrix:
- IMAGE:
@@ -386,6 +429,7 @@ docker_build:
- snmp-exporter
- landing-page
rules:
+ - !reference [.local_job, rules]
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
changes:
- docker/$IMAGE/**/*
@@ -412,12 +456,16 @@ docker_build_device_base:
.run_integration_tests:
allow_failure: true
stage: integration
- image: docker:latest
+ extends: .local_job
+ rules:
+ - !reference [.local_job, rules]
+ - if: $CI
needs:
- run_unit_tests
- docker_build_device_base
tags:
- integration_tests
+ image: docker:latest
variables:
JUMPPAD_HOME: $CI_PROJECT_DIR
before_script:
@@ -454,19 +502,34 @@ docker_build_device_base:
- .jumppad/logs/
run_integration_test_core:
- extends: .run_integration_tests
+ extends:
+ - .run_integration_tests
+ - .local_job
+ rules:
+ - !reference [.local_job, rules]
+ - if: $CI
script:
# Do not remove 'bash' or statement will be ignored by primitive docker shell
- bash -e $CI_PROJECT_DIR/sbin/run_integration_test.sh --no-build --save-logs --module="tango" --station=cs
run_integration_test_remote:
- extends: .run_integration_tests
+ extends:
+ - .run_integration_tests
+ - .local_job
+ rules:
+ - !reference [.local_job, rules]
+ - if: $CI
script:
# Do not remove 'bash' or statement will be ignored by primitive docker shell
- bash -e $CI_PROJECT_DIR/sbin/run_integration_test.sh --no-build --save-logs --module="tango" --station=rs
run_service_test_docker:
- extends: .run_integration_tests
+ extends:
+ - .run_integration_tests
+ - .local_job
+ rules:
+ - !reference [.local_job, rules]
+ - if: $CI
script:
# Do not remove 'bash' or statement will be ignored by primitive docker shell
- bash -e $CI_PROJECT_DIR/sbin/run_integration_test.sh --no-build --skip-tests --module="services" --station=cs
@@ -474,6 +537,10 @@ run_service_test_docker:
run_multi_project_integration_test:
allow_failure: true
stage: integration
+ extends: .local_job
+ rules:
+ - !reference [.local_job, rules]
+ - if: $CI
needs:
- docker_build_device_base
variables:
@@ -491,12 +558,14 @@ run_multi_project_integration_test:
publish_on_gitlab:
stage: publish
environment: gitlab
- needs:
- - package_files
+ extends: .local_job
rules:
+ - !reference [.local_job, rules]
- if: '$CI_COMMIT_TAG && $CI_COMMIT_REF_PROTECTED == "true"'
- if: $CI_COMMIT_TAG
when: manual
+ needs:
+ - package_files
variables:
TWINE_PASSWORD: ${CI_JOB_TOKEN}
TWINE_USERNAME: gitlab-ci-token
@@ -512,7 +581,9 @@ publish_on_gitlab:
release_job:
stage: publish
image: registry.gitlab.com/gitlab-org/release-cli:latest
+ extends: .local_job
rules:
+ - !reference [.local_job, rules]
- if: '$CI_COMMIT_TAG && $CI_COMMIT_REF_PROTECTED == "true"'
before_script:
- echo "running release_job before_script"
@@ -522,43 +593,40 @@ release_job:
tag_name: '$CI_COMMIT_TAG'
description: '$CI_COMMIT_TAG - $CI_COMMIT_TAG_MESSAGE'
-deploy_nomad:
- extends: .components
+deploy_nomad_manual:
stage: deploy
+ extends:
+ - .components
+ - .local_job
rules:
+ - !reference [.local_job, rules]
- if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG
when: manual
needs:
- docker_build
- docker_build_device_base
- - render_levant
- dependencies:
- - render_levant
+ variables:
+ PARENT_PIPELINE_ID: $CI_PIPELINE_ID
+ trigger:
+ include: .deploy.gitlab-ci.yml
+ forward:
+ pipeline_variables: true
parallel:
matrix:
- STATION:
- cs001
- cs032
- rs307
- image:
- name: hashicorp/nomad
- entrypoint: [ "" ]
- environment:
- name: $STATION
- # override default before_script, job won't have Python available
- before_script:
- - uname
- script:
- - |
- if [ "${STATION}" == "dts-lab" ]; then
- # dts-lab test station
- HOSTNAME="dts-lab.lofar.net"
- else
- # core/remote station
- HOSTNAME="monitor.control.lofar"
- fi
- for COMPONENT in ${COMPONENTS}; do
- echo "Running station ${STATION} component ${COMPONENT}"
- nomad job run -address="http://${HOSTNAME}:4646" jobs/${STATION}/${COMPONENT}.nomad
- done
+# Deploy on $STATION as set by a multi-project pipeline
+deploy_nomad_auto:
+ stage: deploy
+ extends: .components
+ rules:
+ - if: $CI_PIPELINE_SOURCE == "pipeline"
+ variables:
+ PARENT_PIPELINE_ID: $CI_PIPELINE_ID
+ trigger:
+ include: .deploy.gitlab-ci.yml
+ forward:
+ pipeline_variables: true