From 71f35f690a661770cc8d95db50ef061b7c2a8066 Mon Sep 17 00:00:00 2001 From: lukken <lukken@astron.nl> Date: Wed, 18 Jan 2023 13:00:00 +0000 Subject: [PATCH 1/6] L2SS-752: Seperate jobs for merge and master in CI/CD --- .gitlab-ci.yml | 127 +++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 113 insertions(+), 14 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a660cf080..b4c8701fe 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -557,15 +557,24 @@ docker_build_image_hdbppts_es: script: # Do not remove 'bash' or statement will be ignored by primitive docker shell - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh hdbppts-es $tag -newline_at_eof: +.newline_at_eof: stage: linting before_script: - pip3 install -r tangostationcontrol/test-requirements.txt script: # TODO(Corne): Ignore shell files in submodules more cleanly - flake8 --filename *.sh,*.conf,*.md,*.yml --select=W292 --exclude docker-compose/tango-prometheus-exporter,.tox,.egg-info,docker +newline_at_eof_merge: + extends: .newline_at_eof + only: + refs: + - merge_requests +newline_at_eof_master: + extends: .newline_at_eof + rules: + - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG -run_black: +.run_black: stage: linting script: - cd tangostationcontrol @@ -576,38 +585,95 @@ run_black: when: always paths: - tangostationcontrol/black.patch +run_black_merge: + extends: .run_black + only: + refs: + - merge_requests +run_black_master: + extends: .run_black + rules: + - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG -run_flake8: +.run_flake8: stage: linting script: - cd tangostationcontrol - tox -e pep8 +run_flake8_merge: + extends: .run_flake8 + only: + refs: + - merge_requests +run_flake8_master: + extends: .run_flake8 + rules: + - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG -run_pylint: +.run_pylint: stage: linting allow_failure: true script: - cd tangostationcontrol - tox -e pylint +run_pylint_merge: + extends: .run_pylint + only: + refs: + - merge_requests +run_pylint_master: + extends: .run_pylint + rules: + - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG -bandit: +.bandit: stage: static-analysis script: - cd tangostationcontrol - tox -e bandit -xenon: +bandit_merge: + extends: .bandit + only: + refs: + - merge_requests +bandit_master: + extends: .bandit + rules: + - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG + +.xenon: stage: static-analysis allow_failure: true script: - cd tangostationcontrol - tox -e xenon -shellcheck: +xenon_merge: + extends: .xenon + only: + refs: + - merge_requests +xenon_master: + extends: .xenon + rules: + - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG + +.shellcheck: stage: static-analysis script: # TODO(Corne): L2SS-962: Ignore shell files in submodules - shellcheck --version - shellcheck **/*.sh -sphinx-documentation: +shellcheck_merge: + extends: .shellcheck + only: + refs: + - merge_requests +shellcheck_master: + extends: .shellcheck + rules: + - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG + +.sphinx_documentation: stage: documentation script: - cd tangostationcontrol @@ -615,7 +681,17 @@ sphinx-documentation: artifacts: paths: - tangostationcontrol/docs/build/ -unit_test: +sphinx_documentation_merge: + extends: .sphinx_documentation + only: + refs: + - merge_requests +sphinx_documentation_master: + extends: .sphinx_documentation + rules: + - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG + +.unit_test: stage: unit-tests script: - cd tangostationcontrol @@ -629,7 +705,17 @@ unit_test: paths: - tangostationcontrol/cover/* - tangostationcontrol/.coverage -integration_test_docker: +unit_test_merge: + extends: .unit_test + only: + refs: + - merge_requests +unit_test_master: + extends: .unit_test + rules: + - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG + +.integration_test_docker: stage: integration-tests image: docker:latest tags: @@ -640,10 +726,6 @@ integration_test_docker: DOCKER_TLS_CERTDIR: "/certs" needs: - unit_test - artifacts: - when: always - paths: - - log/ before_script: - | if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" && -z "$CI_COMMIT_TAG" ]]; then @@ -682,14 +764,31 @@ integration_test_docker: docker logs "${container}" >& "log/${container}.log" done PGPASSWORD=password pg_dump --host=docker --username=postgres hdb 2>log/archiver-timescale-dump.log | gzip > log/archiver-timescale-dump.txt.gz + artifacts: + when: always + paths: + - log/ +integration_test_docker_merge: + extends: .integration_test_docker + only: + refs: + - merge_requests +integration_test_docker_master: + extends: .integration_test_docker + rules: + - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG + wheel_packaging: stage: packaging + rules: + - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG artifacts: paths: - tangostationcontrol/dist/*.whl script: - cd tangostationcontrol - tox -e build + .base_deploy: stage: deploy image: ubuntu:bionic -- GitLab From 8107a6676e303f81c7206054275e1b8bb3186184 Mon Sep 17 00:00:00 2001 From: lukken <lukken@astron.nl> Date: Wed, 25 Jan 2023 10:42:56 +0100 Subject: [PATCH 2/6] L2SS-752: Fix incorrect needs dependency --- .gitlab-ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b4c8701fe..114fc73f1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -724,8 +724,6 @@ unit_test_master: - name: docker:dind variables: DOCKER_TLS_CERTDIR: "/certs" - needs: - - unit_test before_script: - | if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" && -z "$CI_COMMIT_TAG" ]]; then @@ -770,11 +768,15 @@ unit_test_master: - log/ integration_test_docker_merge: extends: .integration_test_docker + needs: + - unit_test_merge only: refs: - merge_requests integration_test_docker_master: extends: .integration_test_docker + needs: + - unit_test_master rules: - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG -- GitLab From 241b87d5a03c8aa34c6ee39826a90c6efd611ade Mon Sep 17 00:00:00 2001 From: lukken <lukken@astron.nl> Date: Wed, 25 Jan 2023 11:06:59 +0100 Subject: [PATCH 3/6] L2SS-752: Separate prepare trigger --- .gitlab-ci.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 114fc73f1..3b3103339 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,12 +20,23 @@ stages: - packaging - deploy -trigger_prepare: +.trigger_prepare: stage: prepare trigger: strategy: depend include: .prepare.gitlab-ci.yml +trigger_prepare_merge: + extends: .trigger_prepare + only: + refs: + - merge_requests + +trigger_prepare_master: + extends: .trigger_prepare + rules: + - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG + # See docker-compose/README.md for docker image behavior and explanation .base_docker_images: stage: images -- GitLab From d24173170282913e01e2e547dd673efc103822ed Mon Sep 17 00:00:00 2001 From: lukken <lukken@astron.nl> Date: Wed, 25 Jan 2023 11:11:48 +0100 Subject: [PATCH 4/6] L2SS-752: Directly trigger include --- .gitlab-ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3b3103339..42fe3698c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,20 +20,20 @@ stages: - packaging - deploy -.trigger_prepare: +trigger_prepare_merge: stage: prepare trigger: strategy: depend include: .prepare.gitlab-ci.yml - -trigger_prepare_merge: - extends: .trigger_prepare only: refs: - merge_requests trigger_prepare_master: - extends: .trigger_prepare + stage: prepare + trigger: + strategy: depend + include: .prepare.gitlab-ci.yml rules: - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG -- GitLab From ed63efa41ff96ca7abb3c2ada68dd055fc9e262b Mon Sep 17 00:00:00 2001 From: lukken <lukken@astron.nl> Date: Wed, 25 Jan 2023 11:51:04 +0100 Subject: [PATCH 5/6] L2SS-752: Update prepare --- .prepare.gitlab-ci.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.prepare.gitlab-ci.yml b/.prepare.gitlab-ci.yml index f0a04e41a..fce5f41cf 100644 --- a/.prepare.gitlab-ci.yml +++ b/.prepare.gitlab-ci.yml @@ -1,7 +1,7 @@ stages: - build -build_ci_runner_image: +.build_ci_runner_image: stage: build image: docker:stable services: @@ -21,3 +21,12 @@ build_ci_runner_image: docker image tag $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG $CI_REGISTRY_IMAGE/ci-build-runner:latest docker push $CI_REGISTRY_IMAGE/ci-build-runner:latest fi +build_ci_runner_image_merge: + extends: .build_ci_runner_image + only: + refs: + - merge_requests +build_ci_runner_image_master: + extends: .build_ci_runner_image + rules: + - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG \ No newline at end of file -- GitLab From 61081185490a482fa4c191e3ce5164d32b4bad0d Mon Sep 17 00:00:00 2001 From: lukken <lukken@astron.nl> Date: Wed, 25 Jan 2023 11:57:48 +0100 Subject: [PATCH 6/6] L2SS-752: Do not use trigger prepare --- .gitlab-ci.yml | 33 +++++++++++++++++++++++---------- .prepare.gitlab-ci.yml | 32 -------------------------------- 2 files changed, 23 insertions(+), 42 deletions(-) delete mode 100644 .prepare.gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 42fe3698c..2423bd05a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,20 +20,33 @@ stages: - packaging - deploy -trigger_prepare_merge: +.build_ci_runner_image: stage: prepare - trigger: - strategy: depend - include: .prepare.gitlab-ci.yml + image: docker:stable + services: + - docker:dind + script: + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + - | + if docker pull $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG; then + docker build --cache-from $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG --tag $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG docker-compose/ci-runner + else + docker pull $CI_REGISTRY_IMAGE/ci-build-runner:latest || true + docker build --cache-from $CI_REGISTRY_IMAGE/ci-build-runner:latest --tag $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG docker-compose/ci-runner + fi + - docker push $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG # push the image + - | + if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then + docker image tag $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG $CI_REGISTRY_IMAGE/ci-build-runner:latest + docker push $CI_REGISTRY_IMAGE/ci-build-runner:latest + fi +build_ci_runner_image_merge: + extends: .build_ci_runner_image only: refs: - merge_requests - -trigger_prepare_master: - stage: prepare - trigger: - strategy: depend - include: .prepare.gitlab-ci.yml +build_ci_runner_image_master: + extends: .build_ci_runner_image rules: - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG diff --git a/.prepare.gitlab-ci.yml b/.prepare.gitlab-ci.yml deleted file mode 100644 index fce5f41cf..000000000 --- a/.prepare.gitlab-ci.yml +++ /dev/null @@ -1,32 +0,0 @@ -stages: - - build - -.build_ci_runner_image: - stage: build - image: docker:stable - services: - - docker:dind - script: - - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - - | - if docker pull $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG; then - docker build --cache-from $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG --tag $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG docker-compose/ci-runner - else - docker pull $CI_REGISTRY_IMAGE/ci-build-runner:latest || true - docker build --cache-from $CI_REGISTRY_IMAGE/ci-build-runner:latest --tag $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG docker-compose/ci-runner - fi - - docker push $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG # push the image - - | - if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then - docker image tag $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG $CI_REGISTRY_IMAGE/ci-build-runner:latest - docker push $CI_REGISTRY_IMAGE/ci-build-runner:latest - fi -build_ci_runner_image_merge: - extends: .build_ci_runner_image - only: - refs: - - merge_requests -build_ci_runner_image_master: - extends: .build_ci_runner_image - rules: - - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG \ No newline at end of file -- GitLab