From 71f35f690a661770cc8d95db50ef061b7c2a8066 Mon Sep 17 00:00:00 2001
From: lukken <lukken@astron.nl>
Date: Wed, 18 Jan 2023 13:00:00 +0000
Subject: [PATCH 1/6] L2SS-752: Seperate jobs for merge and master in CI/CD

---
 .gitlab-ci.yml | 127 +++++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 113 insertions(+), 14 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a660cf080..b4c8701fe 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -557,15 +557,24 @@ docker_build_image_hdbppts_es:
   script:
     #    Do not remove 'bash' or statement will be ignored by primitive docker shell
     - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh hdbppts-es $tag
-newline_at_eof:
+.newline_at_eof:
   stage: linting
   before_script:
     - pip3 install -r tangostationcontrol/test-requirements.txt
   script:
     #     TODO(Corne): Ignore shell files in submodules more cleanly
     - flake8 --filename *.sh,*.conf,*.md,*.yml --select=W292 --exclude docker-compose/tango-prometheus-exporter,.tox,.egg-info,docker
+newline_at_eof_merge:
+  extends: .newline_at_eof
+  only:
+    refs:
+      - merge_requests
+newline_at_eof_master:
+  extends: .newline_at_eof
+  rules:
+    - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG
 
-run_black:
+.run_black:
   stage: linting
   script:
     - cd tangostationcontrol
@@ -576,38 +585,95 @@ run_black:
     when: always
     paths:
       - tangostationcontrol/black.patch
+run_black_merge:
+  extends: .run_black
+  only:
+    refs:
+      - merge_requests
+run_black_master:
+  extends: .run_black
+  rules:
+    - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG
 
-run_flake8:
+.run_flake8:
   stage: linting
   script:
     - cd tangostationcontrol
     - tox -e pep8
+run_flake8_merge:
+  extends: .run_flake8
+  only:
+    refs:
+      - merge_requests
+run_flake8_master:
+  extends: .run_flake8
+  rules:
+    - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG
 
-run_pylint:
+.run_pylint:
   stage: linting
   allow_failure: true
   script:
     - cd tangostationcontrol
     - tox -e pylint
+run_pylint_merge:
+  extends: .run_pylint
+  only:
+    refs:
+      - merge_requests
+run_pylint_master:
+  extends: .run_pylint
+  rules:
+    - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG
 
-bandit:
+.bandit:
   stage: static-analysis
   script:
     - cd tangostationcontrol
     - tox -e bandit
-xenon:
+bandit_merge:
+  extends: .bandit
+  only:
+    refs:
+      - merge_requests
+bandit_master:
+  extends: .bandit
+  rules:
+    - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG
+
+.xenon:
   stage: static-analysis
   allow_failure: true
   script:
     - cd tangostationcontrol
     - tox -e xenon
-shellcheck:
+xenon_merge:
+  extends: .xenon
+  only:
+    refs:
+      - merge_requests
+xenon_master:
+  extends: .xenon
+  rules:
+    - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG
+
+.shellcheck:
   stage: static-analysis
   script:
     #     TODO(Corne): L2SS-962: Ignore shell files in submodules
     - shellcheck --version
     - shellcheck **/*.sh
-sphinx-documentation:
+shellcheck_merge:
+  extends: .shellcheck
+  only:
+    refs:
+      - merge_requests
+shellcheck_master:
+  extends: .shellcheck
+  rules:
+    - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG
+
+.sphinx_documentation:
   stage: documentation
   script:
     - cd tangostationcontrol
@@ -615,7 +681,17 @@ sphinx-documentation:
   artifacts:
     paths:
       - tangostationcontrol/docs/build/
-unit_test:
+sphinx_documentation_merge:
+  extends: .sphinx_documentation
+  only:
+    refs:
+      - merge_requests
+sphinx_documentation_master:
+  extends: .sphinx_documentation
+  rules:
+    - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG
+
+.unit_test:
   stage: unit-tests
   script:
     - cd tangostationcontrol
@@ -629,7 +705,17 @@ unit_test:
     paths:
       - tangostationcontrol/cover/*
       - tangostationcontrol/.coverage
-integration_test_docker:
+unit_test_merge:
+  extends: .unit_test
+  only:
+    refs:
+      - merge_requests
+unit_test_master:
+  extends: .unit_test
+  rules:
+    - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG
+
+.integration_test_docker:
   stage: integration-tests
   image: docker:latest
   tags:
@@ -640,10 +726,6 @@ integration_test_docker:
     DOCKER_TLS_CERTDIR: "/certs"
   needs:
     - unit_test
-  artifacts:
-    when: always
-    paths:
-      - log/
   before_script:
     - |
       if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" && -z "$CI_COMMIT_TAG" ]]; then
@@ -682,14 +764,31 @@ integration_test_docker:
         docker logs "${container}" >& "log/${container}.log"
       done
       PGPASSWORD=password pg_dump --host=docker --username=postgres hdb 2>log/archiver-timescale-dump.log | gzip > log/archiver-timescale-dump.txt.gz
+  artifacts:
+    when: always
+    paths:
+      - log/
+integration_test_docker_merge:
+  extends: .integration_test_docker
+  only:
+    refs:
+      - merge_requests
+integration_test_docker_master:
+  extends: .integration_test_docker
+  rules:
+    - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG
+
 wheel_packaging:
   stage: packaging
+  rules:
+    - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG
   artifacts:
     paths:
       - tangostationcontrol/dist/*.whl
   script:
     - cd tangostationcontrol
     - tox -e build
+
 .base_deploy:
   stage: deploy
   image: ubuntu:bionic
-- 
GitLab


From 8107a6676e303f81c7206054275e1b8bb3186184 Mon Sep 17 00:00:00 2001
From: lukken <lukken@astron.nl>
Date: Wed, 25 Jan 2023 10:42:56 +0100
Subject: [PATCH 2/6] L2SS-752: Fix incorrect needs dependency

---
 .gitlab-ci.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b4c8701fe..114fc73f1 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -724,8 +724,6 @@ unit_test_master:
     - name: docker:dind
   variables:
     DOCKER_TLS_CERTDIR: "/certs"
-  needs:
-    - unit_test
   before_script:
     - |
       if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" && -z "$CI_COMMIT_TAG" ]]; then
@@ -770,11 +768,15 @@ unit_test_master:
       - log/
 integration_test_docker_merge:
   extends: .integration_test_docker
+  needs:
+    - unit_test_merge
   only:
     refs:
       - merge_requests
 integration_test_docker_master:
   extends: .integration_test_docker
+  needs:
+    - unit_test_master
   rules:
     - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG
 
-- 
GitLab


From 241b87d5a03c8aa34c6ee39826a90c6efd611ade Mon Sep 17 00:00:00 2001
From: lukken <lukken@astron.nl>
Date: Wed, 25 Jan 2023 11:06:59 +0100
Subject: [PATCH 3/6] L2SS-752: Separate prepare trigger

---
 .gitlab-ci.yml | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 114fc73f1..3b3103339 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -20,12 +20,23 @@ stages:
   - packaging
   - deploy
 
-trigger_prepare:
+.trigger_prepare:
   stage: prepare
   trigger:
     strategy: depend
     include: .prepare.gitlab-ci.yml
 
+trigger_prepare_merge:
+  extends: .trigger_prepare
+  only:
+    refs:
+      - merge_requests
+
+trigger_prepare_master:
+  extends: .trigger_prepare
+  rules:
+    - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG
+
   # See docker-compose/README.md for docker image behavior and explanation
 .base_docker_images:
   stage: images
-- 
GitLab


From d24173170282913e01e2e547dd673efc103822ed Mon Sep 17 00:00:00 2001
From: lukken <lukken@astron.nl>
Date: Wed, 25 Jan 2023 11:11:48 +0100
Subject: [PATCH 4/6] L2SS-752: Directly trigger include

---
 .gitlab-ci.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3b3103339..42fe3698c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -20,20 +20,20 @@ stages:
   - packaging
   - deploy
 
-.trigger_prepare:
+trigger_prepare_merge:
   stage: prepare
   trigger:
     strategy: depend
     include: .prepare.gitlab-ci.yml
-
-trigger_prepare_merge:
-  extends: .trigger_prepare
   only:
     refs:
       - merge_requests
 
 trigger_prepare_master:
-  extends: .trigger_prepare
+  stage: prepare
+  trigger:
+    strategy: depend
+    include: .prepare.gitlab-ci.yml
   rules:
     - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG
 
-- 
GitLab


From ed63efa41ff96ca7abb3c2ada68dd055fc9e262b Mon Sep 17 00:00:00 2001
From: lukken <lukken@astron.nl>
Date: Wed, 25 Jan 2023 11:51:04 +0100
Subject: [PATCH 5/6] L2SS-752: Update prepare

---
 .prepare.gitlab-ci.yml | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/.prepare.gitlab-ci.yml b/.prepare.gitlab-ci.yml
index f0a04e41a..fce5f41cf 100644
--- a/.prepare.gitlab-ci.yml
+++ b/.prepare.gitlab-ci.yml
@@ -1,7 +1,7 @@
 stages:
   - build
 
-build_ci_runner_image:
+.build_ci_runner_image:
   stage: build
   image: docker:stable
   services:
@@ -21,3 +21,12 @@ build_ci_runner_image:
         docker image tag $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG $CI_REGISTRY_IMAGE/ci-build-runner:latest
         docker push $CI_REGISTRY_IMAGE/ci-build-runner:latest
       fi
+build_ci_runner_image_merge:
+  extends: .build_ci_runner_image
+  only:
+    refs:
+      - merge_requests
+build_ci_runner_image_master:
+  extends: .build_ci_runner_image
+  rules:
+    - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG
\ No newline at end of file
-- 
GitLab


From 61081185490a482fa4c191e3ce5164d32b4bad0d Mon Sep 17 00:00:00 2001
From: lukken <lukken@astron.nl>
Date: Wed, 25 Jan 2023 11:57:48 +0100
Subject: [PATCH 6/6] L2SS-752: Do not use trigger prepare

---
 .gitlab-ci.yml         | 33 +++++++++++++++++++++++----------
 .prepare.gitlab-ci.yml | 32 --------------------------------
 2 files changed, 23 insertions(+), 42 deletions(-)
 delete mode 100644 .prepare.gitlab-ci.yml

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 42fe3698c..2423bd05a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -20,20 +20,33 @@ stages:
   - packaging
   - deploy
 
-trigger_prepare_merge:
+.build_ci_runner_image:
   stage: prepare
-  trigger:
-    strategy: depend
-    include: .prepare.gitlab-ci.yml
+  image: docker:stable
+  services:
+    - docker:dind
+  script:
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - |
+      if docker pull $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG; then
+        docker build --cache-from $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG --tag $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG docker-compose/ci-runner
+      else
+        docker pull $CI_REGISTRY_IMAGE/ci-build-runner:latest || true
+        docker build --cache-from $CI_REGISTRY_IMAGE/ci-build-runner:latest --tag $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG docker-compose/ci-runner
+      fi
+    - docker push $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG  # push the image
+    - |
+      if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then
+        docker image tag $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG $CI_REGISTRY_IMAGE/ci-build-runner:latest
+        docker push $CI_REGISTRY_IMAGE/ci-build-runner:latest
+      fi
+build_ci_runner_image_merge:
+  extends: .build_ci_runner_image
   only:
     refs:
       - merge_requests
-
-trigger_prepare_master:
-  stage: prepare
-  trigger:
-    strategy: depend
-    include: .prepare.gitlab-ci.yml
+build_ci_runner_image_master:
+  extends: .build_ci_runner_image
   rules:
     - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG
 
diff --git a/.prepare.gitlab-ci.yml b/.prepare.gitlab-ci.yml
deleted file mode 100644
index fce5f41cf..000000000
--- a/.prepare.gitlab-ci.yml
+++ /dev/null
@@ -1,32 +0,0 @@
-stages:
-  - build
-
-.build_ci_runner_image:
-  stage: build
-  image: docker:stable
-  services:
-    - docker:dind
-  script:
-    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-    - |
-      if docker pull $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG; then
-        docker build --cache-from $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG --tag $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG docker-compose/ci-runner
-      else
-        docker pull $CI_REGISTRY_IMAGE/ci-build-runner:latest || true
-        docker build --cache-from $CI_REGISTRY_IMAGE/ci-build-runner:latest --tag $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG docker-compose/ci-runner
-      fi
-    - docker push $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG  # push the image
-    - |
-      if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then
-        docker image tag $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG $CI_REGISTRY_IMAGE/ci-build-runner:latest
-        docker push $CI_REGISTRY_IMAGE/ci-build-runner:latest
-      fi
-build_ci_runner_image_merge:
-  extends: .build_ci_runner_image
-  only:
-    refs:
-      - merge_requests
-build_ci_runner_image_master:
-  extends: .build_ci_runner_image
-  rules:
-    - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG
\ No newline at end of file
-- 
GitLab