Resolve L2SS-1432 "Improve pipeline execution"

.gitlab-ci.yml

@@ -66,14 +66,6 @@ trigger_prepare:
 ##    Allow docker image script to execute
 #    - chmod u+x $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh
 
-# Template for docker images NOT on tagged or master builds
-.base_docker_images_except:
-  extends: .base_docker_images
-  except:
-    refs:
-      - tags
-      - master
-
 # Template to download all remote images and store them on our image registry
 # (call tag_and_push without arguments)
 .base_docker_store_images:
@@ -103,102 +95,33 @@ docker_store_images_changes:
     changes:
       - docker-compose/.env
 
-# Build and push all our custom images on tagged or master builds
-docker_build_image_all:
+# Build and push custom images on merge request if relevant files changed
+docker_build_image:
   extends: .base_docker_images
+  parallel:
+    matrix:
+      - IMAGE:
+          - lofar-device-base
+          - ec-sim
+          - http-json-schemas
+          - prometheus
+          - tango-prometheus-exporter
+          - itango
+          - grafana
+          - loki
+          - logstash
+          - jupyter-lab
   rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+        - docker-compose/$IMAGE.yml
+        - docker-compose/$IMAGE/*
+        - docker-compose/.env
     - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG
   script:
     #    Do not remove 'bash' or statement will be ignored by primitive docker shell
-    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh lofar-device-base latest
-    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh prometheus latest
-    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh itango latest
-    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh grafana latest
-    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh loki latest
-    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh logstash latest
-    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh jupyter-lab latest
-    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh grafana latest
-    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh prometheus latest
-    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh tango-prometheus-exporter latest
+    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh $IMAGE $tag
 
-# Build and push custom images on merge request if relevant files changed
-docker_build_image_lofar_device_base:
-  extends: .base_docker_images_except
-  only:
-    refs:
-      - merge_requests
-    changes:
-      - docker-compose/lofar-device-base.yml
-      - docker-compose/lofar-device-base/*
-  script:
-    #    Do not remove 'bash' or statement will be ignored by primitive docker shell
-    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh lofar-device-base $tag
-docker_build_image_prometheus:
-  extends: .base_docker_images_except
-  only:
-    refs:
-      - merge_requests
-    changes:
-      - docker-compose/prometheus.yml
-      - docker-compose/prometheus/*
-  script:
-    #    Do not remove 'bash' or statement will be ignored by primitive docker shell
-    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh prometheus $tag
-docker_build_image_itango:
-  extends: .base_docker_images_except
-  only:
-    refs:
-      - merge_requests
-    changes:
-      - docker-compose/itango.yml
-      - docker-compose/itango/*
-  script:
-    #    Do not remove 'bash' or statement will be ignored by primitive docker shell
-    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh itango $tag
-docker_build_image_grafana:
-  extends: .base_docker_images_except
-  only:
-    refs:
-      - merge_requests
-    changes:
-      - docker-compose/grafana.yml
-      - docker-compose/grafana/*
-  script:
-    #    Do not remove 'bash' or statement will be ignored by primitive docker shell
-    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh grafana $tag
-docker_build_image_loki:
-  extends: .base_docker_images_except
-  only:
-    refs:
-      - merge_requests
-    changes:
-      - docker-compose/loki.yml
-      - docker-compose/loki/*
-  script:
-    #    Do not remove 'bash' or statement will be ignored by primitive docker shell
-    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh loki $tag
-docker_build_image_logstash:
-  extends: .base_docker_images_except
-  only:
-    refs:
-      - merge_requests
-    changes:
-      - docker-compose/logstash.yml
-      - docker-compose/logstash/*
-  script:
-    #    Do not remove 'bash' or statement will be ignored by primitive docker shell
-    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh logstash $tag
-docker_build_image_jupyter:
-  extends: .base_docker_images_except
-  only:
-    refs:
-      - merge_requests
-    changes:
-      - docker-compose/jupyter-lab.yml
-      - docker-compose/jupyterlab/*
-  script:
-    #    Do not remove 'bash' or statement will be ignored by primitive docker shell
-    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh jupyter-lab $tag
 newline_at_eof:
   stage: linting
   before_script:
@@ -335,7 +258,8 @@ integration_test_docker:
     #    Do not remove 'bash' or statement will be ignored by primitive docker shell
     - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh pull $tag
     #    Do not remove 'bash' or statement will be ignored by primitive docker shell
-    - bash -e $CI_PROJECT_DIR/sbin/run_integration_test.sh
+    - export TAG="$tag"
+    - bash -e $CI_PROJECT_DIR/sbin/run_integration_test.sh --no-build
   after_script:
     #    Collect output of all containers
     - |

docker-compose/Makefile

@@ -5,6 +5,8 @@
 MAKEPATH := $(abspath $(lastword $(MAKEFILE_LIST)))
 BASEDIR := $(notdir $(patsubst %/,%,$(dir $(MAKEPATH))))
 
+TAG ?= latest
+
 DOCKER_COMPOSE ?= docker compose
 
 DOCKER_COMPOSE_ENV_FILE := $(abspath .env)
@@ -23,6 +25,9 @@ TIMEOUT ?= 300
 
 SCRATCH ?= /tmp
 
+# Disables building of the lofar-base-device, primarily used for integration tests in CI/CD
+NO_BASE ?= 0
+
 # Host name through which others can reach our control interfaces.
 # Needs to be resolvable from the containers and clients.
 ifneq (,$(wildcard /run/WSL))
@@ -146,6 +151,7 @@ else
     endif
 endif
 
+
 DOCKER_COMPOSE_ARGS := DISPLAY=$(DISPLAY) \
     XAUTHORITY=$(XAUTHORITY) \
     TANGO_HOST=$(TANGO_HOST) \
@@ -159,7 +165,8 @@ DOCKER_COMPOSE_ARGS := DISPLAY=$(DISPLAY) \
     COMPOSE_HTTP_TIMEOUT=180 \
     CONTAINER_EXECUTION_UID=$(shell id -u) \
     DOCKER_GID=$(DOCKER_GID) \
-    TEST_MODULE=$(INTEGRATION_MODULE)
+    TEST_MODULE=$(INTEGRATION_MODULE) \
+    TAG=$(TAG)
 
 
 .PHONY: up base base-nocache down minimal context run integration start stop restart build build-nocache status clean pull help await
@@ -169,7 +176,9 @@ pull: ## pull the images from the Docker hub
 	$(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) pull --ignore-pull-failures
 
 base: context ## Build base lofar device image
+ifneq ($(NO_BASE),1)
 	$(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) build --progress=plain lofar-device-base
+endif
 
 base-nocache: context ## Rebuild base lofar device image
 	$(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) build --no-cache --progress=plain lofar-device-base

docker-compose/device-antennafield.yml

@@ -17,7 +17,7 @@ version: '2.1'
 
 services:
   device-antennafield:
-    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
     hostname: device-antennafield
     container_name: device-antennafield
     logging:

docker-compose/device-aps.yml

@@ -8,7 +8,7 @@ version: '2.1'
 
 services:
   device-aps:
-    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
     container_name: device-aps
     hostname: device-aps
     logging:

docker-compose/device-apsct.yml

@@ -16,7 +16,7 @@ version: '2.1'
 
 services:
   device-apsct:
-    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
     hostname: device-apsct
     container_name: device-apsct
     logging:

docker-compose/device-apspu.yml

@@ -16,7 +16,7 @@ version: '2.1'
 
 services:
   device-apspu:
-    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
     hostname: device-apspu
     container_name: device-apspu
     logging:

docker-compose/device-beamlet.yml

@@ -16,7 +16,7 @@ version: '2.1'
 
 services:
   device-beamlet:
-    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
     hostname: device-beamlet
     container_name: device-beamlet
     logging:

docker-compose/device-boot.yml

@@ -15,7 +15,7 @@ version: '2.1'
 
 services:
   device-boot:
-    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
     hostname: device-boot
     container_name: device-boot
     logging:

docker-compose/device-bst.yml

@@ -16,7 +16,7 @@ version: '2.1'
 
 services:
   device-bst:
-    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
     hostname: device-bst
     container_name: device-bst
     logging:

docker-compose/device-calibration.yml

@@ -16,7 +16,7 @@ version: '2.1'
 
 services:
   device-calibration:
-    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
     hostname: device-calibration
     container_name: device-calibration
     logging:

docker-compose/device-ccd.yml

@@ -16,7 +16,7 @@ version: '2.1'
 
 services:
   device-ccd:
-    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
     hostname: device-ccd
     container_name: device-ccd
     logging:

docker-compose/device-configuration.yml

@@ -16,7 +16,7 @@ version: '2.1'
 
 services:
   device-configuration:
-    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
     hostname: device-configuration
     container_name: device-configuration
     logging:

docker-compose/device-digitalbeam.yml

@@ -16,7 +16,7 @@ version: '2.1'
 
 services:
   device-digitalbeam:
-    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
     hostname: device-digitalbeam
     container_name: device-digitalbeam
     logging:

docker-compose/device-docker.yml

@@ -16,7 +16,7 @@ version: '2.1'
 
 services:
   device-docker:
-    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
     hostname: device-docker
     container_name: device-docker
     logging:

docker-compose/device-ec.yml

@@ -11,7 +11,7 @@ volumes:
 
 services:
   device-ec:
-    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
     hostname: device-ec
     container_name: device-ec
     logging:

docker-compose/device-observation-control.yml

@@ -15,7 +15,7 @@ version: '2.1'
 
 services:
   device-observation-control:
-    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
     hostname: device-observation-control
     container_name: device-observation-control
     logging:

docker-compose/device-observation.yml

@@ -14,7 +14,7 @@ version: '2.1'
 
 services:
   device-observation:
-    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
     hostname: device-observation
     container_name: device-observation
     logging:

docker-compose/device-pcon.yml

@@ -11,7 +11,7 @@ volumes:
 
 services:
   device-pcon:
-    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
     hostname: device-pcon
     container_name: device-pcon
     logging:

docker-compose/device-psoc.yml

@@ -11,7 +11,7 @@ volumes:
 
 services:
   device-psoc:
-    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
     hostname: device-psoc
     container_name: device-psoc
     logging:

docker-compose/device-recvh.yml

@@ -8,7 +8,7 @@ version: '2.1'
 
 services:
   device-recvh:
-    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
     hostname: device-recvh
     container_name: device-recvh
     logging: