From 1919b5ed9b0c6d927b35b5cb7e94075ba4228376 Mon Sep 17 00:00:00 2001
From: Hannes Feldt <feldt@astron.nl>
Date: Thu, 13 Jul 2023 09:30:39 +0000
Subject: [PATCH] Resolve L2SS-1432 "Improve pipeline execution"
---
.gitlab-ci.yml | 122 ++++--------------
docker-compose/Makefile | 11 +-
docker-compose/device-antennafield.yml | 2 +-
docker-compose/device-aps.yml | 2 +-
docker-compose/device-apsct.yml | 2 +-
docker-compose/device-apspu.yml | 2 +-
docker-compose/device-beamlet.yml | 2 +-
docker-compose/device-boot.yml | 2 +-
docker-compose/device-bst.yml | 2 +-
docker-compose/device-calibration.yml | 2 +-
docker-compose/device-ccd.yml | 2 +-
docker-compose/device-configuration.yml | 2 +-
docker-compose/device-digitalbeam.yml | 2 +-
docker-compose/device-docker.yml | 2 +-
docker-compose/device-ec.yml | 2 +-
docker-compose/device-observation-control.yml | 2 +-
docker-compose/device-observation.yml | 2 +-
docker-compose/device-pcon.yml | 2 +-
docker-compose/device-psoc.yml | 2 +-
docker-compose/device-recvh.yml | 2 +-
docker-compose/device-recvl.yml | 2 +-
docker-compose/device-sdp.yml | 2 +-
docker-compose/device-sdpfirmware.yml | 2 +-
docker-compose/device-sst.yml | 2 +-
docker-compose/device-station-manager.yml | 2 +-
docker-compose/device-temperature-manager.yml | 2 +-
docker-compose/device-tilebeam.yml | 2 +-
docker-compose/device-unb2.yml | 2 +-
docker-compose/device-xst.yml | 2 +-
docker-compose/ec-sim.yml | 1 +
docker-compose/grafana.yml | 2 +-
docker-compose/http-json-schemas.yml | 1 +
docker-compose/integration-test.yml | 7 +-
docker-compose/itango.yml | 1 +
docker-compose/itango/Dockerfile | 3 +-
docker-compose/jupyter-lab.yml | 2 +-
docker-compose/lofar-device-base.yml | 2 +-
docker-compose/lofar-device-base/Dockerfile | 21 ++-
docker-compose/logstash.yml | 2 +-
docker-compose/loki.yml | 3 +-
docker-compose/prometheus.yml | 2 +-
sbin/run_integration_test.sh | 35 +++--
sbin/tag_and_push_docker_image.sh | 14 +-
43 files changed, 124 insertions(+), 159 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 4ed1753ec..a0e10271b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -66,14 +66,6 @@ trigger_prepare:
## Allow docker image script to execute
# - chmod u+x $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh
-# Template for docker images NOT on tagged or master builds
-.base_docker_images_except:
- extends: .base_docker_images
- except:
- refs:
- - tags
- - master
-
# Template to download all remote images and store them on our image registry
# (call tag_and_push without arguments)
.base_docker_store_images:
@@ -103,102 +95,33 @@ docker_store_images_changes:
changes:
- docker-compose/.env
-# Build and push all our custom images on tagged or master builds
-docker_build_image_all:
+# Build and push custom images on merge request if relevant files changed
+docker_build_image:
extends: .base_docker_images
+ parallel:
+ matrix:
+ - IMAGE:
+ - lofar-device-base
+ - ec-sim
+ - http-json-schemas
+ - prometheus
+ - tango-prometheus-exporter
+ - itango
+ - grafana
+ - loki
+ - logstash
+ - jupyter-lab
rules:
+ - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+ changes:
+ - docker-compose/$IMAGE.yml
+ - docker-compose/$IMAGE/*
+ - docker-compose/.env
- if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG
script:
# Do not remove 'bash' or statement will be ignored by primitive docker shell
- - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh lofar-device-base latest
- - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh prometheus latest
- - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh itango latest
- - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh grafana latest
- - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh loki latest
- - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh logstash latest
- - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh jupyter-lab latest
- - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh grafana latest
- - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh prometheus latest
- - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh tango-prometheus-exporter latest
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh $IMAGE $tag
-# Build and push custom images on merge request if relevant files changed
-docker_build_image_lofar_device_base:
- extends: .base_docker_images_except
- only:
- refs:
- - merge_requests
- changes:
- - docker-compose/lofar-device-base.yml
- - docker-compose/lofar-device-base/*
- script:
- # Do not remove 'bash' or statement will be ignored by primitive docker shell
- - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh lofar-device-base $tag
-docker_build_image_prometheus:
- extends: .base_docker_images_except
- only:
- refs:
- - merge_requests
- changes:
- - docker-compose/prometheus.yml
- - docker-compose/prometheus/*
- script:
- # Do not remove 'bash' or statement will be ignored by primitive docker shell
- - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh prometheus $tag
-docker_build_image_itango:
- extends: .base_docker_images_except
- only:
- refs:
- - merge_requests
- changes:
- - docker-compose/itango.yml
- - docker-compose/itango/*
- script:
- # Do not remove 'bash' or statement will be ignored by primitive docker shell
- - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh itango $tag
-docker_build_image_grafana:
- extends: .base_docker_images_except
- only:
- refs:
- - merge_requests
- changes:
- - docker-compose/grafana.yml
- - docker-compose/grafana/*
- script:
- # Do not remove 'bash' or statement will be ignored by primitive docker shell
- - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh grafana $tag
-docker_build_image_loki:
- extends: .base_docker_images_except
- only:
- refs:
- - merge_requests
- changes:
- - docker-compose/loki.yml
- - docker-compose/loki/*
- script:
- # Do not remove 'bash' or statement will be ignored by primitive docker shell
- - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh loki $tag
-docker_build_image_logstash:
- extends: .base_docker_images_except
- only:
- refs:
- - merge_requests
- changes:
- - docker-compose/logstash.yml
- - docker-compose/logstash/*
- script:
- # Do not remove 'bash' or statement will be ignored by primitive docker shell
- - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh logstash $tag
-docker_build_image_jupyter:
- extends: .base_docker_images_except
- only:
- refs:
- - merge_requests
- changes:
- - docker-compose/jupyter-lab.yml
- - docker-compose/jupyterlab/*
- script:
- # Do not remove 'bash' or statement will be ignored by primitive docker shell
- - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh jupyter-lab $tag
newline_at_eof:
stage: linting
before_script:
@@ -335,7 +258,8 @@ integration_test_docker:
# Do not remove 'bash' or statement will be ignored by primitive docker shell
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh pull $tag
# Do not remove 'bash' or statement will be ignored by primitive docker shell
- - bash -e $CI_PROJECT_DIR/sbin/run_integration_test.sh
+ - export TAG="$tag"
+ - bash -e $CI_PROJECT_DIR/sbin/run_integration_test.sh --no-build
after_script:
# Collect output of all containers
- |
diff --git a/docker-compose/Makefile b/docker-compose/Makefile
index 92240df54..910348603 100644
--- a/docker-compose/Makefile
+++ b/docker-compose/Makefile
@@ -5,6 +5,8 @@
MAKEPATH := $(abspath $(lastword $(MAKEFILE_LIST)))
BASEDIR := $(notdir $(patsubst %/,%,$(dir $(MAKEPATH))))
+TAG ?= latest
+
DOCKER_COMPOSE ?= docker compose
DOCKER_COMPOSE_ENV_FILE := $(abspath .env)
@@ -23,6 +25,9 @@ TIMEOUT ?= 300
SCRATCH ?= /tmp
+# Disables building of the lofar-base-device, primarily used for integration tests in CI/CD
+NO_BASE ?= 0
+
# Host name through which others can reach our control interfaces.
# Needs to be resolvable from the containers and clients.
ifneq (,$(wildcard /run/WSL))
@@ -146,6 +151,7 @@ else
endif
endif
+
DOCKER_COMPOSE_ARGS := DISPLAY=$(DISPLAY) \
XAUTHORITY=$(XAUTHORITY) \
TANGO_HOST=$(TANGO_HOST) \
@@ -159,7 +165,8 @@ DOCKER_COMPOSE_ARGS := DISPLAY=$(DISPLAY) \
COMPOSE_HTTP_TIMEOUT=180 \
CONTAINER_EXECUTION_UID=$(shell id -u) \
DOCKER_GID=$(DOCKER_GID) \
- TEST_MODULE=$(INTEGRATION_MODULE)
+ TEST_MODULE=$(INTEGRATION_MODULE) \
+ TAG=$(TAG)
.PHONY: up base base-nocache down minimal context run integration start stop restart build build-nocache status clean pull help await
@@ -169,7 +176,9 @@ pull: ## pull the images from the Docker hub
$(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) pull --ignore-pull-failures
base: context ## Build base lofar device image
+ifneq ($(NO_BASE),1)
$(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) build --progress=plain lofar-device-base
+endif
base-nocache: context ## Rebuild base lofar device image
$(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) build --no-cache --progress=plain lofar-device-base
diff --git a/docker-compose/device-antennafield.yml b/docker-compose/device-antennafield.yml
index 21461e4ec..705e43b51 100644
--- a/docker-compose/device-antennafield.yml
+++ b/docker-compose/device-antennafield.yml
@@ -17,7 +17,7 @@ version: '2.1'
services:
device-antennafield:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-antennafield
container_name: device-antennafield
logging:
diff --git a/docker-compose/device-aps.yml b/docker-compose/device-aps.yml
index 6094efb52..23690af22 100644
--- a/docker-compose/device-aps.yml
+++ b/docker-compose/device-aps.yml
@@ -8,7 +8,7 @@ version: '2.1'
services:
device-aps:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
container_name: device-aps
hostname: device-aps
logging:
diff --git a/docker-compose/device-apsct.yml b/docker-compose/device-apsct.yml
index 921c4cfa5..86c24b6a6 100644
--- a/docker-compose/device-apsct.yml
+++ b/docker-compose/device-apsct.yml
@@ -16,7 +16,7 @@ version: '2.1'
services:
device-apsct:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-apsct
container_name: device-apsct
logging:
diff --git a/docker-compose/device-apspu.yml b/docker-compose/device-apspu.yml
index 88752625b..ac585e6c4 100644
--- a/docker-compose/device-apspu.yml
+++ b/docker-compose/device-apspu.yml
@@ -16,7 +16,7 @@ version: '2.1'
services:
device-apspu:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-apspu
container_name: device-apspu
logging:
diff --git a/docker-compose/device-beamlet.yml b/docker-compose/device-beamlet.yml
index b998eaf7e..7c734e0e3 100644
--- a/docker-compose/device-beamlet.yml
+++ b/docker-compose/device-beamlet.yml
@@ -16,7 +16,7 @@ version: '2.1'
services:
device-beamlet:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-beamlet
container_name: device-beamlet
logging:
diff --git a/docker-compose/device-boot.yml b/docker-compose/device-boot.yml
index 77585f8da..0ca0d4854 100644
--- a/docker-compose/device-boot.yml
+++ b/docker-compose/device-boot.yml
@@ -15,7 +15,7 @@ version: '2.1'
services:
device-boot:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-boot
container_name: device-boot
logging:
diff --git a/docker-compose/device-bst.yml b/docker-compose/device-bst.yml
index e199b3dba..e1fd18600 100644
--- a/docker-compose/device-bst.yml
+++ b/docker-compose/device-bst.yml
@@ -16,7 +16,7 @@ version: '2.1'
services:
device-bst:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-bst
container_name: device-bst
logging:
diff --git a/docker-compose/device-calibration.yml b/docker-compose/device-calibration.yml
index 7c7388660..387b02286 100644
--- a/docker-compose/device-calibration.yml
+++ b/docker-compose/device-calibration.yml
@@ -16,7 +16,7 @@ version: '2.1'
services:
device-calibration:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-calibration
container_name: device-calibration
logging:
diff --git a/docker-compose/device-ccd.yml b/docker-compose/device-ccd.yml
index 1621c7257..950800e33 100644
--- a/docker-compose/device-ccd.yml
+++ b/docker-compose/device-ccd.yml
@@ -16,7 +16,7 @@ version: '2.1'
services:
device-ccd:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-ccd
container_name: device-ccd
logging:
diff --git a/docker-compose/device-configuration.yml b/docker-compose/device-configuration.yml
index d3dbeb273..1a923f15b 100644
--- a/docker-compose/device-configuration.yml
+++ b/docker-compose/device-configuration.yml
@@ -16,7 +16,7 @@ version: '2.1'
services:
device-configuration:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-configuration
container_name: device-configuration
logging:
diff --git a/docker-compose/device-digitalbeam.yml b/docker-compose/device-digitalbeam.yml
index e6ba98a0e..b093ee63e 100644
--- a/docker-compose/device-digitalbeam.yml
+++ b/docker-compose/device-digitalbeam.yml
@@ -16,7 +16,7 @@ version: '2.1'
services:
device-digitalbeam:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-digitalbeam
container_name: device-digitalbeam
logging:
diff --git a/docker-compose/device-docker.yml b/docker-compose/device-docker.yml
index 9ef0917b0..a9080eeb5 100644
--- a/docker-compose/device-docker.yml
+++ b/docker-compose/device-docker.yml
@@ -16,7 +16,7 @@ version: '2.1'
services:
device-docker:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-docker
container_name: device-docker
logging:
diff --git a/docker-compose/device-ec.yml b/docker-compose/device-ec.yml
index 48e015f2a..f944c751a 100644
--- a/docker-compose/device-ec.yml
+++ b/docker-compose/device-ec.yml
@@ -11,7 +11,7 @@ volumes:
services:
device-ec:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-ec
container_name: device-ec
logging:
diff --git a/docker-compose/device-observation-control.yml b/docker-compose/device-observation-control.yml
index baf3b68c7..b5b7f9fe5 100644
--- a/docker-compose/device-observation-control.yml
+++ b/docker-compose/device-observation-control.yml
@@ -15,7 +15,7 @@ version: '2.1'
services:
device-observation-control:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-observation-control
container_name: device-observation-control
logging:
diff --git a/docker-compose/device-observation.yml b/docker-compose/device-observation.yml
index 9fa593a96..d016f2071 100644
--- a/docker-compose/device-observation.yml
+++ b/docker-compose/device-observation.yml
@@ -14,7 +14,7 @@ version: '2.1'
services:
device-observation:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-observation
container_name: device-observation
logging:
diff --git a/docker-compose/device-pcon.yml b/docker-compose/device-pcon.yml
index e5b9d1d30..e833bb55d 100644
--- a/docker-compose/device-pcon.yml
+++ b/docker-compose/device-pcon.yml
@@ -11,7 +11,7 @@ volumes:
services:
device-pcon:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-pcon
container_name: device-pcon
logging:
diff --git a/docker-compose/device-psoc.yml b/docker-compose/device-psoc.yml
index ab8ee4388..998a0ad2f 100644
--- a/docker-compose/device-psoc.yml
+++ b/docker-compose/device-psoc.yml
@@ -11,7 +11,7 @@ volumes:
services:
device-psoc:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-psoc
container_name: device-psoc
logging:
diff --git a/docker-compose/device-recvh.yml b/docker-compose/device-recvh.yml
index 0f16da4f6..feb0ce875 100644
--- a/docker-compose/device-recvh.yml
+++ b/docker-compose/device-recvh.yml
@@ -8,7 +8,7 @@ version: '2.1'
services:
device-recvh:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-recvh
container_name: device-recvh
logging:
diff --git a/docker-compose/device-recvl.yml b/docker-compose/device-recvl.yml
index f6c905623..810505463 100644
--- a/docker-compose/device-recvl.yml
+++ b/docker-compose/device-recvl.yml
@@ -8,7 +8,7 @@ version: '2.1'
services:
device-recvl:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-recvl
container_name: device-recvl
logging:
diff --git a/docker-compose/device-sdp.yml b/docker-compose/device-sdp.yml
index 52bb3d72c..02ce3f99e 100644
--- a/docker-compose/device-sdp.yml
+++ b/docker-compose/device-sdp.yml
@@ -16,7 +16,7 @@ version: '2.1'
services:
device-sdp:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-sdp
container_name: device-sdp
logging:
diff --git a/docker-compose/device-sdpfirmware.yml b/docker-compose/device-sdpfirmware.yml
index 731c975ab..5fd34bf0f 100644
--- a/docker-compose/device-sdpfirmware.yml
+++ b/docker-compose/device-sdpfirmware.yml
@@ -16,7 +16,7 @@ version: '2.1'
services:
device-sdpfirmware:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-sdpfirmware
container_name: device-sdpfirmware
logging:
diff --git a/docker-compose/device-sst.yml b/docker-compose/device-sst.yml
index b1712b91d..a6ba4f2b2 100644
--- a/docker-compose/device-sst.yml
+++ b/docker-compose/device-sst.yml
@@ -16,7 +16,7 @@ version: '2.1'
services:
device-sst:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-sst
container_name: device-sst
logging:
diff --git a/docker-compose/device-station-manager.yml b/docker-compose/device-station-manager.yml
index f35e7f1e9..044de2d17 100644
--- a/docker-compose/device-station-manager.yml
+++ b/docker-compose/device-station-manager.yml
@@ -11,7 +11,7 @@ volumes:
services:
device-station-manager:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-station-manager
container_name: device-station-manager
logging:
diff --git a/docker-compose/device-temperature-manager.yml b/docker-compose/device-temperature-manager.yml
index 14c8f3bb6..35e2defff 100644
--- a/docker-compose/device-temperature-manager.yml
+++ b/docker-compose/device-temperature-manager.yml
@@ -11,7 +11,7 @@ volumes:
services:
device-temperature-manager:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-temperature-manager
container_name: device-temperature-manager
logging:
diff --git a/docker-compose/device-tilebeam.yml b/docker-compose/device-tilebeam.yml
index ecad1f162..c67cb289c 100644
--- a/docker-compose/device-tilebeam.yml
+++ b/docker-compose/device-tilebeam.yml
@@ -11,7 +11,7 @@ volumes:
services:
device-tilebeam:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-tilebeam
container_name: device-tilebeam
logging:
diff --git a/docker-compose/device-unb2.yml b/docker-compose/device-unb2.yml
index 9377bc0c6..82ef38460 100644
--- a/docker-compose/device-unb2.yml
+++ b/docker-compose/device-unb2.yml
@@ -16,7 +16,7 @@ version: '2.1'
services:
device-unb2:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-unb2
container_name: device-unb2
logging:
diff --git a/docker-compose/device-xst.yml b/docker-compose/device-xst.yml
index 104472ebf..0890e91d7 100644
--- a/docker-compose/device-xst.yml
+++ b/docker-compose/device-xst.yml
@@ -16,7 +16,7 @@ version: '2.1'
services:
device-xst:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-xst
container_name: device-xst
logging:
diff --git a/docker-compose/ec-sim.yml b/docker-compose/ec-sim.yml
index 650e27ffa..bfcd7ad79 100644
--- a/docker-compose/ec-sim.yml
+++ b/docker-compose/ec-sim.yml
@@ -10,6 +10,7 @@ version: '2.1'
services:
ec-sim:
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/ec-sim:${TAG}
build:
context: ec-sim
hostname: ec-sim
diff --git a/docker-compose/grafana.yml b/docker-compose/grafana.yml
index 64b13e0bd..ade1e00cb 100644
--- a/docker-compose/grafana.yml
+++ b/docker-compose/grafana.yml
@@ -15,7 +15,7 @@ version: '2.1'
services:
grafana:
- image: grafana
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/grafana:${TAG}
build:
context: grafana
hostname: grafana
diff --git a/docker-compose/http-json-schemas.yml b/docker-compose/http-json-schemas.yml
index 314b2a47f..46db5c88a 100644
--- a/docker-compose/http-json-schemas.yml
+++ b/docker-compose/http-json-schemas.yml
@@ -12,6 +12,7 @@ version: '2.1'
services:
http-json-schemas:
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/http-json-schemas:${TAG}
build:
context: http-json-schemas
hostname: http-json-schemas
diff --git a/docker-compose/integration-test.yml b/docker-compose/integration-test.yml
index c6465cc75..8b2cc0a33 100644
--- a/docker-compose/integration-test.yml
+++ b/docker-compose/integration-test.yml
@@ -10,6 +10,7 @@ version: '2.1'
services:
integration-test:
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/ci-build-runner:${TAG}
build:
context: .
dockerfile: ci-runner/Dockerfile
@@ -35,7 +36,7 @@ services:
- --
- tox -e integration --
command:
-# Allow for arguments to be passed that wil be put after the entrypoint
-# tox is configured to take these arguments as integration test directory
-# specifications.
+ # Allow for arguments to be passed that wil be put after the entrypoint
+ # tox is configured to take these arguments as integration test directory
+ # specifications.
- ""
diff --git a/docker-compose/itango.yml b/docker-compose/itango.yml
index dde5295ea..0f38388f2 100644
--- a/docker-compose/itango.yml
+++ b/docker-compose/itango.yml
@@ -16,6 +16,7 @@ version: '2.1'
services:
itango:
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/itango:${TAG}
build:
context: itango
args:
diff --git a/docker-compose/itango/Dockerfile b/docker-compose/itango/Dockerfile
index b0d7038e1..b29e962cd 100644
--- a/docker-compose/itango/Dockerfile
+++ b/docker-compose/itango/Dockerfile
@@ -1,7 +1,8 @@
ARG SOURCE_IMAGE
FROM ${SOURCE_IMAGE}
-RUN sudo apt-get -y update && \
+RUN --mount=type=cache,target=/var/cache/apt \
+ sudo apt-get -y update && \
sudo apt-get -y upgrade && \
sudo apt-get -y install apt-file apt-transport-https apt-utils aptitude && \
sudo apt-get -y install htop iftop iproute2 mc most net-tools tcpdump && \
diff --git a/docker-compose/jupyter-lab.yml b/docker-compose/jupyter-lab.yml
index d0ee565e8..4dfdbd852 100644
--- a/docker-compose/jupyter-lab.yml
+++ b/docker-compose/jupyter-lab.yml
@@ -14,7 +14,7 @@ version: '2.1'
services:
jupyter-lab:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/jupyter-lab
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/jupyter-lab:${TAG}
build:
context: jupyter-lab
args:
diff --git a/docker-compose/lofar-device-base.yml b/docker-compose/lofar-device-base.yml
index 777bb167b..9bb7d5768 100644
--- a/docker-compose/lofar-device-base.yml
+++ b/docker-compose/lofar-device-base.yml
@@ -17,7 +17,7 @@ version: '2.1'
services:
lofar-device-base:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
build:
context: .
dockerfile: lofar-device-base/Dockerfile
diff --git a/docker-compose/lofar-device-base/Dockerfile b/docker-compose/lofar-device-base/Dockerfile
index 7b49a6cfc..82df78ca8 100644
--- a/docker-compose/lofar-device-base/Dockerfile
+++ b/docker-compose/lofar-device-base/Dockerfile
@@ -1,13 +1,20 @@
ARG SOURCE_IMAGE
FROM ${SOURCE_IMAGE}
-RUN sudo apt-get update
-RUN sudo apt-get install -y git && sudo apt-get clean
-RUN sudo apt-get install -y g++ gcc && sudo apt-get clean
-RUN sudo apt-get install -y shellcheck graphviz && sudo apt-get clean
-RUN sudo apt-get install -y python3-dev libboost-python-dev pkg-config && sudo apt-get clean
-RUN sudo apt-get install -y rsync && sudo apt-get clean
-RUN sudo apt-get install -y netcat && sudo apt-get clean
+RUN --mount=type=cache,target=/var/cache/apt \
+ sudo apt-get update
+RUN --mount=type=cache,target=/var/cache/apt \
+ sudo apt-get install -y git
+RUN --mount=type=cache,target=/var/cache/apt \
+ sudo apt-get install -y g++ gcc
+RUN --mount=type=cache,target=/var/cache/apt \
+ sudo apt-get install -y shellcheck graphviz
+RUN --mount=type=cache,target=/var/cache/apt \
+ sudo apt-get install -y python3-dev libboost-python-dev pkg-config
+RUN --mount=type=cache,target=/var/cache/apt \
+ sudo apt-get install -y rsync
+RUN --mount=type=cache,target=/var/cache/apt \
+ sudo apt-get install -y netcat
# Manually install all requirements from the .txt as part of the base image
# This reduces runtime overhead as well as preventing issues around dependency
diff --git a/docker-compose/logstash.yml b/docker-compose/logstash.yml
index 9a20d0616..d44e0c52b 100644
--- a/docker-compose/logstash.yml
+++ b/docker-compose/logstash.yml
@@ -9,7 +9,7 @@ version: '2.1'
services:
logstash:
- image: logstash
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/logstash:${TAG}
build:
context: logstash
args:
diff --git a/docker-compose/loki.yml b/docker-compose/loki.yml
index 58a10e5e0..d2165ea14 100644
--- a/docker-compose/loki.yml
+++ b/docker-compose/loki.yml
@@ -9,10 +9,11 @@
version: "2.1"
volumes:
- loki-data: {}
+ loki-data: { }
services:
loki:
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/loki:${TAG}
build:
context: loki
hostname: loki
diff --git a/docker-compose/prometheus.yml b/docker-compose/prometheus.yml
index 4ba6b7029..a1ce03c21 100644
--- a/docker-compose/prometheus.yml
+++ b/docker-compose/prometheus.yml
@@ -14,7 +14,7 @@ volumes:
services:
prometheus:
- image: prometheus
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/prometheus:${TAG}
build:
context: prometheus
hostname: prometheus
diff --git a/sbin/run_integration_test.sh b/sbin/run_integration_test.sh
index 101e79e3d..ce817b134 100755
--- a/sbin/run_integration_test.sh
+++ b/sbin/run_integration_test.sh
@@ -11,8 +11,11 @@ function usage {
stage of the integration test one after the other. Between each stage the
dsconfig is updated accordingly."
echo ""
- echo "./$(basename "$0") -h
+ echo "./$(basename "$0") -h --help
displays this help message"
+ echo ""
+ echo "./$(basename "$0") --no-build
+ disables building of docker images"
}
# Configure the config database, restart containers and run a specific
@@ -39,19 +42,29 @@ function integration_test {
}
# list of arguments expected in the input
-optstring=":h"
+optstring_long="help,no-build"
+optstring="hnb"
+
+options=$(getopt -l ${optstring_long} -o ${optstring} -- "$@")
+
+eval set -- "$options"
-while getopts ${optstring} arg; do
- case ${arg} in
- h)
+while true; do
+ case ${1} in
+ -h|--help)
usage
exit 0
;;
- ?)
- echo "Invalid option: -${OPTARG}."
- exit 2
+ --no-build)
+ echo "Disable docker compose build step"
+ export no_build=1
+ export NO_BASE=${no_build}
;;
+ --)
+ shift
+ break;;
esac
+ shift
done
if [ -z "$LOFAR20_DIR" ]; then
@@ -66,7 +79,7 @@ fi
cd "$LOFAR20_DIR/docker-compose" || exit 1
# Start the database server first
-make build databaseds dsconfig
+[ -n "${no_build}" ] || make build databaseds dsconfig
make start databaseds dsconfig
# Give dsconfig and databaseds time to start
@@ -84,8 +97,8 @@ SIMULATORS=(sdptr-sim recvh-sim recvl-sim unb2-sim apsct-sim apspu-sim ccd-sim e
# take really long to finish, especially grafana / jupyter / prometheus.
# jupyter is physically large > 2.5gb and overlayfs is really slow.
-make build "${SIMULATORS[@]}"
-make build logstash integration-test http-json-schemas
+[ -n "${no_build}" ] || make build "${SIMULATORS[@]}"
+[ -n "${no_build}" ] || make build logstash integration-test http-json-schemas
# Start and stop sequence
make stop http-json-schemas
diff --git a/sbin/tag_and_push_docker_image.sh b/sbin/tag_and_push_docker_image.sh
index 9ee815ee1..613c2fa5f 100755
--- a/sbin/tag_and_push_docker_image.sh
+++ b/sbin/tag_and_push_docker_image.sh
@@ -69,14 +69,17 @@ REMOTE_IMAGES=(
LOCAL_IMAGES=(
"logstash logstash y"
"lofar-device-base lofar-device-base y"
+ "http-json-schemas http-json-schemas y"
- "itango docker-compose_itango y"
+ "ec-sim ec-sim y"
+
+ "itango itango n"
"grafana grafana n" "prometheus prometheus n"
+ "loki loki n"
"jupyter-lab jupyter-lab n"
"integration-test docker-compose_integration-test n"
"tango-prometheus-exporter tango-prometheus-exporter n"
- "tango-prometheus-fast-exporter docker-compose_tango-prometheus-fast-exporter n"
)
# LOCAL_IMAGES verifier, this has been going wrong a couple of times
@@ -151,10 +154,13 @@ if [ -n "${1+x}" ] && [ "${1}" != "pull" ]; then
# If tag is not latest, than it is not a tagged master build and we can
# pull the latest image as cache (if it already exists).
if [ "${tag}" != "latest" ]; then
- docker pull "${local_url}:latest" || true
+ docker pull "${local_url}:${tag}" || docker pull "${local_url}:latest" || true
+ if [ -n "$(docker image inspect \""${local_url}":"${tag}"\")" ]; then
+ docker tag "${local_url}:latest" "${local_url}:${tag}"
+ fi
fi
- make build "${1}" || exit 1
+ TAG="$tag" NO_BASE=1 make build "${1}" || exit 1
docker tag "${2}" "${local_url}:${tag}" || docker tag "${2/_/-}" "${local_url}:${tag}"
docker push "${local_url}:${tag}"
fi
--
GitLab