diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 4ed1753ec570508e03976ab24ed310731b2fcb64..a0e10271b8c080eea1f28a95b64454bf8a907003 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -66,14 +66,6 @@ trigger_prepare:
## Allow docker image script to execute
# - chmod u+x $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh
-# Template for docker images NOT on tagged or master builds
-.base_docker_images_except:
- extends: .base_docker_images
- except:
- refs:
- - tags
- - master
-
# Template to download all remote images and store them on our image registry
# (call tag_and_push without arguments)
.base_docker_store_images:
@@ -103,102 +95,33 @@ docker_store_images_changes:
changes:
- docker-compose/.env
-# Build and push all our custom images on tagged or master builds
-docker_build_image_all:
+# Build and push custom images on merge request if relevant files changed
+docker_build_image:
extends: .base_docker_images
+ parallel:
+ matrix:
+ - IMAGE:
+ - lofar-device-base
+ - ec-sim
+ - http-json-schemas
+ - prometheus
+ - tango-prometheus-exporter
+ - itango
+ - grafana
+ - loki
+ - logstash
+ - jupyter-lab
rules:
+ - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+ changes:
+ - docker-compose/$IMAGE.yml
+ - docker-compose/$IMAGE/*
+ - docker-compose/.env
- if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG
script:
# Do not remove 'bash' or statement will be ignored by primitive docker shell
- - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh lofar-device-base latest
- - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh prometheus latest
- - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh itango latest
- - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh grafana latest
- - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh loki latest
- - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh logstash latest
- - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh jupyter-lab latest
- - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh grafana latest
- - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh prometheus latest
- - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh tango-prometheus-exporter latest
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh $IMAGE $tag
-# Build and push custom images on merge request if relevant files changed
-docker_build_image_lofar_device_base:
- extends: .base_docker_images_except
- only:
- refs:
- - merge_requests
- changes:
- - docker-compose/lofar-device-base.yml
- - docker-compose/lofar-device-base/*
- script:
- # Do not remove 'bash' or statement will be ignored by primitive docker shell
- - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh lofar-device-base $tag
-docker_build_image_prometheus:
- extends: .base_docker_images_except
- only:
- refs:
- - merge_requests
- changes:
- - docker-compose/prometheus.yml
- - docker-compose/prometheus/*
- script:
- # Do not remove 'bash' or statement will be ignored by primitive docker shell
- - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh prometheus $tag
-docker_build_image_itango:
- extends: .base_docker_images_except
- only:
- refs:
- - merge_requests
- changes:
- - docker-compose/itango.yml
- - docker-compose/itango/*
- script:
- # Do not remove 'bash' or statement will be ignored by primitive docker shell
- - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh itango $tag
-docker_build_image_grafana:
- extends: .base_docker_images_except
- only:
- refs:
- - merge_requests
- changes:
- - docker-compose/grafana.yml
- - docker-compose/grafana/*
- script:
- # Do not remove 'bash' or statement will be ignored by primitive docker shell
- - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh grafana $tag
-docker_build_image_loki:
- extends: .base_docker_images_except
- only:
- refs:
- - merge_requests
- changes:
- - docker-compose/loki.yml
- - docker-compose/loki/*
- script:
- # Do not remove 'bash' or statement will be ignored by primitive docker shell
- - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh loki $tag
-docker_build_image_logstash:
- extends: .base_docker_images_except
- only:
- refs:
- - merge_requests
- changes:
- - docker-compose/logstash.yml
- - docker-compose/logstash/*
- script:
- # Do not remove 'bash' or statement will be ignored by primitive docker shell
- - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh logstash $tag
-docker_build_image_jupyter:
- extends: .base_docker_images_except
- only:
- refs:
- - merge_requests
- changes:
- - docker-compose/jupyter-lab.yml
- - docker-compose/jupyterlab/*
- script:
- # Do not remove 'bash' or statement will be ignored by primitive docker shell
- - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh jupyter-lab $tag
newline_at_eof:
stage: linting
before_script:
@@ -335,7 +258,8 @@ integration_test_docker:
# Do not remove 'bash' or statement will be ignored by primitive docker shell
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh pull $tag
# Do not remove 'bash' or statement will be ignored by primitive docker shell
- - bash -e $CI_PROJECT_DIR/sbin/run_integration_test.sh
+ - export TAG="$tag"
+ - bash -e $CI_PROJECT_DIR/sbin/run_integration_test.sh --no-build
after_script:
# Collect output of all containers
- |
diff --git a/docker-compose/Makefile b/docker-compose/Makefile
index 92240df541107275120d87cfff9f40fa2c03110a..910348603a80f224b1c03a42ceef65f3547a8fdb 100644
--- a/docker-compose/Makefile
+++ b/docker-compose/Makefile
@@ -5,6 +5,8 @@
MAKEPATH := $(abspath $(lastword $(MAKEFILE_LIST)))
BASEDIR := $(notdir $(patsubst %/,%,$(dir $(MAKEPATH))))
+TAG ?= latest
+
DOCKER_COMPOSE ?= docker compose
DOCKER_COMPOSE_ENV_FILE := $(abspath .env)
@@ -23,6 +25,9 @@ TIMEOUT ?= 300
SCRATCH ?= /tmp
+# Disables building of the lofar-base-device, primarily used for integration tests in CI/CD
+NO_BASE ?= 0
+
# Host name through which others can reach our control interfaces.
# Needs to be resolvable from the containers and clients.
ifneq (,$(wildcard /run/WSL))
@@ -146,6 +151,7 @@ else
endif
endif
+
DOCKER_COMPOSE_ARGS := DISPLAY=$(DISPLAY) \
XAUTHORITY=$(XAUTHORITY) \
TANGO_HOST=$(TANGO_HOST) \
@@ -159,7 +165,8 @@ DOCKER_COMPOSE_ARGS := DISPLAY=$(DISPLAY) \
COMPOSE_HTTP_TIMEOUT=180 \
CONTAINER_EXECUTION_UID=$(shell id -u) \
DOCKER_GID=$(DOCKER_GID) \
- TEST_MODULE=$(INTEGRATION_MODULE)
+ TEST_MODULE=$(INTEGRATION_MODULE) \
+ TAG=$(TAG)
.PHONY: up base base-nocache down minimal context run integration start stop restart build build-nocache status clean pull help await
@@ -169,7 +176,9 @@ pull: ## pull the images from the Docker hub
$(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) pull --ignore-pull-failures
base: context ## Build base lofar device image
+ifneq ($(NO_BASE),1)
$(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) build --progress=plain lofar-device-base
+endif
base-nocache: context ## Rebuild base lofar device image
$(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) build --no-cache --progress=plain lofar-device-base
diff --git a/docker-compose/device-antennafield.yml b/docker-compose/device-antennafield.yml
index 21461e4ec2943e2c6ff1af7472d9b9e57ba9d37a..705e43b517718da08e682e72e2ebdaed3504310c 100644
--- a/docker-compose/device-antennafield.yml
+++ b/docker-compose/device-antennafield.yml
@@ -17,7 +17,7 @@ version: '2.1'
services:
device-antennafield:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-antennafield
container_name: device-antennafield
logging:
diff --git a/docker-compose/device-aps.yml b/docker-compose/device-aps.yml
index 6094efb52cc2c9c6b957403dd842164c55ad2626..23690af220e129e34730c6bcd62c95d98520e937 100644
--- a/docker-compose/device-aps.yml
+++ b/docker-compose/device-aps.yml
@@ -8,7 +8,7 @@ version: '2.1'
services:
device-aps:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
container_name: device-aps
hostname: device-aps
logging:
diff --git a/docker-compose/device-apsct.yml b/docker-compose/device-apsct.yml
index 921c4cfa5a7d2261e4a192bb810f724992743e0d..86c24b6a69b8462d61fe480fb8c39083c982c130 100644
--- a/docker-compose/device-apsct.yml
+++ b/docker-compose/device-apsct.yml
@@ -16,7 +16,7 @@ version: '2.1'
services:
device-apsct:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-apsct
container_name: device-apsct
logging:
diff --git a/docker-compose/device-apspu.yml b/docker-compose/device-apspu.yml
index 88752625b15f670ee734b22368d11526014c6909..ac585e6c480c957546d55b6c95ff7852e026ca2a 100644
--- a/docker-compose/device-apspu.yml
+++ b/docker-compose/device-apspu.yml
@@ -16,7 +16,7 @@ version: '2.1'
services:
device-apspu:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-apspu
container_name: device-apspu
logging:
diff --git a/docker-compose/device-beamlet.yml b/docker-compose/device-beamlet.yml
index b998eaf7ed0a1fa934757e9fa985203c31e6de06..7c734e0e3ed33c6ec2e25fdeb6df91b3dead4f96 100644
--- a/docker-compose/device-beamlet.yml
+++ b/docker-compose/device-beamlet.yml
@@ -16,7 +16,7 @@ version: '2.1'
services:
device-beamlet:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-beamlet
container_name: device-beamlet
logging:
diff --git a/docker-compose/device-boot.yml b/docker-compose/device-boot.yml
index 77585f8da9817d218e3a31d77902d63299dc3137..0ca0d485483cffe81b4ca12a9abb4fa21e3635f3 100644
--- a/docker-compose/device-boot.yml
+++ b/docker-compose/device-boot.yml
@@ -15,7 +15,7 @@ version: '2.1'
services:
device-boot:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-boot
container_name: device-boot
logging:
diff --git a/docker-compose/device-bst.yml b/docker-compose/device-bst.yml
index e199b3dbad3f7248be0be739b46fc023523f8262..e1fd186004c2d6016751f08778306d66a530eb3f 100644
--- a/docker-compose/device-bst.yml
+++ b/docker-compose/device-bst.yml
@@ -16,7 +16,7 @@ version: '2.1'
services:
device-bst:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-bst
container_name: device-bst
logging:
diff --git a/docker-compose/device-calibration.yml b/docker-compose/device-calibration.yml
index 7c738866057256cc583159a15540bae8aea5b707..387b02286501e90f3087d71abf3b24486ef0913f 100644
--- a/docker-compose/device-calibration.yml
+++ b/docker-compose/device-calibration.yml
@@ -16,7 +16,7 @@ version: '2.1'
services:
device-calibration:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-calibration
container_name: device-calibration
logging:
diff --git a/docker-compose/device-ccd.yml b/docker-compose/device-ccd.yml
index 1621c7257cbe4d94c6a587dd90108c2e12b5eede..950800e33372a84d3d1f5204cb8aaea1bf3b8d99 100644
--- a/docker-compose/device-ccd.yml
+++ b/docker-compose/device-ccd.yml
@@ -16,7 +16,7 @@ version: '2.1'
services:
device-ccd:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-ccd
container_name: device-ccd
logging:
diff --git a/docker-compose/device-configuration.yml b/docker-compose/device-configuration.yml
index d3dbeb273868136986202993936c8854142ccc2d..1a923f15b33585c8f2c397e7ac0afa5855ac464f 100644
--- a/docker-compose/device-configuration.yml
+++ b/docker-compose/device-configuration.yml
@@ -16,7 +16,7 @@ version: '2.1'
services:
device-configuration:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-configuration
container_name: device-configuration
logging:
diff --git a/docker-compose/device-digitalbeam.yml b/docker-compose/device-digitalbeam.yml
index e6ba98a0e5b84e26425b7b92bf8e91067111af52..b093ee63e050f20205b6b2197fbe87f45daf92d6 100644
--- a/docker-compose/device-digitalbeam.yml
+++ b/docker-compose/device-digitalbeam.yml
@@ -16,7 +16,7 @@ version: '2.1'
services:
device-digitalbeam:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-digitalbeam
container_name: device-digitalbeam
logging:
diff --git a/docker-compose/device-docker.yml b/docker-compose/device-docker.yml
index 9ef0917b09deba02b1dd5b80b657694317910660..a9080eeb5263437fe42d22e91dc6c2509f213596 100644
--- a/docker-compose/device-docker.yml
+++ b/docker-compose/device-docker.yml
@@ -16,7 +16,7 @@ version: '2.1'
services:
device-docker:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-docker
container_name: device-docker
logging:
diff --git a/docker-compose/device-ec.yml b/docker-compose/device-ec.yml
index 48e015f2a603b03f35e23b8469e127f5dbf9d970..f944c751acb3ae72cba986a63b4f7e5816d68e04 100644
--- a/docker-compose/device-ec.yml
+++ b/docker-compose/device-ec.yml
@@ -11,7 +11,7 @@ volumes:
services:
device-ec:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-ec
container_name: device-ec
logging:
diff --git a/docker-compose/device-observation-control.yml b/docker-compose/device-observation-control.yml
index baf3b68c75fa473c792d339bf537e20bd23627a8..b5b7f9fe50defba4487766a576e6213e6fb4ac66 100644
--- a/docker-compose/device-observation-control.yml
+++ b/docker-compose/device-observation-control.yml
@@ -15,7 +15,7 @@ version: '2.1'
services:
device-observation-control:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-observation-control
container_name: device-observation-control
logging:
diff --git a/docker-compose/device-observation.yml b/docker-compose/device-observation.yml
index 9fa593a967a1a73413de306f349c53f9aca4558a..d016f2071957c5ba189cd298537e16a19ee6b8a2 100644
--- a/docker-compose/device-observation.yml
+++ b/docker-compose/device-observation.yml
@@ -14,7 +14,7 @@ version: '2.1'
services:
device-observation:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-observation
container_name: device-observation
logging:
diff --git a/docker-compose/device-pcon.yml b/docker-compose/device-pcon.yml
index e5b9d1d3035b8026d155bec20e225642e5116e3a..e833bb55d4e35eb842cf91a42897dcf1ceb6e2aa 100644
--- a/docker-compose/device-pcon.yml
+++ b/docker-compose/device-pcon.yml
@@ -11,7 +11,7 @@ volumes:
services:
device-pcon:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-pcon
container_name: device-pcon
logging:
diff --git a/docker-compose/device-psoc.yml b/docker-compose/device-psoc.yml
index ab8ee4388e98a0c465b8f74ebb08d5f643458bc9..998a0ad2fda23aab5232ebd40632392fad353f08 100644
--- a/docker-compose/device-psoc.yml
+++ b/docker-compose/device-psoc.yml
@@ -11,7 +11,7 @@ volumes:
services:
device-psoc:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-psoc
container_name: device-psoc
logging:
diff --git a/docker-compose/device-recvh.yml b/docker-compose/device-recvh.yml
index 0f16da4f666b3c58012054eab5f5b2c1d80cee73..feb0ce875303bc84aec7c6eec1465d45958e7685 100644
--- a/docker-compose/device-recvh.yml
+++ b/docker-compose/device-recvh.yml
@@ -8,7 +8,7 @@ version: '2.1'
services:
device-recvh:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-recvh
container_name: device-recvh
logging:
diff --git a/docker-compose/device-recvl.yml b/docker-compose/device-recvl.yml
index f6c905623f455510c515021e786b0e9a307c3277..810505463033f32f2afcc4a029a8ecb3f106c0c5 100644
--- a/docker-compose/device-recvl.yml
+++ b/docker-compose/device-recvl.yml
@@ -8,7 +8,7 @@ version: '2.1'
services:
device-recvl:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-recvl
container_name: device-recvl
logging:
diff --git a/docker-compose/device-sdp.yml b/docker-compose/device-sdp.yml
index 52bb3d72c1a340a69878b72fbbb66e76bacef8bc..02ce3f99ebc945f93ec340e7da14a2e318f41608 100644
--- a/docker-compose/device-sdp.yml
+++ b/docker-compose/device-sdp.yml
@@ -16,7 +16,7 @@ version: '2.1'
services:
device-sdp:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-sdp
container_name: device-sdp
logging:
diff --git a/docker-compose/device-sdpfirmware.yml b/docker-compose/device-sdpfirmware.yml
index 731c975abe7f02499c14a2d1daf4e250a36dfd85..5fd34bf0f42bfc3df1ae14e7d50134d1f0faa62b 100644
--- a/docker-compose/device-sdpfirmware.yml
+++ b/docker-compose/device-sdpfirmware.yml
@@ -16,7 +16,7 @@ version: '2.1'
services:
device-sdpfirmware:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-sdpfirmware
container_name: device-sdpfirmware
logging:
diff --git a/docker-compose/device-sst.yml b/docker-compose/device-sst.yml
index b1712b91dfea646aa4d59f7f280176d9d6803cd8..a6ba4f2b2dc7b32f328cd137e1d8e50159dc58d9 100644
--- a/docker-compose/device-sst.yml
+++ b/docker-compose/device-sst.yml
@@ -16,7 +16,7 @@ version: '2.1'
services:
device-sst:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-sst
container_name: device-sst
logging:
diff --git a/docker-compose/device-station-manager.yml b/docker-compose/device-station-manager.yml
index f35e7f1e97307f54044b5d1bc587720f4271c42f..044de2d17f7cf878a8e4449cb3029bae71d09197 100644
--- a/docker-compose/device-station-manager.yml
+++ b/docker-compose/device-station-manager.yml
@@ -11,7 +11,7 @@ volumes:
services:
device-station-manager:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-station-manager
container_name: device-station-manager
logging:
diff --git a/docker-compose/device-temperature-manager.yml b/docker-compose/device-temperature-manager.yml
index 14c8f3bb6c7b7db0c16009629eb153812a46180a..35e2defff2f45a67ddc42f632dc4565536617fe8 100644
--- a/docker-compose/device-temperature-manager.yml
+++ b/docker-compose/device-temperature-manager.yml
@@ -11,7 +11,7 @@ volumes:
services:
device-temperature-manager:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-temperature-manager
container_name: device-temperature-manager
logging:
diff --git a/docker-compose/device-tilebeam.yml b/docker-compose/device-tilebeam.yml
index ecad1f16270c50885bd870b582d0eb847fa472dd..c67cb289cabfec8f73c0cb586e8633ca401b8489 100644
--- a/docker-compose/device-tilebeam.yml
+++ b/docker-compose/device-tilebeam.yml
@@ -11,7 +11,7 @@ volumes:
services:
device-tilebeam:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-tilebeam
container_name: device-tilebeam
logging:
diff --git a/docker-compose/device-unb2.yml b/docker-compose/device-unb2.yml
index 9377bc0c6cc13c07c5e290320b1471bcc425f2a4..82ef3846066fd78efe7eae2c26f15ddbd2d62d31 100644
--- a/docker-compose/device-unb2.yml
+++ b/docker-compose/device-unb2.yml
@@ -16,7 +16,7 @@ version: '2.1'
services:
device-unb2:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-unb2
container_name: device-unb2
logging:
diff --git a/docker-compose/device-xst.yml b/docker-compose/device-xst.yml
index 104472ebff28c9f7fe7fc763df9b30b5bb02d78d..0890e91d748ebf14754a51b5b927a38987716084 100644
--- a/docker-compose/device-xst.yml
+++ b/docker-compose/device-xst.yml
@@ -16,7 +16,7 @@ version: '2.1'
services:
device-xst:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
hostname: device-xst
container_name: device-xst
logging:
diff --git a/docker-compose/ec-sim.yml b/docker-compose/ec-sim.yml
index 650e27ffa32cf445d3eab11be8dab650c156cca6..bfcd7ad79e5a278e7bb23a95b0bb69734435d780 100644
--- a/docker-compose/ec-sim.yml
+++ b/docker-compose/ec-sim.yml
@@ -10,6 +10,7 @@ version: '2.1'
services:
ec-sim:
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/ec-sim:${TAG}
build:
context: ec-sim
hostname: ec-sim
diff --git a/docker-compose/grafana.yml b/docker-compose/grafana.yml
index 64b13e0bd3c752746d46ac9d002350622d92f20b..ade1e00cbda1cf9fc512c45c0b967bb730cac254 100644
--- a/docker-compose/grafana.yml
+++ b/docker-compose/grafana.yml
@@ -15,7 +15,7 @@ version: '2.1'
services:
grafana:
- image: grafana
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/grafana:${TAG}
build:
context: grafana
hostname: grafana
diff --git a/docker-compose/http-json-schemas.yml b/docker-compose/http-json-schemas.yml
index 314b2a47f67b93986fc70ce78ed120c4b4aa9502..46db5c88a5598f7bb8a7c61700dc53c97482f5e4 100644
--- a/docker-compose/http-json-schemas.yml
+++ b/docker-compose/http-json-schemas.yml
@@ -12,6 +12,7 @@ version: '2.1'
services:
http-json-schemas:
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/http-json-schemas:${TAG}
build:
context: http-json-schemas
hostname: http-json-schemas
diff --git a/docker-compose/integration-test.yml b/docker-compose/integration-test.yml
index c6465cc75b0447cb210f9d2065a82ac089c20f42..8b2cc0a33f72865ee65c0cd66d5bc1af036fe8f2 100644
--- a/docker-compose/integration-test.yml
+++ b/docker-compose/integration-test.yml
@@ -10,6 +10,7 @@ version: '2.1'
services:
integration-test:
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/ci-build-runner:${TAG}
build:
context: .
dockerfile: ci-runner/Dockerfile
@@ -35,7 +36,7 @@ services:
- --
- tox -e integration --
command:
-# Allow for arguments to be passed that wil be put after the entrypoint
-# tox is configured to take these arguments as integration test directory
-# specifications.
+ # Allow for arguments to be passed that wil be put after the entrypoint
+ # tox is configured to take these arguments as integration test directory
+ # specifications.
- ""
diff --git a/docker-compose/itango.yml b/docker-compose/itango.yml
index dde5295ea2b458fae1379f2ba7aa90ecb52da5fc..0f38388f25e8652d240001addea62994dd7efcf4 100644
--- a/docker-compose/itango.yml
+++ b/docker-compose/itango.yml
@@ -16,6 +16,7 @@ version: '2.1'
services:
itango:
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/itango:${TAG}
build:
context: itango
args:
diff --git a/docker-compose/itango/Dockerfile b/docker-compose/itango/Dockerfile
index b0d7038e12d8cb59cdf6d52101559a1c40238415..b29e962cd966601680b2c1cc2213f30922f860b8 100644
--- a/docker-compose/itango/Dockerfile
+++ b/docker-compose/itango/Dockerfile
@@ -1,7 +1,8 @@
ARG SOURCE_IMAGE
FROM ${SOURCE_IMAGE}
-RUN sudo apt-get -y update && \
+RUN --mount=type=cache,target=/var/cache/apt \
+ sudo apt-get -y update && \
sudo apt-get -y upgrade && \
sudo apt-get -y install apt-file apt-transport-https apt-utils aptitude && \
sudo apt-get -y install htop iftop iproute2 mc most net-tools tcpdump && \
diff --git a/docker-compose/jupyter-lab.yml b/docker-compose/jupyter-lab.yml
index d0ee565e8b60678a721270c23e21262c7ee869ff..4dfdbd852744acaa8510b640b6a4092cb87376a8 100644
--- a/docker-compose/jupyter-lab.yml
+++ b/docker-compose/jupyter-lab.yml
@@ -14,7 +14,7 @@ version: '2.1'
services:
jupyter-lab:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/jupyter-lab
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/jupyter-lab:${TAG}
build:
context: jupyter-lab
args:
diff --git a/docker-compose/lofar-device-base.yml b/docker-compose/lofar-device-base.yml
index 777bb167bfb69d42ffa8f332d306d9e7af12e766..9bb7d5768ac630cae34d308453f21bbc2b893311 100644
--- a/docker-compose/lofar-device-base.yml
+++ b/docker-compose/lofar-device-base.yml
@@ -17,7 +17,7 @@ version: '2.1'
services:
lofar-device-base:
- image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base:${TAG}
build:
context: .
dockerfile: lofar-device-base/Dockerfile
diff --git a/docker-compose/lofar-device-base/Dockerfile b/docker-compose/lofar-device-base/Dockerfile
index 7b49a6cfcd2db4b786d7f5698267a672c6790a01..82df78ca809cd5fd2061ea7163b9138e611eab52 100644
--- a/docker-compose/lofar-device-base/Dockerfile
+++ b/docker-compose/lofar-device-base/Dockerfile
@@ -1,13 +1,20 @@
ARG SOURCE_IMAGE
FROM ${SOURCE_IMAGE}
-RUN sudo apt-get update
-RUN sudo apt-get install -y git && sudo apt-get clean
-RUN sudo apt-get install -y g++ gcc && sudo apt-get clean
-RUN sudo apt-get install -y shellcheck graphviz && sudo apt-get clean
-RUN sudo apt-get install -y python3-dev libboost-python-dev pkg-config && sudo apt-get clean
-RUN sudo apt-get install -y rsync && sudo apt-get clean
-RUN sudo apt-get install -y netcat && sudo apt-get clean
+RUN --mount=type=cache,target=/var/cache/apt \
+ sudo apt-get update
+RUN --mount=type=cache,target=/var/cache/apt \
+ sudo apt-get install -y git
+RUN --mount=type=cache,target=/var/cache/apt \
+ sudo apt-get install -y g++ gcc
+RUN --mount=type=cache,target=/var/cache/apt \
+ sudo apt-get install -y shellcheck graphviz
+RUN --mount=type=cache,target=/var/cache/apt \
+ sudo apt-get install -y python3-dev libboost-python-dev pkg-config
+RUN --mount=type=cache,target=/var/cache/apt \
+ sudo apt-get install -y rsync
+RUN --mount=type=cache,target=/var/cache/apt \
+ sudo apt-get install -y netcat
# Manually install all requirements from the .txt as part of the base image
# This reduces runtime overhead as well as preventing issues around dependency
diff --git a/docker-compose/logstash.yml b/docker-compose/logstash.yml
index 9a20d061622b1bc78ece7858c8b63193164b4a66..d44e0c52b4f6fca74cee7c9b00ceec6e9355da38 100644
--- a/docker-compose/logstash.yml
+++ b/docker-compose/logstash.yml
@@ -9,7 +9,7 @@ version: '2.1'
services:
logstash:
- image: logstash
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/logstash:${TAG}
build:
context: logstash
args:
diff --git a/docker-compose/loki.yml b/docker-compose/loki.yml
index 58a10e5e07a016a3065ea63380503f68306131e1..d2165ea14ba542a062f35057eaafadd84ba228cb 100644
--- a/docker-compose/loki.yml
+++ b/docker-compose/loki.yml
@@ -9,10 +9,11 @@
version: "2.1"
volumes:
- loki-data: {}
+ loki-data: { }
services:
loki:
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/loki:${TAG}
build:
context: loki
hostname: loki
diff --git a/docker-compose/prometheus.yml b/docker-compose/prometheus.yml
index 4ba6b702991ef89efab3edc489cfc640c5fb22b4..a1ce03c21c31e12d04e1c204ecda118d077db649 100644
--- a/docker-compose/prometheus.yml
+++ b/docker-compose/prometheus.yml
@@ -14,7 +14,7 @@ volumes:
services:
prometheus:
- image: prometheus
+ image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/prometheus:${TAG}
build:
context: prometheus
hostname: prometheus
diff --git a/sbin/run_integration_test.sh b/sbin/run_integration_test.sh
index 101e79e3d4c2728a4e7ede548bb7d8516883d04c..ce817b134f80b71aab5f6e5722d6622a3990c287 100755
--- a/sbin/run_integration_test.sh
+++ b/sbin/run_integration_test.sh
@@ -11,8 +11,11 @@ function usage {
stage of the integration test one after the other. Between each stage the
dsconfig is updated accordingly."
echo ""
- echo "./$(basename "$0") -h
+ echo "./$(basename "$0") -h --help
displays this help message"
+ echo ""
+ echo "./$(basename "$0") --no-build
+ disables building of docker images"
}
# Configure the config database, restart containers and run a specific
@@ -39,19 +42,29 @@ function integration_test {
}
# list of arguments expected in the input
-optstring=":h"
+optstring_long="help,no-build"
+optstring="hnb"
+
+options=$(getopt -l ${optstring_long} -o ${optstring} -- "$@")
+
+eval set -- "$options"
-while getopts ${optstring} arg; do
- case ${arg} in
- h)
+while true; do
+ case ${1} in
+ -h|--help)
usage
exit 0
;;
- ?)
- echo "Invalid option: -${OPTARG}."
- exit 2
+ --no-build)
+ echo "Disable docker compose build step"
+ export no_build=1
+ export NO_BASE=${no_build}
;;
+ --)
+ shift
+ break;;
esac
+ shift
done
if [ -z "$LOFAR20_DIR" ]; then
@@ -66,7 +79,7 @@ fi
cd "$LOFAR20_DIR/docker-compose" || exit 1
# Start the database server first
-make build databaseds dsconfig
+[ -n "${no_build}" ] || make build databaseds dsconfig
make start databaseds dsconfig
# Give dsconfig and databaseds time to start
@@ -84,8 +97,8 @@ SIMULATORS=(sdptr-sim recvh-sim recvl-sim unb2-sim apsct-sim apspu-sim ccd-sim e
# take really long to finish, especially grafana / jupyter / prometheus.
# jupyter is physically large > 2.5gb and overlayfs is really slow.
-make build "${SIMULATORS[@]}"
-make build logstash integration-test http-json-schemas
+[ -n "${no_build}" ] || make build "${SIMULATORS[@]}"
+[ -n "${no_build}" ] || make build logstash integration-test http-json-schemas
# Start and stop sequence
make stop http-json-schemas
diff --git a/sbin/tag_and_push_docker_image.sh b/sbin/tag_and_push_docker_image.sh
index 9ee815ee16653a2ee64b565afe5f3476d85d8f51..613c2fa5fadf7f19a26aedbac3bd31214db14607 100755
--- a/sbin/tag_and_push_docker_image.sh
+++ b/sbin/tag_and_push_docker_image.sh
@@ -69,14 +69,17 @@ REMOTE_IMAGES=(
LOCAL_IMAGES=(
"logstash logstash y"
"lofar-device-base lofar-device-base y"
+ "http-json-schemas http-json-schemas y"
- "itango docker-compose_itango y"
+ "ec-sim ec-sim y"
+
+ "itango itango n"
"grafana grafana n" "prometheus prometheus n"
+ "loki loki n"
"jupyter-lab jupyter-lab n"
"integration-test docker-compose_integration-test n"
"tango-prometheus-exporter tango-prometheus-exporter n"
- "tango-prometheus-fast-exporter docker-compose_tango-prometheus-fast-exporter n"
)
# LOCAL_IMAGES verifier, this has been going wrong a couple of times
@@ -151,10 +154,13 @@ if [ -n "${1+x}" ] && [ "${1}" != "pull" ]; then
# If tag is not latest, than it is not a tagged master build and we can
# pull the latest image as cache (if it already exists).
if [ "${tag}" != "latest" ]; then
- docker pull "${local_url}:latest" || true
+ docker pull "${local_url}:${tag}" || docker pull "${local_url}:latest" || true
+ if [ -n "$(docker image inspect \""${local_url}":"${tag}"\")" ]; then
+ docker tag "${local_url}:latest" "${local_url}:${tag}"
+ fi
fi
- make build "${1}" || exit 1
+ TAG="$tag" NO_BASE=1 make build "${1}" || exit 1
docker tag "${2}" "${local_url}:${tag}" || docker tag "${2/_/-}" "${local_url}:${tag}"
docker push "${local_url}:${tag}"
fi