From 2585211ce10adda398a4bf4eb9ccd3ed68f89e4e Mon Sep 17 00:00:00 2001
From: lukken <lukken@astron.nl>
Date: Wed, 24 Aug 2022 11:52:50 +0000
Subject: [PATCH] L2SS-897: Reduce docker context by copying files through Make
---
.gitignore | 1 +
bin/start-ds.sh | 7 +++++--
docker-compose/Makefile | 9 +++++++--
docker-compose/device-antennafield.yml | 4 ++--
docker-compose/device-apsct.yml | 4 ++--
docker-compose/device-apspu.yml | 4 ++--
docker-compose/device-beamlet.yml | 4 ++--
docker-compose/device-boot.yml | 4 ++--
docker-compose/device-bst.yml | 4 ++--
docker-compose/device-digitalbeam.yml | 4 ++--
docker-compose/device-docker.yml | 4 ++--
docker-compose/device-observation-control.yml | 4 ++--
docker-compose/device-observation.yml | 4 ++--
docker-compose/device-pcon.yml | 4 ++--
docker-compose/device-psoc.yml | 4 ++--
docker-compose/device-recv.yml | 4 ++--
docker-compose/device-sdp.yml | 4 ++--
docker-compose/device-sst.yml | 4 ++--
docker-compose/device-temperature-manager.yml | 4 ++--
docker-compose/device-tilebeam.yml | 4 ++--
docker-compose/device-unb2.yml | 4 ++--
docker-compose/device-xst.yml | 4 ++--
docker-compose/integration-test.yml | 4 ++--
docker-compose/lofar-device-base.yml | 4 ++--
docker-compose/lofar-device-base/Dockerfile | 11 +++++++----
25 files changed, 62 insertions(+), 50 deletions(-)
diff --git a/.gitignore b/.gitignore
index cfd4dc461..f77736405 100644
--- a/.gitignore
+++ b/.gitignore
@@ -28,3 +28,4 @@ tangostationcontrol/docs/build
**/.eggs
docker-compose/alerta-web/alerta-secrets.json
+docker-compose/tmp
diff --git a/bin/start-ds.sh b/bin/start-ds.sh
index 8a5baba50..2753e299a 100755
--- a/bin/start-ds.sh
+++ b/bin/start-ds.sh
@@ -31,8 +31,11 @@ if [[ $TANGOSTATIONCONTROL ]]; then
exit 2
else
# Install the package, exit 1 if it fails
- cd tangostationcontrol || exit 1
- pip install --upgrade --force-reinstall ./
+ # pip install ./ will _NOT_ install dependencies in requirements.txt!
+ rm -rf /tmp/tangostationcontrol
+ cp -R /opt/lofar/tango/tangostationcontrol /tmp/
+ cd /tmp/tangostationcontrol || exit 1
+ pip -vvv install --upgrade --force-reinstall ./
fi
# Return to the stored the directory, this preserves the working_dir argument in
diff --git a/docker-compose/Makefile b/docker-compose/Makefile
index e0689cc7c..a69733e84 100644
--- a/docker-compose/Makefile
+++ b/docker-compose/Makefile
@@ -153,7 +153,7 @@ DOCKER_COMPOSE_ARGS := DISPLAY=$(DISPLAY) \
DOCKER_GID=$(DOCKER_GID)
-.PHONY: up down minimal run integration start stop restart build build-nocache status clean pull help
+.PHONY: up down minimal context run integration start stop restart build build-nocache status clean pull help
.DEFAULT_GOAL := help
pull: ## pull the images from the Docker hub
@@ -183,13 +183,18 @@ ifneq ($(NETWORK_MODE),host)
docker network inspect 9000-$(NETWORK_MODE) &> /dev/null && ([ $$? -eq 0 ] && docker network rm 9000-$(NETWORK_MODE)) || true
endif
-minimal: ## start the base TANGO system
+minimal: context ## start the base TANGO system
ifneq ($(NETWORK_MODE),host)
docker network inspect $(NETWORK_MODE) &> /dev/null || ([ $$? -ne 0 ] && docker network create $(NETWORK_MODE))
docker network inspect 9000-$(NETWORK_MODE) &> /dev/null || ([ $$? -ne 0 ] && docker network create 9000-$(NETWORK_MODE) -o com.docker.network.driver.mtu=9000)
endif
+
$(DOCKER_COMPOSE_ARGS) docker-compose -f tango.yml -f networks.yml up --no-recreate -d
+context: ## Move the necessary files to create minimal docker context
+ @mkdir -p tmp
+ @cp ../tangostationcontrol/requirements.txt tmp/
+
bootstrap: pull build # first start, initialise from scratch
$(MAKE) start elk-configure-host # configure host kernel for elk container
$(MAKE) start dsconfig # boot up containers to load configurations
diff --git a/docker-compose/device-antennafield.yml b/docker-compose/device-antennafield.yml
index d33dacac0..0e16043f1 100644
--- a/docker-compose/device-antennafield.yml
+++ b/docker-compose/device-antennafield.yml
@@ -19,8 +19,8 @@ services:
# build explicitly, as docker-compose does not understand a local image
# being shared among services.
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-antennafield
diff --git a/docker-compose/device-apsct.yml b/docker-compose/device-apsct.yml
index cb43adf7b..8addefff2 100644
--- a/docker-compose/device-apsct.yml
+++ b/docker-compose/device-apsct.yml
@@ -18,8 +18,8 @@ services:
# build explicitly, as docker-compose does not understand a local image
# being shared among services.
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-apsct
diff --git a/docker-compose/device-apspu.yml b/docker-compose/device-apspu.yml
index 6613b2109..55a2d5a9a 100644
--- a/docker-compose/device-apspu.yml
+++ b/docker-compose/device-apspu.yml
@@ -18,8 +18,8 @@ services:
# build explicitly, as docker-compose does not understand a local image
# being shared among services.
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-apspu
diff --git a/docker-compose/device-beamlet.yml b/docker-compose/device-beamlet.yml
index 6d067c154..4e80eba8f 100644
--- a/docker-compose/device-beamlet.yml
+++ b/docker-compose/device-beamlet.yml
@@ -18,8 +18,8 @@ services:
# build explicitly, as docker-compose does not understand a local image
# being shared among services.
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-beamlet
diff --git a/docker-compose/device-boot.yml b/docker-compose/device-boot.yml
index cbeb91653..4d4108d9f 100644
--- a/docker-compose/device-boot.yml
+++ b/docker-compose/device-boot.yml
@@ -17,8 +17,8 @@ services:
# build explicitly, as docker-compose does not understand a local image
# being shared among services.
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-boot
diff --git a/docker-compose/device-bst.yml b/docker-compose/device-bst.yml
index c06514b4f..92522527c 100644
--- a/docker-compose/device-bst.yml
+++ b/docker-compose/device-bst.yml
@@ -18,8 +18,8 @@ services:
# build explicitly, as docker-compose does not understand a local image
# being shared among services.
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-bst
diff --git a/docker-compose/device-digitalbeam.yml b/docker-compose/device-digitalbeam.yml
index 32847ca8d..c5b45e9ba 100644
--- a/docker-compose/device-digitalbeam.yml
+++ b/docker-compose/device-digitalbeam.yml
@@ -18,8 +18,8 @@ services:
# build explicitly, as docker-compose does not understand a local image
# being shared among services.
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-digitalbeam
diff --git a/docker-compose/device-docker.yml b/docker-compose/device-docker.yml
index 2be9467ea..db33c9aed 100644
--- a/docker-compose/device-docker.yml
+++ b/docker-compose/device-docker.yml
@@ -18,8 +18,8 @@ services:
# build explicitly, as docker-compose does not understand a local image
# being shared among services.
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-docker
diff --git a/docker-compose/device-observation-control.yml b/docker-compose/device-observation-control.yml
index 197d192a5..7025b8b3a 100644
--- a/docker-compose/device-observation-control.yml
+++ b/docker-compose/device-observation-control.yml
@@ -17,8 +17,8 @@ services:
# build explicitly, as docker-compose does not understand a local image
# being shared among services.
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-observation-control
diff --git a/docker-compose/device-observation.yml b/docker-compose/device-observation.yml
index ee8f3a165..3379e41e1 100644
--- a/docker-compose/device-observation.yml
+++ b/docker-compose/device-observation.yml
@@ -16,8 +16,8 @@ services:
# build explicitly, as docker-compose does not understand a local image
# being shared among services.
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-observation
diff --git a/docker-compose/device-pcon.yml b/docker-compose/device-pcon.yml
index aa20c1498..d90b9e1f7 100644
--- a/docker-compose/device-pcon.yml
+++ b/docker-compose/device-pcon.yml
@@ -13,8 +13,8 @@ services:
# build explicitly, as docker-compose does not understand a local image
# being shared among services.
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-pcon
diff --git a/docker-compose/device-psoc.yml b/docker-compose/device-psoc.yml
index b7b9738b0..bfa2bb7c4 100644
--- a/docker-compose/device-psoc.yml
+++ b/docker-compose/device-psoc.yml
@@ -13,8 +13,8 @@ services:
# build explicitly, as docker-compose does not understand a local image
# being shared among services.
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-psoc
diff --git a/docker-compose/device-recv.yml b/docker-compose/device-recv.yml
index 10126cae3..3c79a0a14 100644
--- a/docker-compose/device-recv.yml
+++ b/docker-compose/device-recv.yml
@@ -18,8 +18,8 @@ services:
# build explicitly, as docker-compose does not understand a local image
# being shared among services.
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-recv
diff --git a/docker-compose/device-sdp.yml b/docker-compose/device-sdp.yml
index 2e9c77312..144630c88 100644
--- a/docker-compose/device-sdp.yml
+++ b/docker-compose/device-sdp.yml
@@ -18,8 +18,8 @@ services:
# build explicitly, as docker-compose does not understand a local image
# being shared among services.
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-sdp
diff --git a/docker-compose/device-sst.yml b/docker-compose/device-sst.yml
index 54c221f3b..e6b0edb75 100644
--- a/docker-compose/device-sst.yml
+++ b/docker-compose/device-sst.yml
@@ -18,8 +18,8 @@ services:
# build explicitly, as docker-compose does not understand a local image
# being shared among services.
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-sst
diff --git a/docker-compose/device-temperature-manager.yml b/docker-compose/device-temperature-manager.yml
index 4729bd631..d1b20359b 100644
--- a/docker-compose/device-temperature-manager.yml
+++ b/docker-compose/device-temperature-manager.yml
@@ -13,8 +13,8 @@ services:
# build explicitly, as docker-compose does not understand a local image
# being shared among services.
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-temperature-manager
diff --git a/docker-compose/device-tilebeam.yml b/docker-compose/device-tilebeam.yml
index 20e44c90c..7919beded 100644
--- a/docker-compose/device-tilebeam.yml
+++ b/docker-compose/device-tilebeam.yml
@@ -13,8 +13,8 @@ services:
# build explicitly, as docker-compose does not understand a local image
# being shared among services.
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-tilebeam
diff --git a/docker-compose/device-unb2.yml b/docker-compose/device-unb2.yml
index 9a7505f0b..2c05d6e66 100644
--- a/docker-compose/device-unb2.yml
+++ b/docker-compose/device-unb2.yml
@@ -18,8 +18,8 @@ services:
# build explicitly, as docker-compose does not understand a local image
# being shared among services.
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-unb2
diff --git a/docker-compose/device-xst.yml b/docker-compose/device-xst.yml
index 1ed7a1fe3..6f49e17f6 100644
--- a/docker-compose/device-xst.yml
+++ b/docker-compose/device-xst.yml
@@ -18,8 +18,8 @@ services:
# build explicitly, as docker-compose does not understand a local image
# being shared among services.
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}device-xst
diff --git a/docker-compose/integration-test.yml b/docker-compose/integration-test.yml
index 346bbbcc5..6e7e5407d 100644
--- a/docker-compose/integration-test.yml
+++ b/docker-compose/integration-test.yml
@@ -9,8 +9,8 @@ version: '2.1'
services:
integration-test:
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}integration-test
diff --git a/docker-compose/lofar-device-base.yml b/docker-compose/lofar-device-base.yml
index 34f3f1c92..8f3169643 100644
--- a/docker-compose/lofar-device-base.yml
+++ b/docker-compose/lofar-device-base.yml
@@ -17,8 +17,8 @@ services:
lofar-device-base:
image: lofar-device-base
build:
- context: ..
- dockerfile: docker-compose/lofar-device-base/Dockerfile
+ context: .
+ dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}lofar-device-base
diff --git a/docker-compose/lofar-device-base/Dockerfile b/docker-compose/lofar-device-base/Dockerfile
index 39b9d652f..becc95d08 100644
--- a/docker-compose/lofar-device-base/Dockerfile
+++ b/docker-compose/lofar-device-base/Dockerfile
@@ -4,20 +4,23 @@ FROM ${SOURCE_IMAGE}
RUN sudo apt-get update && sudo apt-get install -y git g++ gcc shellcheck graphviz python3-dev && sudo apt-get clean
-COPY docker-compose/lofar-device-base/lofar-requirements.txt /lofar-requirements.txt
+COPY lofar-device-base/lofar-requirements.txt /lofar-requirements.txt
RUN sudo pip3 install -r /lofar-requirements.txt
-COPY tangostationcontrol/requirements.txt /tangostationcontrol-requirements.txt
+# Manually install all requirements from the .txt as part of the base image
+# This reduces runtime overhead as well as preventing issues around dependency
+# installation for development builds (pip install ./ ignores requirements.txt)
+COPY tmp/requirements.txt /tangostationcontrol-requirements.txt
RUN sudo pip3 install -r /tangostationcontrol-requirements.txt
# install and use ephimerides and geodetic ("measures") tables for casacore.
# we install a _stub_ since the tables need to be deployed explicitly from within the software.
RUN sudo mkdir -p /opt/IERS && sudo chmod a+rwx /opt/IERS
ARG IERS_DIRNAME=IERS-1970-01-01T00:00:00-stub
-COPY docker-compose/lofar-device-base/WSRT_Measures_stub /opt/IERS/${IERS_DIRNAME}
+COPY lofar-device-base/WSRT_Measures_stub /opt/IERS/${IERS_DIRNAME}
RUN ln -sfT /opt/IERS/${IERS_DIRNAME} /opt/IERS/current
-COPY docker-compose/lofar-device-base/casarc /home/tango/.casarc
+COPY lofar-device-base/casarc /home/tango/.casarc
ENV TANGO_LOG_PATH=/var/log/tango
RUN sudo mkdir -p /var/log/tango && sudo chmod a+rwx /var/log/tango
--
GitLab