Merge branch 'L2SS-205-docker-images' into 'master'

Docker image building & pushing with CI pipeline caching

Closes L2SS-205

See merge request !183

docker-compose/grafana.yml

@@ -4,6 +4,7 @@
 # Defines:
 #   - grafana: Grafana
 #
+
 version: '2'
 
 #volumes:

docker-compose/integration-test.yml

@@ -11,7 +11,7 @@ services:
     build:
         context: itango
         args:
-            SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
+            SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}integration-test
     networks:
       - control

docker-compose/itango.yml

@@ -17,7 +17,7 @@ services:
     build:
         context: itango
         args:
-            SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
+            SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}itango
     networks:
       - control

docker-compose/jive.yml

@@ -18,7 +18,7 @@ version: '2'
 
 services:
   jive:
-    image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-java:${TANGO_JAVA_VERSION}
+    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-java:${TANGO_JAVA_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}jive
     network_mode: host
     volumes:

docker-compose/jupyter.yml

@@ -7,6 +7,7 @@
 # Defines:
 #   - jupyter: Jupyter Notebook with iTango support
 #
+
 version: '2'
 
 services:
@@ -15,7 +16,7 @@ services:
         context: jupyter
         args:
             CONTAINER_EXECUTION_UID: ${CONTAINER_EXECUTION_UID}
-            SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
+            SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}jupyter
     networks:
       - control

docker-compose/lofar-device-base.yml

@@ -10,6 +10,7 @@
 # Requires:
 #   - tango.yml
 #
+
 version: '2'
 
 services:
@@ -18,7 +19,7 @@ services:
     build:
         context: lofar-device-base
         args:
-            SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
+            SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}lofar-device-base
     # These parameters are just visual queues, you have to define them again
     # in derived docker-compose files!

docker-compose/lofar-device-base/lofar-requirements.txt

-# Do not put tangostationcontrol dependencies here
-astropy
-
-# requirements to build tangocontrol 
+# Do not put tangostationcontrol dependencies here, only setup.py / __init__.py
 GitPython >= 3.1.24 # BSD

docker-compose/logviewer.yml

@@ -12,7 +12,7 @@ version: '2'
 
 services:
   logviewer:
-    image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-java:${TANGO_JAVA_VERSION}
+    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-java:${TANGO_JAVA_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}logviewer
     networks:
       - control

docker-compose/pogo.yml

@@ -20,7 +20,7 @@ volumes:
 
 services:
   pogo:
-    image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-pogo:${TANGO_POGO_VERSION}
+    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-pogo:${TANGO_POGO_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}pogo
     networks:
       - control

docker-compose/prometheus.yml

@@ -4,6 +4,7 @@
 # Defines:
 #   - prometheus: Prometheus
 #
+
 version: '2'
 
 services:

docker-compose/pypcc-sim-base/Dockerfile

-FROM ubuntu:20.04
+ARG LOCAL_DOCKER_REGISTRY_HOST
+ARG LOCAL_DOCKER_REGISTRY_LOFAR
 
-COPY requirements.txt /requirements.txt
+FROM ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_LOFAR}/pypcc:latest
 
-RUN apt-get update && apt-get install -y python3 python3-pip python3-yaml git && \
-    pip3 install -r requirements.txt && \
-    git clone --depth 1 --branch master https://git.astron.nl/lofar2.0/pypcc
-
-WORKDIR /pypcc
-CMD ["python3","pypcc2.py","--simulator","--port","4843"]
+CMD ["python3", "pypcc2.py", "--simulator", "--port","4843"]

docker-compose/pypcc-sim-base/requirements.txt

-opcua
-numpy
-recordclass>=0.16,<0.16.1
\ No newline at end of file

docker-compose/recv-sim.yml

@@ -10,6 +10,9 @@ services:
   recv-sim:
     build:
         context: pypcc-sim-base
+        args:
+         - LOCAL_DOCKER_REGISTRY_HOST=${LOCAL_DOCKER_REGISTRY_HOST}
+         - LOCAL_DOCKER_REGISTRY_LOFAR=${LOCAL_DOCKER_REGISTRY_LOFAR}
     container_name: ${CONTAINER_NAME_PREFIX}recv-sim
     networks:
       - control

docker-compose/rest.yml

@@ -13,7 +13,7 @@ version: '2'
 
 services:
   rest:
-    image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-rest:${TANGO_REST_VERSION}
+    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-rest:${TANGO_REST_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}tango-rest
     networks:
       - control

docker-compose/sdptr-sim.yml

@@ -10,6 +10,9 @@ services:
   sdptr-sim:
     build:
         context: sdptr-sim
+        args:
+         - LOCAL_DOCKER_REGISTRY_HOST=${LOCAL_DOCKER_REGISTRY_HOST}
+         - LOCAL_DOCKER_REGISTRY_LOFAR=${LOCAL_DOCKER_REGISTRY_LOFAR}
     container_name: ${CONTAINER_NAME_PREFIX}sdptr-sim
     networks:
       - control

docker-compose/sdptr-sim/Dockerfile

-FROM ubuntu:20.04
+ARG LOCAL_DOCKER_REGISTRY_HOST
+ARG LOCAL_DOCKER_REGISTRY_LOFAR
 
-# Install build tools for sdptr and the C language OPC-UA lib
-RUN apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common && \
-    DEBIAN_FRONTEND=noninteractive add-apt-repository ppa:open62541-team/ppa && \
-    apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y autoconf automake git make g++ build-essential pkg-config libboost-dev libboost-regex-dev libboost-system-dev libboost-program-options-dev libopen62541-1-dev libopen62541-1-tools && \
-    apt-get clean
-
-# Install SDPTR
-RUN cd / && git clone --depth 1 --branch master https://git.astron.nl/lofar2.0/sdptr
-
-RUN cd /sdptr && \
-    autoreconf -v -f -i && \
-    ./configure && \
-    bash -c "make -j `nproc` install"
+FROM ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_LOFAR}/sdptr:latest
 
 COPY simulator.conf /sdptr/src/simulator.conf
 

docker-compose/tango.yml

@@ -15,7 +15,7 @@ volumes:
 
 services:
   tangodb:
-    image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-db:${TANGO_DB_VERSION}
+    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-db:${TANGO_DB_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}tangodb
     networks:
       - control
@@ -37,7 +37,7 @@ services:
     restart: unless-stopped
 
   databaseds:
-    image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-cpp:${TANGO_CPP_VERSION}
+    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-cpp:${TANGO_CPP_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}databaseds
     networks:
       - control

docker-compose/tangotest.yml

@@ -11,7 +11,7 @@ version: '2'
 
 services:
   tangotest:
-    image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-java:${TANGO_JAVA_VERSION}
+    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-java:${TANGO_JAVA_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}tangotest
     networks:
       - control

docker-compose/unb2-sim.yml

@@ -10,6 +10,9 @@ services:
   unb2-sim:
     build:
         context: pypcc-sim-base
+        args:
+         - LOCAL_DOCKER_REGISTRY_HOST=${LOCAL_DOCKER_REGISTRY_HOST}
+         - LOCAL_DOCKER_REGISTRY_LOFAR=${LOCAL_DOCKER_REGISTRY_LOFAR}
     container_name: ${CONTAINER_NAME_PREFIX}unb2-sim
     networks:
       - control

sbin/run_integration_test.sh

@@ -11,8 +11,12 @@ fi
 
 cd "$LOFAR20_DIR/docker-compose" || exit 1
 
-# Make sure builds are recent, and use our building parameters.
-make build
+# Build only the required images, please do not build everything that makes CI
+# take really long to finish, especially grafana / jupyter / prometheus.
+# jupyter is physically large > 2.5gb and overlayfs is really slow.
+make build device-sdp device-recv device-sst device-unb2 device-xst
+make build sdptr-sim recv-sim unb2-sim apsct-sim apspu-sim
+make build databaseds dsconfig elk integration-test
 
 # Start and stop sequence
 make stop device-boot device-docker device-apsct device-apspu device-sdp device-recv device-sst device-unb2 device-xst device-beam sdptr-sim recv-sim unb2-sim apsct-sim apspu-sim