Merge branch 'L2SS-822-cache-archiver-ts-docker-image' into 'master'

Resolve L2SS-822 "Cache archiver ts docker image"

Closes L2SS-822

See merge request !417

.gitlab-ci.yml

@@ -125,6 +125,7 @@ docker_build_image_all:
     - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-xst latest
     - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-temperature-manager latest
     - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh archiver-timescale latest
+    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh hdbpp latest
     - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh hdbppts-cm latest
     - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh hdbppts-es latest
 
@@ -494,6 +495,17 @@ docker_build_image_archiver_timescale:
   script:
 #    Do not remove 'bash' or statement will be ignored by primitive docker shell
     - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh archiver-timescale $tag
+docker_build_image_hdbpp:
+  extends: .base_docker_images_except
+  only:
+    refs:
+      - merge_requests
+    changes:
+      - docker-compose/archiver-timescale.yml
+      - docker-compose/hdbpp/*
+  script:
+#    Do not remove 'bash' or statement will be ignored by primitive docker shell
+    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh hdbpp $tag
 docker_build_image_hdbppts_cm:
   extends: .base_docker_images_except
   only:
@@ -501,7 +513,7 @@ docker_build_image_hdbppts_cm:
       - merge_requests
     changes:
       - docker-compose/archiver-timescale.yml
-      - docker-compose/tango-archiver-ts/*
+      - docker-compose/hdbppts-cm/*
   script:
 #    Do not remove 'bash' or statement will be ignored by primitive docker shell
     - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh hdbppts-cm $tag
@@ -512,7 +524,7 @@ docker_build_image_hdbppts_es:
       - merge_requests
     changes:
       - docker-compose/archiver-timescale.yml
-      - docker-compose/tango-archiver-ts/*
+      - docker-compose/hdbppts-es/*
   script:
 #    Do not remove 'bash' or statement will be ignored by primitive docker shell
     - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh hdbppts-es $tag

docker-compose/.env

@@ -15,6 +15,7 @@ TANGO_POGO_VERSION=9.6.34
 TANGO_REST_VERSION=1.14.6
 TANGO_STARTER_VERSION=2021-05-28
 
+PG_TIMESCALEDB_VERSION=latest-pg12
 PG_SUPERUSER_PASSWORD=password
 PG_HDB_PASSWORD=hdbpp
 MYSQL_ROOT_PASSWORD=secret

docker-compose/README.md

@@ -39,6 +39,23 @@ registry and uploaded to our own using matching tags.
 
 Services, same mechanism as devices.
 
+### HDB++ image updates
+
+The hdbpp Docker image is used as a base image for the hdbppts-cm (ConfigurationManager) and hdbppts-es (EventSubscriber)
+images. If one is developing on a branch and any updates is made in hdbpp/Dockerfile, those won't be automatically picked up from hdbppts-cm and hdbppts-es, because the argument SOURCE_IMAGE in the docker-compose yml file always refers to the remote hdbpp image in the repository. 
+
+A temporary workaround for locally testing on these archiving containers is the following:
+
+- Stop and remove any running hdbpp* container
+- In the archiver-timescale.yml file, replace the hdbppts-cm and hdbppts-es SOURCE_IMAGE tag 'latest' with the branch name 
+(e.g. SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/hdbpp:l2ss-new-branch)
+- Rebuild all the hdbpp* container (make build hdbpp hdbppts-cm hdbppts-es), and then start them
+- Test the new features
+
+After the branch has been correctly developed, tested, the merge has been approved, and the new images have been built on the repository:
+- Put back 'latest' tag on the archiver-timescale.yml file, replacing the branch name
+- Merge on master 
+
 ### Setup and maintenance
 
 All behavioral logic to orchestrate change detection and image pushing can be

docker-compose/archiver-timescale.yml

@@ -8,6 +8,8 @@ services:
     image: timescaledb
     build:
         context: timescaledb
+        args: 
+            SOURCE_IMAGE: timescale/timescaledb:${PG_TIMESCALEDB_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}archiver-timescale
     networks:
       - control
@@ -31,10 +33,30 @@ services:
         tag: "{{.Name}}"
     restart: unless-stopped
   
+  hdbpp:
+    image: hdbpp
+    build:
+        context: hdbpp
+        args:
+            SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-cpp:${TANGO_CPP_VERSION}
+    container_name: ${CONTAINER_NAME_PREFIX}hdbpp
+    networks:
+      - control
+    depends_on:
+      - databaseds
+      - dsconfig
+      - archiver-timescale
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    environment:
+      - TANGO_HOST=${TANGO_HOST}
+  
   hdbppts-cm:
     image: hdbppts-cm
     build:
-        context: tango-archiver-ts
+        context: hdbppts-cm
+        args:
+            SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/hdbpp:latest
     container_name: ${CONTAINER_NAME_PREFIX}hdbppts-cm
     networks:
       - control    
@@ -62,11 +84,14 @@ services:
   hdbppts-es:
     image: hdbppts-es
     build:
-        context: tango-archiver-ts
+        context: hdbppts-es
+        args:
+            SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/hdbpp:latest
     container_name: ${CONTAINER_NAME_PREFIX}hdbppts-es
     networks:
       - control    
     depends_on:
+      - hdbppts-cm
       - databaseds
       - dsconfig
       - archiver-timescale

docker-compose/tango-archiver-ts/Dockerfile → docker-compose/hdbpp/Dockerfile

-#ARG DOCKER_REGISTRY_USER
-#ARG DOCKER_REGISTRY_HOST
-#FROM ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}/tango-cpp:latest
-FROM artefact.skao.int/ska-tango-images-tango-cpp:9.3.9
+ARG SOURCE_IMAGE
+FROM ${SOURCE_IMAGE}
 
 USER root
 
@@ -95,70 +93,6 @@ RUN cd libhdbpp-timescale/build \
     --requires="libpq5" \
     make install  
 
-
-# ----------- HDB++ CONFIGURATION MANAGER ---------------
-
-RUN git clone -b master https://github.com/tango-controls-hdbpp/hdbpp-cm.git
-
-RUN cd hdbpp-cm \
- && mkdir -p build \
- && cd build \
- && cmake .. -DCMAKE_PREFIX_PATH=/usr/local/include/tango \
- && make -j4
-
-RUN cd hdbpp-cm/build \
- && checkinstall \
-    --install=yes \
-    --fstrans=no \
-    --showinstall=no \
-    --backup=no \
-    --type=debian \
-    --pkgsource="https://github.com/tango-controls-hdbpp/hdbpp-cm" \
-    --pkglicense="GPLv3" \
-    --deldesc=no \
-    --nodoc \
-    --strip \
-    --stripso \
-    --maintainer="tango" \
-    --pkgarch=$(dpkg --print-architecture) \
-    --pkgversion="2.0.0" \
-    --pkgrelease="SNAPSHOT" \
-    --pkgname="hdbpp-cm" \
-    --requires="libzmq5,libomniorb4-2,libcos4-2,libomnithread4" \
-    make install
-
-
-# ----------- HDB++ EVENT SUBSCRIBER ---------------
-
-RUN git clone -b master https://github.com/tango-controls-hdbpp/hdbpp-es.git
-
-RUN cd hdbpp-es \
- && mkdir -p build \
- && cd build \
- && cmake .. -DCMAKE_PREFIX_PATH=/usr/local/include/tango -DFETCH_LIBHDBPP=OFF -DLIBHDBPP_BACKEND=timescale -DPostgreSQL_TYPE_INCLUDE_DIR=/usr/local/include/postgresql \
- && make -j4 
- 
-RUN cd hdbpp-es/build \
- && checkinstall \
-    --install=yes \
-    --fstrans=no \
-    --showinstall=no \
-    --backup=no \
-    --type=debian \
-    --pkgsource="https://github.com/tango-controls-hdbpp/hdbpp-es" \
-    --pkglicense="GPLv3" \
-    --deldesc=no \
-    --nodoc \
-    --strip \
-    --stripso \
-    --maintainer="tango" \
-    --pkgarch=$(dpkg --print-architecture) \
-    --pkgversion="2.0.0" \
-    --pkgrelease="SNAPSHOT" \
-    --pkgname="hdbpp-es" \
-    --requires="libzmq5,libomniorb4-2,libcos4-2,libomnithread4" \
-    make install
-
 RUN apt-get update && \
     apt-get install -y \
     build-essential && \
@@ -166,10 +100,5 @@ RUN apt-get update && \
 
 RUN dpkg -i /libhdbpp/build/libhdbpp_2.0.0-SNAPSHOT_amd64.deb
 RUN dpkg -i /libhdbpp-timescale/build/libhdbpp-timescale_2.0.0-SNAPSHOT_amd64.deb
-RUN dpkg -i /hdbpp-cm/build/hdbpp-cm_2.0.0-SNAPSHOT_amd64.deb
-RUN dpkg -i /hdbpp-es/build/hdbpp-es_2.0.0-SNAPSHOT_amd64.deb
 
 RUN ldconfig
-
-RUN mv /usr/local/bin/hdb++cm-srv /usr/local/bin/hdbppcm-srv
-RUN mv /usr/local/bin/hdb++es-srv /usr/local/bin/hdbppes-srv

docker-compose/hdbppts-cm/Dockerfile

+ARG SOURCE_IMAGE
+FROM ${SOURCE_IMAGE}
+
+# ----------- HDB++ CONFIGURATION MANAGER ---------------
+
+RUN git clone -b master https://github.com/tango-controls-hdbpp/hdbpp-cm.git
+
+RUN cd hdbpp-cm \
+ && mkdir -p build \
+ && cd build \
+ && cmake .. -DCMAKE_PREFIX_PATH=/usr/local/include/tango \
+ && make -j4
+
+RUN cd hdbpp-cm/build \
+ && checkinstall \
+    --install=yes \
+    --fstrans=no \
+    --showinstall=no \
+    --backup=no \
+    --type=debian \
+    --pkgsource="https://github.com/tango-controls-hdbpp/hdbpp-cm" \
+    --pkglicense="GPLv3" \
+    --deldesc=no \
+    --nodoc \
+    --strip \
+    --stripso \
+    --maintainer="tango" \
+    --pkgarch=$(dpkg --print-architecture) \
+    --pkgversion="2.0.0" \
+    --pkgrelease="SNAPSHOT" \
+    --pkgname="hdbpp-cm" \
+    --requires="libzmq5,libomniorb4-2,libcos4-2,libomnithread4" \
+    make install
+
+RUN apt-get update && \
+    apt-get install -y \
+    build-essential && \
+    apt-get clean
+
+RUN dpkg -i /libhdbpp/build/libhdbpp_2.0.0-SNAPSHOT_amd64.deb
+RUN dpkg -i /libhdbpp-timescale/build/libhdbpp-timescale_2.0.0-SNAPSHOT_amd64.deb
+RUN dpkg -i /hdbpp-cm/build/hdbpp-cm_2.0.0-SNAPSHOT_amd64.deb
+
+RUN ldconfig
+
+RUN mv /usr/local/bin/hdb++cm-srv /usr/local/bin/hdbppcm-srv

docker-compose/hdbppts-es/Dockerfile

+ARG SOURCE_IMAGE
+FROM ${SOURCE_IMAGE}
+
+# ----------- HDB++ EVENT SUBSCRIBER ---------------
+
+RUN git clone -b master https://github.com/tango-controls-hdbpp/hdbpp-es.git
+
+RUN cd hdbpp-es \
+ && mkdir -p build \
+ && cd build \
+ && cmake .. -DCMAKE_PREFIX_PATH=/usr/local/include/tango -DFETCH_LIBHDBPP=OFF -DLIBHDBPP_BACKEND=timescale -DPostgreSQL_TYPE_INCLUDE_DIR=/usr/local/include/postgresql \
+ && make -j4 
+ 
+RUN cd hdbpp-es/build \
+ && checkinstall \
+    --install=yes \
+    --fstrans=no \
+    --showinstall=no \
+    --backup=no \
+    --type=debian \
+    --pkgsource="https://github.com/tango-controls-hdbpp/hdbpp-es" \
+    --pkglicense="GPLv3" \
+    --deldesc=no \
+    --nodoc \
+    --strip \
+    --stripso \
+    --maintainer="tango" \
+    --pkgarch=$(dpkg --print-architecture) \
+    --pkgversion="2.0.0" \
+    --pkgrelease="SNAPSHOT" \
+    --pkgname="hdbpp-es" \
+    --requires="libzmq5,libomniorb4-2,libcos4-2,libomnithread4" \
+    make install
+
+RUN apt-get update && \
+    apt-get install -y \
+    build-essential && \
+    apt-get clean
+
+RUN dpkg -i /libhdbpp/build/libhdbpp_2.0.0-SNAPSHOT_amd64.deb
+RUN dpkg -i /libhdbpp-timescale/build/libhdbpp-timescale_2.0.0-SNAPSHOT_amd64.deb
+RUN dpkg -i /hdbpp-es/build/hdbpp-es_2.0.0-SNAPSHOT_amd64.deb
+
+RUN ldconfig
+
+RUN mv /usr/local/bin/hdb++es-srv /usr/local/bin/hdbppes-srv

docker-compose/timescaledb/Dockerfile

-FROM timescale/timescaledb:latest-pg12
+ARG SOURCE_IMAGE
+FROM ${SOURCE_IMAGE}
 
 # Set admin role to perform DB creation
 COPY resources/01_admin.sh docker-entrypoint-initdb.d/002_admin.sh

sbin/tag_and_push_docker_image.sh

@@ -82,8 +82,8 @@ LOCAL_IMAGES=(
 
   "itango docker-compose_itango y"
 
-  "archiver-timescale timescaledb n" "hdbppts-cm hdbppts-cm n"
-  "hdbppts-es hdbppts-es n"
+  "archiver-timescale timescaledb y" "hdbpp hdbpp y" "hdbppts-cm hdbppts-cm y"
+  "hdbppts-es hdbppts-es y"
 
   "grafana grafana n" "prometheus prometheus n"
   "jupyter docker-compose_jupyter n"