From 21f8bd8ddf13ba5a11bfe63e33f8564aab5697a4 Mon Sep 17 00:00:00 2001 From: stedif Date: Tue, 13 Sep 2022 15:49:48 +0200 Subject: [PATCH 01/15] L2SS-822: add args to archiver yml file --- docker-compose/archiver-timescale.yml | 4 ++++ docker-compose/tango-archiver-ts/Dockerfile | 6 ++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docker-compose/archiver-timescale.yml b/docker-compose/archiver-timescale.yml index cdbf1b03..e88a00eb 100644 --- a/docker-compose/archiver-timescale.yml +++ b/docker-compose/archiver-timescale.yml @@ -35,6 +35,8 @@ services: image: hdbppts-cm build: context: tango-archiver-ts + args: + SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-cpp:${TANGO_CPP_VERSION} container_name: ${CONTAINER_NAME_PREFIX}hdbppts-cm networks: - control @@ -63,6 +65,8 @@ services: image: hdbppts-es build: context: tango-archiver-ts + args: + SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-cpp:${TANGO_CPP_VERSION} container_name: ${CONTAINER_NAME_PREFIX}hdbppts-es networks: - control diff --git a/docker-compose/tango-archiver-ts/Dockerfile b/docker-compose/tango-archiver-ts/Dockerfile index b6f44b0b..57f7bd70 100644 --- a/docker-compose/tango-archiver-ts/Dockerfile +++ b/docker-compose/tango-archiver-ts/Dockerfile @@ -1,7 +1,5 @@ -#ARG DOCKER_REGISTRY_USER -#ARG DOCKER_REGISTRY_HOST -#FROM ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}/tango-cpp:latest -FROM artefact.skao.int/ska-tango-images-tango-cpp:9.3.9 +ARG SOURCE_IMAGE +FROM ${SOURCE_IMAGE} USER root -- GitLab From d305ced3997712f85a091924a53391fa47527ffb Mon Sep 17 00:00:00 2001 From: stedif Date: Wed, 14 Sep 2022 11:39:03 +0200 Subject: [PATCH 02/15] L2SS-822: set flag on yes about using archiver on integration-test --- sbin/tag_and_push_docker_image.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sbin/tag_and_push_docker_image.sh b/sbin/tag_and_push_docker_image.sh index d12bee57..eb6b46d1 100755 --- a/sbin/tag_and_push_docker_image.sh +++ b/sbin/tag_and_push_docker_image.sh @@ -80,8 +80,8 @@ LOCAL_IMAGES=( "itango docker-compose_itango y" - "archiver-timescale timescaledb n" "hdbppts-cm hdbppts-cm n" - "hdbppts-es hdbppts-es n" + "archiver-timescale timescaledb y" "hdbppts-cm hdbppts-cm y" + "hdbppts-es hdbppts-es y" "grafana grafana n" "prometheus prometheus n" "jupyter docker-compose_jupyter n" -- GitLab From ebdb617c06140eeb21af9eef0b769d66ec236ca0 Mon Sep 17 00:00:00 2001 From: stedif Date: Wed, 21 Sep 2022 12:09:03 +0200 Subject: [PATCH 03/15] L2SS-822: parameterize timescaledb source image --- docker-compose/.env | 1 + docker-compose/archiver-timescale.yml | 2 ++ docker-compose/timescaledb/Dockerfile | 3 ++- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docker-compose/.env b/docker-compose/.env index eb952e41..ac4d6ab6 100644 --- a/docker-compose/.env +++ b/docker-compose/.env @@ -15,6 +15,7 @@ TANGO_POGO_VERSION=9.6.34 TANGO_REST_VERSION=1.14.6 TANGO_STARTER_VERSION=2021-05-28 +PG_TIMESCALEDB_VERSION=latest-pg12 PG_SUPERUSER_PASSWORD=password PG_HDB_PASSWORD=hdbpp MYSQL_ROOT_PASSWORD=secret diff --git a/docker-compose/archiver-timescale.yml b/docker-compose/archiver-timescale.yml index e88a00eb..820142ab 100644 --- a/docker-compose/archiver-timescale.yml +++ b/docker-compose/archiver-timescale.yml @@ -8,6 +8,8 @@ services: image: timescaledb build: context: timescaledb + args: + SOURCE_IMAGE: timescale/timescaledb:${PG_TIMESCALEDB_VERSION} container_name: ${CONTAINER_NAME_PREFIX}archiver-timescale networks: - control diff --git a/docker-compose/timescaledb/Dockerfile b/docker-compose/timescaledb/Dockerfile index 656d2ef6..73932cad 100644 --- a/docker-compose/timescaledb/Dockerfile +++ b/docker-compose/timescaledb/Dockerfile @@ -1,4 +1,5 @@ -FROM timescale/timescaledb:latest-pg12 +ARG SOURCE_IMAGE +FROM ${SOURCE_IMAGE} # Set admin role to perform DB creation COPY resources/01_admin.sh docker-entrypoint-initdb.d/002_admin.sh -- GitLab From 0ac35eb0c9c74437687ab531fa8d6bd68017d45b Mon Sep 17 00:00:00 2001 From: stedif Date: Fri, 23 Sep 2022 11:42:42 +0200 Subject: [PATCH 04/15] L2SS-822-introduce-dockerfile-dependencies --- docker-compose/archiver-timescale.yml | 18 +++-- .../{tango-archiver-ts => hdbpp}/Dockerfile | 69 ------------------- docker-compose/hdbppts-cm/Dockerfile | 39 +++++++++++ docker-compose/hdbppts-es/Dockerfile | 39 +++++++++++ 4 files changed, 91 insertions(+), 74 deletions(-) rename docker-compose/{tango-archiver-ts => hdbpp}/Dockerfile (58%) create mode 100644 docker-compose/hdbppts-cm/Dockerfile create mode 100644 docker-compose/hdbppts-es/Dockerfile diff --git a/docker-compose/archiver-timescale.yml b/docker-compose/archiver-timescale.yml index 820142ab..c7fc972d 100644 --- a/docker-compose/archiver-timescale.yml +++ b/docker-compose/archiver-timescale.yml @@ -33,16 +33,23 @@ services: tag: "{{.Name}}" restart: unless-stopped + hdbpp: + image: hdbpp + build: + context: hdbpp + args: + SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-cpp:${TANGO_CPP_VERSION} hdbppts-cm: image: hdbppts-cm build: - context: tango-archiver-ts + context: hdbppts-cm args: - SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-cpp:${TANGO_CPP_VERSION} + SOURCE_IMAGE: hdbpp container_name: ${CONTAINER_NAME_PREFIX}hdbppts-cm networks: - control depends_on: + - hdbpp - databaseds - dsconfig - archiver-timescale @@ -62,17 +69,18 @@ services: syslog-address: udp://${LOG_HOSTNAME}:1514 syslog-format: rfc3164 tag: "{{.Name}}" - hdbppts-es: image: hdbppts-es build: - context: tango-archiver-ts + context: hdbppts-es args: - SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-cpp:${TANGO_CPP_VERSION} + SOURCE_IMAGE: hdbppts-cm container_name: ${CONTAINER_NAME_PREFIX}hdbppts-es networks: - control depends_on: + - hdbpp + - hdbppts-cm - databaseds - dsconfig - archiver-timescale diff --git a/docker-compose/tango-archiver-ts/Dockerfile b/docker-compose/hdbpp/Dockerfile similarity index 58% rename from docker-compose/tango-archiver-ts/Dockerfile rename to docker-compose/hdbpp/Dockerfile index 57f7bd70..cdfbebb2 100644 --- a/docker-compose/tango-archiver-ts/Dockerfile +++ b/docker-compose/hdbpp/Dockerfile @@ -93,70 +93,6 @@ RUN cd libhdbpp-timescale/build \ --requires="libpq5" \ make install - -# ----------- HDB++ CONFIGURATION MANAGER --------------- - -RUN git clone -b master https://github.com/tango-controls-hdbpp/hdbpp-cm.git - -RUN cd hdbpp-cm \ - && mkdir -p build \ - && cd build \ - && cmake .. -DCMAKE_PREFIX_PATH=/usr/local/include/tango \ - && make -j4 - -RUN cd hdbpp-cm/build \ - && checkinstall \ - --install=yes \ - --fstrans=no \ - --showinstall=no \ - --backup=no \ - --type=debian \ - --pkgsource="https://github.com/tango-controls-hdbpp/hdbpp-cm" \ - --pkglicense="GPLv3" \ - --deldesc=no \ - --nodoc \ - --strip \ - --stripso \ - --maintainer="tango" \ - --pkgarch=$(dpkg --print-architecture) \ - --pkgversion="2.0.0" \ - --pkgrelease="SNAPSHOT" \ - --pkgname="hdbpp-cm" \ - --requires="libzmq5,libomniorb4-2,libcos4-2,libomnithread4" \ - make install - - -# ----------- HDB++ EVENT SUBSCRIBER --------------- - -RUN git clone -b master https://github.com/tango-controls-hdbpp/hdbpp-es.git - -RUN cd hdbpp-es \ - && mkdir -p build \ - && cd build \ - && cmake .. -DCMAKE_PREFIX_PATH=/usr/local/include/tango -DFETCH_LIBHDBPP=OFF -DLIBHDBPP_BACKEND=timescale -DPostgreSQL_TYPE_INCLUDE_DIR=/usr/local/include/postgresql \ - && make -j4 - -RUN cd hdbpp-es/build \ - && checkinstall \ - --install=yes \ - --fstrans=no \ - --showinstall=no \ - --backup=no \ - --type=debian \ - --pkgsource="https://github.com/tango-controls-hdbpp/hdbpp-es" \ - --pkglicense="GPLv3" \ - --deldesc=no \ - --nodoc \ - --strip \ - --stripso \ - --maintainer="tango" \ - --pkgarch=$(dpkg --print-architecture) \ - --pkgversion="2.0.0" \ - --pkgrelease="SNAPSHOT" \ - --pkgname="hdbpp-es" \ - --requires="libzmq5,libomniorb4-2,libcos4-2,libomnithread4" \ - make install - RUN apt-get update && \ apt-get install -y \ build-essential && \ @@ -164,10 +100,5 @@ RUN apt-get update && \ RUN dpkg -i /libhdbpp/build/libhdbpp_2.0.0-SNAPSHOT_amd64.deb RUN dpkg -i /libhdbpp-timescale/build/libhdbpp-timescale_2.0.0-SNAPSHOT_amd64.deb -RUN dpkg -i /hdbpp-cm/build/hdbpp-cm_2.0.0-SNAPSHOT_amd64.deb -RUN dpkg -i /hdbpp-es/build/hdbpp-es_2.0.0-SNAPSHOT_amd64.deb RUN ldconfig - -RUN mv /usr/local/bin/hdb++cm-srv /usr/local/bin/hdbppcm-srv -RUN mv /usr/local/bin/hdb++es-srv /usr/local/bin/hdbppes-srv diff --git a/docker-compose/hdbppts-cm/Dockerfile b/docker-compose/hdbppts-cm/Dockerfile new file mode 100644 index 00000000..7a4d4cd0 --- /dev/null +++ b/docker-compose/hdbppts-cm/Dockerfile @@ -0,0 +1,39 @@ +ARG SOURCE_IMAGE +FROM ${SOURCE_IMAGE} + +# ----------- HDB++ CONFIGURATION MANAGER --------------- + +RUN git clone -b master https://github.com/tango-controls-hdbpp/hdbpp-cm.git + +RUN cd hdbpp-cm \ + && mkdir -p build \ + && cd build \ + && cmake .. -DCMAKE_PREFIX_PATH=/usr/local/include/tango \ + && make -j4 + +RUN cd hdbpp-cm/build \ + && checkinstall \ + --install=yes \ + --fstrans=no \ + --showinstall=no \ + --backup=no \ + --type=debian \ + --pkgsource="https://github.com/tango-controls-hdbpp/hdbpp-cm" \ + --pkglicense="GPLv3" \ + --deldesc=no \ + --nodoc \ + --strip \ + --stripso \ + --maintainer="tango" \ + --pkgarch=$(dpkg --print-architecture) \ + --pkgversion="2.0.0" \ + --pkgrelease="SNAPSHOT" \ + --pkgname="hdbpp-cm" \ + --requires="libzmq5,libomniorb4-2,libcos4-2,libomnithread4" \ + make install + +RUN dpkg -i /hdbpp-cm/build/hdbpp-cm_2.0.0-SNAPSHOT_amd64.deb + +RUN ldconfig + +RUN mv /usr/local/bin/hdb++cm-srv /usr/local/bin/hdbppcm-srv diff --git a/docker-compose/hdbppts-es/Dockerfile b/docker-compose/hdbppts-es/Dockerfile new file mode 100644 index 00000000..09032840 --- /dev/null +++ b/docker-compose/hdbppts-es/Dockerfile @@ -0,0 +1,39 @@ +ARG SOURCE_IMAGE +FROM ${SOURCE_IMAGE} + +# ----------- HDB++ EVENT SUBSCRIBER --------------- + +RUN git clone -b master https://github.com/tango-controls-hdbpp/hdbpp-es.git + +RUN cd hdbpp-es \ + && mkdir -p build \ + && cd build \ + && cmake .. -DCMAKE_PREFIX_PATH=/usr/local/include/tango -DFETCH_LIBHDBPP=OFF -DLIBHDBPP_BACKEND=timescale -DPostgreSQL_TYPE_INCLUDE_DIR=/usr/local/include/postgresql \ + && make -j4 + +RUN cd hdbpp-es/build \ + && checkinstall \ + --install=yes \ + --fstrans=no \ + --showinstall=no \ + --backup=no \ + --type=debian \ + --pkgsource="https://github.com/tango-controls-hdbpp/hdbpp-es" \ + --pkglicense="GPLv3" \ + --deldesc=no \ + --nodoc \ + --strip \ + --stripso \ + --maintainer="tango" \ + --pkgarch=$(dpkg --print-architecture) \ + --pkgversion="2.0.0" \ + --pkgrelease="SNAPSHOT" \ + --pkgname="hdbpp-es" \ + --requires="libzmq5,libomniorb4-2,libcos4-2,libomnithread4" \ + make install + +RUN dpkg -i /hdbpp-es/build/hdbpp-es_2.0.0-SNAPSHOT_amd64.deb + +RUN ldconfig + +RUN mv /usr/local/bin/hdb++es-srv /usr/local/bin/hdbppes-srv -- GitLab From 17b1b20501c093f3911ea18bd0efe4861c51bb26 Mon Sep 17 00:00:00 2001 From: stedif Date: Fri, 23 Sep 2022 18:41:16 +0200 Subject: [PATCH 05/15] L2SS-822: update scripts --- .gitlab-ci.yml | 16 ++++++++++++++-- sbin/run_integration_test.sh | 4 ++-- sbin/tag_and_push_docker_image.sh | 2 +- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8d2e2c7c..4eb432f8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -112,6 +112,7 @@ docker_build_image_all: - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-xst latest - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-temperature-manager latest - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh archiver-timescale latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh hdbpp latest - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh hdbppts-cm latest - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh hdbppts-es latest docker_build_image_elk: @@ -479,6 +480,17 @@ docker_build_image_archiver_timescale: script: # Do not remove 'bash' or statement will be ignored by primitive docker shell - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh archiver-timescale $tag +docker_build_image_hdbpp: + extends: .base_docker_images_except + only: + refs: + - merge_requests + changes: + - docker-compose/archiver-timescale.yml + - docker-compose/hdbpp/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh hdbppts-cm $tag docker_build_image_hdbppts_cm: extends: .base_docker_images_except only: @@ -486,7 +498,7 @@ docker_build_image_hdbppts_cm: - merge_requests changes: - docker-compose/archiver-timescale.yml - - docker-compose/tango-archiver-ts/* + - docker-compose/hdbppts-cm/* script: # Do not remove 'bash' or statement will be ignored by primitive docker shell - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh hdbppts-cm $tag @@ -497,7 +509,7 @@ docker_build_image_hdbppts_es: - merge_requests changes: - docker-compose/archiver-timescale.yml - - docker-compose/tango-archiver-ts/* + - docker-compose/hdbppts-es/* script: # Do not remove 'bash' or statement will be ignored by primitive docker shell - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh hdbppts-es $tag diff --git a/sbin/run_integration_test.sh b/sbin/run_integration_test.sh index 3047557a..d722e1f5 100755 --- a/sbin/run_integration_test.sh +++ b/sbin/run_integration_test.sh @@ -84,7 +84,7 @@ SIMULATORS="sdptr-sim recv-sim unb2-sim apsct-sim apspu-sim ccd-sim" # shellcheck disable=SC2086 make build $DEVICES $SIMULATORS make build elk integration-test -make build archiver-timescale hdbppts-cm hdbppts-es +make build archiver-timescale hdbpp hdbppts-cm hdbppts-es # Start and stop sequence # shellcheck disable=SC2086 @@ -131,4 +131,4 @@ integration_test default integration_test recv_cluster "device-recv device-tilebeam device-antennafield" "${LOFAR20_DIR}/CDB/integrations/recvcluster_ConfigDb.json" -integration_test observations "archiver-timescale hdbppts-cm hdbppts-es" "${LOFAR20_DIR}/CDB/integrations/multiobs_ConfigDb.json" +integration_test observations "archiver-timescale hdbpp hdbppts-cm hdbppts-es" "${LOFAR20_DIR}/CDB/integrations/multiobs_ConfigDb.json" diff --git a/sbin/tag_and_push_docker_image.sh b/sbin/tag_and_push_docker_image.sh index 299dfdda..d4e9d794 100755 --- a/sbin/tag_and_push_docker_image.sh +++ b/sbin/tag_and_push_docker_image.sh @@ -82,7 +82,7 @@ LOCAL_IMAGES=( "itango docker-compose_itango y" - "archiver-timescale timescaledb y" "hdbppts-cm hdbppts-cm y" + "archiver-timescale timescaledb y" "hdbpp hdbpp y" "hdbppts-cm hdbppts-cm y" "hdbppts-es hdbppts-es y" "grafana grafana n" "prometheus prometheus n" -- GitLab From f34e7a0f9e3d42ef4c876b6c26c3b2ecc60377d7 Mon Sep 17 00:00:00 2001 From: stedif Date: Fri, 23 Sep 2022 20:37:05 +0200 Subject: [PATCH 06/15] L2SS-822: bug fix --- .gitlab-ci.yml | 2 +- docker-compose/archiver-timescale.yml | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4eb432f8..653f6746 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -490,7 +490,7 @@ docker_build_image_hdbpp: - docker-compose/hdbpp/* script: # Do not remove 'bash' or statement will be ignored by primitive docker shell - - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh hdbppts-cm $tag + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh hdbpp $tag docker_build_image_hdbppts_cm: extends: .base_docker_images_except only: diff --git a/docker-compose/archiver-timescale.yml b/docker-compose/archiver-timescale.yml index c7fc972d..a2e05c4e 100644 --- a/docker-compose/archiver-timescale.yml +++ b/docker-compose/archiver-timescale.yml @@ -39,6 +39,18 @@ services: context: hdbpp args: SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-cpp:${TANGO_CPP_VERSION} + container_name: ${CONTAINER_NAME_PREFIX}hdbpp + networks: + - control + depends_on: + - databaseds + - dsconfig + - archiver-timescale + extra_hosts: + - "host.docker.internal:host-gateway" + environment: + - TANGO_HOST=${TANGO_HOST} + hdbppts-cm: image: hdbppts-cm build: @@ -69,6 +81,7 @@ services: syslog-address: udp://${LOG_HOSTNAME}:1514 syslog-format: rfc3164 tag: "{{.Name}}" + hdbppts-es: image: hdbppts-es build: -- GitLab From 35031624d5ee02ee74d70e89991cad78442f4335 Mon Sep 17 00:00:00 2001 From: stedif Date: Mon, 26 Sep 2022 10:43:37 +0200 Subject: [PATCH 07/15] L2SS-822: update dockerfiles --- docker-compose/hdbppts-cm/Dockerfile | 7 +++++++ docker-compose/hdbppts-es/Dockerfile | 9 +++++++++ 2 files changed, 16 insertions(+) diff --git a/docker-compose/hdbppts-cm/Dockerfile b/docker-compose/hdbppts-cm/Dockerfile index 7a4d4cd0..c6c5c921 100644 --- a/docker-compose/hdbppts-cm/Dockerfile +++ b/docker-compose/hdbppts-cm/Dockerfile @@ -32,6 +32,13 @@ RUN cd hdbpp-cm/build \ --requires="libzmq5,libomniorb4-2,libcos4-2,libomnithread4" \ make install +RUN apt-get update && \ + apt-get install -y \ + build-essential && \ + apt-get clean + +RUN dpkg -i /libhdbpp/build/libhdbpp_2.0.0-SNAPSHOT_amd64.deb +RUN dpkg -i /libhdbpp-timescale/build/libhdbpp-timescale_2.0.0-SNAPSHOT_amd64.deb RUN dpkg -i /hdbpp-cm/build/hdbpp-cm_2.0.0-SNAPSHOT_amd64.deb RUN ldconfig diff --git a/docker-compose/hdbppts-es/Dockerfile b/docker-compose/hdbppts-es/Dockerfile index 09032840..cd03ffbc 100644 --- a/docker-compose/hdbppts-es/Dockerfile +++ b/docker-compose/hdbppts-es/Dockerfile @@ -32,8 +32,17 @@ RUN cd hdbpp-es/build \ --requires="libzmq5,libomniorb4-2,libcos4-2,libomnithread4" \ make install +RUN apt-get update && \ + apt-get install -y \ + build-essential && \ + apt-get clean + +RUN dpkg -i /libhdbpp/build/libhdbpp_2.0.0-SNAPSHOT_amd64.deb +RUN dpkg -i /libhdbpp-timescale/build/libhdbpp-timescale_2.0.0-SNAPSHOT_amd64.deb +RUN dpkg -i /hdbpp-cm/build/hdbpp-cm_2.0.0-SNAPSHOT_amd64.deb RUN dpkg -i /hdbpp-es/build/hdbpp-es_2.0.0-SNAPSHOT_amd64.deb RUN ldconfig +RUN mv /usr/local/bin/hdb++cm-srv /usr/local/bin/hdbppcm-srv RUN mv /usr/local/bin/hdb++es-srv /usr/local/bin/hdbppes-srv -- GitLab From 83d0e4d00f66f8dff93d2a15117ee8d42877bc2e Mon Sep 17 00:00:00 2001 From: stedif Date: Mon, 26 Sep 2022 12:09:27 +0200 Subject: [PATCH 08/15] L2SS-822: remove dependency in yml --- docker-compose/archiver-timescale.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-compose/archiver-timescale.yml b/docker-compose/archiver-timescale.yml index a2e05c4e..48249ea6 100644 --- a/docker-compose/archiver-timescale.yml +++ b/docker-compose/archiver-timescale.yml @@ -92,7 +92,6 @@ services: networks: - control depends_on: - - hdbpp - hdbppts-cm - databaseds - dsconfig -- GitLab From 4b7cc5dd416514fbd22f9627c8466d71f7685bbb Mon Sep 17 00:00:00 2001 From: stedif Date: Mon, 26 Sep 2022 15:39:54 +0200 Subject: [PATCH 09/15] L2SS-822: fix source image names --- docker-compose/archiver-timescale.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose/archiver-timescale.yml b/docker-compose/archiver-timescale.yml index 48249ea6..9d5d87f4 100644 --- a/docker-compose/archiver-timescale.yml +++ b/docker-compose/archiver-timescale.yml @@ -56,7 +56,7 @@ services: build: context: hdbppts-cm args: - SOURCE_IMAGE: hdbpp + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/hdbpp:latest container_name: ${CONTAINER_NAME_PREFIX}hdbppts-cm networks: - control @@ -87,7 +87,7 @@ services: build: context: hdbppts-es args: - SOURCE_IMAGE: hdbppts-cm + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/hdbppts-cm:latest container_name: ${CONTAINER_NAME_PREFIX}hdbppts-es networks: - control -- GitLab From c3385c328836750acb5c9d897910e55aa7f7c568 Mon Sep 17 00:00:00 2001 From: stedif Date: Mon, 26 Sep 2022 15:58:37 +0200 Subject: [PATCH 10/15] L2SS-822: workaround with image tags --- docker-compose/archiver-timescale.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docker-compose/archiver-timescale.yml b/docker-compose/archiver-timescale.yml index 9d5d87f4..5d0f8c92 100644 --- a/docker-compose/archiver-timescale.yml +++ b/docker-compose/archiver-timescale.yml @@ -37,7 +37,7 @@ services: image: hdbpp build: context: hdbpp - args: + args: SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-cpp:${TANGO_CPP_VERSION} container_name: ${CONTAINER_NAME_PREFIX}hdbpp networks: @@ -55,8 +55,9 @@ services: image: hdbppts-cm build: context: hdbppts-cm - args: - SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/hdbpp:latest + args: + # TODO (Stefano): replace image tag with 'latest' + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/hdbpp:l2ss-822-cache-archiver-ts-docker-image container_name: ${CONTAINER_NAME_PREFIX}hdbppts-cm networks: - control @@ -86,8 +87,9 @@ services: image: hdbppts-es build: context: hdbppts-es - args: - SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/hdbppts-cm:latest + args: + # TODO (Stefano): replace image tag with 'latest' + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/hdbppts-cm:l2ss-822-cache-archiver-ts-docker-image container_name: ${CONTAINER_NAME_PREFIX}hdbppts-es networks: - control -- GitLab From 9e265cc07065fe09aa72a2c4a1e8a15a34b8f959 Mon Sep 17 00:00:00 2001 From: stedif Date: Wed, 28 Sep 2022 10:44:04 +0200 Subject: [PATCH 11/15] L2SS-822: do not build hdbpp --- sbin/run_integration_test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sbin/run_integration_test.sh b/sbin/run_integration_test.sh index d722e1f5..3047557a 100755 --- a/sbin/run_integration_test.sh +++ b/sbin/run_integration_test.sh @@ -84,7 +84,7 @@ SIMULATORS="sdptr-sim recv-sim unb2-sim apsct-sim apspu-sim ccd-sim" # shellcheck disable=SC2086 make build $DEVICES $SIMULATORS make build elk integration-test -make build archiver-timescale hdbpp hdbppts-cm hdbppts-es +make build archiver-timescale hdbppts-cm hdbppts-es # Start and stop sequence # shellcheck disable=SC2086 @@ -131,4 +131,4 @@ integration_test default integration_test recv_cluster "device-recv device-tilebeam device-antennafield" "${LOFAR20_DIR}/CDB/integrations/recvcluster_ConfigDb.json" -integration_test observations "archiver-timescale hdbpp hdbppts-cm hdbppts-es" "${LOFAR20_DIR}/CDB/integrations/multiobs_ConfigDb.json" +integration_test observations "archiver-timescale hdbppts-cm hdbppts-es" "${LOFAR20_DIR}/CDB/integrations/multiobs_ConfigDb.json" -- GitLab From 4ce4fcc4da65260671fff73bec2ff4528a0bcd8f Mon Sep 17 00:00:00 2001 From: stedif Date: Wed, 28 Sep 2022 10:45:17 +0200 Subject: [PATCH 12/15] L2SS-822: fix archiver-timescale yml --- docker-compose/archiver-timescale.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-compose/archiver-timescale.yml b/docker-compose/archiver-timescale.yml index 5d0f8c92..5654716d 100644 --- a/docker-compose/archiver-timescale.yml +++ b/docker-compose/archiver-timescale.yml @@ -62,7 +62,6 @@ services: networks: - control depends_on: - - hdbpp - databaseds - dsconfig - archiver-timescale -- GitLab From da096efb9f56ccc56efe90fbf73d673917fbdce0 Mon Sep 17 00:00:00 2001 From: stedif Date: Wed, 28 Sep 2022 11:01:39 +0200 Subject: [PATCH 13/15] L2SS-822: exclude ConfManager code from EventSubscriber container --- docker-compose/archiver-timescale.yml | 2 +- docker-compose/hdbppts-es/Dockerfile | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/docker-compose/archiver-timescale.yml b/docker-compose/archiver-timescale.yml index 5654716d..ff46d103 100644 --- a/docker-compose/archiver-timescale.yml +++ b/docker-compose/archiver-timescale.yml @@ -88,7 +88,7 @@ services: context: hdbppts-es args: # TODO (Stefano): replace image tag with 'latest' - SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/hdbppts-cm:l2ss-822-cache-archiver-ts-docker-image + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/hdbpp:l2ss-822-cache-archiver-ts-docker-image container_name: ${CONTAINER_NAME_PREFIX}hdbppts-es networks: - control diff --git a/docker-compose/hdbppts-es/Dockerfile b/docker-compose/hdbppts-es/Dockerfile index cd03ffbc..bd74e1b9 100644 --- a/docker-compose/hdbppts-es/Dockerfile +++ b/docker-compose/hdbppts-es/Dockerfile @@ -39,10 +39,8 @@ RUN apt-get update && \ RUN dpkg -i /libhdbpp/build/libhdbpp_2.0.0-SNAPSHOT_amd64.deb RUN dpkg -i /libhdbpp-timescale/build/libhdbpp-timescale_2.0.0-SNAPSHOT_amd64.deb -RUN dpkg -i /hdbpp-cm/build/hdbpp-cm_2.0.0-SNAPSHOT_amd64.deb RUN dpkg -i /hdbpp-es/build/hdbpp-es_2.0.0-SNAPSHOT_amd64.deb RUN ldconfig -RUN mv /usr/local/bin/hdb++cm-srv /usr/local/bin/hdbppcm-srv RUN mv /usr/local/bin/hdb++es-srv /usr/local/bin/hdbppes-srv -- GitLab From b292294ad7b1a35e45c9c6606f4d2ecab40444ec Mon Sep 17 00:00:00 2001 From: stedif Date: Wed, 28 Sep 2022 13:01:19 +0200 Subject: [PATCH 14/15] L2SS-822: add documentation --- docker-compose/README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docker-compose/README.md b/docker-compose/README.md index d76a75b0..c68719dc 100644 --- a/docker-compose/README.md +++ b/docker-compose/README.md @@ -39,6 +39,23 @@ registry and uploaded to our own using matching tags. Services, same mechanism as devices. +### HDB++ image updates + +The hdbpp Docker image is used as a base image for the hdbppts-cm (ConfigurationManager) and hdbppts-es (EventSubscriber) +images. If one is developing on a branch and any updates is made in hdbpp/Dockerfile, those won't be automatically picked up from hdbppts-cm and hdbppts-es, because the argument SOURCE_IMAGE in the docker-compose yml file always refers to the remote hdbpp image in the repository. + +A temporary workaround for locally testing on these archiving containers is the following: + +- Stop and remove any running hdbpp* container +- In the archiver-timescale.yml file, replace the hdbppts-cm and hdbppts-es SOURCE_IMAGE tag 'latest' with the branch name +(e.g. SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/hdbpp:l2ss-new-branch) +- Rebuild all the hdbpp* container (make build hdbpp hdbppts-cm hdbppts-es), and then start them +- Test the new features + +After the branch has been correctly developed, tested, the merge has been approved, and the new images have been built on the repository: +- Put back 'latest' tag on the archiver-timescale.yml file, replacing the branch name +- Merge on master + ### Setup and maintenance All behavioral logic to orchestrate change detection and image pushing can be -- GitLab From 18121f88e182978fe9824991cb7ad814e2770291 Mon Sep 17 00:00:00 2001 From: stedif Date: Tue, 4 Oct 2022 12:13:46 +0200 Subject: [PATCH 15/15] L2SS-822: replace branch tag with latest --- docker-compose/archiver-timescale.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docker-compose/archiver-timescale.yml b/docker-compose/archiver-timescale.yml index ff46d103..405ef868 100644 --- a/docker-compose/archiver-timescale.yml +++ b/docker-compose/archiver-timescale.yml @@ -56,8 +56,7 @@ services: build: context: hdbppts-cm args: - # TODO (Stefano): replace image tag with 'latest' - SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/hdbpp:l2ss-822-cache-archiver-ts-docker-image + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/hdbpp:latest container_name: ${CONTAINER_NAME_PREFIX}hdbppts-cm networks: - control @@ -87,8 +86,7 @@ services: build: context: hdbppts-es args: - # TODO (Stefano): replace image tag with 'latest' - SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/hdbpp:l2ss-822-cache-archiver-ts-docker-image + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/hdbpp:latest container_name: ${CONTAINER_NAME_PREFIX}hdbppts-es networks: - control -- GitLab