diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5e7b26942b89dbc2a42372599da5e6a31275f517..8b1a90b70cc2a34955ca4d2c0588ec323da8a9af 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -125,6 +125,7 @@ docker_build_image_all: - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-xst latest - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-temperature-manager latest - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh archiver-timescale latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh hdbpp latest - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh hdbppts-cm latest - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh hdbppts-es latest @@ -494,6 +495,17 @@ docker_build_image_archiver_timescale: script: # Do not remove 'bash' or statement will be ignored by primitive docker shell - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh archiver-timescale $tag +docker_build_image_hdbpp: + extends: .base_docker_images_except + only: + refs: + - merge_requests + changes: + - docker-compose/archiver-timescale.yml + - docker-compose/hdbpp/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh hdbpp $tag docker_build_image_hdbppts_cm: extends: .base_docker_images_except only: @@ -501,7 +513,7 @@ docker_build_image_hdbppts_cm: - merge_requests changes: - docker-compose/archiver-timescale.yml - - docker-compose/tango-archiver-ts/* + - docker-compose/hdbppts-cm/* script: # Do not remove 'bash' or statement will be ignored by primitive docker shell - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh hdbppts-cm $tag @@ -512,7 +524,7 @@ docker_build_image_hdbppts_es: - merge_requests changes: - docker-compose/archiver-timescale.yml - - docker-compose/tango-archiver-ts/* + - docker-compose/hdbppts-es/* script: # Do not remove 'bash' or statement will be ignored by primitive docker shell - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh hdbppts-es $tag diff --git a/docker-compose/.env b/docker-compose/.env index 50751ff6a5ed2a57225e747b56a30b97c28beba6..e58211d21a920dcbafb41e5103891e039b03fac0 100644 --- a/docker-compose/.env +++ b/docker-compose/.env @@ -15,6 +15,7 @@ TANGO_POGO_VERSION=9.6.34 TANGO_REST_VERSION=1.14.6 TANGO_STARTER_VERSION=2021-05-28 +PG_TIMESCALEDB_VERSION=latest-pg12 PG_SUPERUSER_PASSWORD=password PG_HDB_PASSWORD=hdbpp MYSQL_ROOT_PASSWORD=secret diff --git a/docker-compose/README.md b/docker-compose/README.md index d76a75b0c79dad1574d9d24912a671a39998abee..c68719dc3e38e3b89f92a810664e59995aca288c 100644 --- a/docker-compose/README.md +++ b/docker-compose/README.md @@ -39,6 +39,23 @@ registry and uploaded to our own using matching tags. Services, same mechanism as devices. +### HDB++ image updates + +The hdbpp Docker image is used as a base image for the hdbppts-cm (ConfigurationManager) and hdbppts-es (EventSubscriber) +images. If one is developing on a branch and any updates is made in hdbpp/Dockerfile, those won't be automatically picked up from hdbppts-cm and hdbppts-es, because the argument SOURCE_IMAGE in the docker-compose yml file always refers to the remote hdbpp image in the repository. + +A temporary workaround for locally testing on these archiving containers is the following: + +- Stop and remove any running hdbpp* container +- In the archiver-timescale.yml file, replace the hdbppts-cm and hdbppts-es SOURCE_IMAGE tag 'latest' with the branch name +(e.g. SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/hdbpp:l2ss-new-branch) +- Rebuild all the hdbpp* container (make build hdbpp hdbppts-cm hdbppts-es), and then start them +- Test the new features + +After the branch has been correctly developed, tested, the merge has been approved, and the new images have been built on the repository: +- Put back 'latest' tag on the archiver-timescale.yml file, replacing the branch name +- Merge on master + ### Setup and maintenance All behavioral logic to orchestrate change detection and image pushing can be diff --git a/docker-compose/archiver-timescale.yml b/docker-compose/archiver-timescale.yml index cdbf1b03788117308fa97227e3d83b452a77395a..405ef8687b14872a3f3980dac905d0cee855fa1f 100644 --- a/docker-compose/archiver-timescale.yml +++ b/docker-compose/archiver-timescale.yml @@ -8,6 +8,8 @@ services: image: timescaledb build: context: timescaledb + args: + SOURCE_IMAGE: timescale/timescaledb:${PG_TIMESCALEDB_VERSION} container_name: ${CONTAINER_NAME_PREFIX}archiver-timescale networks: - control @@ -31,10 +33,30 @@ services: tag: "{{.Name}}" restart: unless-stopped + hdbpp: + image: hdbpp + build: + context: hdbpp + args: + SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-cpp:${TANGO_CPP_VERSION} + container_name: ${CONTAINER_NAME_PREFIX}hdbpp + networks: + - control + depends_on: + - databaseds + - dsconfig + - archiver-timescale + extra_hosts: + - "host.docker.internal:host-gateway" + environment: + - TANGO_HOST=${TANGO_HOST} + hdbppts-cm: image: hdbppts-cm build: - context: tango-archiver-ts + context: hdbppts-cm + args: + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/hdbpp:latest container_name: ${CONTAINER_NAME_PREFIX}hdbppts-cm networks: - control @@ -62,11 +84,14 @@ services: hdbppts-es: image: hdbppts-es build: - context: tango-archiver-ts + context: hdbppts-es + args: + SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/hdbpp:latest container_name: ${CONTAINER_NAME_PREFIX}hdbppts-es networks: - control depends_on: + - hdbppts-cm - databaseds - dsconfig - archiver-timescale diff --git a/docker-compose/tango-archiver-ts/Dockerfile b/docker-compose/hdbpp/Dockerfile similarity index 56% rename from docker-compose/tango-archiver-ts/Dockerfile rename to docker-compose/hdbpp/Dockerfile index b6f44b0ba9e08d06538165be9283fadb18c4cbef..cdfbebb254674583ea3d6c0ae633f171e124f5fb 100644 --- a/docker-compose/tango-archiver-ts/Dockerfile +++ b/docker-compose/hdbpp/Dockerfile @@ -1,7 +1,5 @@ -#ARG DOCKER_REGISTRY_USER -#ARG DOCKER_REGISTRY_HOST -#FROM ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}/tango-cpp:latest -FROM artefact.skao.int/ska-tango-images-tango-cpp:9.3.9 +ARG SOURCE_IMAGE +FROM ${SOURCE_IMAGE} USER root @@ -95,70 +93,6 @@ RUN cd libhdbpp-timescale/build \ --requires="libpq5" \ make install - -# ----------- HDB++ CONFIGURATION MANAGER --------------- - -RUN git clone -b master https://github.com/tango-controls-hdbpp/hdbpp-cm.git - -RUN cd hdbpp-cm \ - && mkdir -p build \ - && cd build \ - && cmake .. -DCMAKE_PREFIX_PATH=/usr/local/include/tango \ - && make -j4 - -RUN cd hdbpp-cm/build \ - && checkinstall \ - --install=yes \ - --fstrans=no \ - --showinstall=no \ - --backup=no \ - --type=debian \ - --pkgsource="https://github.com/tango-controls-hdbpp/hdbpp-cm" \ - --pkglicense="GPLv3" \ - --deldesc=no \ - --nodoc \ - --strip \ - --stripso \ - --maintainer="tango" \ - --pkgarch=$(dpkg --print-architecture) \ - --pkgversion="2.0.0" \ - --pkgrelease="SNAPSHOT" \ - --pkgname="hdbpp-cm" \ - --requires="libzmq5,libomniorb4-2,libcos4-2,libomnithread4" \ - make install - - -# ----------- HDB++ EVENT SUBSCRIBER --------------- - -RUN git clone -b master https://github.com/tango-controls-hdbpp/hdbpp-es.git - -RUN cd hdbpp-es \ - && mkdir -p build \ - && cd build \ - && cmake .. -DCMAKE_PREFIX_PATH=/usr/local/include/tango -DFETCH_LIBHDBPP=OFF -DLIBHDBPP_BACKEND=timescale -DPostgreSQL_TYPE_INCLUDE_DIR=/usr/local/include/postgresql \ - && make -j4 - -RUN cd hdbpp-es/build \ - && checkinstall \ - --install=yes \ - --fstrans=no \ - --showinstall=no \ - --backup=no \ - --type=debian \ - --pkgsource="https://github.com/tango-controls-hdbpp/hdbpp-es" \ - --pkglicense="GPLv3" \ - --deldesc=no \ - --nodoc \ - --strip \ - --stripso \ - --maintainer="tango" \ - --pkgarch=$(dpkg --print-architecture) \ - --pkgversion="2.0.0" \ - --pkgrelease="SNAPSHOT" \ - --pkgname="hdbpp-es" \ - --requires="libzmq5,libomniorb4-2,libcos4-2,libomnithread4" \ - make install - RUN apt-get update && \ apt-get install -y \ build-essential && \ @@ -166,10 +100,5 @@ RUN apt-get update && \ RUN dpkg -i /libhdbpp/build/libhdbpp_2.0.0-SNAPSHOT_amd64.deb RUN dpkg -i /libhdbpp-timescale/build/libhdbpp-timescale_2.0.0-SNAPSHOT_amd64.deb -RUN dpkg -i /hdbpp-cm/build/hdbpp-cm_2.0.0-SNAPSHOT_amd64.deb -RUN dpkg -i /hdbpp-es/build/hdbpp-es_2.0.0-SNAPSHOT_amd64.deb RUN ldconfig - -RUN mv /usr/local/bin/hdb++cm-srv /usr/local/bin/hdbppcm-srv -RUN mv /usr/local/bin/hdb++es-srv /usr/local/bin/hdbppes-srv diff --git a/docker-compose/hdbppts-cm/Dockerfile b/docker-compose/hdbppts-cm/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..c6c5c92105c75b36b5aefb57e89c1a8c00e457f7 --- /dev/null +++ b/docker-compose/hdbppts-cm/Dockerfile @@ -0,0 +1,46 @@ +ARG SOURCE_IMAGE +FROM ${SOURCE_IMAGE} + +# ----------- HDB++ CONFIGURATION MANAGER --------------- + +RUN git clone -b master https://github.com/tango-controls-hdbpp/hdbpp-cm.git + +RUN cd hdbpp-cm \ + && mkdir -p build \ + && cd build \ + && cmake .. -DCMAKE_PREFIX_PATH=/usr/local/include/tango \ + && make -j4 + +RUN cd hdbpp-cm/build \ + && checkinstall \ + --install=yes \ + --fstrans=no \ + --showinstall=no \ + --backup=no \ + --type=debian \ + --pkgsource="https://github.com/tango-controls-hdbpp/hdbpp-cm" \ + --pkglicense="GPLv3" \ + --deldesc=no \ + --nodoc \ + --strip \ + --stripso \ + --maintainer="tango" \ + --pkgarch=$(dpkg --print-architecture) \ + --pkgversion="2.0.0" \ + --pkgrelease="SNAPSHOT" \ + --pkgname="hdbpp-cm" \ + --requires="libzmq5,libomniorb4-2,libcos4-2,libomnithread4" \ + make install + +RUN apt-get update && \ + apt-get install -y \ + build-essential && \ + apt-get clean + +RUN dpkg -i /libhdbpp/build/libhdbpp_2.0.0-SNAPSHOT_amd64.deb +RUN dpkg -i /libhdbpp-timescale/build/libhdbpp-timescale_2.0.0-SNAPSHOT_amd64.deb +RUN dpkg -i /hdbpp-cm/build/hdbpp-cm_2.0.0-SNAPSHOT_amd64.deb + +RUN ldconfig + +RUN mv /usr/local/bin/hdb++cm-srv /usr/local/bin/hdbppcm-srv diff --git a/docker-compose/hdbppts-es/Dockerfile b/docker-compose/hdbppts-es/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..bd74e1b97e41a1d1ffa4be13891435e9abb6252a --- /dev/null +++ b/docker-compose/hdbppts-es/Dockerfile @@ -0,0 +1,46 @@ +ARG SOURCE_IMAGE +FROM ${SOURCE_IMAGE} + +# ----------- HDB++ EVENT SUBSCRIBER --------------- + +RUN git clone -b master https://github.com/tango-controls-hdbpp/hdbpp-es.git + +RUN cd hdbpp-es \ + && mkdir -p build \ + && cd build \ + && cmake .. -DCMAKE_PREFIX_PATH=/usr/local/include/tango -DFETCH_LIBHDBPP=OFF -DLIBHDBPP_BACKEND=timescale -DPostgreSQL_TYPE_INCLUDE_DIR=/usr/local/include/postgresql \ + && make -j4 + +RUN cd hdbpp-es/build \ + && checkinstall \ + --install=yes \ + --fstrans=no \ + --showinstall=no \ + --backup=no \ + --type=debian \ + --pkgsource="https://github.com/tango-controls-hdbpp/hdbpp-es" \ + --pkglicense="GPLv3" \ + --deldesc=no \ + --nodoc \ + --strip \ + --stripso \ + --maintainer="tango" \ + --pkgarch=$(dpkg --print-architecture) \ + --pkgversion="2.0.0" \ + --pkgrelease="SNAPSHOT" \ + --pkgname="hdbpp-es" \ + --requires="libzmq5,libomniorb4-2,libcos4-2,libomnithread4" \ + make install + +RUN apt-get update && \ + apt-get install -y \ + build-essential && \ + apt-get clean + +RUN dpkg -i /libhdbpp/build/libhdbpp_2.0.0-SNAPSHOT_amd64.deb +RUN dpkg -i /libhdbpp-timescale/build/libhdbpp-timescale_2.0.0-SNAPSHOT_amd64.deb +RUN dpkg -i /hdbpp-es/build/hdbpp-es_2.0.0-SNAPSHOT_amd64.deb + +RUN ldconfig + +RUN mv /usr/local/bin/hdb++es-srv /usr/local/bin/hdbppes-srv diff --git a/docker-compose/timescaledb/Dockerfile b/docker-compose/timescaledb/Dockerfile index 656d2ef661a13a5e84eadf7eda2ff6b27684d23c..73932cadd73c5bc0f2ac6d4b0d72d98bcb979d25 100644 --- a/docker-compose/timescaledb/Dockerfile +++ b/docker-compose/timescaledb/Dockerfile @@ -1,4 +1,5 @@ -FROM timescale/timescaledb:latest-pg12 +ARG SOURCE_IMAGE +FROM ${SOURCE_IMAGE} # Set admin role to perform DB creation COPY resources/01_admin.sh docker-entrypoint-initdb.d/002_admin.sh diff --git a/sbin/tag_and_push_docker_image.sh b/sbin/tag_and_push_docker_image.sh index dfba35a67ca1578918a6a5bb42e78aa9a656b967..3e10110ed1ac4dad5e0f6fb0521a314c0ae06fbe 100755 --- a/sbin/tag_and_push_docker_image.sh +++ b/sbin/tag_and_push_docker_image.sh @@ -82,8 +82,8 @@ LOCAL_IMAGES=( "itango docker-compose_itango y" - "archiver-timescale timescaledb n" "hdbppts-cm hdbppts-cm n" - "hdbppts-es hdbppts-es n" + "archiver-timescale timescaledb y" "hdbpp hdbpp y" "hdbppts-cm hdbppts-cm y" + "hdbppts-es hdbppts-es y" "grafana grafana n" "prometheus prometheus n" "jupyter docker-compose_jupyter n"