Merge branch 'L2SS-429-timescaledb-container' into L2SS-406-grafana-archiver

docker-compose/archiver-timescale.yml

+version: '2'
+
+services:
+  archiver-timescale:
+    image: timescaledb
+    build:
+        context: timescaledb
+    container_name: ${CONTAINER_NAME_PREFIX}archiver-timescale
+    networks:
+      - control
+    ports:
+      - "5432:5432/tcp"
+    depends_on:
+      - databaseds
+    environment:
+      - POSTGRES_PASSWORD=password
+      - TANGO_HOST=${TANGO_HOST}
+    logging:
+      driver: syslog
+      options:
+        syslog-address: udp://${LOG_HOSTNAME}:1514
+        syslog-format: rfc3164
+        tag: "{{.Name}}"
+    restart: unless-stopped
+  
+  hdbppts-cm:
+    image: hdbppts-cm
+    build:
+        context: ../docker/tango/tango-archiver-ts
+    networks:
+      - control
+    container_name: ${CONTAINER_NAME_PREFIX}hdbppts-cm
+    depends_on:
+      - databaseds
+      - dsconfig
+      - archiver-timescale
+    environment:
+      - TANGO_HOST=${TANGO_HOST}
+      - HdbManager=archiving/hdbppts/confmanager01
+    command: >
+      /bin/bash -c "
+      wait-for-it.sh archiver-timescale:5432 --timeout=30 --strict --
+      wait-for-it.sh ${TANGO_HOST} --timeout=30 --strict --
+            hdbppcm-srv 02"
+    logging:
+        driver: syslog
+        options:
+          syslog-address: udp://${LOG_HOSTNAME}:1514
+          syslog-format: rfc3164
+          tag: "{{.Name}}"
+  
+  hdbppts-es:
+    image: hdbppts-es
+    build:
+        context: ../docker/tango/tango-archiver-ts
+    networks:
+      - control
+    container_name: ${CONTAINER_NAME_PREFIX}hdbppts-es
+    depends_on:
+      - databaseds
+      - dsconfig
+      - archiver-timescale
+    environment:
+      - TANGO_HOST=${TANGO_HOST}
+      - HdbManager=archiving/hdbppts/confmanager01
+    command: >
+      /bin/bash -c "
+      wait-for-it.sh archiver-timescale:5432 --timeout=30 --strict --
+      wait-for-it.sh ${TANGO_HOST} --timeout=30 --strict --
+            hdbppes-srv 02"
+    logging:
+        driver: syslog
+        options:
+          syslog-address: udp://${LOG_HOSTNAME}:1514
+          syslog-format: rfc3164
+          tag: "{{.Name}}"
+    restart: unless-stopped

docker-compose/archiver.yml

@@ -24,7 +24,7 @@ services:
         syslog-format: rfc3164
         tag: "{{.Name}}"
     restart: unless-stopped
-
+  
   hdbpp-es:
       image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-archiver:2021-05-28
       networks:

docker-compose/timescaledb/Dockerfile

+FROM timescale/timescaledb:latest-pg12
+
+
+COPY resources/01_admin.sql docker-entrypoint-initdb.d/002_admin.sql
+COPY resources/02_hdb_schema.sql docker-entrypoint-initdb.d/003_hdb_schema.sql
+COPY resources/03_hdb_roles.sql docker-entrypoint-initdb.d/004_hdb_roles.sql
+COPY resources/04_hdb_ext_aggregates.sql docker-entrypoint-initdb.d/005_hdb_ext_aggregates.sql
+COPY resources/05_cleanup.sql docker-entrypoint-initdb.d/006_cleanup.sql

docker-compose/timescaledb/README.md

+# TimescaleDB Docker Image
+
+The Dockerfile in this directory allows to create a container with a PostrgreSQL-Timescale DBMS (https://www.timescale.com/), and then initialise it with the DB schema required by the Tango Archiving framework.
+
+The main image is pulled from the official PostgreSQL repository in the Docker Hub (https://hub.docker.com/_/postgres). This image offers several features to customize and extend itself.
+
+## Initialization scripts
+
+If you would like to do additional initialization in an image derived from the Postgres official one, add one or more *.sql, *.sql.gz, or *.sh scripts under /docker-entrypoint-initdb.d (creating the directory if necessary). After the entrypoint calls initdb to create the default postgres user and database, it will run any *.sql files, run any executable *.sh scripts, and source any non-executable *.sh scripts found in that directory to do further initialization before starting the service.
+
+The script files in the directory /docker-entrypoint-initdb.d are sequentially executed following their preempted number in the filename. Hence, the first ones (000_install_timescaledb.sh and 001_timescaledb_tune.sh) are provided by default. 
+The next ones have been pulled from the official Tango repository in order to create the desired DB schema. These files are in the 'resources' directory and they have been pulled from Tango-Hdbpp_Timescale_Project (https://github.com/tango-controls-hdbpp/hdbpp-timescale-project/tree/master/resources/schema):
+- admin.sql creates the admin user that will create the tables
+- hdb_schema.sql creates the standard Tango Archiving DB (This is the only MANDATORY script)
+- hdb_roles.sql creates additional roles
+- hdb_ext_aggregates.sql creates the continuous aggregate views (https://docs.timescale.com/timescaledb/latest/how-to-guides/continuous-aggregates/)
+- cleanup.sql strips the SUPERUSER trait from hdb_admin
+Last updates on these scripts are dated to August 2021 (more info can be found at https://github.com/tango-controls-hdbpp/hdbpp-timescale-project/blob/master/doc/db-schema-config.md)
+
+There is an older version of these Tango-Archiving generating scripts in the directory 'resources_v2'. These files have been pulled from Tango-LibHdbpp-Timescale (https://github.com/tango-controls-hdbpp/libhdbpp-timescale/tree/master/db-schema)
+
+
+

docker-compose/timescaledb/resources/01_admin.sql

+CREATE ROLE hdb_admin WITH LOGIN PASSWORD 'hdbpp';
+ALTER USER hdb_admin CREATEDB;
+ALTER USER hdb_admin CREATEROLE;
+ALTER USER hdb_admin SUPERUSER;

docker-compose/timescaledb/resources/03_hdb_roles.sql

+-- -----------------------------------------------------------------------------
+-- This file is part of the hdbpp-timescale-project
+--
+-- Copyright (C) : 2014-2019
+--   European Synchrotron Radiation Facility
+--   BP 220, Grenoble 38043, FRANCE
+--
+-- libhdb++timescale is free software: you can redistribute it and/or modify
+-- it under the terms of the Lesser GNU General Public License as published by
+-- the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- libhdb++timescale is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the Lesser
+-- GNU General Public License for more details.
+--
+-- You should have received a copy of the Lesser GNU General Public License
+-- along with libhdb++timescale.  If not, see <http://www.gnu.org/licenses/>.
+-- -----------------------------------------------------------------------------
+
+-- Setup roles to access the hdb database
+CREATE ROLE readonly;
+CREATE ROLE readwrite;
+
+-- Permissions - readonly
+GRANT CONNECT ON DATABASE hdb TO readonly;
+GRANT USAGE ON SCHEMA public TO readonly;
+GRANT SELECT ON ALL TABLES IN SCHEMA public TO readonly;
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO readonly;
+
+-- Permissions - readwrite
+GRANT CONNECT ON DATABASE hdb TO readwrite;
+GRANT USAGE ON SCHEMA public TO readwrite;
+GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO readwrite;
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO readwrite;
+GRANT USAGE ON ALL SEQUENCES IN SCHEMA public TO readwrite;
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT USAGE ON SEQUENCES TO readwrite;
+GRANT ALL ON SCHEMA public TO readwrite;
+GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO readwrite;
+GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO readwrite;

docker-compose/timescaledb/resources/05_cleanup.sql

+ALTER USER hdb_admin NOSUPERUSER;

docker-compose/timescaledb/resources_v2/cluster.sql

+ALTER TABLE att_scalar_devboolean CLUSTER ON att_scalar_devboolean_att_conf_id_data_time_idx;
+ALTER TABLE att_array_devboolean CLUSTER ON att_array_devboolean_att_conf_id_data_time_idx;
+ALTER TABLE att_scalar_devuchar CLUSTER ON att_scalar_devuchar_att_conf_id_data_time_idx;
+ALTER TABLE att_array_devuchar CLUSTER ON att_array_devuchar_att_conf_id_data_time_idx;
+ALTER TABLE att_scalar_devshort CLUSTER ON att_scalar_devshort_att_conf_id_data_time_idx;
+ALTER TABLE att_array_devshort CLUSTER ON att_array_devshort_att_conf_id_data_time_idx;
+ALTER TABLE att_scalar_devushort CLUSTER ON att_scalar_devushort_att_conf_id_data_time_idx;
+ALTER TABLE att_array_devushort CLUSTER ON att_array_devushort_att_conf_id_data_time_idx;
+ALTER TABLE att_scalar_devlong CLUSTER ON att_scalar_devlong_att_conf_id_data_time_idx;
+ALTER TABLE att_array_devlong CLUSTER ON att_array_devlong_att_conf_id_data_time_idx;
+ALTER TABLE att_scalar_devulong CLUSTER ON att_scalar_devulong_att_conf_id_data_time_idx;
+ALTER TABLE att_array_devulong CLUSTER ON att_array_devulong_att_conf_id_data_time_idx;
+ALTER TABLE att_scalar_devlong64 CLUSTER ON att_scalar_devlong64_att_conf_id_data_time_idx;
+ALTER TABLE att_array_devlong64 CLUSTER ON att_array_devlong64_att_conf_id_data_time_idx;
+ALTER TABLE att_scalar_devulong64 CLUSTER ON att_scalar_devulong64_att_conf_id_data_time_idx;
+ALTER TABLE att_array_devulong64 CLUSTER ON att_array_devulong64_att_conf_id_data_time_idx;
+ALTER TABLE att_scalar_devfloat CLUSTER ON att_scalar_devfloat_att_conf_id_data_time_idx;
+ALTER TABLE att_array_devfloat CLUSTER ON att_array_devfloat_att_conf_id_data_time_idx;
+ALTER TABLE att_scalar_devdouble CLUSTER ON att_scalar_devdouble_att_conf_id_data_time_idx;
+ALTER TABLE att_array_devdouble CLUSTER ON att_array_devdouble_att_conf_id_data_time_idx;
+ALTER TABLE att_scalar_devstring CLUSTER ON att_scalar_devstring_att_conf_id_data_time_idx;
+ALTER TABLE att_array_devstring CLUSTER ON att_array_devstring_att_conf_id_data_time_idx;
+ALTER TABLE att_scalar_devstate CLUSTER ON att_scalar_devstate_att_conf_id_data_time_idx;
+ALTER TABLE att_array_devstate CLUSTER ON att_array_devstate_att_conf_id_data_time_idx;
+ALTER TABLE att_scalar_devencoded CLUSTER ON att_scalar_devencoded_att_conf_id_data_time_idx;
+ALTER TABLE att_array_devencoded CLUSTER ON att_array_devencoded_att_conf_id_data_time_idx;
+ALTER TABLE att_scalar_devenum CLUSTER ON att_scalar_devenum_att_conf_id_data_time_idx;
+ALTER TABLE att_array_devenum CLUSTER ON att_array_devenum_att_conf_id_data_time_idx;
+
+CLUSTER att_scalar_devboolean;
+CLUSTER att_array_devboolean;
+CLUSTER att_scalar_devuchar;
+CLUSTER att_array_devuchar;
+CLUSTER att_scalar_devshort;
+CLUSTER att_array_devshort;
+CLUSTER att_scalar_devushort;
+CLUSTER att_array_devushort;
+CLUSTER att_scalar_devlong;
+CLUSTER att_array_devlong;
+CLUSTER att_scalar_devulong;
+CLUSTER att_array_devulong;
+CLUSTER att_scalar_devlong64;
+CLUSTER att_array_devlong64;
+CLUSTER att_scalar_devulong64;
+CLUSTER att_array_devulong64;
+CLUSTER att_scalar_devfloat;
+CLUSTER att_array_devfloat;
+CLUSTER att_scalar_devdouble;
+CLUSTER att_array_devdouble;
+CLUSTER att_scalar_devstring;
+CLUSTER att_array_devstring;
+CLUSTER att_scalar_devstate;
+CLUSTER att_array_devstate;
+CLUSTER att_scalar_devencoded;
+CLUSTER att_array_devencoded;
+CLUSTER att_scalar_devenum;
+CLUSTER att_array_devenum;

docker-compose/timescaledb/resources_v2/users.sql

+-- Roles
+CREATE ROLE readonly;
+CREATE ROLE readwrite;
+
+-- Permissions - readonly
+GRANT CONNECT ON DATABASE hdb TO readonly;
+GRANT USAGE ON SCHEMA public TO readonly;
+GRANT SELECT ON ALL TABLES IN SCHEMA public TO readonly;
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO readonly;
+
+-- Permissions - readwrite
+GRANT CONNECT ON DATABASE hdb TO readwrite;
+GRANT USAGE ON SCHEMA public TO readwrite;
+GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO readwrite;
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO readwrite;
+GRANT USAGE ON ALL SEQUENCES IN SCHEMA public TO readwrite;
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT USAGE ON SEQUENCES TO readwrite;
+GRANT ALL ON SCHEMA public TO readwrite;
+GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO readwrite;
+GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO readwrite;
+
+-- Users
+CREATE ROLE hdb_cfg_man WITH LOGIN PASSWORD 'hdbpp';
+GRANT readwrite TO hdb_cfg_man;
+
+CREATE ROLE hdb_event_sub WITH LOGIN PASSWORD 'hdbpp';
+GRANT readwrite TO hdb_event_sub;
+
+CREATE ROLE hdb_java_reporter WITH LOGIN PASSWORD 'hdbpp';
+GRANT readonly TO hdb_java_reporter;

docker/tango/tango-archiver-ts/Dockerfile

+#ARG DOCKER_REGISTRY_USER
+#ARG DOCKER_REGISTRY_HOST
+#FROM ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}/tango-cpp:latest
+FROM artefact.skao.int/ska-tango-images-tango-cpp:9.3.9
+
+USER root
+
+RUN apt-get update && \
+    apt-get install -y ca-certificates
+
+RUN echo "deb http://deb.debian.org/debian buster-backports main contrib non-free" >> /etc/apt/sources.list && \
+    more /etc/apt/sources.list && \
+    apt-get update && \
+    apt-get install -y \
+        checkinstall \
+        git \
+        cmake \
+        make \
+        g++ \
+        libomniorb4-dev \
+        libzmq3-dev \
+        libcos4-dev \
+        mariadb-server \
+        libmariadb-dev-compat libmariadb-dev \
+        libmariadbclient-dev \
+        postgresql \
+        postgresql-contrib \
+        libpq5 \
+        libpqxx-6.2 \
+        libpq-dev \
+        libpqxx-dev       
+        
+RUN git clone -b v2.0.0 https://github.com/Diego91RA/libhdbpp.git
+
+RUN cd libhdbpp \
+ && mkdir build \
+ && cd build \
+ && cmake .. -DCMAKE_INCLUDE_PATH=/usr/local/include/tango \
+ && make -j4
+
+RUN cd libhdbpp/build \
+ && checkinstall \
+    --install=yes \
+    --fstrans=no \
+    --showinstall=no \
+    --backup=no \
+    --type=debian \
+    --pkgsource="https://github.com/tango-controls-hdbpp/libhdbpp" \
+    --pkglicense="LGPLv3" \
+    --deldesc=no \
+    --nodoc \
+    --strip \
+    --stripso \
+    --maintainer="tango" \
+    --pkgarch=$(dpkg --print-architecture) \
+    --pkgversion="2.0.0" \
+    --pkgrelease="SNAPSHOT" \
+    --pkgname="libhdbpp" \
+    --requires="libzmq5,libomniorb4-2,libcos4-2,libomnithread4" \
+    make install
+
+RUN git clone -b master --recurse-submodules https://github.com/Diego91RA/libhdbpp-timescale.git
+RUN cd libhdbpp-timescale && git checkout 85c03657d4518e876869b322d6a50c46875a1300
+
+RUN cd libhdbpp-timescale \
+ && mkdir -p build \
+ && cd build \
+ && cmake .. -DCMAKE_PREFIX_PATH=/usr/local/include/tango -DPostgreSQL_TYPE_INCLUDE_DIR=/usr/local/include/postgresql \
+ && make -j4
+ 
+RUN cd libhdbpp-timescale/build \
+ && checkinstall \
+    --install=yes \
+    --fstrans=no \
+    --showinstall=no \
+    --backup=no \
+    --type=debian \
+    --pkgsource="https://github.com/tango-controls-hdbpp/libhdbpp-timescale" \
+    --pkglicense="LGPLv3" \
+    --deldesc=no \
+    --nodoc \
+    --strip \
+    --stripso \
+    --maintainer="tango" \
+    --pkgarch=$(dpkg --print-architecture) \
+    --pkgversion="2.0.0" \
+    --pkgrelease="SNAPSHOT" \
+    --pkgname="libhdbpp-timescale" \
+    --requires="libpq5" \
+    make install  
+
+RUN git clone -b v2.0.0 https://github.com/Diego91RA/hdbpp-cm.git
+
+RUN cd hdbpp-cm \
+ && mkdir -p build \
+ && cd build \
+ && cmake .. -DCMAKE_PREFIX_PATH=/usr/local/include/tango \
+ && make -j4
+
+RUN cd hdbpp-cm/build \
+ && checkinstall \
+    --install=yes \
+    --fstrans=no \
+    --showinstall=no \
+    --backup=no \
+    --type=debian \
+    --pkgsource="https://github.com/tango-controls-hdbpp/hdbpp-cm" \
+    --pkglicense="GPLv3" \
+    --deldesc=no \
+    --nodoc \
+    --strip \
+    --stripso \
+    --maintainer="tango" \
+    --pkgarch=$(dpkg --print-architecture) \
+    --pkgversion="2.0.0" \
+    --pkgrelease="SNAPSHOT" \
+    --pkgname="hdbpp-cm" \
+    --requires="libzmq5,libomniorb4-2,libcos4-2,libomnithread4" \
+    make install
+
+RUN git clone -b master https://github.com/Diego91RA/hdbpp-es.git
+RUN cd hdbpp-es && git checkout 473cbfbd7af76851bbf6ca2fcf5e4880e9f8e437
+
+RUN cd hdbpp-es \
+ && mkdir -p build \
+ && cd build \
+ && cmake .. -DCMAKE_PREFIX_PATH=/usr/local/include/tango -DFETCH_LIBHDBPP=OFF -DLIBHDBPP_BACKEND=timescale -DPostgreSQL_TYPE_INCLUDE_DIR=/usr/local/include/postgresql \
+ && make -j4 
+ 
+RUN cd hdbpp-es/build \
+ && checkinstall \
+    --install=yes \
+    --fstrans=no \
+    --showinstall=no \
+    --backup=no \
+    --type=debian \
+    --pkgsource="https://github.com/tango-controls-hdbpp/hdbpp-es" \
+    --pkglicense="GPLv3" \
+    --deldesc=no \
+    --nodoc \
+    --strip \
+    --stripso \
+    --maintainer="tango" \
+    --pkgarch=$(dpkg --print-architecture) \
+    --pkgversion="2.0.0" \
+    --pkgrelease="SNAPSHOT" \
+    --pkgname="hdbpp-es" \
+    --requires="libzmq5,libomniorb4-2,libcos4-2,libomnithread4" \
+    make install
+
+RUN apt-get update && \
+    apt-get install -y \
+    build-essential && \
+    apt-get clean
+
+RUN dpkg -i /libhdbpp/build/libhdbpp_2.0.0-SNAPSHOT_amd64.deb
+RUN dpkg -i /libhdbpp-timescale/build/libhdbpp-timescale_2.0.0-SNAPSHOT_amd64.deb
+RUN dpkg -i /hdbpp-cm/build/hdbpp-cm_2.0.0-SNAPSHOT_amd64.deb
+RUN dpkg -i /hdbpp-es/build/hdbpp-es_2.0.0-SNAPSHOT_amd64.deb
+
+RUN ldconfig
+
+RUN mv /usr/local/bin/hdb++cm-srv /usr/local/bin/hdbppcm-srv
+RUN mv /usr/local/bin/hdb++es-srv /usr/local/bin/hdbppes-srv

docker/tango/tango-archiver/data/archiver-devices.json

@@ -13,6 +13,19 @@
                         }
                     }
                 }
+            },
+            "02": {
+                "HdbEventSubscriber": {
+                    "archiving/hdbppts/eventsubscriber01": {
+                        "attribute_properties": {},
+                        "properties": {
+                            "CheckPeriodicTimeoutDelay": ["5"],
+                            "PollingThreadPeriod": ["3"],
+                            "LibConfiguration": ["connect_string= user=postgres password=password host=archiver-timescale port=5432 dbname=hdb","host=archiver-timescale","libname=libhdb++timescale.so","dbname=hdb","port=5432", "user=postgres", "password=password"],
+                            "polled_attr": []
+                        }
+                    }
+                }
             }
         },
         "hdbppcm-srv": {
@@ -28,6 +41,19 @@
                         }
                     }
                 }
+            },
+            "02": {
+                "HdbConfigurationManager": {
+                    "archiving/hdbppts/confmanager01": {
+                        "attribute_properties": {},
+                        "properties": {
+                            "ArchiverList": ["archiving/hdbppts/eventsubscriber01"],
+                            "MaxSearchSize": ["1000"],
+                            "LibConfiguration": ["connect_string= user=postgres password=password host=archiver-timescale port=5432 dbname=hdb","host=archiver-timescale","libname=libhdb++timescale.so","dbname=hdb","port=5432", "user=postgres", "password=password"],
+                            "polled_attr": []
+                        }
+                    }
+                }
             }
         }
     }