L2SS-647: add timescaledb extensions and bug fix

docker-compose/timescaledb/Dockerfile

 FROM timescale/timescaledb:latest-pg12
 
-
+# Set admin role to perform DB creation
 COPY resources/01_admin.sh docker-entrypoint-initdb.d/002_admin.sh
+# Create DB schema (tables, indexes, etc.)
 COPY resources/02_hdb_schema.sql docker-entrypoint-initdb.d/003_hdb_schema.sql
+# Create DB roles
 COPY resources/03_hdb_roles.sql docker-entrypoint-initdb.d/004_hdb_roles.sql
-COPY resources/04_hdb_ext_aggregates.sql docker-entrypoint-initdb.d/005_hdb_ext_aggregates.sql
-COPY resources/05_lofar_func.sh docker-entrypoint-initdb.d/006_lofar_func.sh
-COPY resources/06_lofar_views.sql docker-entrypoint-initdb.d/007_lofar_views.sql
-COPY resources/07_cleanup.sql docker-entrypoint-initdb.d/008_cleanup.sql
+# Create further roles
+COPY resources/04_hdb_ext_users.sql docker-entrypoint-initdb.d/005_hdb_ext_users.sql
+# Create timescaledb aggregates
+COPY resources/05_hdb_ext_aggregates.sql docker-entrypoint-initdb.d/006_hdb_ext_aggregates.sql
+COPY resources/06_hdb_ext_arrays_aggregates_helper.sql docker-entrypoint-initdb.d/007_hdb_ext_arrays_aggregates_helper.sql
+COPY resources/07_hdb_ext_arrays_aggregates.sql docker-entrypoint-initdb.d/008_hdb_ext_arrays_aggregates.sql
+# Add compress policy
+COPY resources/08_hdb_ext_compress_policy.sql docker-entrypoint-initdb.d/009_hdb_ext_compress_policy.sql
+# Add further functions
+COPY resources/09_hdb_ext_import.sql docker-entrypoint-initdb.d/010_hdb_ext_import.sql
+# Add reorder policy
+COPY resources/10_hdb_ext_reorder_policy.sql docker-entrypoint-initdb.d/011_hdb_ext_reorder_policy.sql
+# Add LOFAR functions and views
+COPY resources/11_lofar_func.sh docker-entrypoint-initdb.d/012_lofar_func.sh
+COPY resources/12_lofar_views.sql docker-entrypoint-initdb.d/013_lofar_views.sql
+# Cleanup admin role
+COPY resources/13_cleanup.sql docker-entrypoint-initdb.d/014_cleanup.sql

docker-compose/timescaledb/resources/04_hdb_ext_users.sql

+-- -----------------------------------------------------------------------------
+-- This file is part of the hdbpp-timescale-project
+--
+-- Copyright (C) : 2014-2019
+--   European Synchrotron Radiation Facility
+--   BP 220, Grenoble 38043, FRANCE
+--
+-- libhdb++timescale is free software: you can redistribute it and/or modify
+-- it under the terms of the Lesser GNU General Public License as published by
+-- the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- libhdb++timescale is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the Lesser
+-- GNU General Public License for more details.
+--
+-- You should have received a copy of the Lesser GNU General Public License
+-- along with libhdb++timescale.  If not, see <http://www.gnu.org/licenses/>.
+-- -----------------------------------------------------------------------------
+
+\c hdb
+
+-- Some useful users for a basic system
+CREATE ROLE hdb_cfg_man WITH LOGIN PASSWORD 'hdbpp';
+GRANT readwrite TO hdb_cfg_man;
+
+CREATE ROLE hdb_event_sub WITH LOGIN PASSWORD 'hdbpp';
+GRANT readwrite TO hdb_event_sub;
+
+CREATE ROLE hdb_data_reporter WITH LOGIN PASSWORD 'hdbpp';
+GRANT readonly TO hdb_data_reporter;

docker-compose/timescaledb/resources/04_hdb_ext_aggregates.sql → docker-compose/timescaledb/resources/05_hdb_ext_aggregates.sql

@@ -1290,3 +1290,4 @@ GRANT SELECT ON cagg_scalar_devushort_1day TO readonly;
 -- DROP VIEW cagg_scalar_devushort_1hour CASCADE;
 -- DROP VIEW cagg_scalar_devushort_8hour CASCADE;
 -- DROP VIEW cagg_scalar_devushort_1day CASCADE;
+

docker-compose/timescaledb/resources/08_hdb_ext_compress_policy.sql

+-- -----------------------------------------------------------------------------
+-- This file is part of the hdbpp-timescale-project
+--
+-- Copyright (C) : 2014-2019
+--   European Synchrotron Radiation Facility
+--   BP 220, Grenoble 38043, FRANCE
+--
+-- libhdb++timescale is free software: you can redistribute it and/or modify
+-- it under the terms of the Lesser GNU General Public License as published by
+-- the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- libhdb++timescale is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the Lesser
+-- GNU General Public License for more details.
+--
+-- You should have received a copy of the Lesser GNU General Public License
+-- along with libhdb++timescale.  If not, see <http://www.gnu.org/licenses/>.
+-- -----------------------------------------------------------------------------
+\c hdb
+-- Compress chunk policy
+-- Allow compression on the table
+ALTER TABLE att_scalar_devboolean SET(timescaledb.compress, timescaledb.compress_segmentby = 'att_conf_id, att_error_desc_id', timescaledb.compress_orderby = 'data_time DESC');
+ALTER TABLE att_scalar_devdouble SET(timescaledb.compress, timescaledb.compress_segmentby = 'att_conf_id, att_error_desc_id', timescaledb.compress_orderby = 'data_time DESC');
+ALTER TABLE att_scalar_devfloat SET(timescaledb.compress, timescaledb.compress_segmentby = 'att_conf_id, att_error_desc_id', timescaledb.compress_orderby = 'data_time DESC');
+ALTER TABLE att_scalar_devencoded SET(timescaledb.compress, timescaledb.compress_segmentby = 'att_conf_id, att_error_desc_id', timescaledb.compress_orderby = 'data_time DESC');
+ALTER TABLE att_scalar_devenum SET(timescaledb.compress, timescaledb.compress_segmentby = 'att_conf_id, att_error_desc_id', timescaledb.compress_orderby = 'data_time DESC');
+ALTER TABLE att_scalar_devstate SET(timescaledb.compress, timescaledb.compress_segmentby = 'att_conf_id, att_error_desc_id', timescaledb.compress_orderby = 'data_time DESC');
+ALTER TABLE att_scalar_devstring SET(timescaledb.compress, timescaledb.compress_segmentby = 'att_conf_id, att_error_desc_id', timescaledb.compress_orderby = 'data_time DESC');
+ALTER TABLE att_scalar_devuchar SET(timescaledb.compress, timescaledb.compress_segmentby = 'att_conf_id, att_error_desc_id', timescaledb.compress_orderby = 'data_time DESC');
+ALTER TABLE att_scalar_devulong SET(timescaledb.compress, timescaledb.compress_segmentby = 'att_conf_id, att_error_desc_id', timescaledb.compress_orderby = 'data_time DESC');
+ALTER TABLE att_scalar_devulong64 SET(timescaledb.compress, timescaledb.compress_segmentby = 'att_conf_id, att_error_desc_id', timescaledb.compress_orderby = 'data_time DESC');
+ALTER TABLE att_scalar_devlong64 SET(timescaledb.compress, timescaledb.compress_segmentby = 'att_conf_id, att_error_desc_id', timescaledb.compress_orderby = 'data_time DESC');
+ALTER TABLE att_scalar_devlong SET(timescaledb.compress, timescaledb.compress_segmentby = 'att_conf_id, att_error_desc_id', timescaledb.compress_orderby = 'data_time DESC');
+ALTER TABLE att_scalar_devushort SET(timescaledb.compress, timescaledb.compress_segmentby = 'att_conf_id, att_error_desc_id', timescaledb.compress_orderby = 'data_time DESC');
+ALTER TABLE att_scalar_devshort SET(timescaledb.compress, timescaledb.compress_segmentby = 'att_conf_id, att_error_desc_id', timescaledb.compress_orderby = 'data_time DESC');
+
+ALTER TABLE att_array_devboolean SET(timescaledb.compress, timescaledb.compress_segmentby = 'att_conf_id, att_error_desc_id', timescaledb.compress_orderby = 'data_time DESC');
+ALTER TABLE att_array_devdouble SET(timescaledb.compress, timescaledb.compress_segmentby = 'att_conf_id, att_error_desc_id', timescaledb.compress_orderby = 'data_time DESC');
+ALTER TABLE att_array_devfloat SET(timescaledb.compress, timescaledb.compress_segmentby = 'att_conf_id, att_error_desc_id', timescaledb.compress_orderby = 'data_time DESC');
+ALTER TABLE att_array_devencoded SET(timescaledb.compress, timescaledb.compress_segmentby = 'att_conf_id, att_error_desc_id', timescaledb.compress_orderby = 'data_time DESC');
+ALTER TABLE att_array_devenum SET(timescaledb.compress, timescaledb.compress_segmentby = 'att_conf_id, att_error_desc_id', timescaledb.compress_orderby = 'data_time DESC');
+ALTER TABLE att_array_devstate SET(timescaledb.compress, timescaledb.compress_segmentby = 'att_conf_id, att_error_desc_id', timescaledb.compress_orderby = 'data_time DESC');
+ALTER TABLE att_array_devstring SET(timescaledb.compress, timescaledb.compress_segmentby = 'att_conf_id, att_error_desc_id', timescaledb.compress_orderby = 'data_time DESC');
+ALTER TABLE att_array_devuchar SET(timescaledb.compress, timescaledb.compress_segmentby = 'att_conf_id, att_error_desc_id', timescaledb.compress_orderby = 'data_time DESC');
+ALTER TABLE att_array_devulong SET(timescaledb.compress, timescaledb.compress_segmentby = 'att_conf_id, att_error_desc_id', timescaledb.compress_orderby = 'data_time DESC');
+ALTER TABLE att_array_devulong64 SET(timescaledb.compress, timescaledb.compress_segmentby = 'att_conf_id, att_error_desc_id', timescaledb.compress_orderby = 'data_time DESC');
+ALTER TABLE att_array_devlong64 SET(timescaledb.compress, timescaledb.compress_segmentby = 'att_conf_id, att_error_desc_id', timescaledb.compress_orderby = 'data_time DESC');
+ALTER TABLE att_array_devlong SET(timescaledb.compress, timescaledb.compress_segmentby = 'att_conf_id, att_error_desc_id', timescaledb.compress_orderby = 'data_time DESC');
+ALTER TABLE att_array_devushort SET(timescaledb.compress, timescaledb.compress_segmentby = 'att_conf_id, att_error_desc_id', timescaledb.compress_orderby = 'data_time DESC');
+ALTER TABLE att_array_devshort SET(timescaledb.compress, timescaledb.compress_segmentby = 'att_conf_id, att_error_desc_id', timescaledb.compress_orderby = 'data_time DESC');
+
+DO $$ BEGIN
+    IF (SELECT extversion>'2.0.0' FROM pg_extension where extname = 'timescaledb') THEN
+        -- If using timescaledb v2
+        PERFORM add_compression_policy('att_scalar_devboolean', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compression_policy('att_scalar_devdouble', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compression_policy('att_scalar_devfloat', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compression_policy('att_scalar_devencoded', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compression_policy('att_scalar_devenum', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compression_policy('att_scalar_devstate', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compression_policy('att_scalar_devstring', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compression_policy('att_scalar_devuchar', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compression_policy('att_scalar_devulong', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compression_policy('att_scalar_devulong64', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compression_policy('att_scalar_devlong64', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compression_policy('att_scalar_devlong', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compression_policy('att_scalar_devushort', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compression_policy('att_scalar_devshort', INTERVAL '200d', if_not_exists => true);
+
+        PERFORM add_compression_policy('att_array_devboolean', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compression_policy('att_array_devdouble', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compression_policy('att_array_devfloat', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compression_policy('att_array_devencoded', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compression_policy('att_array_devenum', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compression_policy('att_array_devstate', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compression_policy('att_array_devstring', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compression_policy('att_array_devuchar', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compression_policy('att_array_devulong', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compression_policy('att_array_devulong64', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compression_policy('att_array_devlong64', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compression_policy('att_array_devlong', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compression_policy('att_array_devushort', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compression_policy('att_array_devshort', INTERVAL '200d', if_not_exists => true);
+    ELSE
+        -- If using timescaledb v1.7
+        PERFORM add_compress_chunks_policy('att_scalar_devboolean', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compress_chunks_policy('att_scalar_devdouble', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compress_chunks_policy('att_scalar_devfloat', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compress_chunks_policy('att_scalar_devencoded', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compress_chunks_policy('att_scalar_devenum', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compress_chunks_policy('att_scalar_devstate', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compress_chunks_policy('att_scalar_devstring', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compress_chunks_policy('att_scalar_devuchar', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compress_chunks_policy('att_scalar_devulong', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compress_chunks_policy('att_scalar_devulong64', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compress_chunks_policy('att_scalar_devlong64', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compress_chunks_policy('att_scalar_devlong', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compress_chunks_policy('att_scalar_devushort', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compress_chunks_policy('att_scalar_devshort', INTERVAL '200d', if_not_exists => true);
+
+        PERFORM add_compress_chunks_policy('att_array_devboolean', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compress_chunks_policy('att_array_devdouble', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compress_chunks_policy('att_array_devfloat', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compress_chunks_policy('att_array_devencoded', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compress_chunks_policy('att_array_devenum', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compress_chunks_policy('att_array_devstate', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compress_chunks_policy('att_array_devstring', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compress_chunks_policy('att_array_devuchar', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compress_chunks_policy('att_array_devulong', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compress_chunks_policy('att_array_devulong64', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compress_chunks_policy('att_array_devlong64', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compress_chunks_policy('att_array_devlong', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compress_chunks_policy('att_array_devushort', INTERVAL '200d', if_not_exists => true);
+        PERFORM add_compress_chunks_policy('att_array_devshort', INTERVAL '200d', if_not_exists => true);
+    END IF;
+END $$;

docker-compose/timescaledb/resources/09_hdb_ext_import.sql

+\c hdb
+
+CREATE OR REPLACE FUNCTION expand_name() RETURNS TRIGGER AS $$
+DECLARE
+    len integer;
+BEGIN
+    IF (NEW.cs_name <> '' AND NEW.domain <> '' AND NEW.family <> '' AND NEW.member <> '' AND NEW.name <> '') IS NOT TRUE THEN
+        len = (SELECT cardinality((SELECT regexp_split_to_array(NEW.att_name, E'/'))));
+        NEW.name := (SELECT split_part(NEW.att_name, '/', len));
+        NEW.member := (SELECT split_part(NEW.att_name, '/', len - 1));
+        NEW.family := (SELECT split_part(NEW.att_name, '/', len - 2));
+        NEW.domain := (SELECT split_part(NEW.att_name, '/', len - 3));
+        NEW.cs_name := (SELECT split_part(NEW.att_name, '/', len - 4));
+    END IF;
+    RETURN NEW;
+END
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER expand_name_trigger BEFORE INSERT ON att_conf FOR EACH ROW EXECUTE PROCEDURE expand_name();

docker-compose/timescaledb/resources/10_hdb_ext_reorder_policy.sql

+-- -----------------------------------------------------------------------------
+-- This file is part of the hdbpp-timescale-project
+--
+-- Copyright (C) : 2014-2019
+--   European Synchrotron Radiation Facility
+--   BP 220, Grenoble 38043, FRANCE
+--
+-- libhdb++timescale is free software: you can redistribute it and/or modify
+-- it under the terms of the Lesser GNU General Public License as published by
+-- the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- libhdb++timescale is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the Lesser
+-- GNU General Public License for more details.
+--
+-- You should have received a copy of the Lesser GNU General Public License
+-- along with libhdb++timescale.  If not, see <http://www.gnu.org/licenses/>.
+-- -----------------------------------------------------------------------------
+
+\c hdb
+-- Reorder chunk policy
+
+SELECT add_reorder_policy('att_scalar_devboolean', 'att_scalar_devboolean_att_conf_id_data_time_idx', if_not_exists => true);
+SELECT add_reorder_policy('att_scalar_devdouble', 'att_scalar_devdouble_att_conf_id_data_time_idx', if_not_exists => true);
+SELECT add_reorder_policy('att_scalar_devfloat', 'att_scalar_devfloat_att_conf_id_data_time_idx', if_not_exists => true);
+SELECT add_reorder_policy('att_scalar_devencoded', 'att_scalar_devencoded_att_conf_id_data_time_idx', if_not_exists => true);
+SELECT add_reorder_policy('att_scalar_devenum', 'att_scalar_devenum_att_conf_id_data_time_idx', if_not_exists => true);
+SELECT add_reorder_policy('att_scalar_devstate', 'att_scalar_devstate_att_conf_id_data_time_idx', if_not_exists => true);
+SELECT add_reorder_policy('att_scalar_devstring', 'att_scalar_devstring_att_conf_id_data_time_idx', if_not_exists => true);
+SELECT add_reorder_policy('att_scalar_devuchar', 'att_scalar_devuchar_att_conf_id_data_time_idx', if_not_exists => true);
+SELECT add_reorder_policy('att_scalar_devulong', 'att_scalar_devulong_att_conf_id_data_time_idx', if_not_exists => true);
+SELECT add_reorder_policy('att_scalar_devulong64', 'att_scalar_devulong64_att_conf_id_data_time_idx', if_not_exists => true);
+SELECT add_reorder_policy('att_scalar_devlong64', 'att_scalar_devlong64_att_conf_id_data_time_idx', if_not_exists => true);
+SELECT add_reorder_policy('att_scalar_devlong', 'att_scalar_devlong_att_conf_id_data_time_idx', if_not_exists => true);
+SELECT add_reorder_policy('att_scalar_devushort', 'att_scalar_devushort_att_conf_id_data_time_idx', if_not_exists => true);
+SELECT add_reorder_policy('att_scalar_devshort', 'att_scalar_devshort_att_conf_id_data_time_idx', if_not_exists => true);
+
+SELECT add_reorder_policy('att_array_devboolean', 'att_array_devboolean_att_conf_id_data_time_idx', if_not_exists => true);
+SELECT add_reorder_policy('att_array_devdouble', 'att_array_devdouble_att_conf_id_data_time_idx', if_not_exists => true);
+SELECT add_reorder_policy('att_array_devfloat', 'att_array_devfloat_att_conf_id_data_time_idx', if_not_exists => true);
+SELECT add_reorder_policy('att_array_devencoded', 'att_array_devencoded_att_conf_id_data_time_idx', if_not_exists => true);
+SELECT add_reorder_policy('att_array_devenum', 'att_array_devenum_att_conf_id_data_time_idx', if_not_exists => true);
+SELECT add_reorder_policy('att_array_devstate', 'att_array_devstate_att_conf_id_data_time_idx', if_not_exists => true);
+SELECT add_reorder_policy('att_array_devstring', 'att_array_devstring_att_conf_id_data_time_idx', if_not_exists => true);
+SELECT add_reorder_policy('att_array_devuchar', 'att_array_devuchar_att_conf_id_data_time_idx', if_not_exists => true);
+SELECT add_reorder_policy('att_array_devulong', 'att_array_devulong_att_conf_id_data_time_idx', if_not_exists => true);
+SELECT add_reorder_policy('att_array_devulong64', 'att_array_devulong64_att_conf_id_data_time_idx', if_not_exists => true);
+SELECT add_reorder_policy('att_array_devlong64', 'att_array_devlong64_att_conf_id_data_time_idx', if_not_exists => true);
+SELECT add_reorder_policy('att_array_devlong', 'att_array_devlong_att_conf_id_data_time_idx', if_not_exists => true);
+SELECT add_reorder_policy('att_array_devushort', 'att_array_devushort_att_conf_id_data_time_idx', if_not_exists => true);
+SELECT add_reorder_policy('att_array_devshort', 'att_array_devshort_att_conf_id_data_time_idx', if_not_exists => true);