From 4f67bcc9fdc6059185c60b8f373a0d8d1cd1549a Mon Sep 17 00:00:00 2001
From: Jan David Mol <mol@astron.nl>
Date: Wed, 23 Mar 2022 17:07:56 +0100
Subject: [PATCH] L2SS-691: Added lofar query support for images

---
 .../timescaledb/resources/10_lofar_views.sh   | 427 ++++++++++++++++++
 1 file changed, 427 insertions(+)
 create mode 100644 docker-compose/timescaledb/resources/10_lofar_views.sh

diff --git a/docker-compose/timescaledb/resources/10_lofar_views.sh b/docker-compose/timescaledb/resources/10_lofar_views.sh
new file mode 100644
index 000000000..7d7fb8cec
--- /dev/null
+++ b/docker-compose/timescaledb/resources/10_lofar_views.sh
@@ -0,0 +1,427 @@
+#!/bin/bash
+psql << EOF
+\c hdb
+
+-- NOTE: We concatenate domain/family/member here, which means we can't index
+--       the resulting column. However, queries also supply the attribute name,
+--       which we can index. The scan on the device name is then limited to
+--        entries which have the same attribute name across devices.
+
+CREATE OR REPLACE VIEW lofar_scalar_double AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    0 AS x,
+    value_r as value
+  FROM att_scalar_devdouble att
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
+CREATE OR REPLACE VIEW lofar_array_devboolean AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    array_element.idx - 1 AS x,
+    array_element.val as value
+  FROM att_array_devboolean att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
+CREATE OR REPLACE VIEW lofar_array_devuchar AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    array_element.idx - 1 AS x,
+    array_element.val as value
+  FROM att_array_devuchar att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
+CREATE OR REPLACE VIEW lofar_array_devshort AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    array_element.idx - 1 AS x,
+    array_element.val as value
+  FROM att_array_devshort att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
+CREATE OR REPLACE VIEW lofar_array_devushort AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    array_element.idx - 1 AS x,
+    array_element.val as value
+  FROM att_array_devushort att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
+CREATE OR REPLACE VIEW lofar_array_devlong AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    array_element.idx - 1 AS x,
+    array_element.val as value
+  FROM att_array_devlong att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
+CREATE OR REPLACE VIEW lofar_array_devulong AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    array_element.idx - 1 AS x,
+    array_element.val as value
+  FROM att_array_devulong att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
+CREATE OR REPLACE VIEW lofar_array_devlong64 AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    array_element.idx - 1 AS x,
+    array_element.val as value
+  FROM att_array_devlong64 att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
+CREATE OR REPLACE VIEW lofar_array_devulong64 AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    array_element.idx - 1 AS x,
+    array_element.val as value
+  FROM att_array_devulong64 att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
+CREATE OR REPLACE VIEW lofar_array_devfloat AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    array_element.idx - 1 AS x,
+    array_element.val as value
+  FROM att_array_devfloat att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
+CREATE OR REPLACE VIEW lofar_array_devdouble AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    array_element.idx - 1 AS x,
+    array_element.val as value
+  FROM att_array_devdouble att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
+CREATE OR REPLACE VIEW lofar_array_devstring AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    array_element.idx - 1 AS x,
+    array_element.val as value
+  FROM att_array_devstring att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
+CREATE OR REPLACE VIEW lofar_array_devstate AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    array_element.idx - 1 AS x,
+    array_element.val as value
+  FROM att_array_devstate att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
+CREATE OR REPLACE VIEW lofar_array_devencoded AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    array_element.idx - 1 AS x,
+    array_element.val as value
+  FROM att_array_devencoded att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
+CREATE OR REPLACE VIEW lofar_array_devenum AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    array_element.idx - 1 AS x,
+    array_element.val as value
+  FROM att_array_devenum att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
+CREATE OR REPLACE VIEW lofar_image_devboolean AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
+    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
+    array_element.val as value
+  FROM att_image_devboolean att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
+CREATE OR REPLACE VIEW lofar_image_devuchar AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
+    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
+    array_element.val as value
+  FROM att_image_devuchar att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
+CREATE OR REPLACE VIEW lofar_image_devshort AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
+    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
+    array_element.val as value
+  FROM att_image_devshort att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
+CREATE OR REPLACE VIEW lofar_image_devushort AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
+    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
+    array_element.val as value
+  FROM att_image_devushort att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
+CREATE OR REPLACE VIEW lofar_image_devlong AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
+    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
+    array_element.val as value
+  FROM att_image_devlong att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
+CREATE OR REPLACE VIEW lofar_image_devulong AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
+    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
+    array_element.val as value
+  FROM att_image_devulong att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
+CREATE OR REPLACE VIEW lofar_image_devlong64 AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
+    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
+    array_element.val as value
+  FROM att_image_devlong64 att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
+CREATE OR REPLACE VIEW lofar_image_devulong64 AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
+    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
+    array_element.val as value
+  FROM att_image_devulong64 att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
+CREATE OR REPLACE VIEW lofar_image_devfloat AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
+    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
+    array_element.val as value
+  FROM att_image_devfloat att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
+CREATE OR REPLACE VIEW lofar_image_devdouble AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
+    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
+    array_element.val as value
+  FROM att_image_devdouble att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
+CREATE OR REPLACE VIEW lofar_image_devstring AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
+    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
+    array_element.val as value
+  FROM att_image_devstring att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
+CREATE OR REPLACE VIEW lofar_image_devstate AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
+    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
+    array_element.val as value
+  FROM att_image_devstate att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
+CREATE OR REPLACE VIEW lofar_image_devencoded AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
+    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
+    array_element.val as value
+  FROM att_image_devencoded att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
+CREATE OR REPLACE VIEW lofar_image_devenum AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
+    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
+    array_element.val as value
+  FROM att_image_devenum att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+EOF
-- 
GitLab