Merge branch 'master' into L2SS-704

docker-compose/device-antennafield.yml

@@ -31,7 +31,7 @@ services:
     networks:
       - control
     ports:
-      - "5714:5714" # unique port for this DS
+      - "5715:5715" # unique port for this DS
     extra_hosts:
       - "host.docker.internal:host-gateway"
     volumes:
@@ -43,5 +43,5 @@ services:
       - bin/start-ds.sh
       # configure CORBA to _listen_ on 0:port, but tell others we're _reachable_ through ${HOSTNAME}:port, since CORBA
       # can't know about our Docker port forwarding
-      - l2ss-antennafield AntennaField STAT -v -ORBendPoint giop:tcp:0:5714 -ORBendPointPublish giop:tcp:${HOSTNAME}:5714
+      - l2ss-antennafield AntennaField STAT -v -ORBendPoint giop:tcp:0:5715 -ORBendPointPublish giop:tcp:${HOSTNAME}:5715
     restart: unless-stopped

docker-compose/device-pdu.yml

+#
+# Requires:
+#   - lofar-device-base.yml
+#
+version: '2'
+
+volumes:
+  iers-data: {}
+
+services:
+  device-pdu:
+    image: device-pdu
+    # build explicitly, as docker-compose does not understand a local image
+    # being shared among services.
+    build:
+        context: ..
+        dockerfile: docker-compose/lofar-device-base/Dockerfile
+        args:
+            SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
+    container_name: ${CONTAINER_NAME_PREFIX}device-pdu
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "100m"
+        max-file: "10"
+    networks:
+      - control
+    ports:
+      - "5714:5714" # unique port for this DS
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    volumes:
+      - ..:/opt/lofar/tango:rw
+    environment:
+      - TANGO_HOST=${TANGO_HOST}
+    working_dir: /opt/lofar/tango
+    entrypoint:
+      - bin/start-ds.sh
+      # configure CORBA to _listen_ on 0:port, but tell others we're _reachable_ through ${HOSTNAME}:port, since CORBA
+      # can't know about our Docker port forwarding
+      - l2ss-pdu PDU STAT -v -ORBendPoint giop:tcp:0:5714 -ORBendPointPublish giop:tcp:${HOSTNAME}:5714
+    restart: unless-stopped

docker-compose/grafana.yml

@@ -24,6 +24,8 @@ services:
     #  - grafana-configs:/etc/grafana
     ports:
       - "3000:3000"
+    environment:
+      - GF_SERVER_DOMAIN=${HOSTNAME}
     logging:
       driver: syslog
       options:

docker-compose/grafana/Dockerfile

@@ -3,6 +3,7 @@ FROM grafana/grafana
 # Install some plugins
 RUN grafana-cli plugins install briangann-datatable-panel
 RUN grafana-cli plugins install ae3e-plotly-panel
+RUN grafana-cli plugins install yesoreyeram-infinity-datasource
 
 COPY grafana.ini /etc/grafana/
 

docker-compose/grafana/README.md

+# Post configuration
+
+To export all current alert rules, use:
+
+To import rules into a fresh Grafana instance:
+
+  * Obtain an 'editor' API key through the Grafan GUI (cogwheel -> API keys),
+  * Run:
+
+  curl http://localhost:3000/api/alertmanager/grafana/config/api/v1/alerts -H 'Authorization: Bearer (api key)' > alerting.json
+  curl localhost:3000/api/ruler/grafana/api/v1/rules > rules.json
+
+  * Delete the UIDs in alerting.json
+
+To import rules into a fresh Grafana instance:
+
+  * Obtain an 'editor' API key through the Grafan GUI (cogwheel -> API keys),
+  * Run (first without piping to bash):
+
+  python3 import-rules.py -c alerting.json -r rules.json -B key | bash
+

docker-compose/grafana/alerting.json

@@ -2,7 +2,8 @@
   "template_files": {},
   "alertmanager_config": {
     "route": {
-      "receiver": "Alerta"
+      "receiver": "Alerta",
+      "repeat_interval": "10m"
     },
     "templates": null,
     "receivers": [
@@ -10,7 +11,6 @@
         "name": "Alerta",
         "grafana_managed_receiver_configs": [
           {
-            "uid": "ROaAvQEnz",
             "name": "Alerta",
             "type": "webhook",
             "disableResolveMessage": false,

docker-compose/grafana/datasources/alertaui.yaml

+apiVersion: 1
+
+datasources:
+  # <string, required> name of the datasource. Required
+  - name: Alerta UI
+    # <string, required> datasource type. Required
+    type: yesoreyeram-infinity-datasource
+    # <string, required> access mode. proxy or direct (Server or Browser in the UI). Required
+    access: proxy
+    # <int> org id. will default to orgId 1 if not specified
+    orgId: 1
+    # <string> custom UID which can be used to reference this datasource in other parts of the configuration, if not specified will be generated automatically
+    uid: alertaui
+    # <string> url
+    url: http://alerta-web:8080/api
+    # <string> Deprecated, use secureJsonData.password
+    password:
+    # <string> database user, if used
+    user: postgres
+    # <string> database name, if used
+    database: hdb
+    # <bool> enable/disable basic auth
+    basicAuth: false
+    # <string> basic auth username
+    basicAuthUser:
+    # <string> Deprecated, use secureJsonData.basicAuthPassword
+    basicAuthPassword:
+    # <bool> enable/disable with credentials headers
+    withCredentials:
+    # <bool> mark as default datasource. Max one per org
+    isDefault: false
+    # <map> fields that will be converted to json and stored in jsonData
+    jsonData:
+      secureQueryName1: "api-key"
+    # <string> json object of data that will be encrypted.
+    secureJsonData:
+      secureQueryValue1: "demo-key"
+    version: 1
+    # <bool> allow users to edit datasources from the UI.
+    editable: false
+    

docker-compose/grafana/grafana.ini

@@ -58,7 +58,7 @@
 ;static_root_path = public
 
 # enable gzip
-;enable_gzip = false
+enable_gzip = true
 
 # https certs & key file
 ;cert_file =
@@ -867,7 +867,9 @@ enabled = true
 
 [panels]
 # If set to true Grafana will allow script tags in text panels. Not recommended as it enable XSS vulnerabilities.
-;disable_sanitize_html = false
+
+# enable this to allow us to create mash ups with other pages
+disable_sanitize_html = true
 
 [plugins]
 ;enable_alpha = false

docker-compose/grafana/import-rules.py

+#!/usr/bin/python3
+import json
+import os
+import argparse
+
+parser = argparse.ArgumentParser(
+  formatter_class=argparse.RawDescriptionHelpFormatter,
+  description=
+"""
+Generate rule import files and script for Grafana.
+
+This script expands a given rules.json file into individual rules and
+prints the bash commands to import them in Grafana.
+
+To export rules from Grafana, use
+  curl <grafana>/api/ruler/grafana/api/v1/rules > rules.json
+""")
+parser.add_argument(
+    '-c', '--alert-config-file', type=str, required=False, help="Input alertmanager configuration JSON to parse, output of 'curl <grafana>/api/ruler/grafana/api/v1/rules' [%(default)s]")
+parser.add_argument(
+    '-r', '--rules-file', type=str, required=True, help="Input rules JSON to parse, output of 'curl <grafana>/api/ruler/grafana/api/v1/rules' [%(default)s]")
+parser.add_argument(
+    '-o', '--output-dir', type=str, default="rules", help="Directory to store the output [%(default)s]")
+parser.add_argument(
+    '-B', '--authorization-bearer', type=str, default="abcdefghijklmnopqrstuvwxyz", help="Authorization bearer from the Grafana 'editor' API key [%(default)s]")
+parser.add_argument(
+    '-g', '--grafana_url', type=str, default="http://localhost:3000", help="Base URL of Grafana [%(default)s]")
+parser.add_argument(
+    '-u', '--update', default=False, action='store_true', help="Update existing alerts, instead of creating new ones [%(default)s]")
+
+args = parser.parse_args()
+
+if args.alert_config_file:
+  print(f"echo Importing alert configuration file {args.alert_config_file}")
+  print(f"curl -X POST {args.grafana_url}/api/alertmanager/grafana/config/api/v1/alerts -H 'Content-Type: application/json' -H 'Accept: application/json' -H 'Authorization: Bearer {args.authorization_bearer}' -d '@{args.alert_config_file}'")
+  print(f"echo ''")
+
+with open(args.rules_file) as f:
+  data=json.load(f)
+
+  try:
+    os.mkdir(args.output_dir)
+  except FileExistsError as e:
+    pass
+
+  # the rules are of format {"folder": [{alert}, {alert}] }
+  for folder, rules in data.items():
+    try:
+      os.mkdir(f"{args.output_dir}/{folder}")
+    except FileExistsError as e:
+      pass
+
+    # print command to create folder
+    payload = json.dumps({"title": folder})
+    print(f"echo Creating folder {folder}")
+    print(f"curl -X POST {args.grafana_url}/api/folders -H 'Content-Type: application/json' -H 'Accept: application/json' -H 'Authorization: Bearer {args.authorization_bearer}' -d '{payload}'")
+    print(f"echo ''")
+
+    for rule in rules:
+      rule_filename = f"{args.output_dir}/{folder}/{rule['name']}.json"
+
+      if not args.update:
+        # strip rule UIDs
+        for subrule in rule["rules"]:
+          del subrule["grafana_alert"]["uid"]
+
+      # dump this rule
+      with open(rule_filename, "w") as rule_file:
+        json.dump(rule, rule_file)
+
+      # print import statement for this rule
+      print(f"echo Processing rule {folder}/{rule['name']}")
+      print(f"curl -X POST {args.grafana_url}/api/ruler/grafana/api/v1/rules/{folder} -H 'Content-Type: application/json' -H 'Accept: application/json' -H 'Authorization: Bearer {args.authorization_bearer}' -d '@{rule_filename}'")
+      print(f"echo ''")

docker-compose/grafana/rules.json

+{"station":[{"name":"FPGA processing error","interval":"10s","rules":[{"expr":"","for":"20s","labels":{"severity":"major"},"annotations":{"__dashboardUid__":"nC8N_kO7k","__panelId__":"9","summary":"One or more FPGAs are unusable."},"grafana_alert":{"id":1,"orgId":1,"title":"FPGA processing error","condition":"B","data":[{"refId":"A","queryType":"","relativeTimeRange":{"from":600,"to":0},"datasourceUid":"ZqArtG97z","model":{"format":"time_series","group":[],"hide":false,"intervalMs":1000,"maxDataPoints":43200,"metricColumn":"none","rawQuery":true,"rawSql":"SELECT\n  $__timeGroup(data_time, $__interval),\n  x::text,\n  device,\n  name,\n  value\nFROM lofar_array_boolean\nWHERE\n  $__timeFilter(data_time) AND\n  name = 'fpga_error_r'\nORDER BY 1,2","refId":"A","select":[[{"params":["value_r"],"type":"column"}]],"table":"att_scalar_devdouble","timeColumn":"data_time","timeColumnType":"timestamp","where":[{"name":"$__timeFilter","params":[],"type":"macro"}]}},{"refId":"B","queryType":"","relativeTimeRange":{"from":0,"to":0},"datasourceUid":"-100","model":{"conditions":[{"evaluator":{"params":[0],"type":"gt"},"operator":{"type":"and"},"query":{"params":["A"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"-100"},"expression":"A","hide":false,"intervalMs":1000,"maxDataPoints":43200,"reducer":"last","refId":"B","settings":{"mode":"dropNN"},"type":"reduce"}}],"updated":"2022-04-04T18:01:53Z","intervalSeconds":10,"version":3,"uid":"kujybCynk","namespace_uid":"R_jsbCynz","namespace_id":6,"rule_group":"FPGA processing error","no_data_state":"NoData","exec_err_state":"Alerting"}}]}]}
\ No newline at end of file

docker-compose/jupyter/Dockerfile

@@ -5,9 +5,8 @@ FROM ${SOURCE_IMAGE}
 # that are needed for temporary storage the proper owner and access rights.
 ARG CONTAINER_EXECUTION_UID=1000
 
-# Create homedir
-ENV HOME=/home/user
-RUN sudo mkdir -p ${HOME}
+# Create new user with uid but only if uid not used
+RUN sudo adduser --disabled-password --system --uid ${CONTAINER_EXECUTION_UID} --no-create-home --home ${HOME} user || exit 0
 RUN sudo chown ${CONTAINER_EXECUTION_UID} -R ${HOME}
 
 COPY requirements.txt ./

docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py

@@ -8,6 +8,7 @@ xst = DeviceProxy("STAT/XST/1")
 unb2 = DeviceProxy("STAT/UNB2/1")
 boot = DeviceProxy("STAT/Boot/1")
 tilebeam = DeviceProxy("STAT/TileBeam/1")
+pdu = DeviceProxy("STAT/PDU/1")
 beamlet = DeviceProxy("STAT/Beamlet/1")
 digitalbeam = DeviceProxy("STAT/DigitalBeam/1")
 antennafield = DeviceProxy("STAT/AntennaField/1")

docker-compose/pypcc-sim-base/Dockerfile

@@ -3,4 +3,4 @@ ARG LOCAL_DOCKER_REGISTRY_LOFAR
 
 FROM ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_LOFAR}/pypcc:latest
 
-CMD ["python3", "pypcc2.py", "--simulator", "--port","4843"]
+CMD ["hwtr", "--simulator", "--port","4843"]

docker-compose/recv-sim.yml

@@ -21,5 +21,5 @@ services:
         max-file: "10"
     networks:
       - control
-    entrypoint: python3 pypcc2.py --simulator --port 4840 --config RECVTR
+    entrypoint: hwtr --simulator --port 4840 --config RECVTR
     restart: on-failure

docker-compose/timescaledb/resources/10_lofar_views.sh

-#!/bin/bash
-psql << EOF
-\c hdb
-
--- NOTE: We concatenate domain/family/member here, which means we can't index
---       the resulting column. However, queries also supply the attribute name,
---       which we can index. The scan on the device name is then limited to
---        entries which have the same attribute name across devices.
-
-CREATE OR REPLACE VIEW lofar_scalar_double AS
-  SELECT
-    att.data_time AS data_time,
-    CONCAT_WS('/', domain, family, member) AS device,
-    ac.name AS name,
-    0 AS x,
-    value_r as value
-  FROM att_scalar_devdouble att
-  -- add the device information
-  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
-  WHERE att.value_r IS NOT NULL;
-
-CREATE OR REPLACE VIEW lofar_array_devboolean AS
-  SELECT
-    att.data_time AS data_time,
-    CONCAT_WS('/', domain, family, member) AS device,
-    ac.name AS name,
-    array_element.idx - 1 AS x,
-    array_element.val as value
-  FROM att_array_devboolean att
-  -- add array values, and their index
-  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
-  -- add the device information
-  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
-  WHERE att.value_r IS NOT NULL;
-
-CREATE OR REPLACE VIEW lofar_array_devuchar AS
-  SELECT
-    att.data_time AS data_time,
-    CONCAT_WS('/', domain, family, member) AS device,
-    ac.name AS name,
-    array_element.idx - 1 AS x,
-    array_element.val as value
-  FROM att_array_devuchar att
-  -- add array values, and their index
-  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
-  -- add the device information
-  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
-  WHERE att.value_r IS NOT NULL;
-
-CREATE OR REPLACE VIEW lofar_array_devshort AS
-  SELECT
-    att.data_time AS data_time,
-    CONCAT_WS('/', domain, family, member) AS device,
-    ac.name AS name,
-    array_element.idx - 1 AS x,
-    array_element.val as value
-  FROM att_array_devshort att
-  -- add array values, and their index
-  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
-  -- add the device information
-  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
-  WHERE att.value_r IS NOT NULL;
-
-CREATE OR REPLACE VIEW lofar_array_devushort AS
-  SELECT
-    att.data_time AS data_time,
-    CONCAT_WS('/', domain, family, member) AS device,
-    ac.name AS name,
-    array_element.idx - 1 AS x,
-    array_element.val as value
-  FROM att_array_devushort att
-  -- add array values, and their index
-  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
-  -- add the device information
-  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
-  WHERE att.value_r IS NOT NULL;
-
-CREATE OR REPLACE VIEW lofar_array_devlong AS
-  SELECT
-    att.data_time AS data_time,
-    CONCAT_WS('/', domain, family, member) AS device,
-    ac.name AS name,
-    array_element.idx - 1 AS x,
-    array_element.val as value
-  FROM att_array_devlong att
-  -- add array values, and their index
-  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
-  -- add the device information
-  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
-  WHERE att.value_r IS NOT NULL;
-
-CREATE OR REPLACE VIEW lofar_array_devulong AS
-  SELECT
-    att.data_time AS data_time,
-    CONCAT_WS('/', domain, family, member) AS device,
-    ac.name AS name,
-    array_element.idx - 1 AS x,
-    array_element.val as value
-  FROM att_array_devulong att
-  -- add array values, and their index
-  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
-  -- add the device information
-  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
-  WHERE att.value_r IS NOT NULL;
-
-CREATE OR REPLACE VIEW lofar_array_devlong64 AS
-  SELECT
-    att.data_time AS data_time,
-    CONCAT_WS('/', domain, family, member) AS device,
-    ac.name AS name,
-    array_element.idx - 1 AS x,
-    array_element.val as value
-  FROM att_array_devlong64 att
-  -- add array values, and their index
-  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
-  -- add the device information
-  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
-  WHERE att.value_r IS NOT NULL;
-
-CREATE OR REPLACE VIEW lofar_array_devulong64 AS
-  SELECT
-    att.data_time AS data_time,
-    CONCAT_WS('/', domain, family, member) AS device,
-    ac.name AS name,
-    array_element.idx - 1 AS x,
-    array_element.val as value
-  FROM att_array_devulong64 att
-  -- add array values, and their index
-  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
-  -- add the device information
-  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
-  WHERE att.value_r IS NOT NULL;
-
-CREATE OR REPLACE VIEW lofar_array_devfloat AS
-  SELECT
-    att.data_time AS data_time,
-    CONCAT_WS('/', domain, family, member) AS device,
-    ac.name AS name,
-    array_element.idx - 1 AS x,
-    array_element.val as value
-  FROM att_array_devfloat att
-  -- add array values, and their index
-  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
-  -- add the device information
-  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
-  WHERE att.value_r IS NOT NULL;
-
-CREATE OR REPLACE VIEW lofar_array_devdouble AS
-  SELECT
-    att.data_time AS data_time,
-    CONCAT_WS('/', domain, family, member) AS device,
-    ac.name AS name,
-    array_element.idx - 1 AS x,
-    array_element.val as value
-  FROM att_array_devdouble att
-  -- add array values, and their index
-  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
-  -- add the device information
-  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
-  WHERE att.value_r IS NOT NULL;
-
-CREATE OR REPLACE VIEW lofar_array_devstring AS
-  SELECT
-    att.data_time AS data_time,
-    CONCAT_WS('/', domain, family, member) AS device,
-    ac.name AS name,
-    array_element.idx - 1 AS x,
-    array_element.val as value
-  FROM att_array_devstring att
-  -- add array values, and their index
-  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
-  -- add the device information
-  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
-  WHERE att.value_r IS NOT NULL;
-
-CREATE OR REPLACE VIEW lofar_array_devstate AS
-  SELECT
-    att.data_time AS data_time,
-    CONCAT_WS('/', domain, family, member) AS device,
-    ac.name AS name,
-    array_element.idx - 1 AS x,
-    array_element.val as value
-  FROM att_array_devstate att
-  -- add array values, and their index
-  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
-  -- add the device information
-  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
-  WHERE att.value_r IS NOT NULL;
-
-CREATE OR REPLACE VIEW lofar_array_devencoded AS
-  SELECT
-    att.data_time AS data_time,
-    CONCAT_WS('/', domain, family, member) AS device,
-    ac.name AS name,
-    array_element.idx - 1 AS x,
-    array_element.val as value
-  FROM att_array_devencoded att
-  -- add array values, and their index
-  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
-  -- add the device information
-  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
-  WHERE att.value_r IS NOT NULL;
-
-CREATE OR REPLACE VIEW lofar_array_devenum AS
-  SELECT
-    att.data_time AS data_time,
-    CONCAT_WS('/', domain, family, member) AS device,
-    ac.name AS name,
-    array_element.idx - 1 AS x,
-    array_element.val as value
-  FROM att_array_devenum att
-  -- add array values, and their index
-  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
-  -- add the device information
-  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
-  WHERE att.value_r IS NOT NULL;
-
-CREATE OR REPLACE VIEW lofar_image_devboolean AS
-  SELECT
-    att.data_time AS data_time,
-    CONCAT_WS('/', domain, family, member) AS device,
-    ac.name AS name,
-    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
-    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
-    array_element.val as value
-  FROM att_image_devboolean att
-  -- add array values, and their index
-  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
-  -- add the device information
-  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
-  WHERE att.value_r IS NOT NULL;
-
-CREATE OR REPLACE VIEW lofar_image_devuchar AS
-  SELECT
-    att.data_time AS data_time,
-    CONCAT_WS('/', domain, family, member) AS device,
-    ac.name AS name,
-    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
-    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
-    array_element.val as value
-  FROM att_image_devuchar att
-  -- add array values, and their index
-  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
-  -- add the device information
-  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
-  WHERE att.value_r IS NOT NULL;
-
-CREATE OR REPLACE VIEW lofar_image_devshort AS
-  SELECT
-    att.data_time AS data_time,
-    CONCAT_WS('/', domain, family, member) AS device,
-    ac.name AS name,
-    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
-    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
-    array_element.val as value
-  FROM att_image_devshort att
-  -- add array values, and their index
-  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
-  -- add the device information
-  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
-  WHERE att.value_r IS NOT NULL;
-
-CREATE OR REPLACE VIEW lofar_image_devushort AS
-  SELECT
-    att.data_time AS data_time,
-    CONCAT_WS('/', domain, family, member) AS device,
-    ac.name AS name,
-    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
-    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
-    array_element.val as value
-  FROM att_image_devushort att
-  -- add array values, and their index
-  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
-  -- add the device information
-  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
-  WHERE att.value_r IS NOT NULL;
-
-CREATE OR REPLACE VIEW lofar_image_devlong AS
-  SELECT
-    att.data_time AS data_time,
-    CONCAT_WS('/', domain, family, member) AS device,
-    ac.name AS name,
-    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
-    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
-    array_element.val as value
-  FROM att_image_devlong att
-  -- add array values, and their index
-  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
-  -- add the device information
-  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
-  WHERE att.value_r IS NOT NULL;
-
-CREATE OR REPLACE VIEW lofar_image_devulong AS
-  SELECT
-    att.data_time AS data_time,
-    CONCAT_WS('/', domain, family, member) AS device,
-    ac.name AS name,
-    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
-    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
-    array_element.val as value
-  FROM att_image_devulong att
-  -- add array values, and their index
-  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
-  -- add the device information
-  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
-  WHERE att.value_r IS NOT NULL;
-
-CREATE OR REPLACE VIEW lofar_image_devlong64 AS
-  SELECT
-    att.data_time AS data_time,
-    CONCAT_WS('/', domain, family, member) AS device,
-    ac.name AS name,
-    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
-    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
-    array_element.val as value
-  FROM att_image_devlong64 att
-  -- add array values, and their index
-  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
-  -- add the device information
-  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
-  WHERE att.value_r IS NOT NULL;
-
-CREATE OR REPLACE VIEW lofar_image_devulong64 AS
-  SELECT
-    att.data_time AS data_time,
-    CONCAT_WS('/', domain, family, member) AS device,
-    ac.name AS name,
-    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
-    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
-    array_element.val as value
-  FROM att_image_devulong64 att
-  -- add array values, and their index
-  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
-  -- add the device information
-  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
-  WHERE att.value_r IS NOT NULL;
-
-CREATE OR REPLACE VIEW lofar_image_devfloat AS
-  SELECT
-    att.data_time AS data_time,
-    CONCAT_WS('/', domain, family, member) AS device,
-    ac.name AS name,
-    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
-    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
-    array_element.val as value
-  FROM att_image_devfloat att
-  -- add array values, and their index
-  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
-  -- add the device information
-  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
-  WHERE att.value_r IS NOT NULL;
-
-CREATE OR REPLACE VIEW lofar_image_devdouble AS
-  SELECT
-    att.data_time AS data_time,
-    CONCAT_WS('/', domain, family, member) AS device,
-    ac.name AS name,
-    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
-    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
-    array_element.val as value
-  FROM att_image_devdouble att
-  -- add array values, and their index
-  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
-  -- add the device information
-  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
-  WHERE att.value_r IS NOT NULL;
-
-CREATE OR REPLACE VIEW lofar_image_devstring AS
-  SELECT
-    att.data_time AS data_time,
-    CONCAT_WS('/', domain, family, member) AS device,
-    ac.name AS name,
-    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
-    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
-    array_element.val as value
-  FROM att_image_devstring att
-  -- add array values, and their index
-  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
-  -- add the device information
-  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
-  WHERE att.value_r IS NOT NULL;
-
-CREATE OR REPLACE VIEW lofar_image_devstate AS
-  SELECT
-    att.data_time AS data_time,
-    CONCAT_WS('/', domain, family, member) AS device,
-    ac.name AS name,
-    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
-    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
-    array_element.val as value
-  FROM att_image_devstate att
-  -- add array values, and their index
-  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
-  -- add the device information
-  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
-  WHERE att.value_r IS NOT NULL;
-
-CREATE OR REPLACE VIEW lofar_image_devencoded AS
-  SELECT
-    att.data_time AS data_time,
-    CONCAT_WS('/', domain, family, member) AS device,
-    ac.name AS name,
-    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
-    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
-    array_element.val as value
-  FROM att_image_devencoded att
-  -- add array values, and their index
-  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
-  -- add the device information
-  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
-  WHERE att.value_r IS NOT NULL;
-
-CREATE OR REPLACE VIEW lofar_image_devenum AS
-  SELECT
-    att.data_time AS data_time,
-    CONCAT_WS('/', domain, family, member) AS device,
-    ac.name AS name,
-    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
-    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
-    array_element.val as value
-  FROM att_image_devenum att
-  -- add array values, and their index
-  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
-  -- add the device information
-  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
-  WHERE att.value_r IS NOT NULL;
-EOF

docker-compose/timescaledb/resources/13_lofar_views.sql

@@ -7,6 +7,21 @@
 
 -- DOUBLE --
 
+CREATE OR REPLACE VIEW lofar_image_double AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
+    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
+    array_element.val as value
+  FROM att_image_devdouble att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
 CREATE OR REPLACE VIEW lofar_array_double AS
   SELECT
     att.data_time AS data_time,
@@ -34,13 +49,28 @@ CREATE OR REPLACE VIEW lofar_array_double AS
   
 -- BOOLEAN --
 
+CREATE OR REPLACE VIEW lofar_image_boolean AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
+    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
+    CASE WHEN array_element.val THEN 1 ELSE 0 END AS value
+  FROM att_image_devboolean att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+ 
 CREATE OR REPLACE VIEW lofar_array_boolean AS
   SELECT
     att.data_time AS data_time,
     CONCAT_WS('/', domain, family, member) AS device,
     ac.name AS name,
     array_element.idx - 1 AS x,
-    array_element.val as value
+    CASE WHEN array_element.val THEN 1 ELSE 0 END AS value
   FROM att_array_devboolean att
   -- add array values, and their index
   JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
@@ -53,13 +83,29 @@ CREATE OR REPLACE VIEW lofar_array_double AS
     att.data_time AS data_time,
     CONCAT_WS('/', domain, family, member) AS device,
     ac.name AS name,
-    value_r as value
+    CASE WHEN value_r THEN 1 ELSE 0 END AS value
   FROM att_scalar_devboolean att
   -- add the device information
   JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
   WHERE att.value_r IS NOT NULL;
  
 -- UCHAR --
+
+CREATE OR REPLACE VIEW lofar_image_uchar AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
+    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
+    array_element.val as value
+  FROM att_image_devuchar att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
 CREATE OR REPLACE VIEW lofar_array_uchar AS
   SELECT
     att.data_time AS data_time,
@@ -86,6 +132,22 @@ CREATE OR REPLACE VIEW lofar_array_double AS
   WHERE att.value_r IS NOT NULL;
  
 -- SHORT --
+
+CREATE OR REPLACE VIEW lofar_image_short AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
+    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
+    array_element.val as value
+  FROM att_image_devshort att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
 CREATE OR REPLACE VIEW lofar_array_short AS
   SELECT
     att.data_time AS data_time,
@@ -112,6 +174,22 @@ CREATE OR REPLACE VIEW lofar_array_double AS
   WHERE att.value_r IS NOT NULL;
  
 -- USHORT --
+
+CREATE OR REPLACE VIEW lofar_image_ushort AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
+    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
+    array_element.val as value
+  FROM att_image_devushort att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
 CREATE OR REPLACE VIEW lofar_array_ushort AS
   SELECT
     att.data_time AS data_time,
@@ -138,6 +216,22 @@ CREATE OR REPLACE VIEW lofar_array_double AS
   WHERE att.value_r IS NOT NULL;
  
 -- LONG -- 
+
+CREATE OR REPLACE VIEW lofar_image_long AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
+    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
+    array_element.val as value
+  FROM att_image_devlong att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
 CREATE OR REPLACE VIEW lofar_array_long AS
   SELECT
     att.data_time AS data_time,
@@ -164,6 +258,22 @@ CREATE OR REPLACE VIEW lofar_array_double AS
   WHERE att.value_r IS NOT NULL;
  
 -- ULONG -- 
+
+CREATE OR REPLACE VIEW lofar_image_ulong AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
+    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
+    array_element.val as value
+  FROM att_image_devulong att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
 CREATE OR REPLACE VIEW lofar_array_ulong AS
   SELECT
     att.data_time AS data_time,
@@ -190,6 +300,22 @@ CREATE OR REPLACE VIEW lofar_array_double AS
   WHERE att.value_r IS NOT NULL;
  
 -- LONG64 --
+
+CREATE OR REPLACE VIEW lofar_image_long64 AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
+    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
+    array_element.val as value
+  FROM att_image_devlong64 att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
 CREATE OR REPLACE VIEW lofar_array_long64 AS
   SELECT
     att.data_time AS data_time,
@@ -216,6 +342,22 @@ CREATE OR REPLACE VIEW lofar_array_double AS
   WHERE att.value_r IS NOT NULL;
  
 -- ULONG64 --
+
+CREATE OR REPLACE VIEW lofar_image_ulong64 AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
+    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
+    array_element.val as value
+  FROM att_image_devulong64 att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
 CREATE OR REPLACE VIEW lofar_array_ulong64 AS
   SELECT
     att.data_time AS data_time,
@@ -242,6 +384,22 @@ CREATE OR REPLACE VIEW lofar_array_double AS
   WHERE att.value_r IS NOT NULL;
  
 -- FLOAT -- 
+
+CREATE OR REPLACE VIEW lofar_image_float AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
+    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
+    array_element.val as value
+  FROM att_image_devfloat att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
 CREATE OR REPLACE VIEW lofar_array_float AS
   SELECT
     att.data_time AS data_time,
@@ -268,6 +426,22 @@ CREATE OR REPLACE VIEW lofar_array_double AS
   WHERE att.value_r IS NOT NULL;
  
 -- STRING --
+
+CREATE OR REPLACE VIEW lofar_image_string AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
+    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
+    array_element.val as value
+  FROM att_image_devstring att
+-- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
 CREATE OR REPLACE VIEW lofar_array_string AS
   SELECT
     att.data_time AS data_time,
@@ -294,6 +468,22 @@ CREATE OR REPLACE VIEW lofar_array_double AS
   WHERE att.value_r IS NOT NULL;
  
 -- STATE --
+
+CREATE OR REPLACE VIEW lofar_image_state AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
+    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
+    array_element.val as value
+  FROM att_image_devstate att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
 CREATE OR REPLACE VIEW lofar_array_state AS
   SELECT
     att.data_time AS data_time,
@@ -320,6 +510,22 @@ CREATE OR REPLACE VIEW lofar_array_double AS
   WHERE att.value_r IS NOT NULL;
  
 -- ENCODED -- 
+
+CREATE OR REPLACE VIEW lofar_image_encoded AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
+    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
+    array_element.val as value
+  FROM att_image_devencoded att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
 CREATE OR REPLACE VIEW lofar_array_encoded AS
   SELECT
     att.data_time AS data_time,
@@ -346,6 +552,22 @@ CREATE OR REPLACE VIEW lofar_array_double AS
   WHERE att.value_r IS NOT NULL;
  
 -- ENUM --
+
+CREATE OR REPLACE VIEW lofar_image_enum AS
+  SELECT
+    att.data_time AS data_time,
+    CONCAT_WS('/', domain, family, member) AS device,
+    ac.name AS name,
+    (array_element.idx - 1) / ARRAY_LENGTH(att.value_r, 1) AS x,
+    (array_element.idx - 1) % ARRAY_LENGTH(att.value_r, 1) AS y,
+    array_element.val as value
+  FROM att_image_devenum att
+  -- add array values, and their index
+  JOIN LATERAL UNNEST(att.value_r) WITH ORDINALITY AS array_element(val,idx) ON TRUE
+  -- add the device information
+  JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
+  WHERE att.value_r IS NOT NULL;
+
 CREATE OR REPLACE VIEW lofar_array_enum AS
   SELECT
     att.data_time AS data_time,
@@ -370,8 +592,3 @@ CREATE OR REPLACE VIEW lofar_scalar_enum AS
   -- add the device information
   JOIN att_conf ac ON att.att_conf_id = ac.att_conf_id
   WHERE att.value_r IS NOT NULL;
- 
- 
- 
- 
- 
\ No newline at end of file

docker-compose/unb2-sim.yml

@@ -21,5 +21,5 @@ services:
         max-file: "10"
     networks:
       - control
-    entrypoint: python3 pypcc2.py --simulator --port 4841 --config UNB2TR
+    entrypoint: hwtr --simulator --port 4841 --config UNB2TR
     restart: on-failure

sbin/run_integration_test.sh

@@ -20,12 +20,13 @@ sleep 1 # dsconfig container must be up and running...
 # shellcheck disable=SC2016
 echo '/usr/local/bin/wait-for-it.sh ${TANGO_HOST} --strict --timeout=300 -- true' | make run dsconfig bash -
 
-DEVICES="device-boot device-apsct device-apspu device-sdp device-recv device-sst device-unb2 device-xst device-beamlet device-digitalbeam device-tilebeam device-antennafield"
+DEVICES="device-boot device-apsct device-apspu device-sdp device-recv device-sst device-unb2 device-xst device-beamlet device-digitalbeam device-tilebeam device-pdu device-antennafield"
 SIMULATORS="sdptr-sim recv-sim unb2-sim apsct-sim apspu-sim"
 
 # Build only the required images, please do not build everything that makes CI
 # take really long to finish, especially grafana / jupyter / prometheus.
 # jupyter is physically large > 2.5gb and overlayfs is really slow.
+
 # shellcheck disable=SC2086
 make build $DEVICES $SIMULATORS
 make build elk integration-test

tangostationcontrol/docs/source/alerting.rst

+Alerting
+==================
+
+We use the following setup to forward alarms:
+
+- The Tango Controls `hdbpp subsystem <https://tango-controls.readthedocs.io/en/latest/administration/services/hdbpp/hdb++-design-guidelines.html>`_ archives data-value changes into a `TimescaleDB <http://timescale.com>`_ database,
+- Grafana allows `Alert rules <https://grafana.com/docs/grafana/latest/alerting/>`_ to be configured, which poll TimescaleDB and generate an *alert* when the configured condition is met. It also maintains a list of currently firing alerts,
+- `Alerta <https://alerta.io/>`_ is the *alert manager*: itreceives these alerts, manages duplicates, and maintains alerts until the operator explicitly acknowledges them. It thus also has a list of alerts that fired in the past.
+
+Archiving attributes
+```````````````````````
+
+The attributes of interest will have to be *archived* periodically to be able to see them in Grafana, and thus to be able to define alerts for them. In Tango Controls, there is an *configuration manager* that provides an interface to manage what is archived, and one or more *event subscribers* to subscribe to event changes and forward them to the archive database.
+
+The ``tangoncontrols.toolkit.archiver.Archiver`` class provides an easy interface to the archiver. It uses the ``device/attribute`` notation for attributes, f.e. ``STAT/SDP/1/FPGA_error_R``. Some of the functions it provides:
+
+:add_attribute_to_archiver(attribute, polling_period, event_period): Register the given attribute every ``polling_period`` ms. Also attribute on changes with a maximum rate of ``event_period`` ms.
+
+:remove_attribute_from_archiver(attribute): Unregister the given attribute.
+
+:start_archiving_attribute(attribute): Start archiving the given attribute.
+
+:stop_archiving_attribute(attribute): Stop archiving the given attribute.
+
+:get_attribute_errors(attribute): Return any errors detected while trying to archive the attribute.
+
+:get_subscriber_errors(): Return any errors detected by the subscribers.
+
+So a useful idiom to archive an individual attribute is::
+
+  from tangostationcontrol.archiver import Archiver
+
+  archiver = Archiver()
+  attribute = "STAT/SDP/1/FPGA_error_R"
+  archiver.add_attribute_to_archiver(attribute, 1000, 1000)
+  archiver.start_archiving_attribute(attribute)
+
+.. note:: The archive subscriber gets confused if attributes it archives disappear from the monitoring database. This can cause an archive subscriber to stall. To fix this, get a proxy to the event subscriber, f.e. ``DeviceProxy("archiving/hdbppts/eventsubscriber01")``, and remove the offending attribute(s) from thr ``ArchivingList`` property using ``proxy.get_property("ArchivingList")`` and ``proxy.put_property({"ArchivingList": [...])``.
+
+Inspecting the database
+`````````````````````````
+
+The archived attributes end up in a `TimescaleDB <http://timescale.com>`_ database, exposed on port 5432, with credentials ``postgres/pasword``. Key tables are:
+
+:att_conf: Describes which attributes are registered. Note that any device and attribute names are in lower case.
+
+:att_scalar_devXXX: Contains the attribute history for scalar attributes of type XXX.
+
+:att_array_devXXX: Contains the attribute history for 1D array attributes of type XXX.
+
+:att_image_devXXX: Contains the attribute history for 2D array attributes of type XXX.
+
+Each of the attribute history tables contains entries for any recorded value changes, but also for changes in ``quality`` (0=ok, >0=issues), and any error ``att_error_desc_id``. Futhermore, we provide specialised views which combine tables into more readable information:
+
+:lofar_scalar_XXX: View on the attribute history for scalar attributes of type XXX.
+
+:lofar_array_XXX: View on the attribute history for 1D array attributes of type XXX. Each array element is returned in its own row, with ``x`` denoting the index.
+
+:lofar_image_XXX: View on the attribute history for 2D array attributes of type XXX. Each array element is returned in its own row, with ``x`` and ``y`` denoting the indices.
+
+A typical selection could thus look like::
+
+  SELECT
+    date_time AS time, device, name, x, value
+  FROM lofar_array_boolean
+  WHERE device = 'stat/sdp/1' AND name = 'fpga_error_r'
+  ORDER BY time DESC
+  LIMIT 16
+
+Attributes in Grafana
+````````````````````````
+
+The Grafana instance (http://localhost:3000) is linked to TimescaleDB by default. The query for plotting an attribute requires some Grafana-specific macros to select the exact data points Grafana requires::
+
+  SELECT
+    $__timeGroup(data_time, $__interval),
+    x::text, device, name,
+    value
+  FROM lofar_array_boolean
+  WHERE
+    $__timeFilter(data_time) AND name = 'fpga_error_r'
+  ORDER BY 1,2
+
+The fields ``x``, ``device``, and ``name`` are retrieved as *string*, as that makes them labels to the query, which Grafana then uses to identify the different metrics for each array element.
+
+.. hint:: Grafana orders labels alphabetically. To order the ``x`` element properly, one could use the ``TO_CHAR(x, '00')`` function instead of ``x::text`` to prepend values with 0.
+
+Setting up alerts
+```````````````````
+
+We use the `Grafana 8+ alerts <https://grafana.com/docs/grafana/latest/alerting/>`_ to monitor our system, and the alerts are to be forwarded to our Alerta instance. Both our default set of alerts and this forwarding has to be post-configured after installation:
+
+- Go to Grafana (http://localhost:3000) and sign in with an administration account (default: admin/admin),
+- Go to ``(cogwheel) -> API keys`` and create an ``editor`` API key. Copy the resulting hash,
+- Go to the ``docker-compose/grafana/`` source directory, and run::
+
+  ./import-rules.py -c alerting.json -r rules.json -B <apikey> | bash
+
+.. hint:: Whether Grafana can send alerts to Alerta can be tested by sending a `test alert <http://localhost:3000/alerting/notifications/receivers/Alerta/edit?alertmanager=grafana>`_.
+
+The following enhancements are useful to configure for the alerts:
+
+- You'll want to alert on a query, followed by a ``Reduce`` step with Function ``Last`` and Mode ``Drop Non-numeric Value``. This triggers the alert on the latest value(s), but keeps the individual array elements separated,
+- In ``Add details``, the ``Dashboard UID`` and ``Panel ID`` annotations are useful to configure to where you want the user to go, as Grafana will generate hyperlinks from them. To obtain a dashboard uid, go to ``Dashboards -> Browse`` and check out its URL. For the panel id, view a panel and check the URL,
+- In ``Add details``, the ``Summary`` annotation will be used as the alert description,
+- In ``Custom labels``, add ``severity = major`` to raise the severity of the alert (default: warning). See also the `supported values <https://docs.alerta.io/webui/configuration.html#severity-colors>`_.
+
+Alerta dashboard
+``````````````````
+
+The Alerta dashboard (http://localhost:8081) provides an overview of received alerts, which stay in the list until the alert condition disappears, and the alert is explicitly acknowledged or deleted:
+
+- *Acknowledging* an alert silences it for a day,
+- *Shelving* an alert silences it for 2 hours, and removes it from more overviews,
+- *Watching* an alert means receiving browser notifications on changes,
+- *Deleting* an alert removes it until Grafana sends it again (default: 10 minutes).
+
+See ``docker-compose/alerta-web/alertad.conf`` for these settings.
+
+Several installed plugins enhance the received events:
+
+- ``slack`` plugin forwards alerts to Slack (see below),
+- Our own ``grafana`` plugin parses Grafana-specific fields and adds them to the alert,
+- Our own ``lofar`` plugin parses and generates LOFAR-specific fields.
+
+Slack integration
+```````````````````
+
+Our Alerta setup is configured to send alerts to Slack. To set this up, you need to:
+
+- Create a Slack App: https://api.slack.com/apps?new_app=1
+- Under ``OAuth & Permissions``, add the following ``OAuth Scope``: ``chat:write``,
+- Install the App in your Workspace,
+- Copy the ``OAuth Token``.
+
+.. hint:: To obtain the ``OAuth Token`` later on, go to https://api.slack.com/apps, click on your App, and look under ``Install App``.
+
+Now, edit ``docker-compose/alerta-web/alerta-secrets.json``:
+
+.. literalinclude:: ../../../docker-compose/alerta-web/alerta-secrets.json
+
+The ``SLACK_TOKEN`` is the ``OAuth Token``, and the ``SLACK_CHANNEL`` is the channel in which to post the alerts.
+
+Any further tweaking can be done by modifying ``docker-compose/alerta-web/alertad.conf``.
+
+Debugging hints
+````````````````````````
+
+- Grafana sends alerts to Alerta using the *Prometheus AlertManager* format, and thus uses the Prometheus webhook to do so. To see what Grafana emits, configure it to send to your custom https://hookbin.com/ endpoint,
+- Grafana by default resends firing alerts every 4 hours, and we set this to 10 minutes. This means that if an alert was succesfully sent but lost (or deleted), it takes that long to get it back. For debugging, you may want to lower this to f.e. 10 seconds in the ``Alerting -> Notification policies`` settings of Grafana,
+- Alerta has a plugin system which allows easily modifying the attributes of an alert (see ``docker-compose/alerta-web`` and https://github.com/alerta/alerta-contrib). To see which attributes an alert has, simply go to the alert in the web GUI, press *Copy*, and paste in your editor,
+- Alerta allows a ``DEBUG=True`` parameter in ``docker-compose/alerta-web/alertad.conf`` to generate debug output.