Merge branch 'L2SS-1002-create-docker-image' into 'master'

Migrate grafana to new pre-build docker image

Closes L2SS-1002

See merge request !450

.gitmodules

 [submodule "tangostationcontrol/tangostationcontrol/toolkit/libhdbpp-python"]
 	path = tangostationcontrol/tangostationcontrol/toolkit/libhdbpp-python
 	url = https://gitlab.com/tango-controls/hdbpp/libhdbpp-python.git
-[submodule "docker-compose/grafana/dashboards"]
-	path = docker-compose/grafana/dashboards
-	url = https://git.astron.nl/lofar2.0/grafana-station-dashboards.git

docker-compose/grafana/Dockerfile

-FROM grafana/grafana
+FROM git.astron.nl:5000/lofar2.0/grafana-station-dashboards:latest
 
-# Install some plugins
-RUN grafana-cli plugins install briangann-datatable-panel
-RUN grafana-cli plugins install ae3e-plotly-panel
-RUN grafana-cli plugins install yesoreyeram-infinity-datasource
-RUN grafana-cli plugins install aceiot-svg-panel
-RUN grafana-cli plugins install yesoreyeram-boomtable-panel
-RUN grafana-cli plugins install orchestracities-map-panel
-
-RUN wget https://algenty.github.io/flowcharting-repository/archives/agenty-flowcharting-panel-1.0.0b-SNAPSHOT.zip -O /tmp/agenty-flowcharting-panel.zip
-RUN cd /var/lib/grafana/plugins/ && unzip /tmp/agenty-flowcharting-panel.zip && mv grafana-flowcharting agenty-flowcharting-panel
-
-COPY grafana.ini /etc/grafana/
-
-# Add default configuration through provisioning (see https://grafana.com/docs/grafana/latest/administration/provisioning)
-COPY datasources /etc/grafana/provisioning/datasources/
-COPY dashboards /var/lib/grafana/dashboards/
 COPY stationcontrol-dashboards.yaml /etc/grafana/provisioning/dashboards/

docker-compose/grafana/alerting.json

-{
-  "template_files": {},
-  "alertmanager_config": {
-    "route": {
-      "receiver": "Alerta",
-      "repeat_interval": "10m"
-    },
-    "templates": null,
-    "receivers": [
-      {
-        "name": "Alerta",
-        "grafana_managed_receiver_configs": [
-          {
-            "name": "Alerta",
-            "type": "webhook",
-            "disableResolveMessage": false,
-            "settings": {
-              "url": "http://alerta-server:8080/api/webhooks/prometheus?api-key=demo-key"
-            },
-            "secureFields": {}
-          }
-        ]
-      }
-    ]
-  }
-}

dashboards @ faf5cbb2

-Subproject commit faf5cbb2fc7981ca4430a9e341145ce66d304851

docker-compose/grafana/datasources/alertaui.yaml

-apiVersion: 1
-
-datasources:
-  # <string, required> name of the datasource. Required
-  - name: Alerta UI
-    # <string, required> datasource type. Required
-    type: yesoreyeram-infinity-datasource
-    # <string, required> access mode. proxy or direct (Server or Browser in the UI). Required
-    access: proxy
-    # <int> org id. will default to orgId 1 if not specified
-    orgId: 1
-    # <string> custom UID which can be used to reference this datasource in other parts of the configuration, if not specified will be generated automatically
-    uid: alertaui
-    # <string> url
-    url: http://alerta-server:8080/api
-    # <string> Deprecated, use secureJsonData.password
-    password:
-    # <string> database user, if used
-    user: postgres
-    # <string> database name, if used
-    database: hdb
-    # <bool> enable/disable basic auth
-    basicAuth: false
-    # <string> basic auth username
-    basicAuthUser:
-    # <string> Deprecated, use secureJsonData.basicAuthPassword
-    basicAuthPassword:
-    # <bool> enable/disable with credentials headers
-    withCredentials:
-    # <bool> mark as default datasource. Max one per org
-    isDefault: false
-    # <map> fields that will be converted to json and stored in jsonData
-    jsonData:
-      secureQueryName1: "api-key"
-    # <string> json object of data that will be encrypted.
-    secureJsonData:
-      secureQueryValue1: "demo-key"
-    version: 1
-    # <bool> allow users to edit datasources from the UI.
-    editable: false
-    

docker-compose/grafana/datasources/grafanaapi.yaml

-apiVersion: 1
-
-datasources:
-  # <string, required> name of the datasource. Required
-  - name: Grafana API
-    # <string, required> datasource type. Required
-    type: yesoreyeram-infinity-datasource
-    # <string, required> access mode. proxy or direct (Server or Browser in the UI). Required
-    access: proxy
-    # <int> org id. will default to orgId 1 if not specified
-    orgId: 1
-    # <string> custom UID which can be used to reference this datasource in other parts of the configuration, if not specified will be generated automatically
-    uid: grafanaapi
-    # <string> url
-    url: http://localhost:3000/api
-    # <string> Deprecated, use secureJsonData.password
-    password:
-    # <string> database user, if used
-    user: postgres
-    # <string> database name, if used
-    database: hdb
-    # <bool> enable/disable basic auth
-    basicAuth: false
-    # <string> basic auth username
-    basicAuthUser:
-    # <string> Deprecated, use secureJsonData.basicAuthPassword
-    basicAuthPassword:
-    # <bool> enable/disable with credentials headers
-    withCredentials:
-    # <bool> mark as default datasource. Max one per org
-    isDefault: false
-    # <map> fields that will be converted to json and stored in jsonData
-    version: 1
-    # <bool> allow users to edit datasources from the UI.
-    editable: false
-    

docker-compose/grafana/datasources/prometheus.yaml

-apiVersion: 1
-
-datasources:
-  # <string, required> name of the datasource. Required
-  - name: Prometheus
-    # <string, required> datasource type. Required
-    type: prometheus
-    # <string, required> access mode. proxy or direct (Server or Browser in the UI). Required
-    access: proxy
-    # <int> org id. will default to orgId 1 if not specified
-    orgId: 1
-    # <string> custom UID which can be used to reference this datasource in other parts of the configuration, if not specified will be generated automatically
-    uid: prometheus
-    # <string> url
-    url: http://prometheus:9090
-    # <string> Deprecated, use secureJsonData.password
-    password:
-    # <string> database user, if used
-    user:
-    # <string> database name, if used
-    database:
-    # <bool> enable/disable basic auth
-    basicAuth: false
-    # <string> basic auth username
-    basicAuthUser:
-    # <string> Deprecated, use secureJsonData.basicAuthPassword
-    basicAuthPassword:
-    # <bool> enable/disable with credentials headers
-    withCredentials:
-    # <bool> mark as default datasource. Max one per org
-    isDefault: true
-    # <map> fields that will be converted to json and stored in jsonData
-    jsonData:
-      httpMethod: POST
-    # <string> json object of data that will be encrypted.
-    secureJsonData:
-    version: 1
-    # <bool> allow users to edit datasources from the UI.
-    editable: false

docker-compose/grafana/import-rules.py

-#!/usr/bin/python3
-import json
-import os
-import argparse
-
-parser = argparse.ArgumentParser(
-  formatter_class=argparse.RawDescriptionHelpFormatter,
-  description=
-"""
-Generate rule import files and script for Grafana.
-
-This script expands a given rules.json file into individual rules and
-prints the bash commands to import them in Grafana.
-
-To export rules from Grafana, use
-  curl <grafana>/api/ruler/grafana/api/v1/rules > rules.json
-""")
-parser.add_argument(
-    '-c', '--alert-config-file', type=str, required=False, help="Input alertmanager configuration JSON to parse, output of 'curl <grafana>/api/ruler/grafana/api/v1/rules' [%(default)s]")
-parser.add_argument(
-    '-r', '--rules-file', type=str, required=True, help="Input rules JSON to parse, output of 'curl <grafana>/api/ruler/grafana/api/v1/rules' [%(default)s]")
-parser.add_argument(
-    '-o', '--output-dir', type=str, default="rules", help="Directory to store the output [%(default)s]")
-parser.add_argument(
-    '-B', '--authorization-bearer', type=str, default="abcdefghijklmnopqrstuvwxyz", help="Authorization bearer from the Grafana 'editor' API key [%(default)s]")
-parser.add_argument(
-    '-g', '--grafana_url', type=str, default="http://localhost:3000", help="Base URL of Grafana [%(default)s]")
-parser.add_argument(
-    '-u', '--update', default=False, action='store_true', help="Update existing alerts, instead of creating new ones [%(default)s]")
-
-args = parser.parse_args()
-
-if args.alert_config_file:
-  print(f"echo Importing alert configuration file {args.alert_config_file}")
-  print(f"curl -X POST {args.grafana_url}/api/alertmanager/grafana/config/api/v1/alerts -H 'Content-Type: application/json' -H 'Accept: application/json' -H 'Authorization: Bearer {args.authorization_bearer}' -d '@{args.alert_config_file}'")
-  print(f"echo ''")
-
-with open(args.rules_file) as f:
-  data=json.load(f)
-
-  try:
-    os.mkdir(args.output_dir)
-  except FileExistsError as e:
-    pass
-
-  # the rules are of format {"folder": [{alert}, {alert}] }
-  for folder, rules in data.items():
-    try:
-      os.mkdir(f"{args.output_dir}/{folder}")
-    except FileExistsError as e:
-      pass
-
-    # print command to create folder
-    payload = json.dumps({"title": folder})
-    print(f"echo Creating folder {folder}")
-    print(f"curl -X POST {args.grafana_url}/api/folders -H 'Content-Type: application/json' -H 'Accept: application/json' -H 'Authorization: Bearer {args.authorization_bearer}' -d '{payload}'")
-    print(f"echo ''")
-
-    for rule in rules:
-      rule_filename = f"{args.output_dir}/{folder}/{rule['name']}.json"
-
-      if not args.update:
-        # strip rule UIDs
-        for subrule in rule["rules"]:
-          del subrule["grafana_alert"]["uid"]
-
-      # dump this rule
-      with open(rule_filename, "w") as rule_file:
-        json.dump(rule, rule_file)
-
-      # print import statement for this rule
-      print(f"echo Processing rule {folder}/{rule['name']}")
-      print(f"curl -X POST {args.grafana_url}/api/ruler/grafana/api/v1/rules/{folder} -H 'Content-Type: application/json' -H 'Accept: application/json' -H 'Authorization: Bearer {args.authorization_bearer}' -d '@{rule_filename}'")
-      print(f"echo ''")

docker-compose/grafana/rules.json

-{"station":[{"name":"FPGA processing error","interval":"10s","rules":[{"expr":"","for":"20s","labels":{"severity":"major"},"annotations":{"__dashboardUid__":"nC8N_kO7k","__panelId__":"9","summary":"One or more FPGAs are unusable."},"grafana_alert":{"id":1,"orgId":1,"title":"FPGA processing error","condition":"B","data":[{"refId":"A","queryType":"","relativeTimeRange":{"from":600,"to":0},"datasourceUid":"timescaledb","model":{"format":"time_series","group":[],"hide":false,"intervalMs":1000,"maxDataPoints":43200,"metricColumn":"none","rawQuery":true,"rawSql":"SELECT\n  $__timeGroup(data_time, $__interval),\n  x::text,\n  device,\n  name,\n  value\nFROM lofar_array_boolean\nWHERE\n  $__timeFilter(data_time) AND\n  name = 'fpga_error_r'\nORDER BY 1,2","refId":"A","select":[[{"params":["value_r"],"type":"column"}]],"table":"att_scalar_devdouble","timeColumn":"data_time","timeColumnType":"timestamp","where":[{"name":"$__timeFilter","params":[],"type":"macro"}]}},{"refId":"B","queryType":"","relativeTimeRange":{"from":0,"to":0},"datasourceUid":"-100","model":{"conditions":[{"evaluator":{"params":[0],"type":"gt"},"operator":{"type":"and"},"query":{"params":["A"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"-100"},"expression":"A","hide":false,"intervalMs":1000,"maxDataPoints":43200,"reducer":"last","refId":"B","settings":{"mode":"dropNN"},"type":"reduce"}}],"updated":"2022-04-04T18:01:53Z","intervalSeconds":10,"version":3,"uid":"kujybCynk","namespace_uid":"R_jsbCynz","namespace_id":6,"rule_group":"FPGA processing error","no_data_state":"NoData","exec_err_state":"Alerting"}}]}]}
\ No newline at end of file