diff --git a/.gitignore b/.gitignore
index 00941bb5af067040269d23ab91d781bfb6cd5bc7..cfd4dc461a50e0a01b60ca0f88152e9ca9a2d787 100644
--- a/.gitignore
+++ b/.gitignore
@@ -27,3 +27,4 @@ tangostationcontrol/docs/build
**/pending_log_messages.db
**/.eggs
+docker-compose/alerta-web/alerta-secrets.json
diff --git a/docker-compose/alerta-web/Dockerfile b/docker-compose/alerta-web/Dockerfile
index 090c14354a37260d75a506cabd53226d9f35865b..80431da39da9ddb7ff0c28997660163234eb6d57 100644
--- a/docker-compose/alerta-web/Dockerfile
+++ b/docker-compose/alerta-web/Dockerfile
@@ -1,8 +1,14 @@
FROM alerta/alerta-web
+RUN bash -c 'source /venv/bin/activate; pip install git+https://github.com/alerta/alerta-contrib.git#subdirectory=plugins/slack'
+RUN bash -c 'source /venv/bin/activate; pip install git+https://github.com/alerta/alerta-contrib.git#subdirectory=plugins/jira'
+
+COPY grafana-plugin /tmp/grafana-plugin
+RUN bash -c 'source /venv/bin/activate; pip install /tmp/grafana-plugin'
+
+COPY lofar-plugin /tmp/lofar-plugin
+RUN bash -c 'source /venv/bin/activate; pip install /tmp/lofar-plugin'
+
COPY alertad.conf /app/alertad.conf
COPY alerta.conf /app/alerta.conf
COPY config.json /web/config.json
-
-RUN pip install git+https://github.com/alerta/alerta-contrib.git#subdirectory=plugins/slack
-
diff --git a/docker-compose/alerta-web/README.md b/docker-compose/alerta-web/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..8900026226cb6e3ee9c987792f24b44d8beff374
--- /dev/null
+++ b/docker-compose/alerta-web/README.md
@@ -0,0 +1,23 @@
+You need:
+
+* Your own Slack App:
+ * Give it channel write rights
+ * Get the OAuth token
+ * Install it in your slack
+ * Invite the app into your channel
+ * Feed the OAuth token to the config
+ * Add it to alerta-secrets.json
+* Grafana:
+ * By default, grafana resends alarms every 4h, configure this in the notification settings to faster resend deleted alarms for testing
+ * Add alerts by hand
+ * add "Summary" as alert text
+ * add label "severity": "major"/"minor"/etc (see https://docs.alerta.io/webui/configuration.html#severity-colors)
+
+* Create alerta-secrets.json in this directory:
+
+Example alerta-secrets.json:
+
+{
+ "SLACK_TOKEN": "xoxb-...",
+ "SLACK_CHANNEL": "#lofar20-alerta"
+}
diff --git a/docker-compose/alerta-web/alerta-secrets.json b/docker-compose/alerta-web/alerta-secrets.json
new file mode 100644
index 0000000000000000000000000000000000000000..8fb44d7b830c3090408fb3bd576fa297e0e2dcc9
--- /dev/null
+++ b/docker-compose/alerta-web/alerta-secrets.json
@@ -0,0 +1,4 @@
+{
+ "SLACK_TOKEN": "xoxb-get-this-from-your-slack-app",
+ "SLACK_CHANNEL": "#your-channel"
+}
diff --git a/docker-compose/alerta-web/alertad.conf b/docker-compose/alerta-web/alertad.conf
index 96c25994054d7ffffc1ca720c46ca734abf9be63..a9e3e93c6ed20e12751714c5680aa8a7c469003e 100644
--- a/docker-compose/alerta-web/alertad.conf
+++ b/docker-compose/alerta-web/alertad.conf
@@ -1,4 +1,56 @@
DEBUG = True
SECRET = "T=&7xvF2S&x7w_JAcq$h1x5ocfA)8H2i"
-PLUGINS = ['reject', 'blackout', 'normalise', 'enhance', 'slack']
+# Allow non-admin views
+CUSTOMER_VIEWS = True
+
+# Never timeout alerts
+ALERT_TIMEOUT = 0
+# Auto unack after a day
+ACK_TIMEOUT = 24 * 3600
+# Auto unshelve after 2 hours
+SHELVE_TIMEOUT = 2 * 3600
+
+# Use custom date formats
+DATE_FORMAT_MEDIUM_DATE = "dd DD/MM HH:mm"
+DATE_FORMAT_LONG_DATE = "yyyy-MM-DD HH:mm:ss.sss"
+
+# Default columns to list
+COLUMNS = ['severity', 'status', 'lastReceiveTime', 'environment', 'resource', 'lofarDevice', 'lofarAttribute', 'event', 'text']
+
+# ------------------------------------
+# Plugin configuration
+# ------------------------------------
+
+PLUGINS = ['reject', 'blackout', 'acked_by', 'enhance', 'grafana', 'lofar', 'slack']
+
+# Slack plugin settings, see https://github.com/alerta/alerta-contrib/tree/master/plugins/slack
+import os, json
+
+with open("/run/secrets/alerta-secrets") as secrets_file:
+ secrets = json.load(secrets_file)
+
+SLACK_WEBHOOK_URL = 'https://slack.com/api/chat.postMessage'
+SLACK_TOKEN = secrets["SLACK_TOKEN"]
+SLACK_CHANNEL = secrets["SLACK_CHANNEL"]
+SLACK_ATTACHMENTS = True
+DASHBOARD_URL = os.environ.get("DASHBOARD_URL", "")
+
+# for the Slack message configuration syntax, see https://api.slack.com/methods/chat.postMessage
+# and https://app.slack.com/block-kit-builder
+SLACK_PAYLOAD = {
+ "channel": "{{ channel }}",
+ "emoji": ":fire:",
+ "text": "*{{ alert.severity|capitalize }}* :: _{{ alert.event }}_\n\n```{{ alert.text }}```",
+ "attachments": [{
+ "color": "{{ color }}",
+ "fields": [
+ {"title": "Device", "value": "{{ alert.attributes.lofarDevice }}", "short": True },
+ {"title": "Attribute", "value": "{{ alert.attributes.lofarAttribute }}", "short": True },
+ {"title": "Environment", "value": "{{ alert.environment }}", "short": True },
+ {"title": "Status", "value": "{{ status|capitalize }}", "short": True },
+ {"title": "Dashboards", "value": "<{{ config.DASHBOARD_URL }}/#/alert/{{ alert.id }}|Alerta>\nGrafana <{{ alert.attributes.grafanaDashboardUrl }}|Dashboard> <{{ alert.attributes.grafanaPanelUrl }}|Panel>", "short": True },
+ {"title": "Configure", "value": "Grafana <{{ alert.attributes.grafanaAlertUrl }}|Edit> <{{ alert.attributes.grafanaSilenceUrl }}|Silence>", "short": True },
+ ],
+ }]
+}
diff --git a/docker-compose/alerta-web/grafana-plugin/alerta_grafana.py b/docker-compose/alerta-web/grafana-plugin/alerta_grafana.py
new file mode 100644
index 0000000000000000000000000000000000000000..917cddae1e57191151e8405eaeb277c330859745
--- /dev/null
+++ b/docker-compose/alerta-web/grafana-plugin/alerta_grafana.py
@@ -0,0 +1,32 @@
+import os
+import json
+import logging
+
+from alerta.plugins import PluginBase
+
+LOG = logging.getLogger()
+
+
+class EnhanceGrafana(PluginBase):
+ """
+ Plugin for parsing alerts coming from Grafana
+ """
+
+ def pre_receive(self, alert, **kwargs):
+ # Parse Grafana-specific fields
+ alert.attributes['grafanaStatus'] = alert.raw_data.get('status', '')
+ alert.attributes['grafanaPanelUrl'] = alert.raw_data.get('panelURL', '')
+ alert.attributes['grafanaDashboardUrl'] = alert.raw_data.get('dashboardURL', '')
+ alert.attributes['grafanaAlertUrl'] = alert.raw_data.get('generatorURL', '')
+ alert.attributes['grafanaSilenceUrl'] = alert.raw_data.get('silenceURL', '')
+
+ return alert
+
+ def post_receive(self, alert, **kwargs):
+ return
+
+ def status_change(self, alert, status, text, **kwargs):
+ return
+
+ def take_action(self, alert, action, text, **kwargs):
+ raise NotImplementedError
diff --git a/docker-compose/alerta-web/grafana-plugin/setup.py b/docker-compose/alerta-web/grafana-plugin/setup.py
new file mode 100644
index 0000000000000000000000000000000000000000..cb06d95919fde788f299e8318bfc23ef01dbfb79
--- /dev/null
+++ b/docker-compose/alerta-web/grafana-plugin/setup.py
@@ -0,0 +1,24 @@
+
+from setuptools import setup, find_packages
+
+version = '1.0.0'
+
+setup(
+ name="alerta-grafana",
+ version=version,
+ description='Alerta plugin for enhancing Grafana alerts',
+ url='https://git.astron.nl/lofar2.0/tango',
+ license='Apache License 2.0',
+ author='Jan David Mol',
+ author_email='mol@astron.nl',
+ packages=find_packages(),
+ py_modules=['alerta_grafana'],
+ include_package_data=True,
+ zip_safe=True,
+ entry_points={
+ 'alerta.plugins': [
+ 'grafana = alerta_grafana:EnhanceGrafana'
+ ]
+ },
+ python_requires='>=3.5'
+)
diff --git a/docker-compose/alerta-web/lofar-plugin/alerta_lofar.py b/docker-compose/alerta-web/lofar-plugin/alerta_lofar.py
new file mode 100644
index 0000000000000000000000000000000000000000..eaaed8614d34be68c7fa217e813760b75fe9161b
--- /dev/null
+++ b/docker-compose/alerta-web/lofar-plugin/alerta_lofar.py
@@ -0,0 +1,38 @@
+import os
+import json
+import logging
+
+from alerta.plugins import PluginBase
+
+LOG = logging.getLogger()
+
+
+class EnhanceLOFAR(PluginBase):
+ """
+ Plugin for enhancing alerts with LOFAR-specific information
+ """
+
+ def pre_receive(self, alert, **kwargs):
+ # Parse LOFAR-specific fields
+ for tag in alert.tags:
+ try:
+ key, value = tag.split("=", 1)
+ except ValueError:
+ continue
+
+ if key == "device":
+ alert.attributes['lofarDevice'] = value
+
+ if key == "name":
+ alert.attributes['lofarAttribute'] = value
+
+ return alert
+
+ def post_receive(self, alert, **kwargs):
+ return
+
+ def status_change(self, alert, status, text, **kwargs):
+ return
+
+ def take_action(self, alert, action, text, **kwargs):
+ raise NotImplementedError
diff --git a/docker-compose/alerta-web/lofar-plugin/setup.py b/docker-compose/alerta-web/lofar-plugin/setup.py
new file mode 100644
index 0000000000000000000000000000000000000000..70ab552180a5ad10a978fb10f1deeb0d87319bb7
--- /dev/null
+++ b/docker-compose/alerta-web/lofar-plugin/setup.py
@@ -0,0 +1,24 @@
+
+from setuptools import setup, find_packages
+
+version = '1.0.0'
+
+setup(
+ name="alerta-lofar",
+ version=version,
+ description='Alerta plugin for enhancing LOFAR alerts',
+ url='https://git.astron.nl/lofar2.0/tango',
+ license='Apache License 2.0',
+ author='Jan David Mol',
+ author_email='mol@astron.nl',
+ packages=find_packages(),
+ py_modules=['alerta_lofar'],
+ include_package_data=True,
+ zip_safe=True,
+ entry_points={
+ 'alerta.plugins': [
+ 'lofar = alerta_lofar:EnhanceLOFAR'
+ ]
+ },
+ python_requires='>=3.5'
+)
diff --git a/docker-compose/alerta-web/rules.json b/docker-compose/alerta-web/rules.json
new file mode 100644
index 0000000000000000000000000000000000000000..ca8df8cf7b01a4bd014387e045a2492d35292300
--- /dev/null
+++ b/docker-compose/alerta-web/rules.json
@@ -0,0 +1 @@
+{"test":[{"name":"test2","interval":"10s","rules":[{"expr":"","for":"20s","labels":{"severity":"major"},"annotations":{"__dashboardUid__":"nC8N_kO7k","__panelId__":"9","summary":"My test alert"},"grafana_alert":{"id":3,"orgId":1,"title":"FPGA processing error 2","condition":"B","data":[{"refId":"A","queryType":"","relativeTimeRange":{"from":600,"to":0},"datasourceUid":"ZqArtG97z","model":{"exemplar":false,"expr":"device_attribute{device=\"stat/sdp/1\",name=\"FPGA_error_R\"}","format":"time_series","group":[],"hide":false,"interval":"","intervalMs":1000,"legendFormat":"","maxDataPoints":43200,"metricColumn":"name","rawQuery":true,"rawSql":"SELECT\n data_time AS \"time\",\n x::text,\n device,\n name,\n case when value then 1 else 0 end AS value\nFROM lofar_array_boolean\nWHERE\n $__timeFilter(data_time) AND\n name = 'fpga_error_r'\nORDER BY 1,2","refId":"A","select":[[{"params":["x"],"type":"column"}],[{"params":["value"],"type":"column"}]],"table":"lofar_array_boolean","timeColumn":"data_time","timeColumnType":"timestamptz","where":[{"name":"$__timeFilter","params":[],"type":"macro"},{"datatype":"text","name":"","params":["name","=","'fpga_error_r'"],"type":"expression"}]}},{"refId":"B","queryType":"","relativeTimeRange":{"from":0,"to":0},"datasourceUid":"-100","model":{"conditions":[{"evaluator":{"params":[0,0],"type":"gt"},"operator":{"type":"and"},"query":{"params":[]},"reducer":{"params":[],"type":"avg"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"A","hide":false,"intervalMs":1000,"maxDataPoints":43200,"reducer":"last","refId":"B","settings":{"mode":"dropNN"},"type":"reduce"}}],"updated":"2022-04-04T14:18:48Z","intervalSeconds":10,"version":1,"uid":"waXdSCynk","namespace_uid":"9DkbdYy7z","namespace_id":6,"rule_group":"test2","no_data_state":"OK","exec_err_state":"Error"}}]},{"name":"test","interval":"10s","rules":[{"expr":"","for":"20s","labels":{"severity":"major"},"annotations":{"__dashboardUid__":"nC8N_kO7k","__panelId__":"9","summary":"My test alert"},"grafana_alert":{"id":2,"orgId":1,"title":"FPGA processing error","condition":"B","data":[{"refId":"A","queryType":"","relativeTimeRange":{"from":600,"to":0},"datasourceUid":"ZqArtG97z","model":{"exemplar":false,"expr":"device_attribute{device=\"stat/sdp/1\",name=\"FPGA_error_R\"}","format":"time_series","group":[],"hide":false,"interval":"","intervalMs":1000,"legendFormat":"","maxDataPoints":43200,"metricColumn":"name","rawQuery":true,"rawSql":"SELECT\n data_time AS \"time\",\n x::text,\n device,\n name,\n case when value then 1 else 0 end AS value\nFROM lofar_array_boolean\nWHERE\n $__timeFilter(data_time) AND\n name = 'fpga_error_r'\nORDER BY 1,2","refId":"A","select":[[{"params":["x"],"type":"column"}],[{"params":["value"],"type":"column"}]],"table":"lofar_array_boolean","timeColumn":"data_time","timeColumnType":"timestamptz","where":[{"name":"$__timeFilter","params":[],"type":"macro"},{"datatype":"text","name":"","params":["name","=","'fpga_error_r'"],"type":"expression"}]}},{"refId":"B","queryType":"","relativeTimeRange":{"from":0,"to":0},"datasourceUid":"-100","model":{"conditions":[{"evaluator":{"params":[0,0],"type":"gt"},"operator":{"type":"and"},"query":{"params":[]},"reducer":{"params":[],"type":"avg"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"A","hide":false,"intervalMs":1000,"maxDataPoints":43200,"reducer":"last","refId":"B","settings":{"mode":"dropNN"},"type":"reduce"}}],"updated":"2022-04-04T14:16:22Z","intervalSeconds":10,"version":1,"uid":"MIt4Ijs7k","namespace_uid":"9DkbdYy7z","namespace_id":6,"rule_group":"test","no_data_state":"OK","exec_err_state":"Error"}}]}]}
\ No newline at end of file
diff --git a/docker-compose/alerta.yml b/docker-compose/alerta.yml
index 6c30bf9dfc018e445b58faa83d807d5788bc3888..696fcb24e77871d0cd94fb9cefb3fc49158138e1 100644
--- a/docker-compose/alerta.yml
+++ b/docker-compose/alerta.yml
@@ -3,6 +3,10 @@ version: '2.1'
volumes:
alerta-postgres-data: {}
+secrets:
+ alerta-secrets:
+ file: alerta-web/alerta-secrets.json
+
services:
alerta-web:
build: alerta-web
@@ -13,9 +17,12 @@ services:
- "8081:8080"
depends_on:
- alerta-db
+ secrets:
+ - alerta-secrets
environment:
- DEBUG=1 # remove this line to turn DEBUG off
- DATABASE_URL=postgres://postgres:postgres@alerta-db:5432/monitoring
+ - DASHBOARD_URL=http://${HOSTNAME}:8081
- AUTH_REQUIRED=True
- ADMIN_USERS=admin #default password: alerta
- ADMIN_KEY=demo-key
diff --git a/docker-compose/grafana/Dockerfile b/docker-compose/grafana/Dockerfile
index e51cce5eeaa0310c1ecd698d8d797e3163ce4457..7eceb9c154c654da53eb0a4b060df945013bf766 100644
--- a/docker-compose/grafana/Dockerfile
+++ b/docker-compose/grafana/Dockerfile
@@ -3,6 +3,7 @@ FROM grafana/grafana
# Install some plugins
RUN grafana-cli plugins install briangann-datatable-panel
RUN grafana-cli plugins install ae3e-plotly-panel
+RUN grafana-cli plugins install yesoreyeram-infinity-datasource
COPY grafana.ini /etc/grafana/
diff --git a/docker-compose/grafana/README.md b/docker-compose/grafana/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..754c00a75abde5600ee65088d057558eabe02352
--- /dev/null
+++ b/docker-compose/grafana/README.md
@@ -0,0 +1,21 @@
+# Post configuration
+
+To export all current alert rules, use:
+
+To import rules into a fresh Grafana instance:
+
+ * Obtain an 'editor' API key through the Grafan GUI (cogwheel -> API keys),
+ * Run:
+
+ curl http://localhost:3000/api/alertmanager/grafana/config/api/v1/alerts -H 'Authorization: Bearer (api key)' > alerting.json
+ curl localhost:3000/api/ruler/grafana/api/v1/rules > rules.json
+
+ * Delete the UIDs in alerting.json
+
+To import rules into a fresh Grafana instance:
+
+ * Obtain an 'editor' API key through the Grafan GUI (cogwheel -> API keys),
+ * Run (first without piping to bash):
+
+ python3 import-rules.py -c alerting.json -r rules.json -B key | bash
+
diff --git a/docker-compose/grafana/grafana.ini b/docker-compose/grafana/grafana.ini
index 6be79d62ce20e1b456caa09292c5ced027c5dccb..acfabe0f10190c2b07ae579d21bd1abfc1891ff3 100644
--- a/docker-compose/grafana/grafana.ini
+++ b/docker-compose/grafana/grafana.ini
@@ -867,7 +867,9 @@ enabled = true
[panels]
# If set to true Grafana will allow script tags in text panels. Not recommended as it enable XSS vulnerabilities.
-;disable_sanitize_html = false
+
+# enable this to allow us to create mash ups with other pages
+disable_sanitize_html = true
[plugins]
;enable_alpha = false
diff --git a/docker-compose/grafana/import-rules.py b/docker-compose/grafana/import-rules.py
new file mode 100755
index 0000000000000000000000000000000000000000..340215ce1e53744aef3a2722f69c3ecdfd28ca82
--- /dev/null
+++ b/docker-compose/grafana/import-rules.py
@@ -0,0 +1,74 @@
+#!/usr/bin/python3
+import json
+import os
+import argparse
+
+parser = argparse.ArgumentParser(
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ description=
+"""
+Generate rule import files and script for Grafana.
+
+This script expands a given rules.json file into individual rules and
+prints the bash commands to import them in Grafana.
+
+To export rules from Grafana, use
+ curl <grafana>/api/ruler/grafana/api/v1/rules > rules.json
+""")
+parser.add_argument(
+ '-c', '--alert-config-file', type=str, required=False, help="Input alertmanager configuration JSON to parse, output of 'curl <grafana>/api/ruler/grafana/api/v1/rules' [%(default)s]")
+parser.add_argument(
+ '-r', '--rules-file', type=str, required=True, help="Input rules JSON to parse, output of 'curl <grafana>/api/ruler/grafana/api/v1/rules' [%(default)s]")
+parser.add_argument(
+ '-o', '--output-dir', type=str, default="rules", help="Directory to store the output [%(default)s]")
+parser.add_argument(
+ '-B', '--authorization-bearer', type=str, default="abcdefghijklmnopqrstuvwxyz", help="Authorization bearer from the Grafana 'editor' API key [%(default)s]")
+parser.add_argument(
+ '-g', '--grafana_url', type=str, default="http://localhost:3000", help="Base URL of Grafana [%(default)s]")
+parser.add_argument(
+ '-u', '--update', default=False, action='store_true', help="Update existing alerts, instead of creating new ones [%(default)s]")
+
+args = parser.parse_args()
+
+if args.alert_config_file:
+ print(f"echo Importing alert configuration file {args.alert_config_file}")
+ print(f"curl -X POST {args.grafana_url}/api/alertmanager/grafana/config/api/v1/alerts -H 'Content-Type: application/json' -H 'Accept: application/json' -H 'Authorization: Bearer {args.authorization_bearer}' -d '@{args.alert_config_file}'")
+ print(f"echo ''")
+
+with open(args.rules_file) as f:
+ data=json.load(f)
+
+ try:
+ os.mkdir(args.output_dir)
+ except FileExistsError as e:
+ pass
+
+ # the rules are of format {"folder": [{alert}, {alert}] }
+ for folder, rules in data.items():
+ try:
+ os.mkdir(f"{args.output_dir}/{folder}")
+ except FileExistsError as e:
+ pass
+
+ # print command to create folder
+ payload = json.dumps({"title": folder})
+ print(f"echo Creating folder {folder}")
+ print(f"curl -X POST {args.grafana_url}/api/folders -H 'Content-Type: application/json' -H 'Accept: application/json' -H 'Authorization: Bearer {args.authorization_bearer}' -d '{payload}'")
+ print(f"echo ''")
+
+ for rule in rules:
+ rule_filename = f"{args.output_dir}/{folder}/{rule['name']}.json"
+
+ if not args.update:
+ # strip rule UIDs
+ for subrule in rule["rules"]:
+ del subrule["grafana_alert"]["uid"]
+
+ # dump this rule
+ with open(rule_filename, "w") as rule_file:
+ json.dump(rule, rule_file)
+
+ # print import statement for this rule
+ print(f"echo Processing rule {folder}/{rule['name']}")
+ print(f"curl -X POST {args.grafana_url}/api/ruler/grafana/api/v1/rules/{folder} -H 'Content-Type: application/json' -H 'Accept: application/json' -H 'Authorization: Bearer {args.authorization_bearer}' -d '@{rule_filename}'")
+ print(f"echo ''")
diff --git a/docker-compose/grafana/rules.json b/docker-compose/grafana/rules.json
new file mode 100644
index 0000000000000000000000000000000000000000..1469233d304ab0ff26d744dc0ac8738bddf03566
--- /dev/null
+++ b/docker-compose/grafana/rules.json
@@ -0,0 +1 @@
+{"station":[{"name":"FPGA processing error","interval":"10s","rules":[{"expr":"","for":"20s","labels":{"severity":"major"},"annotations":{"__dashboardUid__":"nC8N_kO7k","__panelId__":"9","summary":"One or more FPGAs are unusable."},"grafana_alert":{"id":4,"orgId":1,"title":"FPGA processing error","condition":"B","data":[{"refId":"A","queryType":"","relativeTimeRange":{"from":600,"to":0},"datasourceUid":"ZqArtG97z","model":{"format":"time_series","group":[],"hide":false,"intervalMs":1000,"maxDataPoints":43200,"metricColumn":"none","rawQuery":true,"rawSql":"SELECT\n data_time AS \"time\",\n x::text,\n device,\n name,\n case when value then 1 else 0 end AS value\nFROM lofar_array_boolean\nWHERE\n $__timeFilter(data_time) AND\n name = 'fpga_error_r'\nORDER BY 1,2","refId":"A","select":[[{"params":["value_r"],"type":"column"}]],"table":"att_scalar_devdouble","timeColumn":"data_time","timeColumnType":"timestamp","where":[{"name":"$__timeFilter","params":[],"type":"macro"}]}},{"refId":"B","queryType":"","relativeTimeRange":{"from":0,"to":0},"datasourceUid":"-100","model":{"conditions":[{"evaluator":{"params":[0],"type":"gt"},"operator":{"type":"and"},"query":{"params":["A"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"-100"},"expression":"A","hide":false,"intervalMs":1000,"maxDataPoints":43200,"reducer":"last","refId":"B","settings":{"mode":"dropNN"},"type":"reduce"}}],"updated":"2022-04-04T14:29:45Z","intervalSeconds":10,"version":1,"uid":"tzj3Ijynk","namespace_uid":"9DkbdYy7z","namespace_id":6,"rule_group":"FPGA processing error","no_data_state":"NoData","exec_err_state":"Alerting"}}]}]}
diff --git a/tangostationcontrol/docs/source/alerting.rst b/tangostationcontrol/docs/source/alerting.rst
new file mode 100644
index 0000000000000000000000000000000000000000..ec613fe0ff449602e5694f2e7b17117e7b424480
--- /dev/null
+++ b/tangostationcontrol/docs/source/alerting.rst
@@ -0,0 +1,11 @@
+Alerting
+==================
+
+To setup alerting, you first need to post-configure Grafana to populate it with alerting rules, and a policy to forward rules to Grafana:
+
+- Go to Grafana (http://localhost:3000) and sign in with an administration account (default: admin/admin),
+- Go to ``(cogwheel) -> API keys`` and create an ``editor`` API key. Copy the resulting hash,
+- Go to the ``docker-compose/grafana/`` source directory, and run::
+
+ ./import-rules.py -c alerting.json -r rules.json -B <apikey> | bash
+
diff --git a/tangostationcontrol/docs/source/index.rst b/tangostationcontrol/docs/source/index.rst
index c0b64b2a83975abf0636157347354506c0532d9d..d89a0cda0bef89798497ae94263cd0204c4dfe3a 100644
--- a/tangostationcontrol/docs/source/index.rst
+++ b/tangostationcontrol/docs/source/index.rst
@@ -30,6 +30,7 @@ Even without having access to any LOFAR2.0 hardware, you can install the full st
devices/sst-xst
devices/configure
configure_station
+ alerting
signal_chain
beam_tracking
developer
diff --git a/tangostationcontrol/docs/source/installation.rst b/tangostationcontrol/docs/source/installation.rst
index fd01fe45e0b27de2170c23341ab04e1c6b97f900..09877ef26a8c5f2ea71822338910b884e9bd7a3b 100644
--- a/tangostationcontrol/docs/source/installation.rst
+++ b/tangostationcontrol/docs/source/installation.rst
@@ -78,13 +78,3 @@ Configuration
These sections are optional, to configure specific functionality you may or may not want to use.
-Alerta
-````````
-
-If you want Grafana alerts to appear in Alerta, you need to manually configure Grafana to forward them. Import the alert settings manually:
-
-- Go to Grafana (http://localhost:3000) and sign in with an administration account (default: admin/admin),
-- Go to ``Alerting`` and select ``Admin`` in the left menu bar,
-- Copy/paste the following information, and press ``Save``:
-
-.. literalinclude:: ../../../docker-compose/grafana/alerting.json