Merge branch 'L2SS-685-add-alerta-docker' into 'master'

Resolve L2SS-685 "Add alerta docker"

Closes L2SS-685

See merge request !290

.gitignore

@@ -27,3 +27,4 @@ tangostationcontrol/docs/build
 **/pending_log_messages.db
 **/.eggs
 
+docker-compose/alerta-web/alerta-secrets.json

CDB/stations/DTS_ConfigDb.json

@@ -357,6 +357,24 @@
                                 "902",
                                 "902",
                                 "902"
+                            ],
+                            "TR_fpga_mask_RW_default": [
+                                "True",
+                                "True",
+                                "True",
+                                "True",
+                                "False",
+                                "False",
+                                "False",
+                                "False",
+                                "False",
+                                "False",
+                                "False",
+                                "False",
+                                "False",
+                                "False",
+                                "False",
+                                "False"
                             ]
                         }
                     }

docker-compose/alerta-web/Dockerfile

 FROM alerta/alerta-web
 
-RUN pip install git+https://github.com/alerta/alerta-contrib.git#subdirectory=plugins/slack
+RUN bash -c 'source /venv/bin/activate; pip install git+https://github.com/alerta/alerta-contrib.git#subdirectory=plugins/slack'
+RUN bash -c 'source /venv/bin/activate; pip install git+https://github.com/alerta/alerta-contrib.git#subdirectory=plugins/jira'
+
+COPY grafana-plugin /tmp/grafana-plugin
+RUN bash -c 'source /venv/bin/activate; pip install /tmp/grafana-plugin'
+
+COPY lofar-plugin /tmp/lofar-plugin
+RUN bash -c 'source /venv/bin/activate; pip install /tmp/lofar-plugin'
+
+COPY alertad.conf /app/alertad.conf
+COPY alerta.conf /app/alerta.conf
+COPY config.json /web/config.json

docker-compose/alerta-web/README.md

+You need:
+
+* Your own Slack App:
+        * Give it channel write rights
+        * Get the OAuth token
+        * Install it in your slack
+        * Invite the app into your channel
+        * Feed the OAuth token to the config
+        * Add it to alerta-secrets.json
+* Grafana:
+        * By default, grafana resends alarms every 4h, configure this in the notification settings to faster resend deleted alarms for testing
+        * Add alerts by hand
+                * add "Summary" as alert text
+                * add label "severity": "major"/"minor"/etc (see https://docs.alerta.io/webui/configuration.html#severity-colors)
+
+* Create alerta-secrets.json in this directory:
+
+Example alerta-secrets.json:
+
+{
+ "SLACK_TOKEN": "xoxb-...",
+ "SLACK_CHANNEL": "#lofar20-alerta"
+}

docker-compose/alerta-web/alerta-secrets.json

+{
+ "SLACK_TOKEN": "xoxb-get-this-from-your-slack-app",
+ "SLACK_CHANNEL": "#your-channel"
+}

docker-compose/alerta-web/alerta.conf

+[DEFAULT]
+sslverify = no
+output = presto
+endpoint = http://localhost:8080/api
+timezone = Europe/London
+key = NpzX0z_fX8TVKZtXpzop-pi2MhaGnLawKVqbJBoA
+debug = yes

docker-compose/alerta-web/alertad.conf

+DEBUG = True
+SECRET = "T=&7xvF2S&x7w_JAcq$h1x5ocfA)8H2i"
+
+# Allow non-admin views
+CUSTOMER_VIEWS = True
+
+# Never timeout alerts
+ALERT_TIMEOUT = 0
+# Auto unack after a day
+ACK_TIMEOUT = 24 * 3600
+# Auto unshelve after 2 hours
+SHELVE_TIMEOUT = 2 * 3600
+
+# Use custom date formats
+DATE_FORMAT_MEDIUM_DATE = "dd DD/MM HH:mm"
+DATE_FORMAT_LONG_DATE   = "yyyy-MM-DD HH:mm:ss.sss"
+
+# Default overview settings
+COLUMNS = ['severity', 'status', 'createTime', 'lastReceiveTime', 'resource', 'grafanaDashboardHtml', 'grafanaPanelHtml', 'event', 'text']
+DEFAULT_FILTER = {'status': ['open']}
+SORT_LIST_BY = "createTime"
+AUTO_REFRESH_INTERVAL = 5000 # ms
+
+# ------------------------------------
+#    Plugin configuration
+# ------------------------------------
+
+PLUGINS = ['reject', 'blackout', 'acked_by', 'enhance', 'grafana', 'lofar', 'slack']
+
+# Slack plugin settings, see https://github.com/alerta/alerta-contrib/tree/master/plugins/slack
+import os, json
+
+with open("/run/secrets/alerta-secrets") as secrets_file:
+    secrets = json.load(secrets_file)
+
+SLACK_WEBHOOK_URL = 'https://slack.com/api/chat.postMessage'
+SLACK_TOKEN = secrets["SLACK_TOKEN"]
+SLACK_CHANNEL = secrets["SLACK_CHANNEL"]
+SLACK_ATTACHMENTS = True
+BASE_URL = os.environ.get("BASE_URL", "")
+
+# for the Slack message configuration syntax, see https://api.slack.com/methods/chat.postMessage
+# and https://app.slack.com/block-kit-builder
+SLACK_PAYLOAD = {
+  "channel": "{{ channel }}",
+  "emoji": ":fire:",
+  "text": "*{{ alert.severity|capitalize }}* :: *{{ alert.resource }}* :: _{{ alert.event }}_\n\n```{{ alert.text }}```",
+  "attachments": [{
+    "color": "{{ color }}",
+    "fields": [
+      {"title": "Device", "value": "{{ alert.attributes.lofarDevice }}", "short": True },
+      {"title": "Attribute", "value": "{{ alert.attributes.lofarAttribute }}", "short": True },
+      {"title": "Resource", "value": "{{ alert.resource }}", "short": True },
+      {"title": "Status", "value": "{{ status|capitalize }}", "short": True },
+      {"title": "Dashboards", "value": "<{{ config.BASE_URL }}/#/alert/{{ alert.id }}|Alerta>\nGrafana <{{ alert.attributes.grafanaDashboardUrl }}|Dashboard> <{{ alert.attributes.grafanaPanelUrl }}|Panel>", "short": True },
+      {"title": "Configure", "value": "Grafana <{{ alert.attributes.grafanaAlertUrl }}|View> <{{ alert.attributes.grafanaSilenceUrl }}|Silence>", "short": True },
+    ],
+ }]
+}

docker-compose/alerta-web/config.json

+{"endpoint": "/api"}

docker-compose/alerta-web/grafana-plugin/alerta_grafana.py

+import os
+import json
+import logging
+
+from alerta.plugins import PluginBase
+
+LOG = logging.getLogger()
+
+
+class EnhanceGrafana(PluginBase):
+    """
+    Plugin for parsing alerts coming from Grafana
+    """
+
+    def pre_receive(self, alert, **kwargs):
+        # Parse Grafana-specific fields
+        alert.attributes['grafanaStatus']       = alert.raw_data.get('status', '')
+
+        def htmlify(link: str, desc: str) -> str:
+            return f'<a href="{link}" target="_blank">{desc}</a>';
+
+        # User-specified "Panel ID" annotation
+        panelURL = alert.raw_data.get('panelURL', '')
+        if panelURL:
+            alert.attributes['grafanaPanelUrl'] = panelURL
+            alert.attributes['grafanaPanelHtml'] = htmlify(panelURL, "Grafana Panel")
+
+        # User-specified "Dashboard UID" annotation
+        dashboardURL = alert.raw_data.get('dashboardURL', '')
+        if dashboardURL:
+            alert.attributes['grafanaDashboardUrl'] = dashboardURL
+            alert.attributes['grafanaDashboardHtml'] = htmlify(dashboardURL, "Grafana Dashboard")
+
+        alertURL = alert.raw_data.get('generatorURL', '')
+        if alertURL:
+            # expose alert view URL, as user may not have edit rights
+            # Convert from
+            #   http://host:3000/alerting/kujybCynk/edit
+            # to
+            #   http://host:3000/alerting/grafana/kujybCynk/view
+            alertURL = alertURL.replace("/alerting/", "/alerting/grafana/").replace("/edit", "/view")
+
+            alert.attributes['grafanaAlertUrl'] = alertURL
+            alert.attributes['grafanaAlertHtml'] = htmlify(alertURL, "Grafana Alert")
+
+        silenceURL = alert.raw_data.get('silenceURL', '')
+        if silenceURL:
+            alert.attributes['grafanaSilenceUrl'] = silenceURL
+            alert.attributes['grafanaSilenceHtml'] = htmlify(silenceURL, "Grafana Silence Alert")
+
+        return alert
+
+    def post_receive(self, alert, **kwargs):
+        return
+
+    def status_change(self, alert, status, text, **kwargs):
+        return
+
+    def take_action(self, alert, action, text, **kwargs):
+        raise NotImplementedError

docker-compose/alerta-web/grafana-plugin/setup.py

+
+from setuptools import setup, find_packages
+
+version = '1.0.0'
+
+setup(
+    name="alerta-grafana",
+    version=version,
+    description='Alerta plugin for enhancing Grafana alerts',
+    url='https://git.astron.nl/lofar2.0/tango',
+    license='Apache License 2.0',
+    author='Jan David Mol',
+    author_email='mol@astron.nl',
+    packages=find_packages(),
+    py_modules=['alerta_grafana'],
+    include_package_data=True,
+    zip_safe=True,
+    entry_points={
+        'alerta.plugins': [
+            'grafana = alerta_grafana:EnhanceGrafana'
+        ]
+    },
+    python_requires='>=3.5'
+)

docker-compose/alerta-web/lofar-plugin/alerta_lofar.py

+import os
+import json
+import logging
+
+from alerta.plugins import PluginBase
+
+LOG = logging.getLogger()
+
+
+class EnhanceLOFAR(PluginBase):
+    """
+    Plugin for enhancing alerts with LOFAR-specific information
+    """
+
+    def pre_receive(self, alert, **kwargs):
+        # Parse LOFAR-specific fields
+        for tag in alert.tags:
+            try:
+                key, value = tag.split("=", 1)
+            except ValueError:
+                continue
+
+            if key == "device":
+                alert.attributes['lofarDevice'] = value
+
+            if key == "name":
+                alert.attributes['lofarAttribute'] = value
+
+            if key == "station":
+                alert.resource = value
+
+        return alert
+
+    def post_receive(self, alert, **kwargs):
+        return
+
+    def status_change(self, alert, status, text, **kwargs):
+        return
+
+    def take_action(self, alert, action, text, **kwargs):
+        raise NotImplementedError

docker-compose/alerta-web/lofar-plugin/setup.py

+
+from setuptools import setup, find_packages
+
+version = '1.0.0'
+
+setup(
+    name="alerta-lofar",
+    version=version,
+    description='Alerta plugin for enhancing LOFAR alerts',
+    url='https://git.astron.nl/lofar2.0/tango',
+    license='Apache License 2.0',
+    author='Jan David Mol',
+    author_email='mol@astron.nl',
+    packages=find_packages(),
+    py_modules=['alerta_lofar'],
+    include_package_data=True,
+    zip_safe=True,
+    entry_points={
+        'alerta.plugins': [
+            'lofar = alerta_lofar:EnhanceLOFAR'
+        ]
+    },
+    python_requires='>=3.5'
+)

docker-compose/alerta-web/rules.json

+{"test":[{"name":"test2","interval":"10s","rules":[{"expr":"","for":"20s","labels":{"severity":"major"},"annotations":{"__dashboardUid__":"nC8N_kO7k","__panelId__":"9","summary":"My test alert"},"grafana_alert":{"id":3,"orgId":1,"title":"FPGA processing error 2","condition":"B","data":[{"refId":"A","queryType":"","relativeTimeRange":{"from":600,"to":0},"datasourceUid":"ZqArtG97z","model":{"exemplar":false,"expr":"device_attribute{device=\"stat/sdp/1\",name=\"FPGA_error_R\"}","format":"time_series","group":[],"hide":false,"interval":"","intervalMs":1000,"legendFormat":"","maxDataPoints":43200,"metricColumn":"name","rawQuery":true,"rawSql":"SELECT\n  data_time AS \"time\",\n  x::text,\n  device,\n  name,\n  case when value then 1 else 0 end AS value\nFROM lofar_array_boolean\nWHERE\n  $__timeFilter(data_time) AND\n  name = 'fpga_error_r'\nORDER BY 1,2","refId":"A","select":[[{"params":["x"],"type":"column"}],[{"params":["value"],"type":"column"}]],"table":"lofar_array_boolean","timeColumn":"data_time","timeColumnType":"timestamptz","where":[{"name":"$__timeFilter","params":[],"type":"macro"},{"datatype":"text","name":"","params":["name","=","'fpga_error_r'"],"type":"expression"}]}},{"refId":"B","queryType":"","relativeTimeRange":{"from":0,"to":0},"datasourceUid":"-100","model":{"conditions":[{"evaluator":{"params":[0,0],"type":"gt"},"operator":{"type":"and"},"query":{"params":[]},"reducer":{"params":[],"type":"avg"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"A","hide":false,"intervalMs":1000,"maxDataPoints":43200,"reducer":"last","refId":"B","settings":{"mode":"dropNN"},"type":"reduce"}}],"updated":"2022-04-04T14:18:48Z","intervalSeconds":10,"version":1,"uid":"waXdSCynk","namespace_uid":"9DkbdYy7z","namespace_id":6,"rule_group":"test2","no_data_state":"OK","exec_err_state":"Error"}}]},{"name":"test","interval":"10s","rules":[{"expr":"","for":"20s","labels":{"severity":"major"},"annotations":{"__dashboardUid__":"nC8N_kO7k","__panelId__":"9","summary":"My test alert"},"grafana_alert":{"id":2,"orgId":1,"title":"FPGA processing error","condition":"B","data":[{"refId":"A","queryType":"","relativeTimeRange":{"from":600,"to":0},"datasourceUid":"ZqArtG97z","model":{"exemplar":false,"expr":"device_attribute{device=\"stat/sdp/1\",name=\"FPGA_error_R\"}","format":"time_series","group":[],"hide":false,"interval":"","intervalMs":1000,"legendFormat":"","maxDataPoints":43200,"metricColumn":"name","rawQuery":true,"rawSql":"SELECT\n  data_time AS \"time\",\n  x::text,\n  device,\n  name,\n  case when value then 1 else 0 end AS value\nFROM lofar_array_boolean\nWHERE\n  $__timeFilter(data_time) AND\n  name = 'fpga_error_r'\nORDER BY 1,2","refId":"A","select":[[{"params":["x"],"type":"column"}],[{"params":["value"],"type":"column"}]],"table":"lofar_array_boolean","timeColumn":"data_time","timeColumnType":"timestamptz","where":[{"name":"$__timeFilter","params":[],"type":"macro"},{"datatype":"text","name":"","params":["name","=","'fpga_error_r'"],"type":"expression"}]}},{"refId":"B","queryType":"","relativeTimeRange":{"from":0,"to":0},"datasourceUid":"-100","model":{"conditions":[{"evaluator":{"params":[0,0],"type":"gt"},"operator":{"type":"and"},"query":{"params":[]},"reducer":{"params":[],"type":"avg"},"type":"query"}],"datasource":{"type":"__expr__","uid":"__expr__"},"expression":"A","hide":false,"intervalMs":1000,"maxDataPoints":43200,"reducer":"last","refId":"B","settings":{"mode":"dropNN"},"type":"reduce"}}],"updated":"2022-04-04T14:16:22Z","intervalSeconds":10,"version":1,"uid":"MIt4Ijs7k","namespace_uid":"9DkbdYy7z","namespace_id":6,"rule_group":"test","no_data_state":"OK","exec_err_state":"Error"}}]}]}
\ No newline at end of file

docker-compose/alerta.yml

@@ -3,6 +3,10 @@ version: '2.1'
 volumes:
   alerta-postgres-data: {}
 
+secrets:
+  alerta-secrets:
+    file: alerta-web/alerta-secrets.json
+
 services:
   alerta-web:
     build: alerta-web
@@ -13,13 +17,16 @@ services:
       - "8081:8080"
     depends_on:
       - alerta-db
+    secrets:
+      - alerta-secrets
     environment:
       - DEBUG=1  # remove this line to turn DEBUG off
       - DATABASE_URL=postgres://postgres:postgres@alerta-db:5432/monitoring
+      - BASE_URL=http://${HOSTNAME}:8081
+      - DASHBOARD_URL=http://${HOSTNAME}:8081
       - AUTH_REQUIRED=True
       - ADMIN_USERS=admin #default password: alerta
       - ADMIN_KEY=demo-key
-      - PLUGINS=reject,blackout,normalise,enhance
     restart: always
 
   alerta-db:

docker-compose/grafana.yml

@@ -24,6 +24,8 @@ services:
     #  - grafana-configs:/etc/grafana
     ports:
       - "3000:3000"
+    environment:
+      - GF_SERVER_DOMAIN=${HOSTNAME}
     logging:
       driver: syslog
       options:

docker-compose/grafana/Dockerfile

@@ -3,6 +3,7 @@ FROM grafana/grafana
 # Install some plugins
 RUN grafana-cli plugins install briangann-datatable-panel
 RUN grafana-cli plugins install ae3e-plotly-panel
+RUN grafana-cli plugins install yesoreyeram-infinity-datasource
 
 COPY grafana.ini /etc/grafana/
 

docker-compose/grafana/README.md

+# Post configuration
+
+To export all current alert rules, use:
+
+To import rules into a fresh Grafana instance:
+
+  * Obtain an 'editor' API key through the Grafan GUI (cogwheel -> API keys),
+  * Run:
+
+  curl http://localhost:3000/api/alertmanager/grafana/config/api/v1/alerts -H 'Authorization: Bearer (api key)' > alerting.json
+  curl localhost:3000/api/ruler/grafana/api/v1/rules > rules.json
+
+  * Delete the UIDs in alerting.json
+
+To import rules into a fresh Grafana instance:
+
+  * Obtain an 'editor' API key through the Grafan GUI (cogwheel -> API keys),
+  * Run (first without piping to bash):
+
+  python3 import-rules.py -c alerting.json -r rules.json -B key | bash
+

docker-compose/grafana/alerting.json

@@ -2,7 +2,8 @@
   "template_files": {},
   "alertmanager_config": {
     "route": {
-      "receiver": "Alerta"
+      "receiver": "Alerta",
+      "repeat_interval": "10m"
     },
     "templates": null,
     "receivers": [
@@ -10,7 +11,6 @@
         "name": "Alerta",
         "grafana_managed_receiver_configs": [
           {
-            "uid": "ROaAvQEnz",
             "name": "Alerta",
             "type": "webhook",
             "disableResolveMessage": false,

docker-compose/grafana/datasources/alertaui.yaml

+apiVersion: 1
+
+datasources:
+  # <string, required> name of the datasource. Required
+  - name: Alerta UI
+    # <string, required> datasource type. Required
+    type: yesoreyeram-infinity-datasource
+    # <string, required> access mode. proxy or direct (Server or Browser in the UI). Required
+    access: proxy
+    # <int> org id. will default to orgId 1 if not specified
+    orgId: 1
+    # <string> custom UID which can be used to reference this datasource in other parts of the configuration, if not specified will be generated automatically
+    uid: alertaui
+    # <string> url
+    url: http://alerta-web:8080/api
+    # <string> Deprecated, use secureJsonData.password
+    password:
+    # <string> database user, if used
+    user: postgres
+    # <string> database name, if used
+    database: hdb
+    # <bool> enable/disable basic auth
+    basicAuth: false
+    # <string> basic auth username
+    basicAuthUser:
+    # <string> Deprecated, use secureJsonData.basicAuthPassword
+    basicAuthPassword:
+    # <bool> enable/disable with credentials headers
+    withCredentials:
+    # <bool> mark as default datasource. Max one per org
+    isDefault: false
+    # <map> fields that will be converted to json and stored in jsonData
+    jsonData:
+      secureQueryName1: "api-key"
+    # <string> json object of data that will be encrypted.
+    secureJsonData:
+      secureQueryValue1: "demo-key"
+    version: 1
+    # <bool> allow users to edit datasources from the UI.
+    editable: false
+