Merge branch 'L2SS-359-add-grafana' into 'master'

Resolve L2SS-359 "Add grafana"

Closes L2SS-359

See merge request !106

.gitmodules

+[submodule "docker-compose/tango-prometheus-exporter/ska-tango-grafana-exporter"]
+	path = docker-compose/tango-prometheus-exporter/ska-tango-grafana-exporter
+	url = https://git.astron.nl/lofar2.0/ska-tango-grafana-exporter.git
+	branch = station-control

docker-compose/grafana.yml

+#
+# Docker compose file that launches Grafana
+#
+# Defines:
+#   - grafana: Grafana
+#
+version: '2'
+
+volumes:
+  grafana-data: {}
+  grafana-configs: {}
+
+services:
+  grafana:
+    image: grafana
+    build:
+        context: grafana
+    container_name: ${CONTAINER_NAME_PREFIX}grafana
+    networks:
+      - control
+    volumes:
+      - grafana-data:/var/lib/grafana
+      - grafana-configs:/etc/grafana
+    ports:
+      - "3000:3000"
+    restart: unless-stopped

docker-compose/grafana/Dockerfile

+FROM grafana/grafana
+
+# To populate the Grafana configuration:
+#
+# Datasources (thanks to https://rmoff.net/2017/08/08/simple-export/import-of-data-sources-in-grafana/):
+#
+# Import: 
+#
+# for i in data_sources/*; do \
+#     curl -X "POST" "http://localhost:3000/api/datasources" \
+#     -H "Content-Type: application/json" \
+#      --user admin:admin \
+#      --data-binary @$i
+# done
+#
+# Export:
+#
+# mkdir -p data_sources && curl -s "http://localhost:3000/api/datasources"  -u admin:admin|jq -c -M '.[]'|split -l 1 - data_sources/
+#
+# Dashboards:
+#
+# Import: http://localhost:3000/dashboard/import
+# Export: "share" icon next to dashboard title -> "Export"
+# 
+

docker-compose/grafana/dashboards/version-information.json

+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "id": 2,
+  "links": [],
+  "panels": [
+    {
+      "datasource": null,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 7,
+      "title": "SC",
+      "type": "row"
+    },
+    {
+      "datasource": "Prometheus",
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "align": "auto",
+            "displayMode": "auto"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "version"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 1533
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "device"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 136
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 8,
+        "x": 0,
+        "y": 1
+      },
+      "id": 9,
+      "options": {
+        "showHeader": true,
+        "sortBy": []
+      },
+      "pluginVersion": "8.1.2",
+      "targets": [
+        {
+          "exemplar": true,
+          "expr": "device_attribute{name=\"version_R\"}",
+          "instant": true,
+          "interval": "",
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Device software versions",
+      "transformations": [
+        {
+          "id": "labelsToFields",
+          "options": {}
+        },
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true,
+              "Value": true,
+              "device": false,
+              "dim_x": true,
+              "dim_y": true,
+              "instance": true,
+              "job": true,
+              "label": true,
+              "name": true,
+              "type": true,
+              "x": true,
+              "y": true
+            },
+            "indexByName": {},
+            "renameByName": {
+              "Time": "time",
+              "Value": "count",
+              "str_value": "version"
+            }
+          }
+        }
+      ],
+      "type": "table"
+    },
+    {
+      "collapsed": false,
+      "datasource": null,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 7
+      },
+      "id": 5,
+      "panels": [],
+      "title": "SDP",
+      "type": "row"
+    },
+    {
+      "datasource": "Prometheus",
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "align": "auto",
+            "displayMode": "auto"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "version"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 1907
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "x"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 114
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 17,
+        "w": 8,
+        "x": 0,
+        "y": 8
+      },
+      "id": 2,
+      "options": {
+        "showHeader": true,
+        "sortBy": []
+      },
+      "pluginVersion": "8.1.2",
+      "targets": [
+        {
+          "exemplar": true,
+          "expr": "device_attribute{device=\"lts/sdp/1\",name=\"FPGA_firmware_version_R\"}",
+          "instant": true,
+          "interval": "",
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Firmware versions",
+      "transformations": [
+        {
+          "id": "labelsToFields",
+          "options": {}
+        },
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true,
+              "Value": true,
+              "device": true,
+              "dim_x": true,
+              "dim_y": true,
+              "instance": true,
+              "job": true,
+              "label": true,
+              "name": true,
+              "str_value": false,
+              "type": true,
+              "y": true
+            },
+            "indexByName": {
+              "Time": 1,
+              "Value": 12,
+              "device": 2,
+              "dim_x": 3,
+              "dim_y": 4,
+              "instance": 5,
+              "job": 6,
+              "label": 7,
+              "name": 8,
+              "str_value": 9,
+              "type": 10,
+              "x": 0,
+              "y": 11
+            },
+            "renameByName": {
+              "Time": "time",
+              "Value": "count",
+              "str_value": "version"
+            }
+          }
+        },
+        {
+          "id": "sortBy",
+          "options": {
+            "fields": {},
+            "sort": [
+              {
+                "field": "x"
+              }
+            ]
+          }
+        }
+      ],
+      "type": "table"
+    },
+    {
+      "datasource": "Prometheus",
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "align": "auto",
+            "displayMode": "auto"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "version"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 1907
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "x"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 114
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 17,
+        "w": 8,
+        "x": 8,
+        "y": 8
+      },
+      "id": 13,
+      "options": {
+        "showHeader": true,
+        "sortBy": []
+      },
+      "pluginVersion": "8.1.2",
+      "targets": [
+        {
+          "exemplar": true,
+          "expr": "device_attribute{device=\"lts/sdp/1\",name=\"FPGA_hardware_version_R\"}",
+          "instant": true,
+          "interval": "",
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Hardware versions",
+      "transformations": [
+        {
+          "id": "labelsToFields",
+          "options": {}
+        },
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true,
+              "Value": true,
+              "device": true,
+              "dim_x": true,
+              "dim_y": true,
+              "instance": true,
+              "job": true,
+              "label": true,
+              "name": true,
+              "str_value": false,
+              "type": true,
+              "y": true
+            },
+            "indexByName": {
+              "Time": 1,
+              "Value": 12,
+              "device": 2,
+              "dim_x": 3,
+              "dim_y": 4,
+              "instance": 5,
+              "job": 6,
+              "label": 7,
+              "name": 8,
+              "str_value": 9,
+              "type": 10,
+              "x": 0,
+              "y": 11
+            },
+            "renameByName": {
+              "Time": "time",
+              "Value": "count",
+              "str_value": "version"
+            }
+          }
+        },
+        {
+          "id": "sortBy",
+          "options": {
+            "fields": {},
+            "sort": [
+              {
+                "field": "x"
+              }
+            ]
+          }
+        }
+      ],
+      "type": "table"
+    },
+    {
+      "datasource": "Prometheus",
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "align": "auto",
+            "displayMode": "auto"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "version"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 497
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 3,
+        "w": 5,
+        "x": 16,
+        "y": 8
+      },
+      "id": 8,
+      "options": {
+        "showHeader": true,
+        "sortBy": []
+      },
+      "pluginVersion": "8.1.2",
+      "targets": [
+        {
+          "exemplar": true,
+          "expr": "count(device_attribute{device=\"lts/sdp/1\",name=\"TR_software_version_R\"}) by (str_value)",
+          "instant": true,
+          "interval": "",
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Translator software versions",
+      "transformations": [
+        {
+          "id": "labelsToFields",
+          "options": {}
+        },
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true,
+              "Value": true
+            },
+            "indexByName": {},
+            "renameByName": {
+              "Time": "time",
+              "Value": "count",
+              "str_value": "version"
+            }
+          }
+        }
+      ],
+      "type": "table"
+    },
+    {
+      "collapsed": false,
+      "datasource": null,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 25
+      },
+      "id": 11,
+      "panels": [],
+      "title": "PCC",
+      "type": "row"
+    },
+    {
+      "datasource": "Prometheus",
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "align": "auto",
+            "displayMode": "auto"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "version"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 497
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "x"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 81
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 32,
+        "w": 7,
+        "x": 0,
+        "y": 26
+      },
+      "id": 12,
+      "options": {
+        "showHeader": true,
+        "sortBy": []
+      },
+      "pluginVersion": "8.1.2",
+      "targets": [
+        {
+          "exemplar": true,
+          "expr": "device_attribute{device=\"lts/pcc/1\",name=\"RCU_version_R\"}",
+          "instant": true,
+          "interval": "",
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "RCU versions",
+      "transformations": [
+        {
+          "id": "labelsToFields",
+          "options": {}
+        },
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true,
+              "Value": true,
+              "device": true,
+              "dim_x": true,
+              "dim_y": true,
+              "instance": true,
+              "job": true,
+              "label": true,
+              "name": true,
+              "type": true,
+              "y": true
+            },
+            "indexByName": {
+              "Time": 1,
+              "Value": 12,
+              "device": 2,
+              "dim_x": 3,
+              "dim_y": 4,
+              "instance": 5,
+              "job": 6,
+              "label": 7,
+              "name": 8,
+              "str_value": 9,
+              "type": 10,
+              "x": 0,
+              "y": 11
+            },
+            "renameByName": {
+              "Time": "time",
+              "Value": "count",
+              "str_value": "version"
+            }
+          }
+        },
+        {
+          "id": "sortBy",
+          "options": {
+            "fields": {},
+            "sort": [
+              {
+                "field": "x"
+              }
+            ]
+          }
+        }
+      ],
+      "type": "table"
+    }
+  ],
+  "schemaVersion": 30,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Version information",
+  "uid": "eR9posS7z",
+  "version": 1
+}

docker-compose/grafana/data_sources/aa

+{"id":4,"uid":"6W2nM-Vnz","orgId":1,"name":"Prometheus","type":"prometheus","typeName":"Prometheus","typeLogoUrl":"public/app/plugins/datasource/prometheus/img/prometheus_logo.svg","access":"proxy","url":"prometheus:9090","password":"","user":"","database":"","basicAuth":false,"isDefault":false,"jsonData":{"httpMethod":"POST"},"readOnly":false}

docker-compose/grafana/data_sources/ab

+{"id":2,"uid":"d5_heb47k","orgId":1,"name":"TangoDB","type":"mysql","typeName":"MySQL","typeLogoUrl":"public/app/plugins/datasource/mysql/img/mysql_logo.svg","access":"proxy","url":"tangodb","password":"","user":"tango","database":"tango","basicAuth":false,"isDefault":true,"jsonData":{"timezone":""},"readOnly":false}

docker-compose/prometheus.yml

+#
+# Docker compose file that launches Prometheus
+#
+# Defines:
+#   - prometheus: Prometheus
+#
+version: '2'
+
+services:
+  prometheus:
+    image: prometheus
+    build:
+        context: prometheus
+    container_name: ${CONTAINER_NAME_PREFIX}prometheus
+    networks:
+      - control
+    ports:
+      - "9090:9090"
+    restart: unless-stopped

docker-compose/prometheus/Dockerfile

+FROM prom/prometheus
+
+COPY prometheus.yml /etc/prometheus/prometheus.yml

docker-compose/prometheus/prometheus.yml

+global:
+  evaluation_interval: 10s
+  scrape_interval: 10s
+  scrape_timeout: 10s
+
+scrape_configs:
+  - job_name: tango
+    static_configs:
+      - targets:
+        - "tango-prometheus-exporter:8000"
+

docker-compose/tango-prometheus-exporter.yml

+#
+# Docker compose file that launches the Tango -> Prometheus adapter
+#
+version: '2'
+
+services:
+  tango-prometheus-exporter:
+    build:
+        context: tango-prometheus-exporter
+    container_name: ${CONTAINER_NAME_PREFIX}tango-prometheus-exporter
+    networks:
+      - control
+    environment:
+      - TANGO_HOST=${TANGO_HOST}
+    ports:
+      - "8000:8000"
+    depends_on:
+      - databaseds
+    restart: unless-stopped

docker-compose/tango-prometheus-exporter/Dockerfile

+FROM tangocs/tango-pytango
+
+USER root
+
+RUN apt-get update && apt-get install curl -y
+
+USER tango
+
+ADD ska-tango-grafana-exporter/exporter/code /code
+RUN pip install -r /code/pip-requirements.txt
+
+WORKDIR /code
+ENV PYTHONPATH '/code/'
+
+CMD ["python" , "/code/collector.py"]

docker-compose/tango-prometheus-exporter/LICENSE

+Copyright 2020 INAF Matteo Di Carlo
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from this
+software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file

docker-compose/tango-prometheus-exporter/Makefile

+NAME:=tango-exporter
+
+VERSION:=1.0.2
+TAG:=$(VERSION)
+
+include ../make/Makefile.mk
\ No newline at end of file

docker-compose/tango-prometheus-exporter/README

+Source: https://gitlab.com/ska-telescope/TANGO-grafana/-/tree/master/

docker-compose/tango-prometheus-exporter/get_metrics.sh

+curl $(kubectl get svc -n tango-grafana -o jsonpath='{.items[?(@.metadata.name=="tango-exporter-service-0")].spec.clusterIP}')/metrics
\ No newline at end of file

ska-tango-grafana-exporter @ c7cadb8f

+Subproject commit c7cadb8f48dc8b36860f23ce421c72e90216a548