Merge branch 'L2SS-1110-nomad_deploy' into 'main'

Resolve L2SS-1110 "Nomad deploy"

Closes L2SS-1110

See merge request !2

.gitlab-ci.yml

+image: docker:latest
+stages:
+  - build
+  - tag-latest
+  - deploy
+
+variables:
+  DOCKER_TLS_CERTDIR: "/certs"
+  IMAGE_TAG: $CI_COMMIT_REF_SLUG-$CI_PIPELINE_ID
+  IMAGE_NAME: $CI_REGISTRY_IMAGE
+
+#.default_rules:
+#  rules:
+#    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+
+build_grafana:
+  stage: build
+  services:
+    - name: docker:dind
+  script:
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - docker pull $IMAGE_NAME/grafana:latest || true
+    - docker build --cache-from $IMAGE_NAME/grafana:latest --tag $IMAGE_NAME/grafana:$IMAGE_TAG grafana-central
+    - docker push $IMAGE_NAME/grafana:$IMAGE_TAG
+
+build_prometheus:
+  stage: build
+  services:
+    - name: docker:dind
+  script:
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - docker pull $IMAGE_NAME/prometheus:latest || true
+    - docker build --cache-from $IMAGE_NAME/prometheus:latest --tag $IMAGE_NAME/prometheus:$IMAGE_TAG prometheus-central
+    - docker push $IMAGE_NAME/prometheus:$IMAGE_TAG
+
+tag-latest_grafana:
+  stage: tag-latest
+  services:
+    - name: docker:dind
+  script:
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - docker tag $IMAGE_NAME/grafana:$IMAGE_TAG $IMAGE_NAME/grafana:latest
+    - docker push $IMAGE_NAME/grafana:latest
+
+tag-latest_prometheus:
+  stage: tag-latest
+  services:
+    - name: docker:dind
+  script:
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - docker tag $IMAGE_NAME/prometheus:$IMAGE_TAG $IMAGE_NAME/prometheus:latest
+    - docker push $IMAGE_NAME/prometheus:latest
+#  rules:
+#    - !reference [.default_rules, rules]
+
+
+
+deploy_dop94:
+  stage: deploy
+  image:
+    name: hashicorp/levant
+    entrypoint: [ "" ]
+  needs: [ build_grafana, build_prometheus ]
+  dependencies: []
+  environment:
+    name: dop94.astron.nl
+    url: http://dop94.astron.nl:3001
+  script:
+    - |
+      levant deploy \
+        -address="http://10.87.2.94:4646" \
+        -var image_tag="${IMAGE_TAG}" \
+        grafana-central.nomad
+
+
+# has no CI/CD yet
+# trigger-operations-central-management:
+#  stage: trigger
+#  trigger: lofar2.0/operations-central-management
+#  rules:
+#    - !reference [.default_rules, rules]

grafana-central.nomad

+job "central-management" {
+  datacenters = ["dc1"]
+  type = "service"
+
+  constraint {
+    attribute = "${attr.kernel.name}"
+    value = "linux"
+  }
+
+  update {
+    stagger = "30s"
+    max_parallel = 1
+  }
+
+  group "grafana" {
+    restart {
+      attempts = 10
+      interval = "5m"
+      delay    = "10s"
+      mode     = "delay"
+    }
+
+
+    network {
+      mode ="bridge"
+      port "http" {
+        static = 3001
+        to     = 3000
+      }
+    }
+
+    service {
+      name = "grafana"
+      port = "3001"
+
+      connect {
+        sidecar_service {
+          proxy {
+            upstreams {
+              destination_name = "prometheus"
+              local_bind_port  = 9090
+            }
+            upstreams {
+              destination_name = "postgres"
+              local_bind_port  = 5432
+            }
+          }
+        }
+      }
+    }
+
+    task "grafana" {
+      driver = "docker"
+      config {
+        image       = "git.astron.nl:5000/lofar2.0/operations-central-management/grafana:[[.image_tag]]"
+        ports       = ["http"]
+        #mount {
+        #  type = "volume"
+        #  target = "/var/lib/grafana"
+        #  source = "grafana-data"
+        #  readonly = false
+        #}
+        #extra_hosts = [
+        #  "prometheus:${NOMAD_IP_prometheus}"
+        #]
+        extra_hosts = [
+          "prometheus:127.0.0.1"
+        ]
+      }
+
+      env {
+        GF_SERVER_DOMAIN = "dop94.astron.nl"
+        GF_DATABASE_TYPE = "postgres"
+        GF_DATABASE_HOST = "localhost:5432"
+        GF_DATABASE_NAME = "grafana"
+        GF_DATABASE_USER = "postgres"
+        GF_DATABASE_PASSWORD = "password"
+
+      }
+
+      resources {
+        cpu    = 500
+        memory = 256
+      }
+    }
+  }
+
+
+  group "postgres" {
+    count = 1
+
+    network {
+      mode ="bridge"
+      port "postgres" {
+        to = 5432
+      }
+    }
+
+    service {
+      name = "postgres"
+      port = "5432"
+      task = "postgres"
+
+      connect {
+        sidecar_service {}
+      }
+    }
+
+    task "postgres" {
+      driver = "docker"
+      config {
+        image = "postgres"
+        ports = ["postgres"]
+        mount {
+          type = "volume"
+          target = "/var/lib/postgresql/data"
+          source = "postgresql-data"
+          readonly = false
+        }
+      }
+
+      env {
+        POSTGRES_DB="grafana"
+        POSTGRES_USER="postgres"
+        POSTGRES_PASSWORD="password"
+      }
+
+      resources {
+        cpu    = 512
+        memory = 512
+      }
+    }
+  }
+
+
+  group "prometheus" {
+
+    network {
+      mode = "bridge"
+
+      port "prometheus" {
+        to = 9090
+      }
+    }
+
+    service {
+      name = "prometheus"
+      port = "9090"
+      connect {
+        sidecar_service {}
+      }
+
+      check {
+        type     = "http"
+        name     = "prometheus_health"
+        port     = "prometheus"
+        path     = "/-/healthy"
+        interval = "20s"
+        timeout  = "30s"
+
+      #  check_restart {
+      #    limit = 3
+      #    grace = "90s"
+      #    ignore_warnings = false
+      #  }
+      }
+    }
+
+    task "prometheus" {
+      driver = "docker"
+      config {
+        image = "git.astron.nl:5000/lofar2.0/operations-central-management/prometheus:[[.image_tag]]"
+        ports = ["prometheus"]
+        mount {
+          type = "volume"
+          target = "/prometheus"
+          source = "prometheus-central-data"
+          readonly = false
+        }
+      }
+
+      resources {
+        cpu    = 500
+        memory = 768
+      }
+    }
+
+  }
+
+}

grafana-central/Dockerfile

-FROM grafana/grafana
-
-# Install some plugins
-RUN grafana-cli plugins install briangann-datatable-panel
-RUN grafana-cli plugins install ae3e-plotly-panel
-RUN grafana-cli plugins install yesoreyeram-infinity-datasource
-RUN grafana-cli plugins install aceiot-svg-panel
-RUN grafana-cli plugins install yesoreyeram-boomtable-panel
-RUN grafana-cli plugins install orchestracities-map-panel
-
-RUN wget https://algenty.github.io/flowcharting-repository/archives/agenty-flowcharting-panel-1.0.0b-SNAPSHOT.zip -O /tmp/agenty-flowcharting-panel.zip
-RUN cd /var/lib/grafana/plugins/ && unzip /tmp/agenty-flowcharting-panel.zip && mv grafana-flowcharting agenty-flowcharting-panel
-
-COPY grafana.ini /etc/grafana/
+FROM git.astron.nl:5000/lofar2.0/grafana-station-dashboards:latest
 
 # Add default configuration through provisioning (see https://grafana.com/docs/grafana/latest/administration/provisioning)
-COPY datasources /etc/grafana/provisioning/datasources/
-COPY dashboards /var/lib/grafana-static/dashboards/
+COPY dashboards/central-monitoring.json /var/lib/grafana/dashboards/central-monitoring.json
 COPY dashboards.yaml /etc/grafana/provisioning/dashboards/
+COPY datasources/prometheus.yaml /etc/grafana/provisioning/datasources/prometheus.yaml

grafana-central/dashboards.yaml

@@ -19,6 +19,6 @@ providers:
     allowUiUpdates: true
     options:
       # <string, required> path to dashboard files on disk. Required when using the 'file' type
-      path: /var/lib/grafana-static/dashboards
+      path: /var/lib/grafana/dashboards
       # <bool> use folder names from filesystem to create folders in Grafana
       foldersFromFilesStructure: true

Station @ 276dc03f

-Subproject commit 276dc03f3fac30fa618e795db1df9cf8f6794cad

grafana-central/datasources/alertaui.yaml

-apiVersion: 1
-
-datasources:
-  # <string, required> name of the datasource. Required
-  - name: Alerta UI
-    # <string, required> datasource type. Required
-    type: yesoreyeram-infinity-datasource
-    # <string, required> access mode. proxy or direct (Server or Browser in the UI). Required
-    access: proxy
-    # <int> org id. will default to orgId 1 if not specified
-    orgId: 1
-    # <string> custom UID which can be used to reference this datasource in other parts of the configuration, if not specified will be generated automatically
-    uid: alertaui
-    # <string> url
-    url: http://alerta-server:8080/api
-    # <string> Deprecated, use secureJsonData.password
-    password:
-    # <string> database user, if used
-    user: postgres
-    # <string> database name, if used
-    database: hdb
-    # <bool> enable/disable basic auth
-    basicAuth: false
-    # <string> basic auth username
-    basicAuthUser:
-    # <string> Deprecated, use secureJsonData.basicAuthPassword
-    basicAuthPassword:
-    # <bool> enable/disable with credentials headers
-    withCredentials:
-    # <bool> mark as default datasource. Max one per org
-    isDefault: false
-    # <map> fields that will be converted to json and stored in jsonData
-    jsonData:
-      secureQueryName1: "api-key"
-    # <string> json object of data that will be encrypted.
-    secureJsonData:
-      secureQueryValue1: "demo-key"
-    version: 1
-    # <bool> allow users to edit datasources from the UI.
-    editable: false
-    

grafana-central/datasources/grafanaapi.yaml

-apiVersion: 1
-
-datasources:
-  # <string, required> name of the datasource. Required
-  - name: Grafana API
-    # <string, required> datasource type. Required
-    type: yesoreyeram-infinity-datasource
-    # <string, required> access mode. proxy or direct (Server or Browser in the UI). Required
-    access: proxy
-    # <int> org id. will default to orgId 1 if not specified
-    orgId: 1
-    # <string> custom UID which can be used to reference this datasource in other parts of the configuration, if not specified will be generated automatically
-    uid: grafanaapi
-    # <string> url
-    url: http://localhost:3000/api
-    # <string> Deprecated, use secureJsonData.password
-    password:
-    # <string> database user, if used
-    user: postgres
-    # <string> database name, if used
-    database: hdb
-    # <bool> enable/disable basic auth
-    basicAuth: false
-    # <string> basic auth username
-    basicAuthUser:
-    # <string> Deprecated, use secureJsonData.basicAuthPassword
-    basicAuthPassword:
-    # <bool> enable/disable with credentials headers
-    withCredentials:
-    # <bool> mark as default datasource. Max one per org
-    isDefault: false
-    # <map> fields that will be converted to json and stored in jsonData
-    version: 1
-    # <bool> allow users to edit datasources from the UI.
-    editable: false
-    

grafana-central/datasources/prometheus.yaml

@@ -12,7 +12,7 @@ datasources:
     # <string> custom UID which can be used to reference this datasource in other parts of the configuration, if not specified will be generated automatically
     uid: prometheus
     # <string> url
-    url: http://prometheus-central:9090
+    url: http://localhost:9090
     # <string> Deprecated, use secureJsonData.password
     password:
     # <string> database user, if used

grafana-central/import-rules.py

-#!/usr/bin/python3
-import json
-import os
-import argparse
-
-parser = argparse.ArgumentParser(
-  formatter_class=argparse.RawDescriptionHelpFormatter,
-  description=
-"""
-Generate rule import files and script for Grafana.
-
-This script expands a given rules.json file into individual rules and
-prints the bash commands to import them in Grafana.
-
-To export rules from Grafana, use
-  curl <grafana>/api/ruler/grafana/api/v1/rules > rules.json
-""")
-parser.add_argument(
-    '-c', '--alert-config-file', type=str, required=False, help="Input alertmanager configuration JSON to parse, output of 'curl <grafana>/api/ruler/grafana/api/v1/rules' [%(default)s]")
-parser.add_argument(
-    '-r', '--rules-file', type=str, required=True, help="Input rules JSON to parse, output of 'curl <grafana>/api/ruler/grafana/api/v1/rules' [%(default)s]")
-parser.add_argument(
-    '-o', '--output-dir', type=str, default="rules", help="Directory to store the output [%(default)s]")
-parser.add_argument(
-    '-B', '--authorization-bearer', type=str, default="abcdefghijklmnopqrstuvwxyz", help="Authorization bearer from the Grafana 'editor' API key [%(default)s]")
-parser.add_argument(
-    '-g', '--grafana_url', type=str, default="http://localhost:3001", help="Base URL of Grafana [%(default)s]")
-parser.add_argument(
-    '-u', '--update', default=False, action='store_true', help="Update existing alerts, instead of creating new ones [%(default)s]")
-
-args = parser.parse_args()
-
-if args.alert_config_file:
-  print(f"echo Importing alert configuration file {args.alert_config_file}")
-  print(f"curl -X POST {args.grafana_url}/api/alertmanager/grafana/config/api/v1/alerts -H 'Content-Type: application/json' -H 'Accept: application/json' -H 'Authorization: Bearer {args.authorization_bearer}' -d '@{args.alert_config_file}'")
-  print(f"echo ''")
-
-with open(args.rules_file) as f:
-  data=json.load(f)
-
-  try:
-    os.mkdir(args.output_dir)
-  except FileExistsError as e:
-    pass
-
-  # the rules are of format {"folder": [{alert}, {alert}] }
-  for folder, rules in data.items():
-    try:
-      os.mkdir(f"{args.output_dir}/{folder}")
-    except FileExistsError as e:
-      pass
-
-    # print command to create folder
-    payload = json.dumps({"title": folder})
-    print(f"echo Creating folder {folder}")
-    print(f"curl -X POST {args.grafana_url}/api/folders -H 'Content-Type: application/json' -H 'Accept: application/json' -H 'Authorization: Bearer {args.authorization_bearer}' -d '{payload}'")
-    print(f"echo ''")
-
-    for rule in rules:
-      rule_filename = f"{args.output_dir}/{folder}/{rule['name']}.json"
-
-      if not args.update:
-        # strip rule UIDs
-        for subrule in rule["rules"]:
-          del subrule["grafana_alert"]["uid"]
-
-      # dump this rule
-      with open(rule_filename, "w") as rule_file:
-        json.dump(rule, rule_file)
-
-      # print import statement for this rule
-      print(f"echo Processing rule {folder}/{rule['name']}")
-      print(f"curl -X POST {args.grafana_url}/api/ruler/grafana/api/v1/rules/{folder} -H 'Content-Type: application/json' -H 'Accept: application/json' -H 'Authorization: Bearer {args.authorization_bearer}' -d '@{rule_filename}'")
-      print(f"echo ''")

grafana-central/rules.json

-{"station":[{"name":"FPGA processing error","interval":"10s","rules":[{"expr":"","for":"20s","labels":{"severity":"major"},"annotations":{"__dashboardUid__":"nC8N_kO7k","__panelId__":"9","summary":"One or more FPGAs are unusable."},"grafana_alert":{"id":1,"orgId":1,"title":"FPGA processing error","condition":"B","data":[{"refId":"A","queryType":"","relativeTimeRange":{"from":600,"to":0},"datasourceUid":"timescaledb","model":{"format":"time_series","group":[],"hide":false,"intervalMs":1000,"maxDataPoints":43200,"metricColumn":"none","rawQuery":true,"rawSql":"SELECT\n  $__timeGroup(data_time, $__interval),\n  x::text,\n  device,\n  name,\n  value\nFROM lofar_array_boolean\nWHERE\n  $__timeFilter(data_time) AND\n  name = 'fpga_error_r'\nORDER BY 1,2","refId":"A","select":[[{"params":["value_r"],"type":"column"}]],"table":"att_scalar_devdouble","timeColumn":"data_time","timeColumnType":"timestamp","where":[{"name":"$__timeFilter","params":[],"type":"macro"}]}},{"refId":"B","queryType":"","relativeTimeRange":{"from":0,"to":0},"datasourceUid":"-100","model":{"conditions":[{"evaluator":{"params":[0],"type":"gt"},"operator":{"type":"and"},"query":{"params":["A"]},"reducer":{"params":[],"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"-100"},"expression":"A","hide":false,"intervalMs":1000,"maxDataPoints":43200,"reducer":"last","refId":"B","settings":{"mode":"dropNN"},"type":"reduce"}}],"updated":"2022-04-04T18:01:53Z","intervalSeconds":10,"version":3,"uid":"kujybCynk","namespace_uid":"R_jsbCynz","namespace_id":6,"rule_group":"FPGA processing error","no_data_state":"NoData","exec_err_state":"Alerting"}}]}]}
\ No newline at end of file

grafana-central/rules/station/FPGA processing error.json

-{"name": "FPGA processing error", "interval": "10s", "rules": [{"expr": "", "for": "20s", "labels": {"severity": "major"}, "annotations": {"__dashboardUid__": "nC8N_kO7k", "__panelId__": "9", "summary": "One or more FPGAs are unusable."}, "grafana_alert": {"id": 1, "orgId": 1, "title": "FPGA processing error", "condition": "B", "data": [{"refId": "A", "queryType": "", "relativeTimeRange": {"from": 600, "to": 0}, "datasourceUid": "ZqArtG97z", "model": {"format": "time_series", "group": [], "hide": false, "intervalMs": 1000, "maxDataPoints": 43200, "metricColumn": "none", "rawQuery": true, "rawSql": "SELECT\n  $__timeGroup(data_time, $__interval),\n  x::text,\n  device,\n  name,\n  value\nFROM lofar_array_boolean\nWHERE\n  $__timeFilter(data_time) AND\n  name = 'fpga_error_r'\nORDER BY 1,2", "refId": "A", "select": [[{"params": ["value_r"], "type": "column"}]], "table": "att_scalar_devdouble", "timeColumn": "data_time", "timeColumnType": "timestamp", "where": [{"name": "$__timeFilter", "params": [], "type": "macro"}]}}, {"refId": "B", "queryType": "", "relativeTimeRange": {"from": 0, "to": 0}, "datasourceUid": "-100", "model": {"conditions": [{"evaluator": {"params": [0], "type": "gt"}, "operator": {"type": "and"}, "query": {"params": ["A"]}, "reducer": {"params": [], "type": "last"}, "type": "query"}], "datasource": {"type": "__expr__", "uid": "-100"}, "expression": "A", "hide": false, "intervalMs": 1000, "maxDataPoints": 43200, "reducer": "last", "refId": "B", "settings": {"mode": "dropNN"}, "type": "reduce"}}], "updated": "2022-04-04T18:01:53Z", "intervalSeconds": 10, "version": 3, "namespace_uid": "R_jsbCynz", "namespace_id": 6, "rule_group": "FPGA processing error", "no_data_state": "NoData", "exec_err_state": "Alerting"}}]}
\ No newline at end of file