L2SS-445: Forward logs from grafana and prometheus to ELK

docker-compose/elk/logstash/conf.d/20-parse-grafana.conf

+filter {
+  if [program] == "grafana" {
+    kv { }
+    mutate {
+      rename => {
+        "t" => "timestamp"
+        "lvl" => "level"
+        "msg" => "message"
+      }
+      uppercase => [ "level" ]
+    }
+    date {
+      match => [ "timestamp", "ISO8601" ]
+    }
+  }
+}

docker-compose/elk/logstash/conf.d/21-parse-prometheus.conf

+filter {
+  if [program] == "prometheus" {
+    kv { }
+    mutate {
+      rename => {
+        "ts" => "timestamp"
+        "msg" => "message"
+      }
+      uppercase => [ "level" ]
+    }
+    date {
+      match => [ "timestamp", "ISO8601" ]
+    }
+  }
+}

docker-compose/grafana.yml

@@ -23,4 +23,10 @@ services:
     #  - grafana-configs:/etc/grafana
     ports:
       - "3000:3000"
+    logging:
+      driver: syslog
+      options:
+        syslog-address: udp://${HOSTNAME}:1514
+        syslog-format: rfc3164
+        tag: "{{.Name}}"
     restart: unless-stopped

docker-compose/prometheus.yml

@@ -16,4 +16,10 @@ services:
       - control
     ports:
       - "9090:9090"
+    logging:
+      driver: syslog
+      options:
+        syslog-address: udp://${HOSTNAME}:1514
+        syslog-format: rfc3164
+        tag: "{{.Name}}"
     restart: unless-stopped