L2SS-970: add logstash-loki container

docker-compose/elk.yml

@@ -44,10 +44,10 @@ services:
     ports:
       - "5601:5601" # kibana
       - "9200:9200" # elasticsearch
-      - "5044:5044" # logstash beats input
-      - "1514:1514/tcp" # logstash syslog input
-      - "1514:1514/udp" # logstash syslog input
-      - "5959:5959" # logstash tcp json input
+      # - "5044:5044" # logstash beats input
+      # - "1514:1514/tcp" # logstash syslog input
+      # - "1514:1514/udp" # logstash syslog input
+      # - "5959:5959" # logstash tcp json input
     depends_on:
       - elk-configure-host
     restart: unless-stopped

docker-compose/logstash-loki.yml

+#
+# Docker compose file that launches LOgstash-output-loki
+#
+# Defines:
+#   - prometheus: Prometheus
+#
+
+version: '2.1'
+
+services:
+  logstash-loki:
+    image: logstash-loki
+    build:
+        context: logstash-loki
+        args: 
+            SOURCE_IMAGE: grafana/logstash-output-loki:main
+    container_name: ${CONTAINER_NAME_PREFIX}logstash-loki
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "100m"
+        max-file: "10"
+    networks:
+      - control
+    ports:
+      - "5044:5044"     # logstash beats input
+      - "1514:1514/tcp" # logstash syslog input
+      - "1514:1514/udp" # logstash syslog input
+      - "5959:5959"     # logstash tcp json input
+      - "9600:9600"

docker-compose/logstash-loki/Dockerfile

+ARG SOURCE_IMAGE
+FROM ${SOURCE_IMAGE}
+
+# Provide our logstash config
+ADD logstash /home/logstash/

docker-compose/logstash-loki/logstash/conf.d/02-beats-input.conf

+input {
+  beats {
+    port => 5044
+    ssl => true
+    ssl_certificate => "/etc/pki/tls/certs/logstash-beats.crt"
+    ssl_key => "/etc/pki/tls/private/logstash-beats.key"
+  }
+}

docker-compose/logstash-loki/logstash/conf.d/03-syslog-input.conf

+input {
+  syslog {
+    port => 1514
+  }
+}

docker-compose/logstash-loki/logstash/conf.d/04-tcp-input.conf

+input {
+  tcp {
+    port => 5959
+    codec => json
+  }
+}

docker-compose/logstash-loki/logstash/conf.d/10-syslog.conf

+filter {
+  if [type] == "syslog" {
+    grok {
+      match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
+      add_field => [ "received_at", "%{@timestamp}" ]
+      add_field => [ "received_from", "%{host}" ]
+    }
+    syslog_pri { }
+    date {
+      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
+    }
+  }
+}

docker-compose/logstash-loki/logstash/conf.d/11-nginx.conf

+filter {
+  if [type] == "nginx-access" {
+    grok {
+      match => { "message" => "%{NGINXACCESS}" }
+    }
+  }
+}

docker-compose/logstash-loki/logstash/conf.d/20-parse-grafana.conf

+filter {
+  if [program] == "grafana" {
+    kv { }
+    mutate {
+      rename => {
+        "t" => "timestamp"
+        "lvl" => "level"
+        "msg" => "message"
+      }
+      uppercase => [ "level" ]
+    }
+    date {
+      match => [ "timestamp", "ISO8601" ]
+    }
+  }
+}

docker-compose/logstash-loki/logstash/conf.d/21-parse-prometheus.conf

+filter {
+  if [program] == "prometheus" {
+    kv { }
+    mutate {
+      rename => {
+        "ts" => "timestamp"
+        "msg" => "message"
+      }
+      uppercase => [ "level" ]
+    }
+    date {
+      match => [ "timestamp", "ISO8601" ]
+    }
+  }
+}

docker-compose/logstash-loki/logstash/conf.d/22-parse-tango-rest.conf

+filter {
+  if [program] == "tango-rest" {
+    grok {
+      match => {
+        "message" => "%{TIMESTAMP_ISO8601:timestamp} %{WORD:level} %{GREEDYDATA:message}"
+      }
+      "overwrite" => [ "timestamp", "level", "message" ]
+    }
+    date {
+      match => [ "timestamp", "YYYY-MM-dd HH:mm:ss,SSS" ]
+      timezone => "UTC"
+    }
+  }
+}

docker-compose/logstash-loki/logstash/conf.d/23-parse-maria-db.conf

+filter {
+  # mark all our mariadb instances
+  grok {
+    match => {
+      "program" => [ "archiver-maria-db", "tangodb" ]
+    }
+    add_tag => [ "mariadb" ]
+  }
+
+  # parse mariadb output
+  if "mariadb" in [tags] {
+    grok {
+      match => {
+        "message" => [
+          "%{TIMESTAMP_ISO8601:timestamp} .%{WORD:level}. %{GREEDYDATA:message}",
+          "%{TIMESTAMP_ISO8601:timestamp} 0 .%{WORD:level}. %{GREEDYDATA:message}"
+        ]
+      }
+      "overwrite" => [ "timestamp", "level", "message" ]
+    }
+    mutate {
+      gsub => [
+        "level", "Note", "Info"
+      ]
+      uppercase => [ "level" ]
+    }
+    date {
+      match => [ "timestamp", "YYYY-MM-dd HH:mm:ssZZ", "YYYY-MM-dd HH:mm:ss", "YYYY-MM-dd  H:mm:ss"  ]
+      timezone => "UTC"
+    }
+  }
+}

docker-compose/logstash-loki/logstash/conf.d/30-output.conf

+output {
+  elasticsearch {
+    hosts => ["localhost"]
+    manage_template => false
+    index => "logstash-%{+YYYY.MM.dd}"
+  }
+  loki {
+    [url => "localhost:3100" | default = none | required=true]
+
+    [tenant_id => string | default = nil | required=false]
+
+    [message_field => string | default = "message" | required=false]
+    
+    [include_fields => array | default = [] | required=false]
+
+    [batch_wait => number | default = 1(s) | required=false]
+
+    [batch_size => number | default = 102400(bytes) | required=false]
+
+    [min_delay => number | default = 1(s) | required=false]
+
+    [max_delay => number | default = 300(s) | required=false]
+
+    [retries => number | default = 10 | required=false]
+
+    [username => string | default = nil | required=false]
+
+    [password => secret | default = nil | required=false]
+
+    [cert => path | default = nil | required=false]
+
+    [key => path | default = nil| required=false]
+
+    [ca_cert => path | default = nil | required=false]
+
+    [insecure_skip_verify => boolean | default = false | required=false]
+  }
+}