L2SS-970: fix logstash pipeline

docker-compose/grafana/datasources/loki.yaml

@@ -12,7 +12,7 @@ datasources:
     # <string> custom UID which can be used to reference this datasource in other parts of the configuration, if not specified will be generated automatically
     uid: loki
     # <string> url
-    url: loki:3100
+    url: http://loki:3100
     # <string> Deprecated, use secureJsonData.password
     password:
     # <string> database user, if used

docker-compose/logstash/Dockerfile

 ARG SOURCE_IMAGE
 FROM ${SOURCE_IMAGE}
 
+# Disable Elastic Search connection
+ENV ELASTIC_CONTAINER=false
+
 # Provide our logstash config
-ADD logstash /etc/logstash/
 COPY loki.conf /home/logstash/
-COPY loki.conf /home/logstash/loki-test.conf
+COPY logstash.yml /usr/share/logstash/config/logstash.yml
+COPY loki.conf /usr/share/logstash/pipeline/logstash.conf

docker-compose/logstash/logstash.yml

+http.host: "0.0.0.0"
+#xpack.monitoring.elasticsearch.hosts: [ "http://loki:3100" ]

docker-compose/logstash/logstash/conf.d/02-beats-input.conf

-input {
-  beats {
-    port => 5044
-    ssl => true
-    ssl_certificate => "/etc/pki/tls/certs/logstash-beats.crt"
-    ssl_key => "/etc/pki/tls/private/logstash-beats.key"
-  }
-}

docker-compose/logstash/logstash/conf.d/03-syslog-input.conf

-input {
-  syslog {
-    port => 1514
-  }
-}

docker-compose/logstash/logstash/conf.d/04-tcp-input.conf

-input {
-  tcp {
-    port => 5959
-    codec => json
-  }
-}

docker-compose/logstash/logstash/conf.d/10-syslog.conf

-filter {
-  if [type] == "syslog" {
-    grok {
-      match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
-      add_field => [ "received_at", "%{@timestamp}" ]
-      add_field => [ "received_from", "%{host}" ]
-    }
-    syslog_pri { }
-    date {
-      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
-    }
-  }
-}

docker-compose/logstash/logstash/conf.d/11-nginx.conf

-filter {
-  if [type] == "nginx-access" {
-    grok {
-      match => { "message" => "%{NGINXACCESS}" }
-    }
-  }
-}

docker-compose/logstash/logstash/conf.d/20-parse-grafana.conf

-filter {
-  if [program] == "grafana" {
-    kv { }
-    mutate {
-      rename => {
-        "t" => "timestamp"
-        "lvl" => "level"
-        "msg" => "message"
-      }
-      uppercase => [ "level" ]
-    }
-    date {
-      match => [ "timestamp", "ISO8601" ]
-    }
-  }
-}

docker-compose/logstash/logstash/conf.d/21-parse-prometheus.conf

-filter {
-  if [program] == "prometheus" {
-    kv { }
-    mutate {
-      rename => {
-        "ts" => "timestamp"
-        "msg" => "message"
-      }
-      uppercase => [ "level" ]
-    }
-    date {
-      match => [ "timestamp", "ISO8601" ]
-    }
-  }
-}

docker-compose/logstash/logstash/conf.d/22-parse-tango-rest.conf

-filter {
-  if [program] == "tango-rest" {
-    grok {
-      match => {
-        "message" => "%{TIMESTAMP_ISO8601:timestamp} %{WORD:level} %{GREEDYDATA:message}"
-      }
-      "overwrite" => [ "timestamp", "level", "message" ]
-    }
-    date {
-      match => [ "timestamp", "YYYY-MM-dd HH:mm:ss,SSS" ]
-      timezone => "UTC"
-    }
-  }
-}

docker-compose/logstash/logstash/conf.d/23-parse-maria-db.conf

-filter {
-  # mark all our mariadb instances
-  grok {
-    match => {
-      "program" => [ "archiver-maria-db", "tangodb" ]
-    }
-    add_tag => [ "mariadb" ]
-  }
-
-  # parse mariadb output
-  if "mariadb" in [tags] {
-    grok {
-      match => {
-        "message" => [
-          "%{TIMESTAMP_ISO8601:timestamp} .%{WORD:level}. %{GREEDYDATA:message}",
-          "%{TIMESTAMP_ISO8601:timestamp} 0 .%{WORD:level}. %{GREEDYDATA:message}"
-        ]
-      }
-      "overwrite" => [ "timestamp", "level", "message" ]
-    }
-    mutate {
-      gsub => [
-        "level", "Note", "Info"
-      ]
-      uppercase => [ "level" ]
-    }
-    date {
-      match => [ "timestamp", "YYYY-MM-dd HH:mm:ssZZ", "YYYY-MM-dd HH:mm:ss", "YYYY-MM-dd  H:mm:ss"  ]
-      timezone => "UTC"
-    }
-  }
-}

docker-compose/logstash/logstash/conf.d/30-output.conf

-output {
-  # elasticsearch {
-  #   hosts => ["localhost"]
-  #   manage_template => false
-  #   index => "logstash-%{+YYYY.MM.dd}"
-  # }
-  loki {
-    url => "http://loki:3100/loki/api/v1/push"
-  }
-}

docker-compose/logstash/loki.conf

 input {
   beats {
     port => 5044
-    ssl => true
-    ssl_certificate => "/etc/pki/tls/certs/logstash-beats.crt"
-    ssl_key => "/etc/pki/tls/private/logstash-beats.key"
+    # ssl => true
+    # ssl_certificate => "/etc/pki/tls/certs/logstash-beats.crt"
+    # ssl_key => "/etc/pki/tls/private/logstash-beats.key"
   }
 }
 
@@ -34,13 +34,13 @@ filter {
   }
 }
 
-filter {
-  if [type] == "nginx-access" {
-    grok {
-      match => { "message" => "%{NGINXACCESS}" }
-    }
-  }
-}
+# filter {
+#   if [type] == "nginx-access" {
+#     grok {
+#       match => { "message" => "%{NGINXACCESS}" }
+#     }
+#   }
+# }
 
 filter {
   if [program] == "grafana" {