From 21d14b35248308b4f80e34c92759d8db20d36bbf Mon Sep 17 00:00:00 2001
From: stedif <stefano.difrischia@inaf.it>
Date: Tue, 4 Oct 2022 17:03:55 +0200
Subject: [PATCH] L2SS-970: add logstash-loki container
---
docker-compose/elk.yml | 8 ++--
docker-compose/logstash-loki.yml | 30 +++++++++++++++
docker-compose/logstash-loki/Dockerfile | 5 +++
.../logstash/conf.d/02-beats-input.conf | 8 ++++
.../logstash/conf.d/03-syslog-input.conf | 5 +++
.../logstash/conf.d/04-tcp-input.conf | 6 +++
.../logstash/conf.d/10-syslog.conf | 13 +++++++
.../logstash/conf.d/11-nginx.conf | 7 ++++
.../logstash/conf.d/20-parse-grafana.conf | 16 ++++++++
.../logstash/conf.d/21-parse-prometheus.conf | 15 ++++++++
.../logstash/conf.d/22-parse-tango-rest.conf | 14 +++++++
.../logstash/conf.d/23-parse-maria-db.conf | 32 ++++++++++++++++
.../logstash/conf.d/30-output.conf | 38 +++++++++++++++++++
13 files changed, 193 insertions(+), 4 deletions(-)
create mode 100644 docker-compose/logstash-loki.yml
create mode 100644 docker-compose/logstash-loki/Dockerfile
create mode 100644 docker-compose/logstash-loki/logstash/conf.d/02-beats-input.conf
create mode 100644 docker-compose/logstash-loki/logstash/conf.d/03-syslog-input.conf
create mode 100644 docker-compose/logstash-loki/logstash/conf.d/04-tcp-input.conf
create mode 100644 docker-compose/logstash-loki/logstash/conf.d/10-syslog.conf
create mode 100644 docker-compose/logstash-loki/logstash/conf.d/11-nginx.conf
create mode 100644 docker-compose/logstash-loki/logstash/conf.d/20-parse-grafana.conf
create mode 100644 docker-compose/logstash-loki/logstash/conf.d/21-parse-prometheus.conf
create mode 100644 docker-compose/logstash-loki/logstash/conf.d/22-parse-tango-rest.conf
create mode 100644 docker-compose/logstash-loki/logstash/conf.d/23-parse-maria-db.conf
create mode 100644 docker-compose/logstash-loki/logstash/conf.d/30-output.conf
diff --git a/docker-compose/elk.yml b/docker-compose/elk.yml
index 786e843ce..d671ba0c7 100644
--- a/docker-compose/elk.yml
+++ b/docker-compose/elk.yml
@@ -44,10 +44,10 @@ services:
ports:
- "5601:5601" # kibana
- "9200:9200" # elasticsearch
- - "5044:5044" # logstash beats input
- - "1514:1514/tcp" # logstash syslog input
- - "1514:1514/udp" # logstash syslog input
- - "5959:5959" # logstash tcp json input
+ # - "5044:5044" # logstash beats input
+ # - "1514:1514/tcp" # logstash syslog input
+ # - "1514:1514/udp" # logstash syslog input
+ # - "5959:5959" # logstash tcp json input
depends_on:
- elk-configure-host
restart: unless-stopped
diff --git a/docker-compose/logstash-loki.yml b/docker-compose/logstash-loki.yml
new file mode 100644
index 000000000..cf28a9689
--- /dev/null
+++ b/docker-compose/logstash-loki.yml
@@ -0,0 +1,30 @@
+#
+# Docker compose file that launches LOgstash-output-loki
+#
+# Defines:
+# - prometheus: Prometheus
+#
+
+version: '2.1'
+
+services:
+ logstash-loki:
+ image: logstash-loki
+ build:
+ context: logstash-loki
+ args:
+ SOURCE_IMAGE: grafana/logstash-output-loki:main
+ container_name: ${CONTAINER_NAME_PREFIX}logstash-loki
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "100m"
+ max-file: "10"
+ networks:
+ - control
+ ports:
+ - "5044:5044" # logstash beats input
+ - "1514:1514/tcp" # logstash syslog input
+ - "1514:1514/udp" # logstash syslog input
+ - "5959:5959" # logstash tcp json input
+ - "9600:9600"
diff --git a/docker-compose/logstash-loki/Dockerfile b/docker-compose/logstash-loki/Dockerfile
new file mode 100644
index 000000000..1b547baec
--- /dev/null
+++ b/docker-compose/logstash-loki/Dockerfile
@@ -0,0 +1,5 @@
+ARG SOURCE_IMAGE
+FROM ${SOURCE_IMAGE}
+
+# Provide our logstash config
+ADD logstash /home/logstash/
diff --git a/docker-compose/logstash-loki/logstash/conf.d/02-beats-input.conf b/docker-compose/logstash-loki/logstash/conf.d/02-beats-input.conf
new file mode 100644
index 000000000..4ab52b370
--- /dev/null
+++ b/docker-compose/logstash-loki/logstash/conf.d/02-beats-input.conf
@@ -0,0 +1,8 @@
+input {
+ beats {
+ port => 5044
+ ssl => true
+ ssl_certificate => "/etc/pki/tls/certs/logstash-beats.crt"
+ ssl_key => "/etc/pki/tls/private/logstash-beats.key"
+ }
+}
diff --git a/docker-compose/logstash-loki/logstash/conf.d/03-syslog-input.conf b/docker-compose/logstash-loki/logstash/conf.d/03-syslog-input.conf
new file mode 100644
index 000000000..b859a357d
--- /dev/null
+++ b/docker-compose/logstash-loki/logstash/conf.d/03-syslog-input.conf
@@ -0,0 +1,5 @@
+input {
+ syslog {
+ port => 1514
+ }
+}
diff --git a/docker-compose/logstash-loki/logstash/conf.d/04-tcp-input.conf b/docker-compose/logstash-loki/logstash/conf.d/04-tcp-input.conf
new file mode 100644
index 000000000..67def0887
--- /dev/null
+++ b/docker-compose/logstash-loki/logstash/conf.d/04-tcp-input.conf
@@ -0,0 +1,6 @@
+input {
+ tcp {
+ port => 5959
+ codec => json
+ }
+}
diff --git a/docker-compose/logstash-loki/logstash/conf.d/10-syslog.conf b/docker-compose/logstash-loki/logstash/conf.d/10-syslog.conf
new file mode 100644
index 000000000..acce463cd
--- /dev/null
+++ b/docker-compose/logstash-loki/logstash/conf.d/10-syslog.conf
@@ -0,0 +1,13 @@
+filter {
+ if [type] == "syslog" {
+ grok {
+ match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
+ add_field => [ "received_at", "%{@timestamp}" ]
+ add_field => [ "received_from", "%{host}" ]
+ }
+ syslog_pri { }
+ date {
+ match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
+ }
+ }
+}
diff --git a/docker-compose/logstash-loki/logstash/conf.d/11-nginx.conf b/docker-compose/logstash-loki/logstash/conf.d/11-nginx.conf
new file mode 100644
index 000000000..d4a45db2d
--- /dev/null
+++ b/docker-compose/logstash-loki/logstash/conf.d/11-nginx.conf
@@ -0,0 +1,7 @@
+filter {
+ if [type] == "nginx-access" {
+ grok {
+ match => { "message" => "%{NGINXACCESS}" }
+ }
+ }
+}
diff --git a/docker-compose/logstash-loki/logstash/conf.d/20-parse-grafana.conf b/docker-compose/logstash-loki/logstash/conf.d/20-parse-grafana.conf
new file mode 100644
index 000000000..37db44fda
--- /dev/null
+++ b/docker-compose/logstash-loki/logstash/conf.d/20-parse-grafana.conf
@@ -0,0 +1,16 @@
+filter {
+ if [program] == "grafana" {
+ kv { }
+ mutate {
+ rename => {
+ "t" => "timestamp"
+ "lvl" => "level"
+ "msg" => "message"
+ }
+ uppercase => [ "level" ]
+ }
+ date {
+ match => [ "timestamp", "ISO8601" ]
+ }
+ }
+}
diff --git a/docker-compose/logstash-loki/logstash/conf.d/21-parse-prometheus.conf b/docker-compose/logstash-loki/logstash/conf.d/21-parse-prometheus.conf
new file mode 100644
index 000000000..b8323625f
--- /dev/null
+++ b/docker-compose/logstash-loki/logstash/conf.d/21-parse-prometheus.conf
@@ -0,0 +1,15 @@
+filter {
+ if [program] == "prometheus" {
+ kv { }
+ mutate {
+ rename => {
+ "ts" => "timestamp"
+ "msg" => "message"
+ }
+ uppercase => [ "level" ]
+ }
+ date {
+ match => [ "timestamp", "ISO8601" ]
+ }
+ }
+}
diff --git a/docker-compose/logstash-loki/logstash/conf.d/22-parse-tango-rest.conf b/docker-compose/logstash-loki/logstash/conf.d/22-parse-tango-rest.conf
new file mode 100644
index 000000000..5df0cd92b
--- /dev/null
+++ b/docker-compose/logstash-loki/logstash/conf.d/22-parse-tango-rest.conf
@@ -0,0 +1,14 @@
+filter {
+ if [program] == "tango-rest" {
+ grok {
+ match => {
+ "message" => "%{TIMESTAMP_ISO8601:timestamp} %{WORD:level} %{GREEDYDATA:message}"
+ }
+ "overwrite" => [ "timestamp", "level", "message" ]
+ }
+ date {
+ match => [ "timestamp", "YYYY-MM-dd HH:mm:ss,SSS" ]
+ timezone => "UTC"
+ }
+ }
+}
diff --git a/docker-compose/logstash-loki/logstash/conf.d/23-parse-maria-db.conf b/docker-compose/logstash-loki/logstash/conf.d/23-parse-maria-db.conf
new file mode 100644
index 000000000..0a23fddd0
--- /dev/null
+++ b/docker-compose/logstash-loki/logstash/conf.d/23-parse-maria-db.conf
@@ -0,0 +1,32 @@
+filter {
+ # mark all our mariadb instances
+ grok {
+ match => {
+ "program" => [ "archiver-maria-db", "tangodb" ]
+ }
+ add_tag => [ "mariadb" ]
+ }
+
+ # parse mariadb output
+ if "mariadb" in [tags] {
+ grok {
+ match => {
+ "message" => [
+ "%{TIMESTAMP_ISO8601:timestamp} .%{WORD:level}. %{GREEDYDATA:message}",
+ "%{TIMESTAMP_ISO8601:timestamp} 0 .%{WORD:level}. %{GREEDYDATA:message}"
+ ]
+ }
+ "overwrite" => [ "timestamp", "level", "message" ]
+ }
+ mutate {
+ gsub => [
+ "level", "Note", "Info"
+ ]
+ uppercase => [ "level" ]
+ }
+ date {
+ match => [ "timestamp", "YYYY-MM-dd HH:mm:ssZZ", "YYYY-MM-dd HH:mm:ss", "YYYY-MM-dd H:mm:ss" ]
+ timezone => "UTC"
+ }
+ }
+}
diff --git a/docker-compose/logstash-loki/logstash/conf.d/30-output.conf b/docker-compose/logstash-loki/logstash/conf.d/30-output.conf
new file mode 100644
index 000000000..06e5fd692
--- /dev/null
+++ b/docker-compose/logstash-loki/logstash/conf.d/30-output.conf
@@ -0,0 +1,38 @@
+output {
+ elasticsearch {
+ hosts => ["localhost"]
+ manage_template => false
+ index => "logstash-%{+YYYY.MM.dd}"
+ }
+ loki {
+ [url => "localhost:3100" | default = none | required=true]
+
+ [tenant_id => string | default = nil | required=false]
+
+ [message_field => string | default = "message" | required=false]
+
+ [include_fields => array | default = [] | required=false]
+
+ [batch_wait => number | default = 1(s) | required=false]
+
+ [batch_size => number | default = 102400(bytes) | required=false]
+
+ [min_delay => number | default = 1(s) | required=false]
+
+ [max_delay => number | default = 300(s) | required=false]
+
+ [retries => number | default = 10 | required=false]
+
+ [username => string | default = nil | required=false]
+
+ [password => secret | default = nil | required=false]
+
+ [cert => path | default = nil | required=false]
+
+ [key => path | default = nil| required=false]
+
+ [ca_cert => path | default = nil | required=false]
+
+ [insecure_skip_verify => boolean | default = false | required=false]
+ }
+}
--
GitLab