From ed06b9a4edf9048bb0d4d0ac4ee3c1345f0e5405 Mon Sep 17 00:00:00 2001
From: stedif <stefano.difrischia@inaf.it>
Date: Thu, 6 Oct 2022 16:20:01 +0200
Subject: [PATCH] L2SS-970: fix logstash pipeline
---
docker-compose/grafana/datasources/loki.yaml | 2 +-
docker-compose/logstash/Dockerfile | 7 ++--
docker-compose/logstash/logstash.yml | 2 ++
.../logstash/conf.d/02-beats-input.conf | 8 -----
.../logstash/conf.d/03-syslog-input.conf | 5 ---
.../logstash/conf.d/04-tcp-input.conf | 6 ----
.../logstash/logstash/conf.d/10-syslog.conf | 13 --------
.../logstash/logstash/conf.d/11-nginx.conf | 7 ----
.../logstash/conf.d/20-parse-grafana.conf | 16 ----------
.../logstash/conf.d/21-parse-prometheus.conf | 15 ---------
.../logstash/conf.d/22-parse-tango-rest.conf | 14 --------
.../logstash/conf.d/23-parse-maria-db.conf | 32 -------------------
.../logstash/logstash/conf.d/30-output.conf | 10 ------
docker-compose/logstash/loki.conf | 20 ++++++------
14 files changed, 18 insertions(+), 139 deletions(-)
create mode 100644 docker-compose/logstash/logstash.yml
delete mode 100644 docker-compose/logstash/logstash/conf.d/02-beats-input.conf
delete mode 100644 docker-compose/logstash/logstash/conf.d/03-syslog-input.conf
delete mode 100644 docker-compose/logstash/logstash/conf.d/04-tcp-input.conf
delete mode 100644 docker-compose/logstash/logstash/conf.d/10-syslog.conf
delete mode 100644 docker-compose/logstash/logstash/conf.d/11-nginx.conf
delete mode 100644 docker-compose/logstash/logstash/conf.d/20-parse-grafana.conf
delete mode 100644 docker-compose/logstash/logstash/conf.d/21-parse-prometheus.conf
delete mode 100644 docker-compose/logstash/logstash/conf.d/22-parse-tango-rest.conf
delete mode 100644 docker-compose/logstash/logstash/conf.d/23-parse-maria-db.conf
delete mode 100644 docker-compose/logstash/logstash/conf.d/30-output.conf
diff --git a/docker-compose/grafana/datasources/loki.yaml b/docker-compose/grafana/datasources/loki.yaml
index 2adc78160..f9108f15f 100644
--- a/docker-compose/grafana/datasources/loki.yaml
+++ b/docker-compose/grafana/datasources/loki.yaml
@@ -12,7 +12,7 @@ datasources:
# <string> custom UID which can be used to reference this datasource in other parts of the configuration, if not specified will be generated automatically
uid: loki
# <string> url
- url: loki:3100
+ url: http://loki:3100
# <string> Deprecated, use secureJsonData.password
password:
# <string> database user, if used
diff --git a/docker-compose/logstash/Dockerfile b/docker-compose/logstash/Dockerfile
index 7e0fff7a9..94fa5ab4b 100644
--- a/docker-compose/logstash/Dockerfile
+++ b/docker-compose/logstash/Dockerfile
@@ -1,7 +1,10 @@
ARG SOURCE_IMAGE
FROM ${SOURCE_IMAGE}
+# Disable Elastic Search connection
+ENV ELASTIC_CONTAINER=false
+
# Provide our logstash config
-ADD logstash /etc/logstash/
COPY loki.conf /home/logstash/
-COPY loki.conf /home/logstash/loki-test.conf
+COPY logstash.yml /usr/share/logstash/config/logstash.yml
+COPY loki.conf /usr/share/logstash/pipeline/logstash.conf
diff --git a/docker-compose/logstash/logstash.yml b/docker-compose/logstash/logstash.yml
new file mode 100644
index 000000000..5f80650fe
--- /dev/null
+++ b/docker-compose/logstash/logstash.yml
@@ -0,0 +1,2 @@
+http.host: "0.0.0.0"
+#xpack.monitoring.elasticsearch.hosts: [ "http://loki:3100" ]
diff --git a/docker-compose/logstash/logstash/conf.d/02-beats-input.conf b/docker-compose/logstash/logstash/conf.d/02-beats-input.conf
deleted file mode 100644
index 4ab52b370..000000000
--- a/docker-compose/logstash/logstash/conf.d/02-beats-input.conf
+++ /dev/null
@@ -1,8 +0,0 @@
-input {
- beats {
- port => 5044
- ssl => true
- ssl_certificate => "/etc/pki/tls/certs/logstash-beats.crt"
- ssl_key => "/etc/pki/tls/private/logstash-beats.key"
- }
-}
diff --git a/docker-compose/logstash/logstash/conf.d/03-syslog-input.conf b/docker-compose/logstash/logstash/conf.d/03-syslog-input.conf
deleted file mode 100644
index b859a357d..000000000
--- a/docker-compose/logstash/logstash/conf.d/03-syslog-input.conf
+++ /dev/null
@@ -1,5 +0,0 @@
-input {
- syslog {
- port => 1514
- }
-}
diff --git a/docker-compose/logstash/logstash/conf.d/04-tcp-input.conf b/docker-compose/logstash/logstash/conf.d/04-tcp-input.conf
deleted file mode 100644
index 67def0887..000000000
--- a/docker-compose/logstash/logstash/conf.d/04-tcp-input.conf
+++ /dev/null
@@ -1,6 +0,0 @@
-input {
- tcp {
- port => 5959
- codec => json
- }
-}
diff --git a/docker-compose/logstash/logstash/conf.d/10-syslog.conf b/docker-compose/logstash/logstash/conf.d/10-syslog.conf
deleted file mode 100644
index acce463cd..000000000
--- a/docker-compose/logstash/logstash/conf.d/10-syslog.conf
+++ /dev/null
@@ -1,13 +0,0 @@
-filter {
- if [type] == "syslog" {
- grok {
- match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
- add_field => [ "received_at", "%{@timestamp}" ]
- add_field => [ "received_from", "%{host}" ]
- }
- syslog_pri { }
- date {
- match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
- }
- }
-}
diff --git a/docker-compose/logstash/logstash/conf.d/11-nginx.conf b/docker-compose/logstash/logstash/conf.d/11-nginx.conf
deleted file mode 100644
index d4a45db2d..000000000
--- a/docker-compose/logstash/logstash/conf.d/11-nginx.conf
+++ /dev/null
@@ -1,7 +0,0 @@
-filter {
- if [type] == "nginx-access" {
- grok {
- match => { "message" => "%{NGINXACCESS}" }
- }
- }
-}
diff --git a/docker-compose/logstash/logstash/conf.d/20-parse-grafana.conf b/docker-compose/logstash/logstash/conf.d/20-parse-grafana.conf
deleted file mode 100644
index 37db44fda..000000000
--- a/docker-compose/logstash/logstash/conf.d/20-parse-grafana.conf
+++ /dev/null
@@ -1,16 +0,0 @@
-filter {
- if [program] == "grafana" {
- kv { }
- mutate {
- rename => {
- "t" => "timestamp"
- "lvl" => "level"
- "msg" => "message"
- }
- uppercase => [ "level" ]
- }
- date {
- match => [ "timestamp", "ISO8601" ]
- }
- }
-}
diff --git a/docker-compose/logstash/logstash/conf.d/21-parse-prometheus.conf b/docker-compose/logstash/logstash/conf.d/21-parse-prometheus.conf
deleted file mode 100644
index b8323625f..000000000
--- a/docker-compose/logstash/logstash/conf.d/21-parse-prometheus.conf
+++ /dev/null
@@ -1,15 +0,0 @@
-filter {
- if [program] == "prometheus" {
- kv { }
- mutate {
- rename => {
- "ts" => "timestamp"
- "msg" => "message"
- }
- uppercase => [ "level" ]
- }
- date {
- match => [ "timestamp", "ISO8601" ]
- }
- }
-}
diff --git a/docker-compose/logstash/logstash/conf.d/22-parse-tango-rest.conf b/docker-compose/logstash/logstash/conf.d/22-parse-tango-rest.conf
deleted file mode 100644
index 5df0cd92b..000000000
--- a/docker-compose/logstash/logstash/conf.d/22-parse-tango-rest.conf
+++ /dev/null
@@ -1,14 +0,0 @@
-filter {
- if [program] == "tango-rest" {
- grok {
- match => {
- "message" => "%{TIMESTAMP_ISO8601:timestamp} %{WORD:level} %{GREEDYDATA:message}"
- }
- "overwrite" => [ "timestamp", "level", "message" ]
- }
- date {
- match => [ "timestamp", "YYYY-MM-dd HH:mm:ss,SSS" ]
- timezone => "UTC"
- }
- }
-}
diff --git a/docker-compose/logstash/logstash/conf.d/23-parse-maria-db.conf b/docker-compose/logstash/logstash/conf.d/23-parse-maria-db.conf
deleted file mode 100644
index 0a23fddd0..000000000
--- a/docker-compose/logstash/logstash/conf.d/23-parse-maria-db.conf
+++ /dev/null
@@ -1,32 +0,0 @@
-filter {
- # mark all our mariadb instances
- grok {
- match => {
- "program" => [ "archiver-maria-db", "tangodb" ]
- }
- add_tag => [ "mariadb" ]
- }
-
- # parse mariadb output
- if "mariadb" in [tags] {
- grok {
- match => {
- "message" => [
- "%{TIMESTAMP_ISO8601:timestamp} .%{WORD:level}. %{GREEDYDATA:message}",
- "%{TIMESTAMP_ISO8601:timestamp} 0 .%{WORD:level}. %{GREEDYDATA:message}"
- ]
- }
- "overwrite" => [ "timestamp", "level", "message" ]
- }
- mutate {
- gsub => [
- "level", "Note", "Info"
- ]
- uppercase => [ "level" ]
- }
- date {
- match => [ "timestamp", "YYYY-MM-dd HH:mm:ssZZ", "YYYY-MM-dd HH:mm:ss", "YYYY-MM-dd H:mm:ss" ]
- timezone => "UTC"
- }
- }
-}
diff --git a/docker-compose/logstash/logstash/conf.d/30-output.conf b/docker-compose/logstash/logstash/conf.d/30-output.conf
deleted file mode 100644
index 7ade7b0f0..000000000
--- a/docker-compose/logstash/logstash/conf.d/30-output.conf
+++ /dev/null
@@ -1,10 +0,0 @@
-output {
- # elasticsearch {
- # hosts => ["localhost"]
- # manage_template => false
- # index => "logstash-%{+YYYY.MM.dd}"
- # }
- loki {
- url => "http://loki:3100/loki/api/v1/push"
- }
-}
diff --git a/docker-compose/logstash/loki.conf b/docker-compose/logstash/loki.conf
index 31d534391..e22f53dbe 100644
--- a/docker-compose/logstash/loki.conf
+++ b/docker-compose/logstash/loki.conf
@@ -1,9 +1,9 @@
input {
beats {
port => 5044
- ssl => true
- ssl_certificate => "/etc/pki/tls/certs/logstash-beats.crt"
- ssl_key => "/etc/pki/tls/private/logstash-beats.key"
+ # ssl => true
+ # ssl_certificate => "/etc/pki/tls/certs/logstash-beats.crt"
+ # ssl_key => "/etc/pki/tls/private/logstash-beats.key"
}
}
@@ -34,13 +34,13 @@ filter {
}
}
-filter {
- if [type] == "nginx-access" {
- grok {
- match => { "message" => "%{NGINXACCESS}" }
- }
- }
-}
+# filter {
+# if [type] == "nginx-access" {
+# grok {
+# match => { "message" => "%{NGINXACCESS}" }
+# }
+# }
+# }
filter {
if [program] == "grafana" {
--
GitLab