diff --git a/docker-compose/elk.yml b/docker-compose/elk.yml index 786e843ce85c16e7604341a7138c5030f1356fed..d671ba0c7708d3ae9cb37956f9bc7884462f7389 100644 --- a/docker-compose/elk.yml +++ b/docker-compose/elk.yml @@ -44,10 +44,10 @@ services: ports: - "5601:5601" # kibana - "9200:9200" # elasticsearch - - "5044:5044" # logstash beats input - - "1514:1514/tcp" # logstash syslog input - - "1514:1514/udp" # logstash syslog input - - "5959:5959" # logstash tcp json input + # - "5044:5044" # logstash beats input + # - "1514:1514/tcp" # logstash syslog input + # - "1514:1514/udp" # logstash syslog input + # - "5959:5959" # logstash tcp json input depends_on: - elk-configure-host restart: unless-stopped diff --git a/docker-compose/logstash-loki.yml b/docker-compose/logstash-loki.yml new file mode 100644 index 0000000000000000000000000000000000000000..cf28a9689691e82539b65c744e94bf591dc2aaff --- /dev/null +++ b/docker-compose/logstash-loki.yml @@ -0,0 +1,30 @@ +# +# Docker compose file that launches LOgstash-output-loki +# +# Defines: +# - prometheus: Prometheus +# + +version: '2.1' + +services: + logstash-loki: + image: logstash-loki + build: + context: logstash-loki + args: + SOURCE_IMAGE: grafana/logstash-output-loki:main + container_name: ${CONTAINER_NAME_PREFIX}logstash-loki + logging: + driver: "json-file" + options: + max-size: "100m" + max-file: "10" + networks: + - control + ports: + - "5044:5044" # logstash beats input + - "1514:1514/tcp" # logstash syslog input + - "1514:1514/udp" # logstash syslog input + - "5959:5959" # logstash tcp json input + - "9600:9600" diff --git a/docker-compose/logstash-loki/Dockerfile b/docker-compose/logstash-loki/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..1b547baec0925581872ebefe79e7506318afc242 --- /dev/null +++ b/docker-compose/logstash-loki/Dockerfile @@ -0,0 +1,5 @@ +ARG SOURCE_IMAGE +FROM ${SOURCE_IMAGE} + +# Provide our logstash config +ADD logstash /home/logstash/ diff --git a/docker-compose/logstash-loki/logstash/conf.d/02-beats-input.conf b/docker-compose/logstash-loki/logstash/conf.d/02-beats-input.conf new file mode 100644 index 0000000000000000000000000000000000000000..4ab52b37081aa46fdda7edd82a0395b9f73a3705 --- /dev/null +++ b/docker-compose/logstash-loki/logstash/conf.d/02-beats-input.conf @@ -0,0 +1,8 @@ +input { + beats { + port => 5044 + ssl => true + ssl_certificate => "/etc/pki/tls/certs/logstash-beats.crt" + ssl_key => "/etc/pki/tls/private/logstash-beats.key" + } +} diff --git a/docker-compose/logstash-loki/logstash/conf.d/03-syslog-input.conf b/docker-compose/logstash-loki/logstash/conf.d/03-syslog-input.conf new file mode 100644 index 0000000000000000000000000000000000000000..b859a357d505d9ea8d59eb3cf39d2af97b76a119 --- /dev/null +++ b/docker-compose/logstash-loki/logstash/conf.d/03-syslog-input.conf @@ -0,0 +1,5 @@ +input { + syslog { + port => 1514 + } +} diff --git a/docker-compose/logstash-loki/logstash/conf.d/04-tcp-input.conf b/docker-compose/logstash-loki/logstash/conf.d/04-tcp-input.conf new file mode 100644 index 0000000000000000000000000000000000000000..67def0887aef58ab0fd5d2d6cb65d5700a4088c7 --- /dev/null +++ b/docker-compose/logstash-loki/logstash/conf.d/04-tcp-input.conf @@ -0,0 +1,6 @@ +input { + tcp { + port => 5959 + codec => json + } +} diff --git a/docker-compose/logstash-loki/logstash/conf.d/10-syslog.conf b/docker-compose/logstash-loki/logstash/conf.d/10-syslog.conf new file mode 100644 index 0000000000000000000000000000000000000000..acce463cd42543ce87c5a51496aec25b06d21fdd --- /dev/null +++ b/docker-compose/logstash-loki/logstash/conf.d/10-syslog.conf @@ -0,0 +1,13 @@ +filter { + if [type] == "syslog" { + grok { + match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" } + add_field => [ "received_at", "%{@timestamp}" ] + add_field => [ "received_from", "%{host}" ] + } + syslog_pri { } + date { + match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ] + } + } +} diff --git a/docker-compose/logstash-loki/logstash/conf.d/11-nginx.conf b/docker-compose/logstash-loki/logstash/conf.d/11-nginx.conf new file mode 100644 index 0000000000000000000000000000000000000000..d4a45db2d8454d982d52056cd035a9c8ef865389 --- /dev/null +++ b/docker-compose/logstash-loki/logstash/conf.d/11-nginx.conf @@ -0,0 +1,7 @@ +filter { + if [type] == "nginx-access" { + grok { + match => { "message" => "%{NGINXACCESS}" } + } + } +} diff --git a/docker-compose/logstash-loki/logstash/conf.d/20-parse-grafana.conf b/docker-compose/logstash-loki/logstash/conf.d/20-parse-grafana.conf new file mode 100644 index 0000000000000000000000000000000000000000..37db44fda67109d7ef8a6beac1193004968a2349 --- /dev/null +++ b/docker-compose/logstash-loki/logstash/conf.d/20-parse-grafana.conf @@ -0,0 +1,16 @@ +filter { + if [program] == "grafana" { + kv { } + mutate { + rename => { + "t" => "timestamp" + "lvl" => "level" + "msg" => "message" + } + uppercase => [ "level" ] + } + date { + match => [ "timestamp", "ISO8601" ] + } + } +} diff --git a/docker-compose/logstash-loki/logstash/conf.d/21-parse-prometheus.conf b/docker-compose/logstash-loki/logstash/conf.d/21-parse-prometheus.conf new file mode 100644 index 0000000000000000000000000000000000000000..b8323625f329af02f9ff33556e408b94ecf7e0b6 --- /dev/null +++ b/docker-compose/logstash-loki/logstash/conf.d/21-parse-prometheus.conf @@ -0,0 +1,15 @@ +filter { + if [program] == "prometheus" { + kv { } + mutate { + rename => { + "ts" => "timestamp" + "msg" => "message" + } + uppercase => [ "level" ] + } + date { + match => [ "timestamp", "ISO8601" ] + } + } +} diff --git a/docker-compose/logstash-loki/logstash/conf.d/22-parse-tango-rest.conf b/docker-compose/logstash-loki/logstash/conf.d/22-parse-tango-rest.conf new file mode 100644 index 0000000000000000000000000000000000000000..5df0cd92bd32625a1eb91220bf4e7a9827799523 --- /dev/null +++ b/docker-compose/logstash-loki/logstash/conf.d/22-parse-tango-rest.conf @@ -0,0 +1,14 @@ +filter { + if [program] == "tango-rest" { + grok { + match => { + "message" => "%{TIMESTAMP_ISO8601:timestamp} %{WORD:level} %{GREEDYDATA:message}" + } + "overwrite" => [ "timestamp", "level", "message" ] + } + date { + match => [ "timestamp", "YYYY-MM-dd HH:mm:ss,SSS" ] + timezone => "UTC" + } + } +} diff --git a/docker-compose/logstash-loki/logstash/conf.d/23-parse-maria-db.conf b/docker-compose/logstash-loki/logstash/conf.d/23-parse-maria-db.conf new file mode 100644 index 0000000000000000000000000000000000000000..0a23fddd078e5e967bc5f791e020faaa20ed632a --- /dev/null +++ b/docker-compose/logstash-loki/logstash/conf.d/23-parse-maria-db.conf @@ -0,0 +1,32 @@ +filter { + # mark all our mariadb instances + grok { + match => { + "program" => [ "archiver-maria-db", "tangodb" ] + } + add_tag => [ "mariadb" ] + } + + # parse mariadb output + if "mariadb" in [tags] { + grok { + match => { + "message" => [ + "%{TIMESTAMP_ISO8601:timestamp} .%{WORD:level}. %{GREEDYDATA:message}", + "%{TIMESTAMP_ISO8601:timestamp} 0 .%{WORD:level}. %{GREEDYDATA:message}" + ] + } + "overwrite" => [ "timestamp", "level", "message" ] + } + mutate { + gsub => [ + "level", "Note", "Info" + ] + uppercase => [ "level" ] + } + date { + match => [ "timestamp", "YYYY-MM-dd HH:mm:ssZZ", "YYYY-MM-dd HH:mm:ss", "YYYY-MM-dd H:mm:ss" ] + timezone => "UTC" + } + } +} diff --git a/docker-compose/logstash-loki/logstash/conf.d/30-output.conf b/docker-compose/logstash-loki/logstash/conf.d/30-output.conf new file mode 100644 index 0000000000000000000000000000000000000000..06e5fd69229d7b5c02e2d8d7446668abcf62ef49 --- /dev/null +++ b/docker-compose/logstash-loki/logstash/conf.d/30-output.conf @@ -0,0 +1,38 @@ +output { + elasticsearch { + hosts => ["localhost"] + manage_template => false + index => "logstash-%{+YYYY.MM.dd}" + } + loki { + [url => "localhost:3100" | default = none | required=true] + + [tenant_id => string | default = nil | required=false] + + [message_field => string | default = "message" | required=false] + + [include_fields => array | default = [] | required=false] + + [batch_wait => number | default = 1(s) | required=false] + + [batch_size => number | default = 102400(bytes) | required=false] + + [min_delay => number | default = 1(s) | required=false] + + [max_delay => number | default = 300(s) | required=false] + + [retries => number | default = 10 | required=false] + + [username => string | default = nil | required=false] + + [password => secret | default = nil | required=false] + + [cert => path | default = nil | required=false] + + [key => path | default = nil| required=false] + + [ca_cert => path | default = nil | required=false] + + [insecure_skip_verify => boolean | default = false | required=false] + } +}