From bf8f553aaa2627786373ec635094bb1a71e2a6ab Mon Sep 17 00:00:00 2001
From: Jan David Mol <mol@astron.nl>
Date: Wed, 31 Mar 2021 18:29:06 +0200
Subject: [PATCH] Add logstash imports for syslog & python, add logstash
logging python module to our devices
---
docker-compose/elk/Dockerfile | 9 +++++++--
.../elk/logstash/conf.d/02-beats-input.conf | 8 ++++++++
.../conf.d/03-syslog-input.conf} | 1 +
.../elk/logstash/conf.d/04-tcp-input.conf | 12 ++++++++++++
docker-compose/elk/logstash/conf.d/10-syslog.conf | 13 +++++++++++++
docker-compose/elk/logstash/conf.d/11-nginx.conf | 7 +++++++
docker-compose/elk/logstash/conf.d/30-output.conf | 7 +++++++
.../conf.d/31-output-file-for-debugging.conf | 5 +++++
.../lofar-device-base/lofar-requirements.txt | 3 ++-
9 files changed, 62 insertions(+), 3 deletions(-)
create mode 100644 docker-compose/elk/logstash/conf.d/02-beats-input.conf
rename docker-compose/elk/{logstash-syslog-input.local.conf => logstash/conf.d/03-syslog-input.conf} (59%)
create mode 100644 docker-compose/elk/logstash/conf.d/04-tcp-input.conf
create mode 100644 docker-compose/elk/logstash/conf.d/10-syslog.conf
create mode 100644 docker-compose/elk/logstash/conf.d/11-nginx.conf
create mode 100644 docker-compose/elk/logstash/conf.d/30-output.conf
create mode 100644 docker-compose/elk/logstash/conf.d/31-output-file-for-debugging.conf
diff --git a/docker-compose/elk/Dockerfile b/docker-compose/elk/Dockerfile
index 709594fef..5f23bc06b 100644
--- a/docker-compose/elk/Dockerfile
+++ b/docker-compose/elk/Dockerfile
@@ -3,5 +3,10 @@ FROM sebp/elk
# Give more time for ElasticSearch to startup on our poor dev laptops
ENV ES_CONNECT_RETRY=60
-# Let logstash parse remote syslog input
-COPY logstash-syslog-input.local.conf /etc/logstash/conf.d/03-syslog-input.conf
+# Provide our logstash config
+ADD logstash /etc/logstash/
+
+# Provide our kibana config
+# See also https://www.elastic.co/guide/en/kibana/7.x/saved-objects-api-import.html
+# and https://github.com/Bitergia/archimedes
+
diff --git a/docker-compose/elk/logstash/conf.d/02-beats-input.conf b/docker-compose/elk/logstash/conf.d/02-beats-input.conf
new file mode 100644
index 000000000..4ab52b370
--- /dev/null
+++ b/docker-compose/elk/logstash/conf.d/02-beats-input.conf
@@ -0,0 +1,8 @@
+input {
+ beats {
+ port => 5044
+ ssl => true
+ ssl_certificate => "/etc/pki/tls/certs/logstash-beats.crt"
+ ssl_key => "/etc/pki/tls/private/logstash-beats.key"
+ }
+}
diff --git a/docker-compose/elk/logstash-syslog-input.local.conf b/docker-compose/elk/logstash/conf.d/03-syslog-input.conf
similarity index 59%
rename from docker-compose/elk/logstash-syslog-input.local.conf
rename to docker-compose/elk/logstash/conf.d/03-syslog-input.conf
index dd9cbb9b3..b859a357d 100644
--- a/docker-compose/elk/logstash-syslog-input.local.conf
+++ b/docker-compose/elk/logstash/conf.d/03-syslog-input.conf
@@ -1,4 +1,5 @@
input {
syslog {
+ port => 1514
}
}
diff --git a/docker-compose/elk/logstash/conf.d/04-tcp-input.conf b/docker-compose/elk/logstash/conf.d/04-tcp-input.conf
new file mode 100644
index 000000000..532cf5a38
--- /dev/null
+++ b/docker-compose/elk/logstash/conf.d/04-tcp-input.conf
@@ -0,0 +1,12 @@
+input {
+ tcp {
+ port => 5959
+ codec => json
+ }
+}
+
+output {
+ file {
+ path => "/tmp/logstash-input.log"
+ }
+}
diff --git a/docker-compose/elk/logstash/conf.d/10-syslog.conf b/docker-compose/elk/logstash/conf.d/10-syslog.conf
new file mode 100644
index 000000000..acce463cd
--- /dev/null
+++ b/docker-compose/elk/logstash/conf.d/10-syslog.conf
@@ -0,0 +1,13 @@
+filter {
+ if [type] == "syslog" {
+ grok {
+ match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
+ add_field => [ "received_at", "%{@timestamp}" ]
+ add_field => [ "received_from", "%{host}" ]
+ }
+ syslog_pri { }
+ date {
+ match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
+ }
+ }
+}
diff --git a/docker-compose/elk/logstash/conf.d/11-nginx.conf b/docker-compose/elk/logstash/conf.d/11-nginx.conf
new file mode 100644
index 000000000..d4a45db2d
--- /dev/null
+++ b/docker-compose/elk/logstash/conf.d/11-nginx.conf
@@ -0,0 +1,7 @@
+filter {
+ if [type] == "nginx-access" {
+ grok {
+ match => { "message" => "%{NGINXACCESS}" }
+ }
+ }
+}
diff --git a/docker-compose/elk/logstash/conf.d/30-output.conf b/docker-compose/elk/logstash/conf.d/30-output.conf
new file mode 100644
index 000000000..1893dd990
--- /dev/null
+++ b/docker-compose/elk/logstash/conf.d/30-output.conf
@@ -0,0 +1,7 @@
+output {
+ elasticsearch {
+ hosts => ["localhost"]
+ manage_template => false
+ index => "logstash-%{+YYYY.MM.dd}"
+ }
+}
diff --git a/docker-compose/elk/logstash/conf.d/31-output-file-for-debugging.conf b/docker-compose/elk/logstash/conf.d/31-output-file-for-debugging.conf
new file mode 100644
index 000000000..d8b21f0db
--- /dev/null
+++ b/docker-compose/elk/logstash/conf.d/31-output-file-for-debugging.conf
@@ -0,0 +1,5 @@
+output {
+ file {
+ path => "/tmp/logstash-input.log"
+ }
+}
diff --git a/docker-compose/lofar-device-base/lofar-requirements.txt b/docker-compose/lofar-device-base/lofar-requirements.txt
index 90d21efe0..7ed18f765 100644
--- a/docker-compose/lofar-device-base/lofar-requirements.txt
+++ b/docker-compose/lofar-device-base/lofar-requirements.txt
@@ -1,2 +1,3 @@
opcua >= 0.98.9
-astropy
+astropy
+python-logstash-async
--
GitLab