diff --git a/docker-compose/elk/Dockerfile b/docker-compose/elk/Dockerfile index 709594fefb492032e27c0dd4f42746e134575f83..5f23bc06beaff27c2ddce6fe28386d7905ad040f 100644 --- a/docker-compose/elk/Dockerfile +++ b/docker-compose/elk/Dockerfile @@ -3,5 +3,10 @@ FROM sebp/elk # Give more time for ElasticSearch to startup on our poor dev laptops ENV ES_CONNECT_RETRY=60 -# Let logstash parse remote syslog input -COPY logstash-syslog-input.local.conf /etc/logstash/conf.d/03-syslog-input.conf +# Provide our logstash config +ADD logstash /etc/logstash/ + +# Provide our kibana config +# See also https://www.elastic.co/guide/en/kibana/7.x/saved-objects-api-import.html +# and https://github.com/Bitergia/archimedes + diff --git a/docker-compose/elk/logstash/conf.d/02-beats-input.conf b/docker-compose/elk/logstash/conf.d/02-beats-input.conf new file mode 100644 index 0000000000000000000000000000000000000000..4ab52b37081aa46fdda7edd82a0395b9f73a3705 --- /dev/null +++ b/docker-compose/elk/logstash/conf.d/02-beats-input.conf @@ -0,0 +1,8 @@ +input { + beats { + port => 5044 + ssl => true + ssl_certificate => "/etc/pki/tls/certs/logstash-beats.crt" + ssl_key => "/etc/pki/tls/private/logstash-beats.key" + } +} diff --git a/docker-compose/elk/logstash-syslog-input.local.conf b/docker-compose/elk/logstash/conf.d/03-syslog-input.conf similarity index 59% rename from docker-compose/elk/logstash-syslog-input.local.conf rename to docker-compose/elk/logstash/conf.d/03-syslog-input.conf index dd9cbb9b346855fdb63e22b5da5cf45edd340c05..b859a357d505d9ea8d59eb3cf39d2af97b76a119 100644 --- a/docker-compose/elk/logstash-syslog-input.local.conf +++ b/docker-compose/elk/logstash/conf.d/03-syslog-input.conf @@ -1,4 +1,5 @@ input { syslog { + port => 1514 } } diff --git a/docker-compose/elk/logstash/conf.d/04-tcp-input.conf b/docker-compose/elk/logstash/conf.d/04-tcp-input.conf new file mode 100644 index 0000000000000000000000000000000000000000..532cf5a388e840ddda2e5cd58922ba2338f09d0d --- /dev/null +++ b/docker-compose/elk/logstash/conf.d/04-tcp-input.conf @@ -0,0 +1,12 @@ +input { + tcp { + port => 5959 + codec => json + } +} + +output { + file { + path => "/tmp/logstash-input.log" + } +} diff --git a/docker-compose/elk/logstash/conf.d/10-syslog.conf b/docker-compose/elk/logstash/conf.d/10-syslog.conf new file mode 100644 index 0000000000000000000000000000000000000000..acce463cd42543ce87c5a51496aec25b06d21fdd --- /dev/null +++ b/docker-compose/elk/logstash/conf.d/10-syslog.conf @@ -0,0 +1,13 @@ +filter { + if [type] == "syslog" { + grok { + match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" } + add_field => [ "received_at", "%{@timestamp}" ] + add_field => [ "received_from", "%{host}" ] + } + syslog_pri { } + date { + match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ] + } + } +} diff --git a/docker-compose/elk/logstash/conf.d/11-nginx.conf b/docker-compose/elk/logstash/conf.d/11-nginx.conf new file mode 100644 index 0000000000000000000000000000000000000000..d4a45db2d8454d982d52056cd035a9c8ef865389 --- /dev/null +++ b/docker-compose/elk/logstash/conf.d/11-nginx.conf @@ -0,0 +1,7 @@ +filter { + if [type] == "nginx-access" { + grok { + match => { "message" => "%{NGINXACCESS}" } + } + } +} diff --git a/docker-compose/elk/logstash/conf.d/30-output.conf b/docker-compose/elk/logstash/conf.d/30-output.conf new file mode 100644 index 0000000000000000000000000000000000000000..1893dd990ff62ce1a6b165fb449dc5afc4442268 --- /dev/null +++ b/docker-compose/elk/logstash/conf.d/30-output.conf @@ -0,0 +1,7 @@ +output { + elasticsearch { + hosts => ["localhost"] + manage_template => false + index => "logstash-%{+YYYY.MM.dd}" + } +} diff --git a/docker-compose/elk/logstash/conf.d/31-output-file-for-debugging.conf b/docker-compose/elk/logstash/conf.d/31-output-file-for-debugging.conf new file mode 100644 index 0000000000000000000000000000000000000000..d8b21f0dbcd6564909f39ccc4b05a1e88b8097ef --- /dev/null +++ b/docker-compose/elk/logstash/conf.d/31-output-file-for-debugging.conf @@ -0,0 +1,5 @@ +output { + file { + path => "/tmp/logstash-input.log" + } +} diff --git a/docker-compose/lofar-device-base/lofar-requirements.txt b/docker-compose/lofar-device-base/lofar-requirements.txt index 90d21efe0e5ac0601204d4f05ce7efcd16dca2de..7ed18f76527891ca48953150977b2f23703c9baa 100644 --- a/docker-compose/lofar-device-base/lofar-requirements.txt +++ b/docker-compose/lofar-device-base/lofar-requirements.txt @@ -1,2 +1,3 @@ opcua >= 0.98.9 -astropy +astropy +python-logstash-async