Add logstash imports for syslog & python, add logstash logging python module to our devices

docker-compose/elk/Dockerfile

@@ -3,5 +3,10 @@ FROM sebp/elk
 # Give more time for ElasticSearch to startup on our poor dev laptops
 ENV ES_CONNECT_RETRY=60
 
-# Let logstash parse remote syslog input
-COPY logstash-syslog-input.local.conf /etc/logstash/conf.d/03-syslog-input.conf
+# Provide our logstash config
+ADD logstash /etc/logstash/
+
+# Provide our kibana config
+# See also https://www.elastic.co/guide/en/kibana/7.x/saved-objects-api-import.html
+# and https://github.com/Bitergia/archimedes
+

docker-compose/elk/logstash/conf.d/02-beats-input.conf

+input {
+  beats {
+    port => 5044
+    ssl => true
+    ssl_certificate => "/etc/pki/tls/certs/logstash-beats.crt"
+    ssl_key => "/etc/pki/tls/private/logstash-beats.key"
+  }
+}

docker-compose/elk/logstash-syslog-input.local.conf → docker-compose/elk/logstash/conf.d/03-syslog-input.conf

 input {
   syslog {
+    port => 1514
   }
 }

docker-compose/elk/logstash/conf.d/04-tcp-input.conf

+input {
+  tcp {
+    port => 5959
+    codec => json
+  }
+}
+
+output {
+  file {
+    path => "/tmp/logstash-input.log"
+  }
+}

docker-compose/elk/logstash/conf.d/10-syslog.conf

+filter {
+  if [type] == "syslog" {
+    grok {
+      match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
+      add_field => [ "received_at", "%{@timestamp}" ]
+      add_field => [ "received_from", "%{host}" ]
+    }
+    syslog_pri { }
+    date {
+      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
+    }
+  }
+}

docker-compose/elk/logstash/conf.d/11-nginx.conf

+filter {
+  if [type] == "nginx-access" {
+    grok {
+      match => { "message" => "%{NGINXACCESS}" }
+    }
+  }
+}

docker-compose/elk/logstash/conf.d/30-output.conf

+output {
+  elasticsearch {
+    hosts => ["localhost"]
+    manage_template => false
+    index => "logstash-%{+YYYY.MM.dd}"
+  }
+}

docker-compose/elk/logstash/conf.d/31-output-file-for-debugging.conf

+output {
+  file {
+    path => "/tmp/logstash-input.log"
+  }
+}

docker-compose/lofar-device-base/lofar-requirements.txt

 opcua >= 0.98.9
-astropy 
+astropy
+python-logstash-async