From 320529ba1fd0fcb61fb0664164435b5311128804 Mon Sep 17 00:00:00 2001
From: stedif <stefano.difrischia@inaf.it>
Date: Thu, 10 Nov 2022 16:45:42 +0100
Subject: [PATCH] L2SS-1052: remove elk entries
---
README.md | 9 ----
bootstrap/sbin/rebuild_system_from_scratch.sh | 17 ++----
docker-compose/Makefile | 1 -
docker-compose/elk-configure-host/Dockerfile | 4 --
docker-compose/elk.yml | 53 -------------------
docker-compose/elk/Dockerfile | 18 -------
.../elk/kibana/default-objects.ndjson | 3 --
.../elk/logstash/conf.d/02-beats-input.conf | 8 ---
.../elk/logstash/conf.d/03-syslog-input.conf | 5 --
.../elk/logstash/conf.d/04-tcp-input.conf | 6 ---
.../elk/logstash/conf.d/10-syslog.conf | 13 -----
.../elk/logstash/conf.d/11-nginx.conf | 7 ---
.../elk/logstash/conf.d/20-parse-grafana.conf | 16 ------
.../logstash/conf.d/21-parse-prometheus.conf | 15 ------
.../logstash/conf.d/22-parse-tango-rest.conf | 14 -----
.../logstash/conf.d/23-parse-maria-db.conf | 32 -----------
.../elk/logstash/conf.d/30-output.conf | 7 ---
.../code/tango-prometheus-client.py | 2 +-
.../devices/docker_device.py | 2 -
19 files changed, 4 insertions(+), 228 deletions(-)
delete mode 100644 docker-compose/elk-configure-host/Dockerfile
delete mode 100644 docker-compose/elk.yml
delete mode 100644 docker-compose/elk/Dockerfile
delete mode 100755 docker-compose/elk/kibana/default-objects.ndjson
delete mode 100644 docker-compose/elk/logstash/conf.d/02-beats-input.conf
delete mode 100644 docker-compose/elk/logstash/conf.d/03-syslog-input.conf
delete mode 100644 docker-compose/elk/logstash/conf.d/04-tcp-input.conf
delete mode 100644 docker-compose/elk/logstash/conf.d/10-syslog.conf
delete mode 100644 docker-compose/elk/logstash/conf.d/11-nginx.conf
delete mode 100644 docker-compose/elk/logstash/conf.d/20-parse-grafana.conf
delete mode 100644 docker-compose/elk/logstash/conf.d/21-parse-prometheus.conf
delete mode 100644 docker-compose/elk/logstash/conf.d/22-parse-tango-rest.conf
delete mode 100644 docker-compose/elk/logstash/conf.d/23-parse-maria-db.conf
delete mode 100644 docker-compose/elk/logstash/conf.d/30-output.conf
diff --git a/README.md b/README.md
index da4288bc3..6903db66c 100644
--- a/README.md
+++ b/README.md
@@ -72,17 +72,8 @@ them down (`make stop <container>`).
Most notably, you will have web interfaces available at:
- http://localhost:8888 (Jupyter Notebook)
- - http://localhost:5601 (ElasticSearch)
- http://localhost:3000 (Grafana).
-To create the appropriate indexes for ElasticSearch the following needs to be
-executed once while ElasticSearch is running:
-
-```sh
-cd docker-compose/elk
-curl -X POST http://localhost:5601/api/saved_objects/_import -H "kbn-xsrf: true" --form file=@kibana/default-objects.ndjson
-```
-
# Development
For development you will need several dependencies including:
diff --git a/bootstrap/sbin/rebuild_system_from_scratch.sh b/bootstrap/sbin/rebuild_system_from_scratch.sh
index 0af4d0b19..cb19cdcde 100755
--- a/bootstrap/sbin/rebuild_system_from_scratch.sh
+++ b/bootstrap/sbin/rebuild_system_from_scratch.sh
@@ -98,15 +98,9 @@ function configure_tango_db()
${HOME_DIR}/sbin/update_ConfigDb.sh ${HOME_DIR}/CDB/LOFAR_ConfigDb.json || true
}
-function configure_elk()
-{
- (cd ${HOME_DIR}/docker-compose && make start elk-configure-host)
-}
-
function start_support_images()
{
- (cd ${HOME_DIR}/docker-compose && make start elk
- make start jupyter)
+ (cd ${HOME_DIR}/docker-compose && make start jupyter)
}
function start_lofar_images()
@@ -157,13 +151,8 @@ echo "-> Configuring the Tango DB for LOFAR2.0..."
configure_tango_db
echo -e "\tDone.\n"
-# Now configure the ELK container.
-echo "-> Configuring the ELK container for this host..."
-configure_elk
-echo -e "\tDone.\n"
-
-# Here I start ELK & Jupyter.
-echo "-> Start LOFAR2.0 support containers (ELK, Jupyter, etc.)..."
+# Here I start Jupyter.
+echo "-> Start LOFAR2.0 support containers (Jupyter, etc.)..."
start_support_images
echo -e "\tDone.\n"
diff --git a/docker-compose/Makefile b/docker-compose/Makefile
index 0d570999b..a725ea8d3 100644
--- a/docker-compose/Makefile
+++ b/docker-compose/Makefile
@@ -203,7 +203,6 @@ context: ## Move the necessary files to create minimal docker context
@cp ../tangostationcontrol/requirements.txt tmp/
bootstrap: pull build # first start, initialise from scratch
- $(MAKE) start elk-configure-host # configure host kernel for elk container
$(MAKE) start dsconfig # boot up containers to load configurations
sleep 5 # wait for dsconfig container to come up
../sbin/update_ConfigDb.sh ../CDB/LOFAR_ConfigDb.json # load default configuration
diff --git a/docker-compose/elk-configure-host/Dockerfile b/docker-compose/elk-configure-host/Dockerfile
deleted file mode 100644
index 269032484..000000000
--- a/docker-compose/elk-configure-host/Dockerfile
+++ /dev/null
@@ -1,4 +0,0 @@
-FROM ubuntu:20.04
-
-# Expand VM settings as required by ELK
-CMD bash -c "if [ -w /proc/sys/vm/max_map_count ] && [ $(cat /proc/sys/vm/max_map_count) -lt 262144 ]; then echo 262144 > /proc/sys/vm/max_map_count; fi"
diff --git a/docker-compose/elk.yml b/docker-compose/elk.yml
deleted file mode 100644
index d671ba0c7..000000000
--- a/docker-compose/elk.yml
+++ /dev/null
@@ -1,53 +0,0 @@
-#
-# Docker compose file that launches an ELK stack.
-# See https://elk-docker.readthedocs.io/
-#
-# Defines:
-# - elk-configure-host: Configures the hosts's kernel to be able to use the ELK stack
-# - elk: ELK stack
-#
-
-version: '2.1'
-
-volumes:
- elk-data: {}
-
-services:
- elk-configure-host:
- image: elk-configure-host
- build:
- context: elk-configure-host
- container_name: ${CONTAINER_NAME_PREFIX}elk-configure-host
- logging:
- driver: "json-file"
- options:
- max-size: "100m"
- max-file: "10"
- networks:
- - control
- privileged: true
-
- elk:
- image: elk
- build:
- context: elk
- container_name: ${CONTAINER_NAME_PREFIX}elk
- logging:
- driver: "json-file"
- options:
- max-size: "100m"
- max-file: "10"
- networks:
- - control
- volumes:
- - elk-data:/var/lib/elasticsearch
- ports:
- - "5601:5601" # kibana
- - "9200:9200" # elasticsearch
- # - "5044:5044" # logstash beats input
- # - "1514:1514/tcp" # logstash syslog input
- # - "1514:1514/udp" # logstash syslog input
- # - "5959:5959" # logstash tcp json input
- depends_on:
- - elk-configure-host
- restart: unless-stopped
diff --git a/docker-compose/elk/Dockerfile b/docker-compose/elk/Dockerfile
deleted file mode 100644
index fda7ed40e..000000000
--- a/docker-compose/elk/Dockerfile
+++ /dev/null
@@ -1,18 +0,0 @@
-FROM sebp/elk
-
-# Give more time for ElasticSearch to startup on our poor dev laptops
-ENV ES_CONNECT_RETRY=60
-
-# Provide our logstash config
-ADD logstash /etc/logstash/
-
-# Provide our kibana config
-#
-# For now:
-# 1. Start the container,
-# 2. Make sure there are log messages in the database (to populate their patterns),
-# 3. Run in this directory
-# $ curl -X POST http://localhost:5601/api/saved_objects/_import -H "kbn-xsrf: true" --form file=@kibana/default-objects.ndjson
-#
-# See also https://www.elastic.co/guide/en/kibana/7.x/saved-objects-api-import.html
-# and https://github.com/Bitergia/archimedes
diff --git a/docker-compose/elk/kibana/default-objects.ndjson b/docker-compose/elk/kibana/default-objects.ndjson
deleted file mode 100755
index 3c9f118b2..000000000
--- a/docker-compose/elk/kibana/default-objects.ndjson
+++ /dev/null
@@ -1,3 +0,0 @@
-{"attributes":{"fieldAttrs":"{\"level\":{\"count\":1},\"message\":{\"count\":1},\"program\":{\"count\":2}}","fields":"[]","timeFieldName":"@timestamp","title":"logstash-*"},"id":"5340ad20-923d-11eb-9cc8-2fc0b321a697","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"type":"index-pattern","updated_at":"2021-03-31T18:16:04.716Z","version":"WzE0NiwxXQ=="}
-{"attributes":{"columns":["extra.device","level","message"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[],"title":"Last hour","version":1},"id":"cf4947d0-924f-11eb-9cc8-2fc0b321a697","migrationVersion":{"search":"7.9.3"},"references":[{"id":"5340ad20-923d-11eb-9cc8-2fc0b321a697","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2021-03-31T18:35:04.269Z","version":"WzE2NywxXQ=="}
-{"exportedCount":2,"missingRefCount":0,"missingReferences":[]}
\ No newline at end of file
diff --git a/docker-compose/elk/logstash/conf.d/02-beats-input.conf b/docker-compose/elk/logstash/conf.d/02-beats-input.conf
deleted file mode 100644
index 4ab52b370..000000000
--- a/docker-compose/elk/logstash/conf.d/02-beats-input.conf
+++ /dev/null
@@ -1,8 +0,0 @@
-input {
- beats {
- port => 5044
- ssl => true
- ssl_certificate => "/etc/pki/tls/certs/logstash-beats.crt"
- ssl_key => "/etc/pki/tls/private/logstash-beats.key"
- }
-}
diff --git a/docker-compose/elk/logstash/conf.d/03-syslog-input.conf b/docker-compose/elk/logstash/conf.d/03-syslog-input.conf
deleted file mode 100644
index b859a357d..000000000
--- a/docker-compose/elk/logstash/conf.d/03-syslog-input.conf
+++ /dev/null
@@ -1,5 +0,0 @@
-input {
- syslog {
- port => 1514
- }
-}
diff --git a/docker-compose/elk/logstash/conf.d/04-tcp-input.conf b/docker-compose/elk/logstash/conf.d/04-tcp-input.conf
deleted file mode 100644
index 67def0887..000000000
--- a/docker-compose/elk/logstash/conf.d/04-tcp-input.conf
+++ /dev/null
@@ -1,6 +0,0 @@
-input {
- tcp {
- port => 5959
- codec => json
- }
-}
diff --git a/docker-compose/elk/logstash/conf.d/10-syslog.conf b/docker-compose/elk/logstash/conf.d/10-syslog.conf
deleted file mode 100644
index acce463cd..000000000
--- a/docker-compose/elk/logstash/conf.d/10-syslog.conf
+++ /dev/null
@@ -1,13 +0,0 @@
-filter {
- if [type] == "syslog" {
- grok {
- match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
- add_field => [ "received_at", "%{@timestamp}" ]
- add_field => [ "received_from", "%{host}" ]
- }
- syslog_pri { }
- date {
- match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
- }
- }
-}
diff --git a/docker-compose/elk/logstash/conf.d/11-nginx.conf b/docker-compose/elk/logstash/conf.d/11-nginx.conf
deleted file mode 100644
index d4a45db2d..000000000
--- a/docker-compose/elk/logstash/conf.d/11-nginx.conf
+++ /dev/null
@@ -1,7 +0,0 @@
-filter {
- if [type] == "nginx-access" {
- grok {
- match => { "message" => "%{NGINXACCESS}" }
- }
- }
-}
diff --git a/docker-compose/elk/logstash/conf.d/20-parse-grafana.conf b/docker-compose/elk/logstash/conf.d/20-parse-grafana.conf
deleted file mode 100644
index 37db44fda..000000000
--- a/docker-compose/elk/logstash/conf.d/20-parse-grafana.conf
+++ /dev/null
@@ -1,16 +0,0 @@
-filter {
- if [program] == "grafana" {
- kv { }
- mutate {
- rename => {
- "t" => "timestamp"
- "lvl" => "level"
- "msg" => "message"
- }
- uppercase => [ "level" ]
- }
- date {
- match => [ "timestamp", "ISO8601" ]
- }
- }
-}
diff --git a/docker-compose/elk/logstash/conf.d/21-parse-prometheus.conf b/docker-compose/elk/logstash/conf.d/21-parse-prometheus.conf
deleted file mode 100644
index b8323625f..000000000
--- a/docker-compose/elk/logstash/conf.d/21-parse-prometheus.conf
+++ /dev/null
@@ -1,15 +0,0 @@
-filter {
- if [program] == "prometheus" {
- kv { }
- mutate {
- rename => {
- "ts" => "timestamp"
- "msg" => "message"
- }
- uppercase => [ "level" ]
- }
- date {
- match => [ "timestamp", "ISO8601" ]
- }
- }
-}
diff --git a/docker-compose/elk/logstash/conf.d/22-parse-tango-rest.conf b/docker-compose/elk/logstash/conf.d/22-parse-tango-rest.conf
deleted file mode 100644
index 5df0cd92b..000000000
--- a/docker-compose/elk/logstash/conf.d/22-parse-tango-rest.conf
+++ /dev/null
@@ -1,14 +0,0 @@
-filter {
- if [program] == "tango-rest" {
- grok {
- match => {
- "message" => "%{TIMESTAMP_ISO8601:timestamp} %{WORD:level} %{GREEDYDATA:message}"
- }
- "overwrite" => [ "timestamp", "level", "message" ]
- }
- date {
- match => [ "timestamp", "YYYY-MM-dd HH:mm:ss,SSS" ]
- timezone => "UTC"
- }
- }
-}
diff --git a/docker-compose/elk/logstash/conf.d/23-parse-maria-db.conf b/docker-compose/elk/logstash/conf.d/23-parse-maria-db.conf
deleted file mode 100644
index 0a23fddd0..000000000
--- a/docker-compose/elk/logstash/conf.d/23-parse-maria-db.conf
+++ /dev/null
@@ -1,32 +0,0 @@
-filter {
- # mark all our mariadb instances
- grok {
- match => {
- "program" => [ "archiver-maria-db", "tangodb" ]
- }
- add_tag => [ "mariadb" ]
- }
-
- # parse mariadb output
- if "mariadb" in [tags] {
- grok {
- match => {
- "message" => [
- "%{TIMESTAMP_ISO8601:timestamp} .%{WORD:level}. %{GREEDYDATA:message}",
- "%{TIMESTAMP_ISO8601:timestamp} 0 .%{WORD:level}. %{GREEDYDATA:message}"
- ]
- }
- "overwrite" => [ "timestamp", "level", "message" ]
- }
- mutate {
- gsub => [
- "level", "Note", "Info"
- ]
- uppercase => [ "level" ]
- }
- date {
- match => [ "timestamp", "YYYY-MM-dd HH:mm:ssZZ", "YYYY-MM-dd HH:mm:ss", "YYYY-MM-dd H:mm:ss" ]
- timezone => "UTC"
- }
- }
-}
diff --git a/docker-compose/elk/logstash/conf.d/30-output.conf b/docker-compose/elk/logstash/conf.d/30-output.conf
deleted file mode 100644
index 1893dd990..000000000
--- a/docker-compose/elk/logstash/conf.d/30-output.conf
+++ /dev/null
@@ -1,7 +0,0 @@
-output {
- elasticsearch {
- hosts => ["localhost"]
- manage_template => false
- index => "logstash-%{+YYYY.MM.dd}"
- }
-}
diff --git a/docker-compose/tango-prometheus-exporter/code/tango-prometheus-client.py b/docker-compose/tango-prometheus-exporter/code/tango-prometheus-client.py
index 256f78bf1..ce1570a32 100644
--- a/docker-compose/tango-prometheus-exporter/code/tango-prometheus-client.py
+++ b/docker-compose/tango-prometheus-exporter/code/tango-prometheus-client.py
@@ -13,7 +13,7 @@ logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=loggin
# log to memory only, with a limit of 600 seconds. this container is not important enough to keep
# all logs around for, and as a scraper must be robust against overloading on all sides
-handler = AsynchronousLogstashHandler("elk", 5959, database_path=None, event_ttl=600)
+handler = AsynchronousLogstashHandler("logstash", 5959, database_path=None, event_ttl=600)
handler.setLevel(logging.INFO)
logger.addHandler(handler)
diff --git a/tangostationcontrol/tangostationcontrol/devices/docker_device.py b/tangostationcontrol/tangostationcontrol/devices/docker_device.py
index 71e13119d..cf747e69e 100644
--- a/tangostationcontrol/tangostationcontrol/devices/docker_device.py
+++ b/tangostationcontrol/tangostationcontrol/devices/docker_device.py
@@ -99,8 +99,6 @@ class Docker(lofar_device):
databaseds_RW = attribute_wrapper(comms_annotation={"container": "databaseds"}, datatype=bool, access=AttrWriteType.READ_WRITE)
dsconfig_R = attribute_wrapper(comms_annotation={"container": "dsconfig"}, datatype=bool)
dsconfig_RW = attribute_wrapper(comms_annotation={"container": "dsconfig"}, datatype=bool, access=AttrWriteType.READ_WRITE)
- elk_R = attribute_wrapper(comms_annotation={"container": "elk"}, datatype=bool)
- elk_RW = attribute_wrapper(comms_annotation={"container": "elk"}, datatype=bool, access=AttrWriteType.READ_WRITE)
grafana_R = attribute_wrapper(comms_annotation={"container": "grafana"}, datatype=bool)
grafana_RW = attribute_wrapper(comms_annotation={"container": "grafana"}, datatype=bool, access=AttrWriteType.READ_WRITE)
logstash_R = attribute_wrapper(comms_annotation={"container": "logstash"}, datatype=bool)
--
GitLab