diff --git a/docker-compose/apsct-sim.yml b/docker-compose/apsct-sim.yml
index ae3c6beabf23bf229a80ddb738ddfca3f72ed04e..d6ef5b9a579976c0952020030a829bef637f087f 100644
--- a/docker-compose/apsct-sim.yml
+++ b/docker-compose/apsct-sim.yml
@@ -15,6 +15,7 @@ services:
args:
- LOCAL_DOCKER_REGISTRY_HOST=${LOCAL_DOCKER_REGISTRY_HOST}
- LOCAL_DOCKER_REGISTRY_LOFAR=${LOCAL_DOCKER_REGISTRY_LOFAR}
+ hostname: apsct-sim
container_name: apsct-sim
logging:
driver: "json-file"
diff --git a/docker-compose/apspu-sim.yml b/docker-compose/apspu-sim.yml
index d0f3dec1813ba2fe3112d5bfc94e89b2e9f59457..ebfddbaa8bb2963441fe93f239256bc8bb98c88e 100644
--- a/docker-compose/apspu-sim.yml
+++ b/docker-compose/apspu-sim.yml
@@ -15,6 +15,7 @@ services:
args:
- LOCAL_DOCKER_REGISTRY_HOST=${LOCAL_DOCKER_REGISTRY_HOST}
- LOCAL_DOCKER_REGISTRY_LOFAR=${LOCAL_DOCKER_REGISTRY_LOFAR}
+ hostname: apspu-sim
container_name: apspu-sim
logging:
driver: "json-file"
diff --git a/docker-compose/ccd-sim.yml b/docker-compose/ccd-sim.yml
index 0ae852469e56e9048b087b824f1ba422a029e17a..1580b222ec4be3f86de7d3df057efcb3a01e6ce6 100644
--- a/docker-compose/ccd-sim.yml
+++ b/docker-compose/ccd-sim.yml
@@ -15,6 +15,7 @@ services:
args:
- LOCAL_DOCKER_REGISTRY_HOST=${LOCAL_DOCKER_REGISTRY_HOST}
- LOCAL_DOCKER_REGISTRY_LOFAR=${LOCAL_DOCKER_REGISTRY_LOFAR}
+ hostname: ccd-sim
container_name: ccd-sim
logging:
driver: "json-file"
diff --git a/docker-compose/device-antennafield.yml b/docker-compose/device-antennafield.yml
index 3dca9bbd9269986a98d648ca0e3287fe50b2edfd..5272302432c4ef5cc69aa2c7236e567234198b46 100644
--- a/docker-compose/device-antennafield.yml
+++ b/docker-compose/device-antennafield.yml
@@ -18,6 +18,7 @@ version: '2.1'
services:
device-antennafield:
image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ hostname: device-antennafield
container_name: device-antennafield
logging:
driver: "json-file"
diff --git a/docker-compose/device-apsct.yml b/docker-compose/device-apsct.yml
index 4580fbbb19914b3195bbc3734c85bf04880913d2..57a90c10c843d56219d45610b6668c8df695787f 100644
--- a/docker-compose/device-apsct.yml
+++ b/docker-compose/device-apsct.yml
@@ -17,6 +17,7 @@ version: '2.1'
services:
device-apsct:
image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ hostname: device-apsct
container_name: device-apsct
logging:
driver: "json-file"
diff --git a/docker-compose/device-apspu.yml b/docker-compose/device-apspu.yml
index a3edc93d0e482db6f394b8ed1ab22478f134ff7a..5e2de1a66a65eafab4443f04415dc5aa77b598f9 100644
--- a/docker-compose/device-apspu.yml
+++ b/docker-compose/device-apspu.yml
@@ -17,6 +17,7 @@ version: '2.1'
services:
device-apspu:
image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ hostname: device-apspu
container_name: device-apspu
logging:
driver: "json-file"
diff --git a/docker-compose/device-beamlet.yml b/docker-compose/device-beamlet.yml
index 3b5b47947acfdd5162234fec36986c4b10c4a367..1cf7dc5a2f265d45be7700e2c368f7a83a67b53c 100644
--- a/docker-compose/device-beamlet.yml
+++ b/docker-compose/device-beamlet.yml
@@ -17,6 +17,7 @@ version: '2.1'
services:
device-beamlet:
image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ hostname: device-beamlet
container_name: device-beamlet
logging:
driver: "json-file"
diff --git a/docker-compose/device-boot.yml b/docker-compose/device-boot.yml
index efc4db80e3f63a586845f9e213ea3d2344554d88..8472267963ef41407cbca5e10a297c304eba7933 100644
--- a/docker-compose/device-boot.yml
+++ b/docker-compose/device-boot.yml
@@ -16,6 +16,7 @@ version: '2.1'
services:
device-boot:
image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ hostname: device-boot
container_name: device-boot
logging:
driver: "json-file"
diff --git a/docker-compose/device-bst.yml b/docker-compose/device-bst.yml
index 0adf4ef9bf4aea04a612803278ead833c91d176a..69d71ff97f2006d943fa535196b760c4c86f0787 100644
--- a/docker-compose/device-bst.yml
+++ b/docker-compose/device-bst.yml
@@ -17,6 +17,7 @@ version: '2.1'
services:
device-bst:
image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ hostname: device-bst
container_name: device-bst
logging:
driver: "json-file"
diff --git a/docker-compose/device-calibration.yml b/docker-compose/device-calibration.yml
index f973c92a38354c6e696d0bd4b71605d1b29d3b45..a024ab8a2a0e9207e3c9ea294cca6a667c705bf0 100644
--- a/docker-compose/device-calibration.yml
+++ b/docker-compose/device-calibration.yml
@@ -17,6 +17,7 @@ version: '2.1'
services:
device-calibration:
image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ hostname: device-calibration
container_name: device-calibration
logging:
driver: "json-file"
diff --git a/docker-compose/device-ccd.yml b/docker-compose/device-ccd.yml
index ba83540bbb45230905278e210577d4284f7bf1dd..4e2810ddaae0b4e599a4ea17520f0e2973a89a3a 100644
--- a/docker-compose/device-ccd.yml
+++ b/docker-compose/device-ccd.yml
@@ -17,6 +17,7 @@ version: '2.1'
services:
device-ccd:
image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ hostname: device-ccd
container_name: device-ccd
logging:
driver: "json-file"
diff --git a/docker-compose/device-configuration.yml b/docker-compose/device-configuration.yml
index caef8603f83b726ab7b3ee15ffe93b784bd83eb7..346a999a5716f48d5134fa9b05339c70f20248ef 100644
--- a/docker-compose/device-configuration.yml
+++ b/docker-compose/device-configuration.yml
@@ -17,6 +17,7 @@ version: '2.1'
services:
device-configuration:
image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ hostname: device-configuration
container_name: device-configuration
logging:
driver: "json-file"
diff --git a/docker-compose/device-digitalbeam.yml b/docker-compose/device-digitalbeam.yml
index 168924db2e5a6af700a2810f72e21112bcfddd12..2ad5c7d24050b3b8f5021df8f627c3fcd38178aa 100644
--- a/docker-compose/device-digitalbeam.yml
+++ b/docker-compose/device-digitalbeam.yml
@@ -17,6 +17,7 @@ version: '2.1'
services:
device-digitalbeam:
image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ hostname: device-digitalbeam
container_name: device-digitalbeam
logging:
driver: "json-file"
diff --git a/docker-compose/device-docker.yml b/docker-compose/device-docker.yml
index 004bea2eb1b014611ed7d4ad224aa99c809687e9..c31130996c9e92291f9b9a521b5fb6ded181184d 100644
--- a/docker-compose/device-docker.yml
+++ b/docker-compose/device-docker.yml
@@ -17,6 +17,7 @@ version: '2.1'
services:
device-docker:
image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ hostname: device-docker
container_name: device-docker
logging:
driver: "json-file"
diff --git a/docker-compose/device-observation-control.yml b/docker-compose/device-observation-control.yml
index 9bd0a7cfa66e200a86634ed6824e02deaa22d383..c4a3e4bf0bb7bc5861c30335dbe7acf49e86266f 100644
--- a/docker-compose/device-observation-control.yml
+++ b/docker-compose/device-observation-control.yml
@@ -16,6 +16,7 @@ version: '2.1'
services:
device-observation-control:
image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ hostname: device-observation-control
container_name: device-observation-control
logging:
driver: "json-file"
diff --git a/docker-compose/device-observation.yml b/docker-compose/device-observation.yml
index 6d29f38875404ab8fb23318777c60527f3b532c6..3d4b24168c96306c67bd4a0c7b91485b8bf77794 100644
--- a/docker-compose/device-observation.yml
+++ b/docker-compose/device-observation.yml
@@ -15,6 +15,7 @@ version: '2.1'
services:
device-observation:
image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ hostname: device-observation
container_name: device-observation
logging:
driver: "json-file"
diff --git a/docker-compose/device-pcon.yml b/docker-compose/device-pcon.yml
index bca66d499241213ba78f900d9a74c7e8a5d36192..5b900769107dc097af35c0efdd49fbf814e417a2 100644
--- a/docker-compose/device-pcon.yml
+++ b/docker-compose/device-pcon.yml
@@ -12,6 +12,7 @@ volumes:
services:
device-pcon:
image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ hostname: device-pcon
container_name: device-pcon
logging:
driver: "json-file"
diff --git a/docker-compose/device-psoc.yml b/docker-compose/device-psoc.yml
index f85b0a1891eccd3d74a8cad784acb5760d18e558..ea3ef27d9af6ae2475601ab2064c0e1752da8d1b 100644
--- a/docker-compose/device-psoc.yml
+++ b/docker-compose/device-psoc.yml
@@ -12,6 +12,7 @@ volumes:
services:
device-psoc:
image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ hostname: device-psoc
container_name: device-psoc
logging:
driver: "json-file"
diff --git a/docker-compose/device-rcu2h.yml b/docker-compose/device-rcu2h.yml
index 4ab84dd9de43be83c7960af8fa4b312bd5dac092..48594945a5a297940f2347cb8e894bf695312362 100644
--- a/docker-compose/device-rcu2h.yml
+++ b/docker-compose/device-rcu2h.yml
@@ -17,6 +17,7 @@ version: '2.1'
services:
device-rcu2h:
image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ hostname: device-rcu2h
container_name: device-rcu2h
logging:
driver: "json-file"
diff --git a/docker-compose/device-rcu2l.yml b/docker-compose/device-rcu2l.yml
index 884f4d5bfbc166680918eeb3d4d6ad82e754f4bf..a499d8b7c81479487b3fa82042d6c3576aba9092 100644
--- a/docker-compose/device-rcu2l.yml
+++ b/docker-compose/device-rcu2l.yml
@@ -17,6 +17,7 @@ version: '2.1'
services:
device-rcu2l:
image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ hostname: device-rcu2l
container_name: device-rcu2l
logging:
driver: "json-file"
diff --git a/docker-compose/device-sdp.yml b/docker-compose/device-sdp.yml
index a3f83c9e59a104abaedd642b1d67d21a412d98a3..f9e45c368cb5998926d80f8fe27183a5091d7fa8 100644
--- a/docker-compose/device-sdp.yml
+++ b/docker-compose/device-sdp.yml
@@ -17,6 +17,7 @@ version: '2.1'
services:
device-sdp:
image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ hostname: device-sdp
container_name: device-sdp
logging:
driver: "json-file"
diff --git a/docker-compose/device-sst.yml b/docker-compose/device-sst.yml
index c3270de50cd5e5681905f59cbd29f8d92bb49435..1e430f5bfe2d7ec15886b63f4f288120e5a98431 100644
--- a/docker-compose/device-sst.yml
+++ b/docker-compose/device-sst.yml
@@ -17,6 +17,7 @@ version: '2.1'
services:
device-sst:
image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ hostname: device-sst
container_name: device-sst
logging:
driver: "json-file"
diff --git a/docker-compose/device-station-manager.yml b/docker-compose/device-station-manager.yml
index 68c525cc71c3e05f7b82fabd796590546b03f773..24c124256b74238495b8288d305dcef44c931f9a 100644
--- a/docker-compose/device-station-manager.yml
+++ b/docker-compose/device-station-manager.yml
@@ -12,6 +12,7 @@ volumes:
services:
device-station-manager:
image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ hostname: device-station-manager
container_name: device-station-manager
logging:
driver: "json-file"
diff --git a/docker-compose/device-temperature-manager.yml b/docker-compose/device-temperature-manager.yml
index 386893d5b423fc918c4b00e81af6cf5ca9de10c2..9bb1a4589ff5effb6ebe2cfae19cbc10b117d273 100644
--- a/docker-compose/device-temperature-manager.yml
+++ b/docker-compose/device-temperature-manager.yml
@@ -12,6 +12,7 @@ volumes:
services:
device-temperature-manager:
image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ hostname: device-temperature-manager
container_name: device-temperature-manager
logging:
driver: "json-file"
diff --git a/docker-compose/device-tilebeam.yml b/docker-compose/device-tilebeam.yml
index df26f30abf7d292c3d3db54af2689f9cf27ade16..3bd65a9ca2a0eb0db3a0b2714199c0d5160fe233 100644
--- a/docker-compose/device-tilebeam.yml
+++ b/docker-compose/device-tilebeam.yml
@@ -12,6 +12,7 @@ volumes:
services:
device-tilebeam:
image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ hostname: device-tilebeam
container_name: device-tilebeam
logging:
driver: "json-file"
diff --git a/docker-compose/device-unb2.yml b/docker-compose/device-unb2.yml
index 8dc893bb0fbc91ae04da2aa43ec572c14c4add0b..ed5a4309808db5edfbab6c063bae3c2abdc5a204 100644
--- a/docker-compose/device-unb2.yml
+++ b/docker-compose/device-unb2.yml
@@ -17,6 +17,7 @@ version: '2.1'
services:
device-unb2:
image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ hostname: device-unb2
container_name: device-unb2
logging:
driver: "json-file"
diff --git a/docker-compose/device-xst.yml b/docker-compose/device-xst.yml
index a0ca21b11f5adcb84a8b99691b477f695465bc69..e064d4eca68c008efd91fe504ea81236e557cecf 100644
--- a/docker-compose/device-xst.yml
+++ b/docker-compose/device-xst.yml
@@ -17,6 +17,7 @@ version: '2.1'
services:
device-xst:
image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/lofar-device-base
+ hostname: device-xst
container_name: device-xst
logging:
driver: "json-file"
diff --git a/docker-compose/grafana.yml b/docker-compose/grafana.yml
index ae60f2800a6a5603d507c0161fae34cdcac2b96d..64b13e0bd3c752746d46ac9d002350622d92f20b 100644
--- a/docker-compose/grafana.yml
+++ b/docker-compose/grafana.yml
@@ -18,6 +18,7 @@ services:
image: grafana
build:
context: grafana
+ hostname: grafana
container_name: grafana
networks:
- control
diff --git a/docker-compose/http-json-schemas.yml b/docker-compose/http-json-schemas.yml
index c8661f3f96a8be7b50c5d55c20d567c7c10f1897..314b2a47f67b93986fc70ce78ed120c4b4aa9502 100644
--- a/docker-compose/http-json-schemas.yml
+++ b/docker-compose/http-json-schemas.yml
@@ -14,12 +14,10 @@ services:
http-json-schemas:
build:
context: http-json-schemas
+ hostname: http-json-schemas
container_name: http-json-schemas
networks:
- control
- # set the hostname, otherwise duplicate device registrations result every
- # time the hostname changes as the container is restarted.
- hostname: http-json-schemas
environment:
- NGINX_HOST=http-json-schemas
- NGINX_PORT=80
diff --git a/docker-compose/integration-test.yml b/docker-compose/integration-test.yml
index 1d771d6d4ccc5a3c558f6b088a6cc29795fc04b5..0908c50d32f0cd3d9b091555c95e892b4209730b 100644
--- a/docker-compose/integration-test.yml
+++ b/docker-compose/integration-test.yml
@@ -15,6 +15,7 @@ services:
dockerfile: ci-runner/Dockerfile
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
+ hostname: integration-test
container_name: integration-test
networks:
- control
diff --git a/docker-compose/itango.yml b/docker-compose/itango.yml
index 6f6f7aeb56e1f25a4bec1fadc2765e4cefd41d18..dde5295ea2b458fae1379f2ba7aa90ecb52da5fc 100644
--- a/docker-compose/itango.yml
+++ b/docker-compose/itango.yml
@@ -20,6 +20,7 @@ services:
context: itango
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
+ hostname: itango
container_name: itango
logging:
driver: "json-file"
diff --git a/docker-compose/jupyter-lab.yml b/docker-compose/jupyter-lab.yml
index c2136156a665cbc1b77a5c1ffa57a116105dd369..11192b32fa0029ecff265808be331072c6abeacb 100644
--- a/docker-compose/jupyter-lab.yml
+++ b/docker-compose/jupyter-lab.yml
@@ -19,6 +19,7 @@ services:
args:
CONTAINER_EXECUTION_UID: ${CONTAINER_EXECUTION_UID}
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
+ hostname: jupyter-lab
container_name: jupyter-lab
logging:
driver: "json-file"
diff --git a/docker-compose/lofar-device-base.yml b/docker-compose/lofar-device-base.yml
index 725fc10ab82a2944d8b633efc0d32099c9ac0f16..777bb167bfb69d42ffa8f332d306d9e7af12e766 100644
--- a/docker-compose/lofar-device-base.yml
+++ b/docker-compose/lofar-device-base.yml
@@ -23,6 +23,7 @@ services:
dockerfile: lofar-device-base/Dockerfile
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
+ hostname: lofar-device-base
container_name: lofar-device-base
# These parameters are just visual queues, you have to define them again
# in derived docker-compose files!
diff --git a/docker-compose/logstash.yml b/docker-compose/logstash.yml
index 283ef0fb0a76cae6eab432725fe26739f1032210..9a20d061622b1bc78ece7858c8b63193164b4a66 100644
--- a/docker-compose/logstash.yml
+++ b/docker-compose/logstash.yml
@@ -14,6 +14,7 @@ services:
context: logstash
args:
SOURCE_IMAGE: grafana/logstash-output-loki:main
+ hostname: logstash
container_name: logstash
logging:
driver: "json-file"
@@ -35,3 +36,17 @@ services:
- "5959:5959" # logstash tcp json input
- "9600:9600"
restart: unless-stopped
+
+ logstash-exporter:
+ image: sequra/logstash_exporter
+ hostname: logstash-exporter
+ container_name: logstash-exporter
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "100m"
+ max-file: "10"
+ networks:
+ - control
+ command: --logstash.endpoint="http://logstash:9600"
+ restart: unless-stopped
diff --git a/docker-compose/logstash/logstash.yml b/docker-compose/logstash/logstash.yml
index 5f80650fe6fc635570fd9f7e4888da17eddf4e70..a5c9f331919bf8ff4f6f16aff14668cd48c9c345 100644
--- a/docker-compose/logstash/logstash.yml
+++ b/docker-compose/logstash/logstash.yml
@@ -1,2 +1,4 @@
http.host: "0.0.0.0"
#xpack.monitoring.elasticsearch.hosts: [ "http://loki:3100" ]
+
+pipeline.ecs_compatibility: disabled
diff --git a/docker-compose/logstash/loki.conf b/docker-compose/logstash/loki.conf
index b0b22e26996f82f5dcdb9561183090abb131dbad..177ea72c8237826b40ef94323382c76f5c92781a 100644
--- a/docker-compose/logstash/loki.conf
+++ b/docker-compose/logstash/loki.conf
@@ -1,5 +1,8 @@
+# For syntax, see https://www.elastic.co/guide/en/logstash/current/filter-plugins.html
+
input {
beats {
+ id => "input_beats"
port => 5044
# TODO (L2SS-748) add SSL encryption
}
@@ -7,12 +10,14 @@ input {
input {
syslog {
+ id => "input_syslog"
port => 1514
}
}
input {
tcp {
+ id => "input_tcp_json"
port => 5959
codec => json
}
@@ -21,12 +26,14 @@ input {
filter {
if [type] == "syslog" {
grok {
+ id => "grok_syslog"
match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
add_field => [ "received_at", "%{@timestamp}" ]
add_field => [ "received_from", "%{host}" ]
}
- syslog_pri { }
+ syslog_pri { id => "syslog_pri" }
date {
+ id => "date_syslog"
match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
}
}
@@ -34,48 +41,35 @@ filter {
filter {
if [program] == "grafana" {
- kv { }
+ kv { id => "kv_grafana" }
mutate {
+ id => "mutate_grafana"
rename => {
"t" => "timestamp"
"lvl" => "level"
- "msg" => "message"
}
uppercase => [ "level" ]
}
- date {
- match => [ "timestamp", "ISO8601" ]
- }
- }
-}
-filter {
- if [program] == "prometheus" {
- kv { }
- mutate {
- rename => {
- "ts" => "timestamp"
- "msg" => "message"
- }
- uppercase => [ "level" ]
- }
date {
- match => [ "timestamp", "ISO8601" ]
+ id => "date_grafana"
+ match => [ "timestamp", "yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS'Z'", "ISO8601" ]
}
}
}
filter {
if [program] == "prometheus" {
- kv { }
+ kv { id => "kv_prometheus" }
mutate {
+ id => "mutate_prometheus"
rename => {
"ts" => "timestamp"
- "msg" => "message"
}
uppercase => [ "level" ]
}
date {
+ id => "date_prometheus"
match => [ "timestamp", "ISO8601" ]
}
}
@@ -84,12 +78,14 @@ filter {
filter {
if [program] == "tango-rest" {
grok {
+ id => "grok_tango-rest"
match => {
"message" => "%{TIMESTAMP_ISO8601:timestamp} %{WORD:level} %{GREEDYDATA:message}"
}
- "overwrite" => [ "timestamp", "level", "message" ]
+ overwrite => [ "timestamp", "level", "message" ]
}
date {
+ id => "date_tango-rest"
match => [ "timestamp", "YYYY-MM-dd HH:mm:ss,SSS" ]
timezone => "UTC"
}
@@ -97,17 +93,10 @@ filter {
}
filter {
- # mark our tangodb instances
- grok {
- match => {
- "program" => ["tangodb" ]
- }
- add_tag => [ "tangodb" ]
- }
-
# parse tangodb output
- if "tangodb" in [tags] {
+ if [program] == "tangodb" {
grok {
+ id => "grok_tangodb"
match => {
"message" => [
"%{TIMESTAMP_ISO8601:timestamp} .%{WORD:level}. %{GREEDYDATA:message}",
@@ -117,21 +106,114 @@ filter {
"overwrite" => [ "timestamp", "level", "message" ]
}
mutate {
+ id => "mutate_tangodb"
gsub => [
"level", "Note", "Info"
]
uppercase => [ "level" ]
}
date {
+ id => "date_tangodb"
match => [ "timestamp", "YYYY-MM-dd HH:mm:ssZZ", "YYYY-MM-dd HH:mm:ss", "YYYY-MM-dd H:mm:ss" ]
timezone => "UTC"
}
}
}
+filter {
+ if [type] == "python-logstash" {
+ # don't archive debug messages
+ if [level] == "DEBUG" {
+ drop { id => "drop_debug_message" }
+ }
+
+ # strip path from program
+ grok {
+ id => "grok_python-logstash-removepath-program"
+ match => {
+ "program" => "(?<program>[^/]+)$"
+ }
+
+ overwrite => [ "program" ]
+ tag_on_failure => [] # be quiet if there are no matches
+ }
+
+ # strip path from source reference
+ grok {
+ id => "grok_python-logstash-removepath-source"
+ match => {
+ "[extra][path]" => "(?<[extra][path]>[^/]+)$"
+ }
+
+ overwrite => [ "[extra][path]" ]
+ tag_on_failure => [] # be quiet if there are no matches
+ }
+
+ # add source reference to each log entry
+ mutate {
+ id => "mutate_python-logstash-sourceref"
+ replace => { "message" => "%{message} [%{[extra][func_name]}() at %{[extra][path]}:%{[extra][line]}]" }
+ }
+
+ # promote some fields out of extra, if present
+ if [extra][device] {
+ mutate {
+ id => "mutate_python-logstash-device"
+ add_field => {
+ "device" => "%{[extra][device]}"
+ }
+ }
+ }
+
+ if [extra][stack_trace] {
+ mutate {
+ id => "mutate_python-logstash-stacktrace"
+ replace => { "message" => "%{message} %{[extra][stack_trace]}" }
+ }
+ }
+ }
+}
+
+# Throttle spam to max 1000/min per program/device.
+# See https://www.elastic.co/guide/en/logstash/current/plugins-filters-throttle.html
+filter {
+ throttle {
+ id => "throttler"
+ before_count => -1
+ after_count => 1000
+ period => 60
+ max_age => 180
+ key => "%{host}-%{program}-%{device}"
+ add_tag => "throttled"
+ }
+
+ if "throttled" in [tags] {
+ drop { id => "drop_throttled" }
+ }
+}
+
output {
loki {
+ id => "output_loki"
url => "http://loki:3100/loki/api/v1/push"
+ message_field => "message"
+
+ # Every unique combination of field values creates a separate stream in loki.
+ # This overloads loki, so only send fields that are key and bank on messages
+ # being regex-ed when querying instead.
+ #
+ # See https://grafana.com/blog/2020/08/27/the-concise-guide-to-labels-in-loki/
+ #
+ # So avoid putting in fields that vary, such as:
+ # * (parts of) timestamps,
+ # * source file references (file, line, etc).
+ # Instead, f.e. put those in the message field with a replace filter.
+ #
+ # To inspect loki's streams, look at the content of /loki/chunks in the loki container:
+ # docker exec -it loki ls /loki/chunks
+ include_fields => [ "host", "program", "level", "device", "tags" ]
}
-}
+ # enable this to see all logs on logstash's stdout. to view, run: docker logs logstash
+ # stdout {}
+}
diff --git a/docker-compose/loki.yml b/docker-compose/loki.yml
index fa114fcdecf72042f97100af47f90673fb2a57ec..58a10e5e07a016a3065ea63380503f68306131e1 100644
--- a/docker-compose/loki.yml
+++ b/docker-compose/loki.yml
@@ -8,9 +8,14 @@
version: "2.1"
+volumes:
+ loki-data: {}
+
services:
loki:
- image: grafana/loki:2.6.0
+ build:
+ context: loki
+ hostname: loki
container_name: loki
logging:
driver: "json-file"
@@ -19,6 +24,8 @@ services:
max-file: "10"
networks:
- control
+ volumes:
+ - loki-data:/loki
ports:
- "3100:3100"
command: -config.file=/etc/loki/local-config.yaml
diff --git a/docker-compose/loki/Dockerfile b/docker-compose/loki/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..466a31876e3fed1a021d9b928a19edb534f9f3e0
--- /dev/null
+++ b/docker-compose/loki/Dockerfile
@@ -0,0 +1,2 @@
+FROM grafana/loki:2.6.0
+COPY local-config.yaml /etc/loki/local-config.yaml
diff --git a/docker-compose/loki/local-config.yaml b/docker-compose/loki/local-config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..acac70bbd45b5159eecdfa64a1d6e744fc515a58
--- /dev/null
+++ b/docker-compose/loki/local-config.yaml
@@ -0,0 +1,41 @@
+auth_enabled: false
+
+server:
+ http_listen_port: 3100
+
+common:
+ path_prefix: /loki
+ storage:
+ filesystem:
+ chunks_directory: /loki/chunks
+ rules_directory: /loki/rules
+ replication_factor: 1
+ ring:
+ kvstore:
+ store: inmemory
+
+schema_config:
+ configs:
+ - from: 2020-10-24
+ store: boltdb-shipper
+ object_store: filesystem
+ schema: v11
+ index:
+ prefix: index_
+ period: 24h
+
+ruler:
+ alertmanager_url: http://localhost:9093
+
+# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration
+# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/
+#
+# Statistics help us better understand how Loki is used, and they show us performance
+# levels for most users. This helps us prioritize features and documentation.
+# For more information on what's sent, look at
+# https://github.com/grafana/loki/blob/main/pkg/usagestats/stats.go
+# Refer to the buildReport method to see what goes into a report.
+#
+# If you would like to disable reporting, uncomment the following lines:
+analytics:
+ reporting_enabled: false
diff --git a/docker-compose/object-storage.yml b/docker-compose/object-storage.yml
index 5ba417b415080e287500149896f727216a66b95c..feebcee6b94deebecbec658ca6af7aaddad7af60 100644
--- a/docker-compose/object-storage.yml
+++ b/docker-compose/object-storage.yml
@@ -11,6 +11,7 @@ version: '2.1'
services:
object-storage:
image: minio/minio
+ hostname: object-storage
container_name: object-storage
logging:
driver: "json-file"
diff --git a/docker-compose/prometheus-node-exporter.yml b/docker-compose/prometheus-node-exporter.yml
index 84fffd2949534aa51bb9adda6dfd5a7d813ad740..f61e4b8450928d5c9b55c7d368b17dd8d818ab45 100644
--- a/docker-compose/prometheus-node-exporter.yml
+++ b/docker-compose/prometheus-node-exporter.yml
@@ -11,6 +11,7 @@ version: '2.1'
services:
prometheus-node-exporter:
image: prom/node-exporter
+ hostname: prometheus-node-exporter
container_name: prometheus-node-exporter
network_mode: host # run on the host to be able to access host network statistics
logging:
diff --git a/docker-compose/prometheus.yml b/docker-compose/prometheus.yml
index f398a9d2a3ad8322a0e6de77073b8be1519b8081..4ba6b702991ef89efab3edc489cfc640c5fb22b4 100644
--- a/docker-compose/prometheus.yml
+++ b/docker-compose/prometheus.yml
@@ -17,6 +17,7 @@ services:
image: prometheus
build:
context: prometheus
+ hostname: prometheus
container_name: prometheus
networks:
- control
diff --git a/docker-compose/prometheus/prometheus.yml b/docker-compose/prometheus/prometheus.yml
index ea073cbf92c0a5c070ca94dea1ccac03d48ff225..987eaf6225be15b7d01590720c9098ccfc90b415 100644
--- a/docker-compose/prometheus/prometheus.yml
+++ b/docker-compose/prometheus/prometheus.yml
@@ -32,3 +32,13 @@ scrape_configs:
- targets: ["grafana:3000"]
labels:
"host": "localhost"
+ - job_name: logstash
+ static_configs:
+ - targets: ["logstash-exporter:9198"]
+ labels:
+ "host": "localhost"
+ - job_name: loki
+ static_configs:
+ - targets: ["loki:3100"]
+ labels:
+ "host": "localhost"
diff --git a/docker-compose/rcu2h-sim.yml b/docker-compose/rcu2h-sim.yml
index e61322aceb083e1aefa5fe02e98a166ad5425eb3..9e705abcceabebdb0f3e069fb3b03c73f10c3eb2 100644
--- a/docker-compose/rcu2h-sim.yml
+++ b/docker-compose/rcu2h-sim.yml
@@ -15,6 +15,7 @@ services:
args:
- LOCAL_DOCKER_REGISTRY_HOST=${LOCAL_DOCKER_REGISTRY_HOST}
- LOCAL_DOCKER_REGISTRY_LOFAR=${LOCAL_DOCKER_REGISTRY_LOFAR}
+ hostname: rcu2h-sim
container_name: rcu2h-sim
logging:
driver: "json-file"
diff --git a/docker-compose/rcu2l-sim.yml b/docker-compose/rcu2l-sim.yml
index 4e6e533768c90d74bc5a69de9bf917a922a34773..6c2092019898f79b740a4a84507724294d923e22 100644
--- a/docker-compose/rcu2l-sim.yml
+++ b/docker-compose/rcu2l-sim.yml
@@ -15,6 +15,7 @@ services:
args:
- LOCAL_DOCKER_REGISTRY_HOST=${LOCAL_DOCKER_REGISTRY_HOST}
- LOCAL_DOCKER_REGISTRY_LOFAR=${LOCAL_DOCKER_REGISTRY_LOFAR}
+ hostname: rcu2l-sim
container_name: rcu2l-sim
logging:
driver: "json-file"
diff --git a/docker-compose/sdptr-sim.yml b/docker-compose/sdptr-sim.yml
index 8168c2fd00d51310e2eafe850833a6d11b3b5fc8..68c91fff0fd39f06c96ca6dbb4e100e8ac44bd89 100644
--- a/docker-compose/sdptr-sim.yml
+++ b/docker-compose/sdptr-sim.yml
@@ -15,6 +15,7 @@ services:
args:
- LOCAL_DOCKER_REGISTRY_HOST=${LOCAL_DOCKER_REGISTRY_HOST}
- LOCAL_DOCKER_REGISTRY_LOFAR=${LOCAL_DOCKER_REGISTRY_LOFAR}
+ hostname: sdptr-sim
container_name: sdptr-sim
logging:
driver: "json-file"
diff --git a/docker-compose/tango-prometheus-exporter.yml b/docker-compose/tango-prometheus-exporter.yml
index 318d4b64d39a16e1cd3815c7c3a0a786b1f7b336..5dfb01d55af66df97160f2e08de3ed9fceb36e6d 100644
--- a/docker-compose/tango-prometheus-exporter.yml
+++ b/docker-compose/tango-prometheus-exporter.yml
@@ -12,6 +12,7 @@ services:
args:
POLICY_FILE: lofar2-policy.json
dockerfile: ./Dockerfile
+ hostname: tango-prometheus-exporter
container_name: tango-prometheus-exporter
logging:
driver: "json-file"
@@ -34,6 +35,7 @@ services:
args:
POLICY_FILE: lofar2-fast-policy.json
dockerfile: ./Dockerfile
+ hostname: tango-prometheus-fast-exporter
container_name: tango-prometheus-fast-exporter
logging:
driver: "json-file"
diff --git a/docker-compose/tango-prometheus-exporter/code/tango-prometheus-client.py b/docker-compose/tango-prometheus-exporter/code/tango-prometheus-client.py
index 7edd8d8d62a9ef5cb9b301711da1f7bea5f3e653..6ee05172f409deab8fb9a51fceb44447341c5e34 100644
--- a/docker-compose/tango-prometheus-exporter/code/tango-prometheus-client.py
+++ b/docker-compose/tango-prometheus-exporter/code/tango-prometheus-client.py
@@ -241,7 +241,7 @@ class CustomCollector(object):
# obtain list of attributes to scrape
attrs_to_scrape = self.policy.attribute_list(device_name, attr_infos.keys())
- logger.info(f"Processing device {device_name} attributes {attrs_to_scrape}")
+ logger.debug(f"Processing device {device_name} attributes {attrs_to_scrape}")
# scrape each attribute
metrics = []
@@ -290,18 +290,18 @@ class CustomCollector(object):
reason = e.args[0].desc.replace("\n", " ")
logger.warning(f"Error processing device {device_name}: {reason}")
- # get the time since the last try, if less than a second, sleep for a bit.
+ # get the time since the last try, if less than a second, sleep for a bit.
retry_wait_time = time.time() - last_exception_time
if retry_wait_time < reconnect_timeout_time:
time.sleep(reconnect_timeout_time - retry_wait_time)
last_exception_time = time.time()
-
+
except Exception as e:
logger.exception(f"Error processing device {device_name}")
finally:
dev_scrape_end = time.time()
- logger.info(
+ logger.debug(
f"Done processing device {device_name}. Took {dev_scrape_end - dev_scrape_begin} seconds."
)
diff --git a/docker-compose/tango-rest.yml b/docker-compose/tango-rest.yml
index 27f106dca2724b1890a42e8206d0e97f78579283..0505cd7bec8febde9c24342b6bba4eb05b79d2c9 100644
--- a/docker-compose/tango-rest.yml
+++ b/docker-compose/tango-rest.yml
@@ -16,12 +16,10 @@ version: '2.1'
services:
tango-rest:
image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-rest:${TANGO_REST_VERSION}
+ hostname: tango-rest
container_name: tango-rest
networks:
- control
- # set the hostname, otherwise duplicate device registrations result every
- # time the hostname changes as the container is restarted.
- hostname: tango-rest
environment:
- TANGO_HOST=${TANGO_HOST}
ports:
diff --git a/docker-compose/tango.yml b/docker-compose/tango.yml
index e4fd11c5b5c2f5558a90d91930d43b9afcdcf8e4..58dfc1915fa772020b7bec50b722c9e31b89d83b 100644
--- a/docker-compose/tango.yml
+++ b/docker-compose/tango.yml
@@ -18,6 +18,7 @@ volumes:
services:
tangodb:
image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-db:${TANGO_DB_VERSION}
+ hostname: tangodb
container_name: tangodb
networks:
- control
@@ -40,6 +41,7 @@ services:
databaseds:
image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-databaseds:${TANGO_DATABASEDS_VERSION}
+ hostname: databaseds
container_name: databaseds
networks:
- control
@@ -76,6 +78,7 @@ services:
context: dsconfig
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-dsconfig:${TANGO_DSCONFIG_VERSION}
+ hostname: dsconfig
container_name: dsconfig
networks:
- control
@@ -96,4 +99,3 @@ services:
syslog-format: rfc3164
tag: "{{.Name}}"
restart: unless-stopped
-
diff --git a/docker-compose/unb2-sim.yml b/docker-compose/unb2-sim.yml
index 40e86946834472cab04b3048fe02ac77cc5ef456..89abb8ce69d70a0aac09f9ba8d30c30a1aa9ec93 100644
--- a/docker-compose/unb2-sim.yml
+++ b/docker-compose/unb2-sim.yml
@@ -15,6 +15,7 @@ services:
args:
- LOCAL_DOCKER_REGISTRY_HOST=${LOCAL_DOCKER_REGISTRY_HOST}
- LOCAL_DOCKER_REGISTRY_LOFAR=${LOCAL_DOCKER_REGISTRY_LOFAR}
+ hostname: unb2-sim
container_name: unb2-sim
logging:
driver: "json-file"
diff --git a/tangostationcontrol/tangostationcontrol/common/lofar_logging.py b/tangostationcontrol/tangostationcontrol/common/lofar_logging.py
index 74041f24b68029a737df94b5b6615b521960a73c..62a9b946681d4a0fd74dd190b5eaea87b2cdd0ae 100644
--- a/tangostationcontrol/tangostationcontrol/common/lofar_logging.py
+++ b/tangostationcontrol/tangostationcontrol/common/lofar_logging.py
@@ -37,11 +37,11 @@ class TangoLoggingHandler(logging.Handler):
def emit(self, record):
try:
- if record.tango_device is None:
+ if record.device is None:
# log record is not related to any device
return
except AttributeError:
- # log record is not annotated with a tango_device
+ # log record is not annotated with a device
return
# determine which log stream to use
@@ -50,11 +50,11 @@ class TangoLoggingHandler(logging.Handler):
# send the log message to Tango
try:
record_msg = record.msg % record.args
- stream(record.tango_device, record.msg, *record.args)
+ stream(record.device, record.msg, *record.args)
except TypeError:
# Tango's logger barfs on mal-formed log lines, f.e. if msg % args is not possible
record_msg = f"{record.msg} {record.args}".replace("%", "%%")
- stream(record.tango_device, record_msg)
+ stream(record.device, record_msg)
self.flush()
@@ -98,10 +98,10 @@ class LogSuppressErrorSpam(logging.Formatter):
class LogAnnotator(logging.Formatter):
"""Annotates log records with:
- record.tango_device: the Tango Device that is executing."""
+ record.device: the Tango Device that is executing."""
@staticmethod
- def get_current_tango_device() -> Device:
+ def get_current_device() -> Device:
"""Return the tango Device we're currently executing for, or None if it can't be detected.
This is derived by traversing the stack and find a Device as 'self'. In some cases,
@@ -115,10 +115,10 @@ class LogAnnotator(logging.Formatter):
def filter(self, record):
# annotate record with currently executing Tango device, if any
- record.tango_device = self.get_current_tango_device()
+ record.device = self.get_current_device()
# construct an identifier we can add for other devices as well
- record.lofar_id = f"tango - {record.tango_device}"
+ record.lofar_id = f"tango - {record.device}"
# annotate record with the current software version
record.software_version = version
@@ -166,7 +166,7 @@ def configure_logger(logger: logging.Logger = None, log_extra=None, debug=False)
hostname = socket.gethostname()
formatter = logging.Formatter(
- fmt="%(asctime)s.%(msecs)d %(levelname)s - %(tango_device)s: %(message)s [%(funcName)s in %(filename)s:%(lineno)d]".format(
+ fmt="%(asctime)s.%(msecs)d %(levelname)s - %(device)s: %(message)s [%(funcName)s in %(filename)s:%(lineno)d]".format(
hostname
),
datefmt="%Y-%m-%dT%H:%M:%S",
@@ -194,7 +194,7 @@ def configure_logger(logger: logging.Logger = None, log_extra=None, debug=False)
)
# configure log messages
- formatter = LogstashFormatter(extra=log_extra, tags=["python", "lofar"])
+ formatter = LogstashFormatter(extra=log_extra, tags=[])
handler.setFormatter(formatter)
handler.addFilter(LogSuppressErrorSpam())
handler.addFilter(LogAnnotator())
diff --git a/tangostationcontrol/test/common/test_lofar_logging.py b/tangostationcontrol/test/common/test_lofar_logging.py
index 4af2e720736eb7424bd223c732114917d3c49804..ff541489cfac030fd96c138c9a557cf01211baa5 100644
--- a/tangostationcontrol/test/common/test_lofar_logging.py
+++ b/tangostationcontrol/test/common/test_lofar_logging.py
@@ -65,7 +65,7 @@ class TestLofarLogging(base.TestCase):
logger = lofar_logging.configure_logger()
logger.info("test")
- self.assertIn("tango_device", self.memory_handler.records[0].__dict__)
+ self.assertIn("device", self.memory_handler.records[0].__dict__)
self.assertIn("software_version", self.memory_handler.records[0].__dict__)
@unittest.skip("Logs are not sent to Tango device currently, to reduce logspam")
@@ -108,7 +108,7 @@ class TestLofarLogging(base.TestCase):
# we can't compare them direclty. Just verify we're talking about the same thing.
test_object.assertEqual(
str(self),
- str(test_record[0].tango_device),
+ str(test_record[0].device),
msg="configure_logging did not detect active Tango device",
)