diff --git a/.gitignore b/.gitignore index d6ac0c2e4e550e7005cae51a6dacd05dd0e5bb1a..556ded73049fefeed09f4d4fda153864377a997c 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ **/AUTHORS **/ChangeLog **/env +!infra/env **/pending_log_messages.db **/vscode-server.tar **/*.orig diff --git a/infra/env/common.yaml b/infra/env/common.yaml index cbe3c1066a518d05507480ff8164b396324d1f1f..cb6647a9f198759ff84b70e6968c0668c495b809 100644 --- a/infra/env/common.yaml +++ b/infra/env/common.yaml @@ -36,6 +36,9 @@ object_storage: name: minioadmin pass: minioadmin +vector: + version: 0.39.X-distroless-static + pypcc: version: v1-0 diff --git a/infra/jobs/station/logging.levant.nomad b/infra/jobs/station/logging.levant.nomad index 307faa48c43f9323f477012540ea773a9639a680..5fc213d1d568749f632c833d076472694a9e7f75 100644 --- a/infra/jobs/station/logging.levant.nomad +++ b/infra/jobs/station/logging.levant.nomad @@ -22,11 +22,7 @@ job "log-scraping" { mode = "delay" } network { - mode = "bridge" - port "api" { - to = 8686 - host_network = "station" - } + mode = "cni/station" } [[ if ne .station "dev" ]] @@ -46,8 +42,7 @@ job "log-scraping" { task "vector" { driver = "docker" config { - image = "timberio/vector:0.37.0.custom.2ca601b-distroless-static" - ports = ["api"] + image = "timberio/vector:[[.vector.version]]" [[ if eq .station "dev" ]] volumes = [ "/var/run/docker.sock:/var/run/docker.sock" @@ -88,6 +83,12 @@ healthchecks.require_healthy = true address = "0.0.0.0:8686" playground = false +[sources.syslog] + type = "syslog" + address = "0.0.0.0:514" + mode = "tcp" + permit_origin = [ "10.99.0.0/16" ] + [sources.docker-local] type = "docker_logs" docker_host = "/var/run/docker.sock" @@ -138,12 +139,12 @@ healthchecks.require_healthy = true # # since without them it is harder to see logs # # of for example crashing containers # type = "console" -# inputs = [ "nomad-flags" ] +# inputs = [ "nomad-flags", "syslog" ] # encoding.codec = "logfmt" # buffer.when_full = "drop_newest" [sinks.loki] type = "loki" - inputs = [ "nomad-flags" ] + inputs = [ "nomad-flags", "syslog" ] endpoint = "http://loki.service.consul:3100" encoding.codec = "json" healthcheck.enabled = true @@ -156,7 +157,7 @@ healthchecks.require_healthy = true [[ if ne .station "dev" ]] [sinks.loki_central] type = "loki" - inputs = [ "nomad-flags" ] + inputs = [ "nomad-flags", "syslog" ] tenant_id = "lofar" endpoint = "http://logs.service.lofar-central.consul:3100" encoding.codec = "json" @@ -186,16 +187,21 @@ healthchecks.require_healthy = true healthcheck.enabled = false EOF } - service { - check { - port = "api" - type = "http" - path = "/health" - interval = "30s" - timeout = "5s" - } - } kill_timeout = "30s" } + + service { + name = "syslog" + port = "514" + address_mode = "alloc" + check { + port = "8686" + type = "http" + path = "/health" + interval = "30s" + timeout = "5s" + } + } + } }