diff --git a/infra/dev/nomad/config/nomad/client.hcl b/infra/dev/nomad/config/nomad/client.hcl index ee27a3bf6217711cf2254637690fb449818bf3ec..d68fc99e0ba6043cc7cbf5eaa5091ada095b027f 100644 --- a/infra/dev/nomad/config/nomad/client.hcl +++ b/infra/dev/nomad/config/nomad/client.hcl @@ -96,3 +96,14 @@ advertise { rpc = "{{ GetInterfaceIP \"eth0\" }}" serf = "{{ GetInterfaceIP \"eth0\" }}" } + +plugin "docker" { + config { + allow_privileged = true + volumes { + enabled = true + } + allow_caps = ["all"] + extra_labels = ["job_name", "job_id", "task_group_name", "task_name", "namespace", "node_name", "node_id"] + } +} diff --git a/infra/dev/services/services.hcl b/infra/dev/services/services.hcl index 77bb8a1e826a374d4f20e687691963ad4a095c6a..ce72e23e5dd9ce345cda1d638dac34108f7dccf2 100644 --- a/infra/dev/services/services.hcl +++ b/infra/dev/services/services.hcl @@ -9,17 +9,17 @@ resource "nomad_job" "monitoring" { } } -# resource "nomad_job" "logging" { -# cluster = variable.nomad_cluster -# -# paths = ["../jobs/station/logging.nomad"] -# depends_on = ["resource.nomad_job.monitoring"] -# -# health_check { -# timeout = "300s" -# jobs = ["log-scraping"] -# } -# } +resource "nomad_job" "logging" { + cluster = variable.nomad_cluster + + paths = ["../jobs/station/logging.nomad"] + depends_on = ["resource.nomad_job.monitoring"] + + health_check { + timeout = "300s" + jobs = ["log-scraping"] + } +} resource "ingress" "grafana" { port = 3000 diff --git a/infra/jobs/station/logging.levant.nomad b/infra/jobs/station/logging.levant.nomad index 8b1797abcdcc24ff078e403ca42a7ac059f95540..bad27a1010d381d0df9363508e7e692457300db5 100644 --- a/infra/jobs/station/logging.levant.nomad +++ b/infra/jobs/station/logging.levant.nomad @@ -84,17 +84,20 @@ healthchecks.require_healthy = true address = "0.0.0.0:8686" playground = false +[[ if ne .station "dev" ]] [sources.syslog-remote] type = "syslog" address = "0.0.0.0:514" mode = "tcp" permit_origin = [ "10.99.0.0/16" ] +[[ end ]] [sources.docker-local] type = "docker_logs" docker_host = "/var/run/docker.sock" exclude_containers = ["vector-"] +[[ if ne .station "dev" ]] [transforms.syslog-parse] inputs = ["syslog-remote"] type = "remap" @@ -123,13 +126,14 @@ healthchecks.require_healthy = true # delete labels that vary too much or are duplicates del(.procid) ''' +[[ end ]] [transforms.docker-parse] inputs = ["docker-local"] type = "remap" source = ''' # parse_key_value only supports \", not a literal \n, so we replace them with actual newlines instead. - .message = replace!(.message,"\\n","\n") + .message = replace!(.message, "\\n", "\n") # parse trying various formats structured = @@ -164,14 +168,19 @@ healthchecks.require_healthy = true ''' [transforms.postprocess] - inputs = ["docker-parse", "syslog-parse"] + [[ if ne .station "dev" ]] + inputs = ["docker-parse", "syslog-parse"] + [[ else ]] + inputs = ["docker-parse"] + [[ end ]] + type = "remap" source = ''' # hard limit on message size to prevent explosion .message = truncate!(.message, limit: 8192, suffix: "...") # standard labels - .labels.level = .level + .labels.level = .level || "info" .labels.source_type = .source_type '''