Add consul exporter

.deploy.gitlab-ci.yml

@@ -32,6 +32,7 @@ deploy_nomad:
         - dsconfig
         - ec-sim
         - jupyter
+        - server-monitoring
         - network-monitoring
         - landing-page
         - rpc-server

.gitlab-ci.yml

@@ -87,7 +87,7 @@ run_shellcheck:
     DEV_STATIONS: dts-lab rs311
 
     # Repeat this list in .deploy.gitlab-ci.yml
-    COMPONENTS: mesh-gateway monitoring logging tango object-storage object-replication sdptr device-server dsconfig ec-sim jupyter network-monitoring landing-page rpc-server
+    COMPONENTS: mesh-gateway monitoring logging tango object-storage object-replication sdptr device-server dsconfig ec-sim jupyter server-monitoring network-monitoring landing-page rpc-server
 
 # Generate the station-specific TangoDB configuration based on LOFAR1 information.
 # Run manually to bootstrap CDB/stations/$station.json for stations

README.md

@@ -150,6 +150,7 @@ Next change the version in the following places:
    through [https://git.astron.nl/lofar2.0/tango/-/tags](Deploy Tags)
 
 # Release Notes
+* 0.51.7 Add metrics for consul services health
 * 0.51.6 Fix powering off antennas
 * 0.51.5 Use MinIO credentials from Vault
 * 0.51.4 Ping FPGAs and PIs as part of monitoring

infra/jobs/station/server-monitoring.levant.nomad

@@ -75,7 +75,6 @@ job "server-monitoring" {
 
       config {
         image   = "quay.io/superq/chrony-exporter:latest"
-        ports   = ["http"]
         # NB: For --collector.serverstats, we need to connect to chrony over the UNIX domain socket,
         # see https://github.com/SuperQ/chrony_exporter/issues/66
         args    = ["--collector.sources", "--chrony.address=10.99.250.250:323", "--web.listen-address=:80"]
@@ -86,4 +85,34 @@ job "server-monitoring" {
       }
     }
   }
+
+  group "consul-exporter" {
+    count = 1
+
+    network {
+      mode = "cni/station"
+    }
+
+    service {
+      tags = ["scrape"]
+      name = "consul-exporter"
+      task = "consul-exporter"
+      port = "9107"
+      address_mode = "alloc"
+    }
+
+    task "consul-exporter" {
+      driver = "docker"
+
+      config {
+        image   = "prom/consul-exporter:latest"
+        args    = ["--consul.server=consul.service.consul:8500"]
+      }
+
+      resources {
+        cpu    = 100
+        memory = 100
+      }
+    }
+  }
 }

infra/station/consul/consul-agent.hcl.j2

@@ -18,11 +18,21 @@ connect {
 
 telemetry {
   prometheus_retention_time = "24h"
+  disable_hostname = true
 }
+
+tls {
+  defaults {
     verify_incoming = true
     verify_outgoing = true
-verify_server_hostname = true
     ca_file = "/host/agent-certs/ca.crt"
+  }
+
+  internal_rpc {
+    verify_server_hostname = true
+  }
+}
+
 auto_encrypt {
   tls = true
 }

infra/station/consul/consul.hcl.j2

@@ -55,4 +55,5 @@ ui_config {
 
 telemetry {
   prometheus_retention_time = "24h"
+  disable_hostname = true
 }

tangostationcontrol/VERSION

-0.51.6
+0.51.7