From a06a4c78efe6fe0c95e955bcced7b3af1871539c Mon Sep 17 00:00:00 2001 From: Jan David Mol <mol@astron.nl> Date: Thu, 24 Apr 2025 05:58:50 +0000 Subject: [PATCH] Tune startup latency --- infra/jobs/station/device-server.levant.nomad | 30 +++++++++++++++++++ infra/jobs/station/jupyter.levant.nomad | 6 ++++ infra/jobs/station/landing-page.levant.nomad | 6 ++++ infra/jobs/station/logging.levant.nomad | 6 ++++ infra/jobs/station/monitoring.levant.nomad | 6 ++++ .../station/object-replication.levant.nomad | 1 + .../jobs/station/object-storage.levant.nomad | 6 ++++ 7 files changed, 61 insertions(+) diff --git a/infra/jobs/station/device-server.levant.nomad b/infra/jobs/station/device-server.levant.nomad index 5d4572eba..75a700a12 100644 --- a/infra/jobs/station/device-server.levant.nomad +++ b/infra/jobs/station/device-server.levant.nomad @@ -92,6 +92,36 @@ job "device-servers" { } } + task "wait-for-db" { + lifecycle { + hook = "prestart" + sidecar = false + } + driver = "docker" + + config { + image = "[[.registry.astron.url]]/busybox:latest" + command = "sh" + args = ["-c", "while ! nc -z ${MYSQL_HOST} ${MYSQL_PORT}; do sleep 1; done"] + } + + template { + wait { + min = "30s" + max = "300s" + } + error_on_missing_key = true + data = <<EOH + {{- range service "tangodb" }} + MYSQL_HOST = "{{ .Address }}" + MYSQL_PORT = "{{ .Port }}" + {{- end }} + EOH + destination = "local/env.txt" + env = true + } + } + task "device-[[ $name ]]" { driver = "docker" diff --git a/infra/jobs/station/jupyter.levant.nomad b/infra/jobs/station/jupyter.levant.nomad index 5213a5b6e..9134cb4d4 100644 --- a/infra/jobs/station/jupyter.levant.nomad +++ b/infra/jobs/station/jupyter.levant.nomad @@ -3,6 +3,12 @@ job "jupyter" { datacenters = ["stat"] type = "service" + reschedule { + unlimited = true + delay = "60s" + delay_function = "constant" + } + group "jupyter-lab" { count = 1 diff --git a/infra/jobs/station/landing-page.levant.nomad b/infra/jobs/station/landing-page.levant.nomad index c24885db7..81de230e6 100644 --- a/infra/jobs/station/landing-page.levant.nomad +++ b/infra/jobs/station/landing-page.levant.nomad @@ -3,6 +3,12 @@ job "landing-page" { datacenters = ["stat"] type = "service" + reschedule { + unlimited = true + delay = "60s" + delay_function = "constant" + } + group "web-server" { count = 1 diff --git a/infra/jobs/station/logging.levant.nomad b/infra/jobs/station/logging.levant.nomad index bad27a101..e5db3aeb8 100644 --- a/infra/jobs/station/logging.levant.nomad +++ b/infra/jobs/station/logging.levant.nomad @@ -7,6 +7,12 @@ job "log-scraping" { type = "service" [[ end ]] + reschedule { + unlimited = true + delay = "30s" + delay_function = "constant" + } + update { min_healthy_time = "10s" healthy_deadline = "5m" diff --git a/infra/jobs/station/monitoring.levant.nomad b/infra/jobs/station/monitoring.levant.nomad index 84290fb20..e39c888cb 100644 --- a/infra/jobs/station/monitoring.levant.nomad +++ b/infra/jobs/station/monitoring.levant.nomad @@ -3,6 +3,12 @@ job "monitoring" { datacenters = ["stat"] type = "service" + reschedule { + unlimited = true + delay = "60s" + delay_function = "constant" + } + group "postgres" { count = 1 diff --git a/infra/jobs/station/object-replication.levant.nomad b/infra/jobs/station/object-replication.levant.nomad index f538020e4..d2d793e02 100644 --- a/infra/jobs/station/object-replication.levant.nomad +++ b/infra/jobs/station/object-replication.levant.nomad @@ -6,6 +6,7 @@ job "object-replication" { cron = "*/5 * * * * *" prohibit_overlap = true } + group "batch" { count = 1 diff --git a/infra/jobs/station/object-storage.levant.nomad b/infra/jobs/station/object-storage.levant.nomad index c7c55538a..aeedb81df 100644 --- a/infra/jobs/station/object-storage.levant.nomad +++ b/infra/jobs/station/object-storage.levant.nomad @@ -3,6 +3,12 @@ job "object-storage" { datacenters = ["stat"] type = "service" + reschedule { + unlimited = true + delay = "30s" + delay_function = "constant" + } + group "minio" { count = 1 -- GitLab