From a06a4c78efe6fe0c95e955bcced7b3af1871539c Mon Sep 17 00:00:00 2001
From: Jan David Mol <mol@astron.nl>
Date: Thu, 24 Apr 2025 05:58:50 +0000
Subject: [PATCH] Tune startup latency

---
 infra/jobs/station/device-server.levant.nomad | 30 +++++++++++++++++++
 infra/jobs/station/jupyter.levant.nomad       |  6 ++++
 infra/jobs/station/landing-page.levant.nomad  |  6 ++++
 infra/jobs/station/logging.levant.nomad       |  6 ++++
 infra/jobs/station/monitoring.levant.nomad    |  6 ++++
 .../station/object-replication.levant.nomad   |  1 +
 .../jobs/station/object-storage.levant.nomad  |  6 ++++
 7 files changed, 61 insertions(+)

diff --git a/infra/jobs/station/device-server.levant.nomad b/infra/jobs/station/device-server.levant.nomad
index 5d4572eba..75a700a12 100644
--- a/infra/jobs/station/device-server.levant.nomad
+++ b/infra/jobs/station/device-server.levant.nomad
@@ -92,6 +92,36 @@ job "device-servers" {
             }
         }
 
+        task "wait-for-db" {
+          lifecycle {
+            hook    = "prestart"
+            sidecar = false
+          }
+          driver = "docker"
+
+          config {
+            image   = "[[.registry.astron.url]]/busybox:latest"
+            command = "sh"
+            args    = ["-c", "while ! nc -z ${MYSQL_HOST} ${MYSQL_PORT}; do sleep 1; done"]
+          }
+
+          template {
+            wait {
+              min     = "30s"
+              max     = "300s"
+            }
+            error_on_missing_key = true
+            data        = <<EOH
+    {{- range service "tangodb" }}
+    MYSQL_HOST     = "{{ .Address }}"
+    MYSQL_PORT     = "{{ .Port }}"
+    {{- end }}
+    EOH
+            destination = "local/env.txt"
+            env         = true
+          }
+        }
+
         task "device-[[ $name ]]" {
             driver = "docker"
 
diff --git a/infra/jobs/station/jupyter.levant.nomad b/infra/jobs/station/jupyter.levant.nomad
index 5213a5b6e..9134cb4d4 100644
--- a/infra/jobs/station/jupyter.levant.nomad
+++ b/infra/jobs/station/jupyter.levant.nomad
@@ -3,6 +3,12 @@ job "jupyter" {
   datacenters = ["stat"]
   type        = "service"
 
+  reschedule {
+    unlimited = true
+    delay = "60s"
+    delay_function = "constant"
+  }
+
   group "jupyter-lab" {
     count = 1
 
diff --git a/infra/jobs/station/landing-page.levant.nomad b/infra/jobs/station/landing-page.levant.nomad
index c24885db7..81de230e6 100644
--- a/infra/jobs/station/landing-page.levant.nomad
+++ b/infra/jobs/station/landing-page.levant.nomad
@@ -3,6 +3,12 @@ job "landing-page" {
   datacenters = ["stat"]
   type        = "service"
 
+  reschedule {
+    unlimited = true
+    delay = "60s"
+    delay_function = "constant"
+  }
+
   group "web-server" {
     count = 1
 
diff --git a/infra/jobs/station/logging.levant.nomad b/infra/jobs/station/logging.levant.nomad
index bad27a101..e5db3aeb8 100644
--- a/infra/jobs/station/logging.levant.nomad
+++ b/infra/jobs/station/logging.levant.nomad
@@ -7,6 +7,12 @@ job "log-scraping" {
     type        = "service"
 [[ end ]]
 
+    reschedule {
+      unlimited = true
+      delay = "30s"
+      delay_function = "constant"
+    }
+
     update {
         min_healthy_time  = "10s"
         healthy_deadline  = "5m"
diff --git a/infra/jobs/station/monitoring.levant.nomad b/infra/jobs/station/monitoring.levant.nomad
index 84290fb20..e39c888cb 100644
--- a/infra/jobs/station/monitoring.levant.nomad
+++ b/infra/jobs/station/monitoring.levant.nomad
@@ -3,6 +3,12 @@ job "monitoring" {
   datacenters = ["stat"]
   type        = "service"
 
+  reschedule {
+    unlimited = true
+    delay = "60s"
+    delay_function = "constant"
+  }
+
   group "postgres" {
     count = 1
 
diff --git a/infra/jobs/station/object-replication.levant.nomad b/infra/jobs/station/object-replication.levant.nomad
index f538020e4..d2d793e02 100644
--- a/infra/jobs/station/object-replication.levant.nomad
+++ b/infra/jobs/station/object-replication.levant.nomad
@@ -6,6 +6,7 @@ job "object-replication" {
     cron             = "*/5 * * * * *"
     prohibit_overlap = true
   }
+
   group "batch" {
     count = 1
 
diff --git a/infra/jobs/station/object-storage.levant.nomad b/infra/jobs/station/object-storage.levant.nomad
index c7c55538a..aeedb81df 100644
--- a/infra/jobs/station/object-storage.levant.nomad
+++ b/infra/jobs/station/object-storage.levant.nomad
@@ -3,6 +3,12 @@ job "object-storage" {
   datacenters = ["stat"]
   type        = "service"
 
+  reschedule {
+    unlimited = true
+    delay = "30s"
+    delay_function = "constant"
+  }
+
   group "minio" {
     count = 1
 
-- 
GitLab