Tune startup latency

infra/jobs/station/device-server.levant.nomad

@@ -92,6 +92,36 @@ job "device-servers" {
             }
         }
 
+        task "wait-for-db" {
+          lifecycle {
+            hook    = "prestart"
+            sidecar = false
+          }
+          driver = "docker"
+
+          config {
+            image   = "[[.registry.astron.url]]/busybox:latest"
+            command = "sh"
+            args    = ["-c", "while ! nc -z ${MYSQL_HOST} ${MYSQL_PORT}; do sleep 1; done"]
+          }
+
+          template {
+            wait {
+              min     = "30s"
+              max     = "300s"
+            }
+            error_on_missing_key = true
+            data        = <<EOH
+    {{- range service "tangodb" }}
+    MYSQL_HOST     = "{{ .Address }}"
+    MYSQL_PORT     = "{{ .Port }}"
+    {{- end }}
+    EOH
+            destination = "local/env.txt"
+            env         = true
+          }
+        }
+
         task "device-[[ $name ]]" {
             driver = "docker"
 

infra/jobs/station/jupyter.levant.nomad

@@ -3,6 +3,12 @@ job "jupyter" {
   datacenters = ["stat"]
   type        = "service"
 
+  reschedule {
+    unlimited = true
+    delay = "60s"
+    delay_function = "constant"
+  }
+
   group "jupyter-lab" {
     count = 1
 

infra/jobs/station/landing-page.levant.nomad

@@ -3,6 +3,12 @@ job "landing-page" {
   datacenters = ["stat"]
   type        = "service"
 
+  reschedule {
+    unlimited = true
+    delay = "60s"
+    delay_function = "constant"
+  }
+
   group "web-server" {
     count = 1
 

infra/jobs/station/logging.levant.nomad

@@ -7,6 +7,12 @@ job "log-scraping" {
     type        = "service"
 [[ end ]]
 
+    reschedule {
+      unlimited = true
+      delay = "30s"
+      delay_function = "constant"
+    }
+
     update {
         min_healthy_time  = "10s"
         healthy_deadline  = "5m"

infra/jobs/station/monitoring.levant.nomad

@@ -3,6 +3,12 @@ job "monitoring" {
   datacenters = ["stat"]
   type        = "service"
 
+  reschedule {
+    unlimited = true
+    delay = "60s"
+    delay_function = "constant"
+  }
+
   group "postgres" {
     count = 1
 

infra/jobs/station/object-replication.levant.nomad

@@ -6,6 +6,7 @@ job "object-replication" {
     cron             = "*/5 * * * * *"
     prohibit_overlap = true
   }
+
   group "batch" {
     count = 1
 

infra/jobs/station/object-storage.levant.nomad

@@ -3,6 +3,12 @@ job "object-storage" {
   datacenters = ["stat"]
   type        = "service"
 
+  reschedule {
+    unlimited = true
+    delay = "30s"
+    delay_function = "constant"
+  }
+
   group "minio" {
     count = 1