add oncall

infra/env.yaml

@@ -23,6 +23,10 @@ monitoring:
     version: latest
   prometheus:
     version: latest
+  redis:
+    version: 7.0.5
+  oncall:
+    version: latest
 
 object_storage:
   minio:

infra/jobs/station/monitoring.levant.nomad

@@ -65,6 +65,25 @@ job "monitoring" {
       }
     }
 
+    service {
+      tags = ["scrape"]
+      name = "oncall"
+      port = "8080"
+      address_mode = "alloc"
+    }
+
+    service {
+      name = "redis"
+      port = "6379"
+      address_mode = "alloc"
+    }
+
+    volume "redis" {
+      type      = "host"
+      read_only = false
+      source    = "monitoring-redis-data"
+    }
+
     task "wait-for-db" {
       lifecycle {
         hook    = "prestart"
@@ -164,6 +183,114 @@ postgres.service.consul:5432:grafana:postgres:password
         memory_max = 8192
       }
     }
+
+    task "oncall" {
+      driver = "docker"
+
+      config {
+        image = "[[.registry.astron.url]]/oncall:[[.monitoring.oncall.version]]"
+        args  = [
+          "sh",
+          "-c",
+          "uwsgi --ini uwsgi.ini"
+        ]
+      }
+
+      env {
+        BROKER_TYPE            = "redis"
+        REDIS_URI              = "redis://redis:6379/0"
+        DJANGO_SETTINGS_MODULE = "settings.hobby"
+        GRAFANA_API_URL        = "http://${NOMAD_ADDR_grafana}/grafana"
+        SECRET_KEY             = "my-little-secret-key-passphrase-thingy"
+        BASE_URL               = "http://${NOMAD_ADDR_oncall}"
+
+        DATABASE_TYPE          = "postgres"
+        DATABASE_NAME          = "oncall"
+        DATABASE_USER          = "postgres"
+        DATABASE_PASSWORD      = "password"
+        DATABASE_HOST          = "${NOMAD_HOST_postgres}"
+        DATABASE_HOST          = "${NOMAD_PORT_postgres}"
+      }
+    }
+
+    task "celery" {
+      driver = "docker"
+
+      config {
+        image = "[[.registry.astron.url]]/oncall:[[.monitoring.oncall.version]]"
+        args  = [
+          "sh",
+          "-c",
+          "./celery_with_exporter.sh"
+        ]
+      }
+
+      volume_mount {
+        volume      = "oncall"
+        destination = "/var/lib/oncall"
+        read_only   = false
+      }
+
+      env {
+        CELERY_WORKER_QUEUE               = "default,critical,long,slack,telegram,webhook,retry,celery,grafana"
+        CELERY_WORKER_CONCURRENCY         = "1"
+        CELERY_WORKER_MAX_TASKS_PER_CHILD = "100"
+        CELERY_WORKER_SHUTDOWN_INTERVAL   = "65m"
+        CELERY_WORKER_BEAT_ENABLED        = "True"
+      }
+    }
+
+    task "redis" {
+      lifecycle {
+        hook    = "prestart"
+        sidecar = false
+      }
+
+      driver = "docker"
+
+      config {
+        image = "[[.registry.astron.url]]/redis:[[.monitoring.redis.version]]"
+      }
+
+      volume_mount {
+        volume      = "redis"
+        destination = "/data"
+        read_only   = false
+      }
+    }
+
+    task "oncall-db-migration" {
+      lifecycle {
+        hook    = "prestart"
+        sidecar = false
+      }
+
+      driver = "docker"
+
+      config {
+        image = "[[.registry.astron.url]]/oncall:[[.monitoring.oncall.version]]"
+        args  = [
+          "python",
+          "manage.py",
+          "migrate",
+          "--noinput"
+        ]
+      }
+
+      env {
+        BROKER_TYPE            = "redis"
+        REDIS_URI              = "redis://redis:6379/0"
+        DJANGO_SETTINGS_MODULE = "settings.hobby"
+        SECRET_KEY             = "my-little-secret-key-passphrase-thingy"
+
+        DATABASE_TYPE          = "postgres"
+        DATABASE_NAME          = "oncall"
+        DATABASE_USER          = "postgres"
+        DATABASE_PASSWORD      = "password"
+        DATABASE_HOST          = "${NOMAD_HOST_postgres}"
+        DATABASE_HOST          = "${NOMAD_PORT_postgres}"
+      }
+    }
   }
   group "prometheus" {
 

infra/station/nomad.yml

@@ -14,6 +14,10 @@
       volume_name: 'docker-data'
       path: '/var/lib/docker'
       size: 100g
+    - role: volume
+      volume_name: 'redis-data'
+      path: '/var/lib/redis-data'
+      size: 100g
     - role: volume
       volume_name: 'station-data'
       path: '/var/lib/station-data'

infra/station/nomad/nomad.hcl.j2

@@ -55,6 +55,10 @@ client {
     path = "/var/lib/station-data/monitoring-prometheus-data"
   }
 
+  host_volume "monitoring-redis-data" {
+    path = "/var/lib/station-data/monitoring-redis-data"
+  }
+
   host_volume "tango-database" {
     path = "/var/lib/station-data/tango-database"
   }