diff --git a/infra/env.yaml b/infra/env.yaml index 29b70b26cba14d93cff0f581498f8c0157f7d491..b336519f1d88ed43113fc818163c5fb6056fb584 100644 --- a/infra/env.yaml +++ b/infra/env.yaml @@ -23,6 +23,10 @@ monitoring: version: latest prometheus: version: latest + redis: + version: 7.0.5 + oncall: + version: latest object_storage: minio: diff --git a/infra/jobs/station/monitoring.levant.nomad b/infra/jobs/station/monitoring.levant.nomad index c8e29d166c1a01cf32bdc5f9816272d997707328..ed35564c6591429405ceab7f343fe3a76cc1ce00 100644 --- a/infra/jobs/station/monitoring.levant.nomad +++ b/infra/jobs/station/monitoring.levant.nomad @@ -65,6 +65,25 @@ job "monitoring" { } } + service { + tags = ["scrape"] + name = "oncall" + port = "8080" + address_mode = "alloc" + } + + service { + name = "redis" + port = "6379" + address_mode = "alloc" + } + + volume "redis" { + type = "host" + read_only = false + source = "monitoring-redis-data" + } + task "wait-for-db" { lifecycle { hook = "prestart" @@ -164,6 +183,114 @@ postgres.service.consul:5432:grafana:postgres:password memory_max = 8192 } } + + task "oncall" { + driver = "docker" + + config { + image = "[[.registry.astron.url]]/oncall:[[.monitoring.oncall.version]]" + args = [ + "sh", + "-c", + "uwsgi --ini uwsgi.ini" + ] + } + + env { + BROKER_TYPE = "redis" + REDIS_URI = "redis://redis:6379/0" + DJANGO_SETTINGS_MODULE = "settings.hobby" + GRAFANA_API_URL = "http://${NOMAD_ADDR_grafana}/grafana" + SECRET_KEY = "my-little-secret-key-passphrase-thingy" + BASE_URL = "http://${NOMAD_ADDR_oncall}" + + DATABASE_TYPE = "postgres" + DATABASE_NAME = "oncall" + DATABASE_USER = "postgres" + DATABASE_PASSWORD = "password" + DATABASE_HOST = "${NOMAD_HOST_postgres}" + DATABASE_HOST = "${NOMAD_PORT_postgres}" + } + } + + task "celery" { + driver = "docker" + + config { + image = "[[.registry.astron.url]]/oncall:[[.monitoring.oncall.version]]" + args = [ + "sh", + "-c", + "./celery_with_exporter.sh" + ] + } + + volume_mount { + volume = "oncall" + destination = "/var/lib/oncall" + read_only = false + } + + env { + CELERY_WORKER_QUEUE = "default,critical,long,slack,telegram,webhook,retry,celery,grafana" + CELERY_WORKER_CONCURRENCY = "1" + CELERY_WORKER_MAX_TASKS_PER_CHILD = "100" + CELERY_WORKER_SHUTDOWN_INTERVAL = "65m" + CELERY_WORKER_BEAT_ENABLED = "True" + } + } + + task "redis" { + lifecycle { + hook = "prestart" + sidecar = false + } + + driver = "docker" + + config { + image = "[[.registry.astron.url]]/redis:[[.monitoring.redis.version]]" + } + + volume_mount { + volume = "redis" + destination = "/data" + read_only = false + } + } + + task "oncall-db-migration" { + lifecycle { + hook = "prestart" + sidecar = false + } + + driver = "docker" + + config { + image = "[[.registry.astron.url]]/oncall:[[.monitoring.oncall.version]]" + args = [ + "python", + "manage.py", + "migrate", + "--noinput" + ] + } + + env { + BROKER_TYPE = "redis" + REDIS_URI = "redis://redis:6379/0" + DJANGO_SETTINGS_MODULE = "settings.hobby" + SECRET_KEY = "my-little-secret-key-passphrase-thingy" + + DATABASE_TYPE = "postgres" + DATABASE_NAME = "oncall" + DATABASE_USER = "postgres" + DATABASE_PASSWORD = "password" + DATABASE_HOST = "${NOMAD_HOST_postgres}" + DATABASE_HOST = "${NOMAD_PORT_postgres}" + } + } } group "prometheus" { diff --git a/infra/station/nomad.yml b/infra/station/nomad.yml index aa9d8c36f16bded8db48ed4a5425502fac8f192a..63f49041c73bc06f67a4304a6b6f5240758b3994 100644 --- a/infra/station/nomad.yml +++ b/infra/station/nomad.yml @@ -14,6 +14,10 @@ volume_name: 'docker-data' path: '/var/lib/docker' size: 100g + - role: volume + volume_name: 'redis-data' + path: '/var/lib/redis-data' + size: 100g - role: volume volume_name: 'station-data' path: '/var/lib/station-data' diff --git a/infra/station/nomad/nomad.hcl.j2 b/infra/station/nomad/nomad.hcl.j2 index b39cace4450c42910986a37951ac0d1d665dc084..3cbdd03269a9c632d6db85ee93e3fbb0a2f8478b 100644 --- a/infra/station/nomad/nomad.hcl.j2 +++ b/infra/station/nomad/nomad.hcl.j2 @@ -55,6 +55,10 @@ client { path = "/var/lib/station-data/monitoring-prometheus-data" } + host_volume "monitoring-redis-data" { + path = "/var/lib/station-data/monitoring-redis-data" + } + host_volume "tango-database" { path = "/var/lib/station-data/tango-database" }