From 87fa573d159082520ae48efb185459804a3b29ad Mon Sep 17 00:00:00 2001 From: Jan David Mol <mol@astron.nl> Date: Tue, 28 May 2024 14:23:22 +0200 Subject: [PATCH] add oncall --- infra/env.yaml | 4 + infra/jobs/station/monitoring.levant.nomad | 127 +++++++++++++++++++++ infra/station/nomad.yml | 4 + infra/station/nomad/nomad.hcl.j2 | 4 + 4 files changed, 139 insertions(+) diff --git a/infra/env.yaml b/infra/env.yaml index 29b70b26c..b336519f1 100644 --- a/infra/env.yaml +++ b/infra/env.yaml @@ -23,6 +23,10 @@ monitoring: version: latest prometheus: version: latest + redis: + version: 7.0.5 + oncall: + version: latest object_storage: minio: diff --git a/infra/jobs/station/monitoring.levant.nomad b/infra/jobs/station/monitoring.levant.nomad index c8e29d166..ed35564c6 100644 --- a/infra/jobs/station/monitoring.levant.nomad +++ b/infra/jobs/station/monitoring.levant.nomad @@ -65,6 +65,25 @@ job "monitoring" { } } + service { + tags = ["scrape"] + name = "oncall" + port = "8080" + address_mode = "alloc" + } + + service { + name = "redis" + port = "6379" + address_mode = "alloc" + } + + volume "redis" { + type = "host" + read_only = false + source = "monitoring-redis-data" + } + task "wait-for-db" { lifecycle { hook = "prestart" @@ -164,6 +183,114 @@ postgres.service.consul:5432:grafana:postgres:password memory_max = 8192 } } + + task "oncall" { + driver = "docker" + + config { + image = "[[.registry.astron.url]]/oncall:[[.monitoring.oncall.version]]" + args = [ + "sh", + "-c", + "uwsgi --ini uwsgi.ini" + ] + } + + env { + BROKER_TYPE = "redis" + REDIS_URI = "redis://redis:6379/0" + DJANGO_SETTINGS_MODULE = "settings.hobby" + GRAFANA_API_URL = "http://${NOMAD_ADDR_grafana}/grafana" + SECRET_KEY = "my-little-secret-key-passphrase-thingy" + BASE_URL = "http://${NOMAD_ADDR_oncall}" + + DATABASE_TYPE = "postgres" + DATABASE_NAME = "oncall" + DATABASE_USER = "postgres" + DATABASE_PASSWORD = "password" + DATABASE_HOST = "${NOMAD_HOST_postgres}" + DATABASE_HOST = "${NOMAD_PORT_postgres}" + } + } + + task "celery" { + driver = "docker" + + config { + image = "[[.registry.astron.url]]/oncall:[[.monitoring.oncall.version]]" + args = [ + "sh", + "-c", + "./celery_with_exporter.sh" + ] + } + + volume_mount { + volume = "oncall" + destination = "/var/lib/oncall" + read_only = false + } + + env { + CELERY_WORKER_QUEUE = "default,critical,long,slack,telegram,webhook,retry,celery,grafana" + CELERY_WORKER_CONCURRENCY = "1" + CELERY_WORKER_MAX_TASKS_PER_CHILD = "100" + CELERY_WORKER_SHUTDOWN_INTERVAL = "65m" + CELERY_WORKER_BEAT_ENABLED = "True" + } + } + + task "redis" { + lifecycle { + hook = "prestart" + sidecar = false + } + + driver = "docker" + + config { + image = "[[.registry.astron.url]]/redis:[[.monitoring.redis.version]]" + } + + volume_mount { + volume = "redis" + destination = "/data" + read_only = false + } + } + + task "oncall-db-migration" { + lifecycle { + hook = "prestart" + sidecar = false + } + + driver = "docker" + + config { + image = "[[.registry.astron.url]]/oncall:[[.monitoring.oncall.version]]" + args = [ + "python", + "manage.py", + "migrate", + "--noinput" + ] + } + + env { + BROKER_TYPE = "redis" + REDIS_URI = "redis://redis:6379/0" + DJANGO_SETTINGS_MODULE = "settings.hobby" + SECRET_KEY = "my-little-secret-key-passphrase-thingy" + + DATABASE_TYPE = "postgres" + DATABASE_NAME = "oncall" + DATABASE_USER = "postgres" + DATABASE_PASSWORD = "password" + DATABASE_HOST = "${NOMAD_HOST_postgres}" + DATABASE_HOST = "${NOMAD_PORT_postgres}" + } + } } group "prometheus" { diff --git a/infra/station/nomad.yml b/infra/station/nomad.yml index aa9d8c36f..63f49041c 100644 --- a/infra/station/nomad.yml +++ b/infra/station/nomad.yml @@ -14,6 +14,10 @@ volume_name: 'docker-data' path: '/var/lib/docker' size: 100g + - role: volume + volume_name: 'redis-data' + path: '/var/lib/redis-data' + size: 100g - role: volume volume_name: 'station-data' path: '/var/lib/station-data' diff --git a/infra/station/nomad/nomad.hcl.j2 b/infra/station/nomad/nomad.hcl.j2 index b39cace44..3cbdd0326 100644 --- a/infra/station/nomad/nomad.hcl.j2 +++ b/infra/station/nomad/nomad.hcl.j2 @@ -55,6 +55,10 @@ client { path = "/var/lib/station-data/monitoring-prometheus-data" } + host_volume "monitoring-redis-data" { + path = "/var/lib/station-data/monitoring-redis-data" + } + host_volume "tango-database" { path = "/var/lib/station-data/tango-database" } -- GitLab