From 38fb61b24ffe45aab0cada62f05f5c1c07e256e1 Mon Sep 17 00:00:00 2001
From: Jan David Mol <mol@astron.nl>
Date: Thu, 1 Aug 2024 07:42:17 +0000
Subject: [PATCH] Serve prometheus metrics on dedicated port per worker
---
SAS/TMSS/backend/src/tmss/settings.py | 4 ++++
SAS/TMSS/deploy/tmss/docker-compose.yml | 2 ++
2 files changed, 6 insertions(+)
diff --git a/SAS/TMSS/backend/src/tmss/settings.py b/SAS/TMSS/backend/src/tmss/settings.py
index 1dda9915872..233f5a3da17 100644
--- a/SAS/TMSS/backend/src/tmss/settings.py
+++ b/SAS/TMSS/backend/src/tmss/settings.py
@@ -80,6 +80,10 @@ DEBUG = strtobool(os.getenv('DEBUG', 'True'))
ALLOWED_HOSTS = os.getenv('ALLOWED_HOSTS').split(',') if os.getenv('ALLOWED_HOSTS') else []
+# Django-prometheus endpoints, of which one per worker will be spawned
+PROMETHEUS_METRICS_EXPORT_PORT_RANGE = range(9001, 9061)
+PROMETHEUS_METRICS_EXPORT_ADDRESS = '' # all addresses
+
# Application definition
INSTALLED_APPS = [
diff --git a/SAS/TMSS/deploy/tmss/docker-compose.yml b/SAS/TMSS/deploy/tmss/docker-compose.yml
index 2b8edfad045..8c04306b68c 100644
--- a/SAS/TMSS/deploy/tmss/docker-compose.yml
+++ b/SAS/TMSS/deploy/tmss/docker-compose.yml
@@ -36,9 +36,11 @@ services:
- USER=lofarsys
- HOME=/localhome/lofarsys
# from https://docs.gunicorn.org/en/stable/run.html, number of worker processes should generally be between 2-4 workers per core in the server
+ # NB: do not enable --preload, as it will cause a clash in port allocation for the prometheus metrics
command: /bin/bash -c 'source /opt/lofar/lofarinit.sh; gunicorn lofar.sas.tmss.tmss.wsgi --worker-class=gevent --workers=60 --timeout 120 --bind=0.0.0.0:8001'
ports:
- 8001:8001
+ - 9001-9061:9001-9061 # django-prometheus endpoint per worker
depends_on:
db_migrate:
condition: service_completed_successfully
--
GitLab