Merge branch 'TMSS-3106-metrics-from-multiple-instances' into 'master'

Serve prometheus metrics on dedicated port per worker

Closes TMSS-3106

See merge request !1456

SAS/TMSS/backend/src/tmss/settings.py

@@ -80,6 +80,10 @@ DEBUG = strtobool(os.getenv('DEBUG', 'True'))
 
 ALLOWED_HOSTS = os.getenv('ALLOWED_HOSTS').split(',') if os.getenv('ALLOWED_HOSTS') else []
 
+# Django-prometheus endpoints, of which one per worker will be spawned
+PROMETHEUS_METRICS_EXPORT_PORT_RANGE = range(9001, 9061)
+PROMETHEUS_METRICS_EXPORT_ADDRESS = ''  # all addresses
+
 # Application definition
 
 INSTALLED_APPS = [

SAS/TMSS/deploy/tmss/docker-compose.yml

@@ -36,9 +36,11 @@ services:
       - USER=lofarsys
       - HOME=/localhome/lofarsys
     # from https://docs.gunicorn.org/en/stable/run.html, number of worker processes should generally be between 2-4 workers per core in the server
+    # NB: do not enable --preload, as it will cause a clash in port allocation for the prometheus metrics
     command: /bin/bash -c 'source /opt/lofar/lofarinit.sh; gunicorn lofar.sas.tmss.tmss.wsgi --worker-class=gevent --workers=60 --timeout 120 --bind=0.0.0.0:8001'
     ports:
       - 8001:8001
+      - 9001-9061:9001-9061 # django-prometheus endpoint per worker
     depends_on:
       db_migrate:
         condition: service_completed_successfully