From 0c1d27c0c09613c6e25e0f9c0fa146f5f6526bbf Mon Sep 17 00:00:00 2001
From: Nico Vermaas <vermaas@astron.nl>
Date: Thu, 21 Apr 2022 11:28:59 +0200
Subject: [PATCH] added functionality to remove monitoring records older than
<MAX_MONITORING_HISTORY_HOURS>
---
atdb/atdb/settings/base.py | 4 +++-
atdb/taskdatabase/models.py | 21 +++++++++++++++++--
atdb/taskdatabase/services/algorithms.py | 6 +-----
.../templates/taskdatabase/index.html | 2 +-
atdb/taskdatabase/views.py | 1 -
5 files changed, 24 insertions(+), 10 deletions(-)
diff --git a/atdb/atdb/settings/base.py b/atdb/atdb/settings/base.py
index 79d03f49..52dcb06f 100644
--- a/atdb/atdb/settings/base.py
+++ b/atdb/atdb/settings/base.py
@@ -198,4 +198,6 @@ ACTIVE_STATUSSES = ['staging','staged','processing','processed','validated','sto
STATUSSES_WITH_DATA = ['staged','fetching','fetched','processing','processed','validated','storing','stored','scrubbing','scrubbed','archiving','archived']
AGGREGATES = ['failed','active','total']
-QUERY_LIMIT_MULTI_CHANGE = 1000
\ No newline at end of file
+QUERY_LIMIT_MULTI_CHANGE = 1000
+MAX_MONITORING_HISTORY_HOURS = 7 * 24
+SERVICES_LATE_WARNING_SECONDS = 1800
\ No newline at end of file
diff --git a/atdb/taskdatabase/models.py b/atdb/taskdatabase/models.py
index 52295661..60ede446 100644
--- a/atdb/taskdatabase/models.py
+++ b/atdb/taskdatabase/models.py
@@ -1,6 +1,8 @@
from django.db import models
from django.urls import reverse
-from django.utils.timezone import datetime
+from django.utils import timezone
+from django.utils.timezone import datetime, timedelta
+from django.conf import settings
# constants
datetime_format_string = '%Y-%m-%dT%H:%M:%SZ'
@@ -232,13 +234,25 @@ class LatestMonitor(models.Model):
enabled = self.metadata['enabled']
return enabled
except:
- return None
+ # only when metadata['enabled']=False' this will be register as false.
+ # to make sure that services are enabled by default
+ return "True"
# the representation of the value in the REST API
def __str__(self):
return str(self.name) + ' - ('+ self.hostname+') - '+str(self.timestamp) + ' - ' + self.status
+def purge_old_records():
+ current_time = timezone.now() # change this
+ try:
+ time_threshold = current_time - timedelta(hours=settings.MAX_MONITORING_HISTORY_HOURS)
+ records_to_delete = Monitor.objects.filter(timestamp__lt=time_threshold).delete()
+ except Exception as e:
+ # if MAX_MONITORING_HISTORY_HOURS is not set, then do nothing and continue
+ pass
+
+
class Monitor(models.Model):
name = models.CharField(max_length=50, default="unknown")
type = models.CharField(max_length=20, default="ldv-service", null=True, blank=True)
@@ -283,6 +297,9 @@ class Monitor(models.Model):
# finally save the Monitor info itself also
super(Monitor, self).save(*args, **kwargs)
+ # and purge the monitoring table to its max
+ purge_old_records()
+
# the representation of the value in the REST API
def __str__(self):
return str(self.name) + ' - ('+ self.hostname+') - '+str(self.timestamp) + ' - ' + self.status
diff --git a/atdb/taskdatabase/services/algorithms.py b/atdb/taskdatabase/services/algorithms.py
index cce3f3a6..62fe1b7b 100644
--- a/atdb/taskdatabase/services/algorithms.py
+++ b/atdb/taskdatabase/services/algorithms.py
@@ -9,10 +9,6 @@ from django.db.models import Q, Sum
import logging
from .common import timeit
-from urllib.request import urlopen
-from django.core.files import File
-from django.core.files.temp import NamedTemporaryFile
-
from ..models import Task, LogEntry, Workflow, Configuration
from django.conf import settings
@@ -298,7 +294,7 @@ def convert_monitor_to_html(request, monitor_data):
service_enabled = str(record.enabled)
# if the heartbeat is 30 minutes late, show '(late)' in red
- if delta.seconds > 1800:
+ if delta.seconds > settings.SERVICES_LATE_WARNING_SECONDS:
line += "<td>" + service_enabled + "</td>"
line += "<td><i>unknown</i></td>"
line += '<td class="error">' + str(record.timestamp.strftime(TIME_FORMAT)) + " - (late)</td>"
diff --git a/atdb/taskdatabase/templates/taskdatabase/index.html b/atdb/taskdatabase/templates/taskdatabase/index.html
index d8b4f2fd..ea9b66c7 100644
--- a/atdb/taskdatabase/templates/taskdatabase/index.html
+++ b/atdb/taskdatabase/templates/taskdatabase/index.html
@@ -34,7 +34,7 @@
{% include 'taskdatabase/pagination.html' %}
</div>
</div>
- <p class="footer"> Version 1.0.0 (19 apr 2021 - 13:00)
+ <p class="footer"> Version 1.0.0 (21 apr 2021 - 13:00)
</div>
diff --git a/atdb/taskdatabase/views.py b/atdb/taskdatabase/views.py
index 22d065d2..f6775feb 100644
--- a/atdb/taskdatabase/views.py
+++ b/atdb/taskdatabase/views.py
@@ -6,7 +6,6 @@ from django.contrib.auth.decorators import login_required
from django.views.generic import ListView
from django.contrib import messages
-from django.http import QueryDict
from rest_framework import generics, pagination
from rest_framework.response import Response
--
GitLab