added functionality to remove monitoring records older than <MAX_MONITORING_HISTORY_HOURS>

atdb/atdb/settings/base.py

@@ -198,4 +198,6 @@ ACTIVE_STATUSSES = ['staging','staged','processing','processed','validated','sto
 STATUSSES_WITH_DATA = ['staged','fetching','fetched','processing','processed','validated','storing','stored','scrubbing','scrubbed','archiving','archived']
 AGGREGATES = ['failed','active','total']
 
-QUERY_LIMIT_MULTI_CHANGE = 1000
+QUERY_LIMIT_MULTI_CHANGE = 1000
\ No newline at end of file
+MAX_MONITORING_HISTORY_HOURS = 7 * 24
+SERVICES_LATE_WARNING_SECONDS = 1800
\ No newline at end of file

atdb/taskdatabase/models.py

 from django.db import models
 from django.urls import reverse
-from django.utils.timezone import datetime
+from django.utils import timezone
+from django.utils.timezone import datetime, timedelta
+from django.conf import settings
 
 # constants
 datetime_format_string = '%Y-%m-%dT%H:%M:%SZ'
@@ -232,13 +234,25 @@ class LatestMonitor(models.Model):
             enabled = self.metadata['enabled']
             return enabled
         except:
-            return None
+            # only when metadata['enabled']=False' this will be register as false.
+            # to make sure that services are enabled by default
+            return "True"
 
     # the representation of the value in the REST API
     def __str__(self):
         return str(self.name) + ' - ('+ self.hostname+') - '+str(self.timestamp) + ' - ' + self.status
 
 
+def purge_old_records():
+    current_time = timezone.now()  # change this
+    try:
+        time_threshold = current_time - timedelta(hours=settings.MAX_MONITORING_HISTORY_HOURS)
+        records_to_delete = Monitor.objects.filter(timestamp__lt=time_threshold).delete()
+    except Exception as e:
+        # if MAX_MONITORING_HISTORY_HOURS is not set, then do nothing and continue
+        pass
+
+
 class Monitor(models.Model):
     name = models.CharField(max_length=50, default="unknown")
     type = models.CharField(max_length=20, default="ldv-service", null=True, blank=True)
@@ -283,6 +297,9 @@ class Monitor(models.Model):
         # finally save the Monitor info itself also
         super(Monitor, self).save(*args, **kwargs)
 
+        # and purge the monitoring table to its max
+        purge_old_records()
+
     # the representation of the value in the REST API
     def __str__(self):
         return str(self.name) + ' - ('+ self.hostname+') - '+str(self.timestamp) + ' - ' + self.status

atdb/taskdatabase/services/algorithms.py

@@ -9,10 +9,6 @@ from django.db.models import Q, Sum
 import logging
 from .common import timeit
 
-from urllib.request import urlopen
-from django.core.files import File
-from django.core.files.temp import NamedTemporaryFile
-
 from ..models import Task, LogEntry, Workflow, Configuration
 from django.conf import settings
 
@@ -298,7 +294,7 @@ def convert_monitor_to_html(request, monitor_data):
                 service_enabled = str(record.enabled)
 
             # if the heartbeat is 30 minutes late, show '(late)' in red
-            if delta.seconds > 1800:
+            if delta.seconds > settings.SERVICES_LATE_WARNING_SECONDS:
                 line += "<td>" + service_enabled + "</td>"
                 line += "<td><i>unknown</i></td>"
                 line += '<td class="error">' + str(record.timestamp.strftime(TIME_FORMAT)) + " - (late)</td>"

atdb/taskdatabase/templates/taskdatabase/index.html

@@ -34,7 +34,7 @@
         {% include 'taskdatabase/pagination.html' %}
        </div>
     </div>
-    <p class="footer"> Version 1.0.0 (19 apr 2021 - 13:00)
+    <p class="footer"> Version 1.0.0 (21 apr 2021 - 13:00)
 
 </div>
 

atdb/taskdatabase/views.py

@@ -6,7 +6,6 @@ from django.contrib.auth.decorators import login_required
 
 from django.views.generic import ListView
 from django.contrib import messages
-from django.http import QueryDict
 
 from rest_framework import generics, pagination
 from rest_framework.response import Response