Merge branch 'SDC-496-service-monitor' into 'master'

https://support.astron.nl/jira/browse/SDC-499

See merge request !180

atdb/taskdatabase/admin.py

 from django.contrib import admin
-from .models import Status, Task, Workflow, LogEntry, Configuration, Job, PostProcessingRule, Monitor
+from .models import Status, Task, Workflow, LogEntry, Configuration, Job, PostProcessingRule, Monitor, LatestMonitor
 
 admin.site.register(Status)
 admin.site.register(Task)
@@ -8,4 +8,5 @@ admin.site.register(LogEntry)
 admin.site.register(Configuration)
 admin.site.register(Job)
 admin.site.register(PostProcessingRule)
-admin.site.register(Monitor)
+admin.site.register(Monitor)
\ No newline at end of file
+admin.site.register(LatestMonitor)
\ No newline at end of file

atdb/taskdatabase/migrations/0011_auto_20220131_1103.py

+# Generated by Django 3.1.4 on 2022-01-31 10:03
+
+import datetime
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('taskdatabase', '0010_auto_20220127_1458'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='LatestMonitor',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(default='unknown', max_length=50)),
+                ('type', models.CharField(blank=True, default='ldv-service', max_length=20, null=True)),
+                ('timestamp', models.DateTimeField(blank=True, default=datetime.datetime.utcnow)),
+                ('hostname', models.CharField(default='unknown', max_length=50)),
+                ('process_id', models.IntegerField(blank=True, null=True)),
+                ('description', models.CharField(blank=True, max_length=255, null=True)),
+                ('status', models.CharField(default='ok', max_length=50, null=True)),
+                ('metadata', models.JSONField(blank=True, null=True)),
+            ],
+        ),
+        migrations.AlterField(
+            model_name='monitor',
+            name='process_id',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='monitor',
+            name='status',
+            field=models.CharField(default='ok', max_length=50, null=True),
+        ),
+    ]

atdb/taskdatabase/models.py

@@ -131,8 +131,7 @@ class PostProcessingRule(models.Model):
         return str(self.aggregation_key)+' - '+str(self.trigger_status)
 
 
-
-class Monitor(models.Model):
+class LatestMonitor(models.Model):
     name = models.CharField(max_length=50, default="unknown")
     type = models.CharField(max_length=20, default="ldv-service", null=True, blank=True)
     timestamp = models.DateTimeField(default=datetime.utcnow, blank=True)
@@ -144,10 +143,10 @@ class Monitor(models.Model):
 
     # the representation of the value in the REST API
     def __str__(self):
-        return str(self.name)+' - '+str(self.timestamp) + ' - ' + self.status
+        return str(self.name) + ' - ('+ self.hostname+') - '+str(self.timestamp) + ' - ' + self.status
 
 
-class LatestMonitor(models.Model):
+class Monitor(models.Model):
     name = models.CharField(max_length=50, default="unknown")
     type = models.CharField(max_length=20, default="ldv-service", null=True, blank=True)
     timestamp = models.DateTimeField(default=datetime.utcnow, blank=True)
@@ -157,6 +156,29 @@ class LatestMonitor(models.Model):
     status = models.CharField(max_length=50, default="ok", null=True)
     metadata = models.JSONField(null=True, blank=True)
 
+    def save(self, *args, **kwargs):
+        # check if this combination of service name + hostname already exists
+        # in the LatestMonitor, and update if it is newer.
+        try:
+            latestMonitor = LatestMonitor.objects.get(name=self.name,hostname=self.hostname)
+            latestMonitor.delete()
+        except:
+            pass
+
+        # this combination of name and hostname didn't yet exist, create it.
+        latestMonitor = LatestMonitor(
+            name=self.name,
+            type=self.type,
+            timestamp=self.timestamp,
+            hostname = self.hostname,
+            process_id = self.process_id,
+            description = self.description,
+            status = self.status,
+            metadata = self.metadata
+        )
+        latestMonitor.save()
+
     # the representation of the value in the REST API
     def __str__(self):
-        return str(self.name)+' - '+str(self.timestamp) + ' - ' + self.status
+        return str(self.name) + ' - ('+ self.hostname+') - '+str(self.timestamp) + ' - ' + self.status
\ No newline at end of file
+

atdb/taskdatabase/serializers.py

 from rest_framework import serializers
-from .models import Status, Task, Workflow, LogEntry, Configuration, Job, PostProcessingRule, Monitor
-
+from .models import Status, Task, Workflow, LogEntry, Configuration, Job, PostProcessingRule, Monitor, LatestMonitor
 
 
 class WorkflowSerializer(serializers.ModelSerializer):
@@ -175,4 +174,12 @@ class MonitorSerializer(serializers.ModelSerializer):
 
     class Meta:
         model = Monitor
-        fields = "__all__"
+        fields = "__all__"
\ No newline at end of file
+
+
+class LatestMonitorSerializer(serializers.ModelSerializer):
+
+    class Meta:
+        model = LatestMonitor
+        fields = "__all__"
+        #read_only_fields = fields
\ No newline at end of file

atdb/taskdatabase/services/algorithms.py

@@ -201,19 +201,39 @@ def convert_config_to_html(querylist):
     return results
 
 
-def convert_monitor_to_html(monitor_data):
+def construct_link_to_monitor_history(request, title, name, hostname):
+    query = "name=" + name + "&hostname=" + hostname
+    try:
+
+        if settings.DEV == True:
+            url = request.build_absolute_uri('/atdb/monitor/?') + query
+        else:
+            # Unclear why 'build_absolute_uri' doesn't return 'https' in production.
+            # Probably because the https is handled fully outside the container by Traefik
+            # and ATDB is not aware of that.
+
+            url = "https://" + request.get_host() + '/atdb/monitor/?' + query
+        link = '<a href="' + url + '" target="_blank">' + title + "</a>"
+    except:
+        pass
+    return link
+
+
+def convert_monitor_to_html(request, monitor_data):
     results = ""
     try:
         for record in monitor_data:
             # iterate through the dict of key/values
-            key = record.name
-            value = record.status
-            filter = record.description
+
+            # http://localhost:8000/atdb/monitor/?name=stager&hostname=localhost
+            link_to_service_history = construct_link_to_monitor_history(request, record.name, record.name, record.hostname)
+
             line = '<tr>'
             if "error" in record.status:
                 line = '<tr class="' + record.status + '" >'
 
-            line += "<td><b>" + str(record.name) + "</b></td>"
+            #line += "<td><b>" + str(record.name) + "</b></td>"
+            line += "<td><b>" + link_to_service_history + "</b></td>"
             line += "<td>" + str(record.hostname) + "</td>"
             line += '<td class="' + record.status + '" >' + str(record.status) + "</td>"
             line += "<td>" + str(record.timestamp) + "</td>"

atdb/taskdatabase/templates/taskdatabase/index.html

@@ -80,7 +80,7 @@
         {% include 'taskdatabase/pagination.html' %}
        </div>
     </div>
-    <p class="footer"> Version 1.0.0 (31 jan 2021 - 10:00)
+    <p class="footer"> Version 1.0.0 (31 jan 2021 - 14:00)
 
 </div>
 

atdb/taskdatabase/urls.py

@@ -54,8 +54,9 @@ urlpatterns = [
     path('postprocessing/', views.PostProcessingRuleListViewAPI.as_view()),
     path('postprocessing/<int:pk>/', views.PostProcessingRuleDetailsViewAPI.as_view(), name='postprocessing-detail-view-api'),
 
-    path('monitor/', views.MonitorListViewAPI.as_view()),
+    path('monitor/', views.MonitorListViewAPI.as_view(),name='monitor-list-view-api'),
     path('monitor/<int:pk>/', views.MonitorDetailsViewAPI.as_view(),name='monitor-detail-view-api'),
+    path('latest_monitor/', views.LatestMonitorListViewAPI.as_view(),name='latest-monitor-detail-view-api'),
 
     # --- custom requests ---
     # /atdb/get_size?status__in=defined,staged

atdb/taskdatabase/views.py

@@ -22,7 +22,7 @@ from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
 from rest_framework.request import Request
 
 from django.conf import settings
-from .models import Task, Status, Workflow, LogEntry, Configuration, Job, PostProcessingRule, Monitor
+from .models import Task, Status, Workflow, LogEntry, Configuration, Job, PostProcessingRule, Monitor, LatestMonitor
 from .tables import TaskTable
 
 from django.db.models import Q
@@ -35,7 +35,8 @@ from .serializers import \
     ConfigurationSerializer, \
     JobSerializer, \
     PostProcessingRuleSerializer, \
-    MonitorSerializer
+    MonitorSerializer, LatestMonitorSerializer
+
 
 from .services import algorithms
 
@@ -138,6 +139,16 @@ class PostProcessingFilter(filters.FilterSet):
             'workflow_to_apply__id': ['exact'],
         }
 
+class MonitorFilter(filters.FilterSet):
+    class Meta:
+        model = Monitor
+
+        fields = {
+            'name': ['exact', 'icontains', 'in'],
+            'hostname': ['exact', 'icontains', 'in'],
+            'process_id': ['exact'],
+            'timestamp': ['icontains'],
+        }
 
 # ---------- Tables2 Views (experimental) -----------
 # implementation with tables2: http://localhost:8000/atdb/tables2
@@ -326,9 +337,10 @@ def WorkflowDetails(request, id):
 
 def ShowMonitoring(request):
     # get the latest value of each unique combination of service name and hostname.
-    distinct_services_per_host = Monitor.objects.all().order_by('name', 'hostname', '-timestamp').distinct('name', 'hostname')
+    #distinct_services_per_host = Monitor.objects.all().order_by('name', 'hostname', '-timestamp').distinct('name', 'hostname')
+    distinct_services_per_host = LatestMonitor.objects.all().order_by('name', 'hostname', '-timestamp').distinct('name', 'hostname')
 
-    monitor_results = algorithms.convert_monitor_to_html(distinct_services_per_host)
+    monitor_results = algorithms.convert_monitor_to_html(request, distinct_services_per_host)
     return render(request, "taskdatabase/monitoring.html", {'monitor_results': monitor_results})
 
 
@@ -545,14 +557,24 @@ class PostProcessingRuleDetailsViewAPI(generics.RetrieveUpdateDestroyAPIView):
     serializer_class = PostProcessingRuleSerializer
 
 
-# example: /atdb/job/
+# example: /atdb/monitor/
 class MonitorListViewAPI(generics.ListCreateAPIView):
     model = Monitor
-    queryset = Monitor.objects.all()
+    queryset = Monitor.objects.all().order_by('-timestamp')
     serializer_class = MonitorSerializer
 
     filter_backends = (filters.DjangoFilterBackend,)
-    # filter_class = PostProcessingFilter
+    filter_class = MonitorFilter
+
+
+# example: /atdb/latest-monitor/
+class LatestMonitorListViewAPI(generics.ListCreateAPIView):
+    model = LatestMonitor
+    queryset = LatestMonitor.objects.all()
+    serializer_class = LatestMonitorSerializer
+
+    filter_backends = (filters.DjangoFilterBackend,)
+    filter_class = MonitorFilter
 
 
 # example: /atdb/job/5/