add quality statistics to summary page

atdb/taskdatabase/models.py

@@ -127,7 +127,7 @@ def calculate_qualities(task):
         """
         try:
             # gather the results of all the calculated_quality values for this sas_id
-            d = {'poor': 0, 'moderate': 0, 'good': 0}
+            qualities = {'poor': 0, 'moderate': 0, 'good': 0}
 
             for task in Task.objects.filter(sas_id=unsaved_task.sas_id):
 
@@ -140,16 +140,16 @@ def calculate_qualities(task):
 
                 try:
                     key = t.calculated_qualities['per_task']
-                    d[key] = d[key] + 1
+                    qualities[key] = qualities[key] + 1
                 except:
                     # ignore the tasks that have no calculated quality (they are probably not 'stored').
                     pass
 
 
-            total = d['poor'] + d['moderate'] + d['good']
+            total = qualities['poor'] + qualities['moderate'] + qualities['good']
             if total > 0:
-                percentage_poor = (d['poor'] / total) * 100
-                percentage_good = (d['good'] / total) * 100
+                percentage_poor = (qualities['poor'] / total) * 100
+                percentage_good = (qualities['good'] / total) * 100
                 quality_sasid = "moderate"
 
                 if percentage_poor >= quality_thresholds['overall_poor']:

atdb/taskdatabase/services/algorithms.py

@@ -3,10 +3,10 @@
     Author: Nico Vermaas - Astron
     Description:  Business logic for ATDB. These functions are called from the views (views.py).
 """
-import os
+import json
 import requests
 import base64
-from datetime import datetime, timedelta
+from datetime import datetime
 from django.db.models import Q, Sum
 import logging
 from .common import timeit
@@ -801,11 +801,13 @@ def construct_summary(task):
     results = ""
     total_size_input = 0
     total_size_output = 0
+    qualities = {'poor': 0, 'moderate': 0, 'good': 0}
 
     sas_id = task.sas_id
     title = "<h4>Summary File for SAS_ID " + task.sas_id + "</h4> "
 
     tasks = Task.objects.filter(sas_id=sas_id)
+
     for task in tasks:
 
         # skip 'suspended' and 'discarded' tasks
@@ -853,8 +855,9 @@ def construct_summary(task):
                     # add calculated quality (if present)
                     calculated_qualities = task.calculated_qualities
                     if calculated_qualities:
-                        line += '<tr><td><b>Calculated Quality</b></td>'
                         task_quality = calculated_qualities['per_task']
+
+                        line += '<tr><td><b>Calculated Quality</b></td>'
                         line += '<td class="' + task_quality + '">' + str(task_quality) + '</td>'
                         line += '</tr>'
 
@@ -881,12 +884,19 @@ def construct_summary(task):
                 except:
                     pass
 
+                try:
+                    key = task.calculated_qualities['per_task']
+                    qualities[key] = qualities[key] + 1
+                except:
+                    # ignore the tasks that have no calculated quality (they are probably not 'stored').
+                    pass
+
                 results += line
 
         except:
             pass
 
-    totals += '<th>Totals</th><th></th><th></th>'
+    totals += '<th>Totals</th><th></th><th width="35%"></th>'
     try:
         totals += '<tr><td colspan="2"><b>Input size</b></td><td>' + str(total_size_input) + '</td></tr>'
         totals += '<tr><td colspan="2"><b>Output size</b><td>' + str(total_size_output) + '</td></tr>'
@@ -898,6 +908,24 @@ def construct_summary(task):
                 sasid_quality = calculated_qualities['per_sasid']
                 totals += '<tr><td colspan="2"><b>Calculated Quality</b></td>'
                 totals += '<td class="' + sasid_quality + '">' + str(sasid_quality) + '</td></tr>'
+
+                totals += '<tr><td colspan="2"><b>Quality Statistics</b></td><td>' + str(qualities) + '</td></tr>'
+
+                try:
+                    quality_thresholds = json.loads(Configuration.objects.get(key='quality_thresholds').value)
+
+                    totals += '<tr>'
+                    totals += '<td><b>RFI thresholds</b></td>'
+                    totals += '<td>Per Task</td><td>M, rfi >'+ str(quality_thresholds['poor']) + '% = P, rfi <=' + str(quality_thresholds['moderate']) + '% = G</td>'
+                    totals += '</tr>'
+
+                    totals += '<tr>'
+                    totals += '<td></td>'
+                    totals += '<td>Per SAS_ID</td><td>M, >'+ str(quality_thresholds['overall_poor']) + '% P = P, >' + str(quality_thresholds['overall_good']) + '% G = G</td>'
+                    totals += '</tr>'
+
+                except:
+                    pass
         except:
             pass
 

atdb/taskdatabase/templates/taskdatabase/index.html

@@ -31,7 +31,7 @@
             {% include 'taskdatabase/pagination.html' %}
         </div>
     </div>
-    <p class="footer"> Version 21 June 2023
+    <p class="footer"> Version 23 June 2023
 </div>
 
 {% include 'taskdatabase/refresh.html' %}

atdb/taskdatabase/views.py

@@ -1082,7 +1082,8 @@ def TaskValidateSasId(request, pk, quality, new_status, page=0):
     sas_id = task.sas_id
     tasks = Task.objects.filter(sas_id=sas_id)
     for task in tasks:
-        if task.status == 'stored':
+
+        if task.status == 'stored' or task.status == 'validated ':
 
             if quality == 'calculated':
                 try: