add quality algorith per task

activated when status is set to stored
quality threshold values read from Configuration

atdb/taskdatabase/models.py

@@ -3,8 +3,7 @@ from django.urls import reverse
 from django.utils import timezone
 from django.utils.timezone import datetime, timedelta
 from django.conf import settings
-
-
+import json
 import logging
 logger = logging.getLogger(__name__)
 
@@ -68,6 +67,41 @@ def convert_summary_to_list_for_template(task):
     return list
 
 
+def calculate_quality(task):
+    """"
+    calculate the quality of this task based on rfi_percent values
+    The threshold values are written from a configuration json blob
+    """
+    try:
+        # read the quality_thresholds from the Configuration table
+        try:
+            quality_thresholds = json.loads(Configuration.objects.get(key='quality_thresholds').value)
+        except:
+            quality_thresholds = {
+                "moderate" : 20,
+                "poor" : 50,
+                "overall_moderate" : 50,
+                "overall_good": 90,
+            }
+
+        summary = task.quality_json["summary"]
+        quality = task.quality
+
+        for key in summary:
+            record = summary[key]
+            rfi_percent = int(record['rfi_percent'])
+            if rfi_percent > 0:
+                quality = "good"
+                if rfi_percent > quality_thresholds['moderate']:
+                    quality = "moderate"
+                elif rfi_percent >  quality_thresholds['poor']:
+                    quality = "poor"
+        return quality
+
+    except Exception as error:
+        logger.info(error)
+
+
 class Task(models.Model):
 
     # Task control properties
@@ -108,12 +142,15 @@ class Task(models.Model):
         return str(self.id) + ' - (' + self.task_type + ') - ' + str(self.sas_id)
 
     def save(self, *args, **kwargs):
-        # nv:1mar2023, temporary hack, set tasks 'on hold' as soon they get to 'stored'
+        # nv:1mar2023, temporary hack, set tasks 'on hold' as soon they get to 'scrubbed'
         # (users forget to do that manually, causing unwanted ingests)
-
         if (self.status != 'scrubbed') & (self.new_status == 'scrubbed'):
             self.resume = False
 
+        # nv:19jun2023, calculate the quality of this task
+        if (self.status != 'stored') & (self.new_status == 'stored'):
+            self.quality = calculate_quality(self)
+
         super(Task, self).save(*args, **kwargs)
 
 

atdb/taskdatabase/services/algorithms.py

@@ -797,7 +797,6 @@ def construct_inspectionplots(task, expand_image="False", source='task_id'):
 
 def construct_summary(task):
 
-    title = ""
     totals = ""
     results = ""
     total_size_input = 0
@@ -815,7 +814,7 @@ def construct_summary(task):
 
         results += '<tr style="background-color:#7EB1C4"><td colspan="3"><b>Task ' + str(task.id) + '</b></td></tr>'
 
-        # find the plots in the quality json structure
+        # find the summary in the quality json structure
         try:
             summary = task.quality_json["summary"]
 
@@ -867,9 +866,12 @@ def construct_summary(task):
             pass
 
     totals += '<th>Totals</th><th></th><th></th>'
-    totals += '<tr><td colspan="2"><b>Input size</b></td><td>' + str(total_size_input) + '</td></tr>'
-    totals += '<tr><td colspan="2"><b>Output size</b><td>' + str(total_size_output) + '</td></tr>'
-    totals += '<tr><td colspan="2"><b>Ratio</b></td><td>' + str(round(total_size_output / total_size_input, 3)) + '</td></tr>'
+    try:
+        totals += '<tr><td colspan="2"><b>Input size</b></td><td>' + str(total_size_input) + '</td></tr>'
+        totals += '<tr><td colspan="2"><b>Output size</b><td>' + str(total_size_output) + '</td></tr>'
+        totals += '<tr><td colspan="2"><b>Ratio</b></td><td>' + str(round(total_size_output / total_size_input, 3)) + '</td></tr>'
+    except:
+        pass
 
     results = title + totals + results
     return results

atdb/taskdatabase/templates/taskdatabase/index.html

@@ -31,7 +31,7 @@
             {% include 'taskdatabase/pagination.html' %}
         </div>
     </div>
-    <p class="footer"> Version 10 May 2023
+    <p class="footer"> Version 19 June 2023
 </div>
 
 {% include 'taskdatabase/refresh.html' %}

atdb/taskdatabase/templates/taskdatabase/validation/headers.html

@@ -4,6 +4,11 @@
         SAS_ID
         <a href="{% url 'sort-tasks' 'sas_id' 'validation' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
     </th>
+    <th>
+        <a href="{% url 'sort-tasks' '-workflow' 'validation' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+        Workflow
+        <a href="{% url 'sort-tasks' 'workflow' 'validation' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+    </th>
     <th>
         <a href="{% url 'sort-tasks' '-project' 'validation' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
         Project

atdb/taskdatabase/templates/taskdatabase/validation/tasks.html

@@ -5,6 +5,13 @@
         <div class="row">
             <tr class="{{ task.status }}">
                 <td>{{ task.sas_id }}</td>
+                <td>
+                <a class="open-modal btn btn-primary btn-sm"
+                       href="{% url 'workflow-details' task.workflow.id %}"
+                       data-popup-url="{% url 'workflow-details' task.workflow.id %}"
+                       target="_blank"><i class="fas fa-project-diagram"></i> {{ task.workflow.id }}
+                    </a></td>
+                </td>
                 <td>{{ task.project }}</td>