From 88af3df9aa7e8bff2c83d61d8e766d1907316f15 Mon Sep 17 00:00:00 2001
From: Vermaas <vermaas@astron.nl>
Date: Mon, 19 Jun 2023 16:19:06 +0200
Subject: [PATCH] add quality algorith per task activated when status is set to
stored quality threshold values read from Configuration
---
atdb/taskdatabase/models.py | 45 +++++++++++++++++--
atdb/taskdatabase/services/algorithms.py | 12 ++---
.../templates/taskdatabase/index.html | 2 +-
.../taskdatabase/validation/headers.html | 5 +++
.../taskdatabase/validation/tasks.html | 7 +++
5 files changed, 61 insertions(+), 10 deletions(-)
diff --git a/atdb/taskdatabase/models.py b/atdb/taskdatabase/models.py
index 4fac9471..eb3b467d 100644
--- a/atdb/taskdatabase/models.py
+++ b/atdb/taskdatabase/models.py
@@ -3,8 +3,7 @@ from django.urls import reverse
from django.utils import timezone
from django.utils.timezone import datetime, timedelta
from django.conf import settings
-
-
+import json
import logging
logger = logging.getLogger(__name__)
@@ -68,6 +67,41 @@ def convert_summary_to_list_for_template(task):
return list
+def calculate_quality(task):
+ """"
+ calculate the quality of this task based on rfi_percent values
+ The threshold values are written from a configuration json blob
+ """
+ try:
+ # read the quality_thresholds from the Configuration table
+ try:
+ quality_thresholds = json.loads(Configuration.objects.get(key='quality_thresholds').value)
+ except:
+ quality_thresholds = {
+ "moderate" : 20,
+ "poor" : 50,
+ "overall_moderate" : 50,
+ "overall_good": 90,
+ }
+
+ summary = task.quality_json["summary"]
+ quality = task.quality
+
+ for key in summary:
+ record = summary[key]
+ rfi_percent = int(record['rfi_percent'])
+ if rfi_percent > 0:
+ quality = "good"
+ if rfi_percent > quality_thresholds['moderate']:
+ quality = "moderate"
+ elif rfi_percent > quality_thresholds['poor']:
+ quality = "poor"
+ return quality
+
+ except Exception as error:
+ logger.info(error)
+
+
class Task(models.Model):
# Task control properties
@@ -108,12 +142,15 @@ class Task(models.Model):
return str(self.id) + ' - (' + self.task_type + ') - ' + str(self.sas_id)
def save(self, *args, **kwargs):
- # nv:1mar2023, temporary hack, set tasks 'on hold' as soon they get to 'stored'
+ # nv:1mar2023, temporary hack, set tasks 'on hold' as soon they get to 'scrubbed'
# (users forget to do that manually, causing unwanted ingests)
-
if (self.status != 'scrubbed') & (self.new_status == 'scrubbed'):
self.resume = False
+ # nv:19jun2023, calculate the quality of this task
+ if (self.status != 'stored') & (self.new_status == 'stored'):
+ self.quality = calculate_quality(self)
+
super(Task, self).save(*args, **kwargs)
diff --git a/atdb/taskdatabase/services/algorithms.py b/atdb/taskdatabase/services/algorithms.py
index abdc4326..73cb0a5f 100644
--- a/atdb/taskdatabase/services/algorithms.py
+++ b/atdb/taskdatabase/services/algorithms.py
@@ -797,7 +797,6 @@ def construct_inspectionplots(task, expand_image="False", source='task_id'):
def construct_summary(task):
- title = ""
totals = ""
results = ""
total_size_input = 0
@@ -815,7 +814,7 @@ def construct_summary(task):
results += '<tr style="background-color:#7EB1C4"><td colspan="3"><b>Task ' + str(task.id) + '</b></td></tr>'
- # find the plots in the quality json structure
+ # find the summary in the quality json structure
try:
summary = task.quality_json["summary"]
@@ -867,9 +866,12 @@ def construct_summary(task):
pass
totals += '<th>Totals</th><th></th><th></th>'
- totals += '<tr><td colspan="2"><b>Input size</b></td><td>' + str(total_size_input) + '</td></tr>'
- totals += '<tr><td colspan="2"><b>Output size</b><td>' + str(total_size_output) + '</td></tr>'
- totals += '<tr><td colspan="2"><b>Ratio</b></td><td>' + str(round(total_size_output / total_size_input, 3)) + '</td></tr>'
+ try:
+ totals += '<tr><td colspan="2"><b>Input size</b></td><td>' + str(total_size_input) + '</td></tr>'
+ totals += '<tr><td colspan="2"><b>Output size</b><td>' + str(total_size_output) + '</td></tr>'
+ totals += '<tr><td colspan="2"><b>Ratio</b></td><td>' + str(round(total_size_output / total_size_input, 3)) + '</td></tr>'
+ except:
+ pass
results = title + totals + results
return results
diff --git a/atdb/taskdatabase/templates/taskdatabase/index.html b/atdb/taskdatabase/templates/taskdatabase/index.html
index fe379d56..b75b486c 100644
--- a/atdb/taskdatabase/templates/taskdatabase/index.html
+++ b/atdb/taskdatabase/templates/taskdatabase/index.html
@@ -31,7 +31,7 @@
{% include 'taskdatabase/pagination.html' %}
</div>
</div>
- <p class="footer"> Version 10 May 2023
+ <p class="footer"> Version 19 June 2023
</div>
{% include 'taskdatabase/refresh.html' %}
diff --git a/atdb/taskdatabase/templates/taskdatabase/validation/headers.html b/atdb/taskdatabase/templates/taskdatabase/validation/headers.html
index 9cd3a073..8d9206ca 100644
--- a/atdb/taskdatabase/templates/taskdatabase/validation/headers.html
+++ b/atdb/taskdatabase/templates/taskdatabase/validation/headers.html
@@ -4,6 +4,11 @@
SAS_ID
<a href="{% url 'sort-tasks' 'sas_id' 'validation' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
</th>
+ <th>
+ <a href="{% url 'sort-tasks' '-workflow' 'validation' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+ Workflow
+ <a href="{% url 'sort-tasks' 'workflow' 'validation' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+ </th>
<th>
<a href="{% url 'sort-tasks' '-project' 'validation' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
Project
diff --git a/atdb/taskdatabase/templates/taskdatabase/validation/tasks.html b/atdb/taskdatabase/templates/taskdatabase/validation/tasks.html
index b4517b9f..981e77c0 100644
--- a/atdb/taskdatabase/templates/taskdatabase/validation/tasks.html
+++ b/atdb/taskdatabase/templates/taskdatabase/validation/tasks.html
@@ -5,6 +5,13 @@
<div class="row">
<tr class="{{ task.status }}">
<td>{{ task.sas_id }}</td>
+ <td>
+ <a class="open-modal btn btn-primary btn-sm"
+ href="{% url 'workflow-details' task.workflow.id %}"
+ data-popup-url="{% url 'workflow-details' task.workflow.id %}"
+ target="_blank"><i class="fas fa-project-diagram"></i> {{ task.workflow.id }}
+ </a></td>
+ </td>
<td>{{ task.project }}</td>
--
GitLab