From 150a7d52d1a2d628f731d5f87eb684bfd0d60d1c Mon Sep 17 00:00:00 2001
From: Vermaas <vermaas@astron.nl>
Date: Thu, 31 Aug 2023 14:02:25 +0200
Subject: [PATCH] added summary info for LINC pipelines also
---
atdb/taskdatabase/services/algorithms.py | 67 ++++++++++++++++++-
.../services/calculated_qualities.py | 5 +-
atdb/taskdatabase/services/common.py | 17 ++++-
.../templates/taskdatabase/index.html | 2 +-
.../taskdatabase/validation/tasks.html | 4 +-
5 files changed, 85 insertions(+), 10 deletions(-)
diff --git a/atdb/taskdatabase/services/algorithms.py b/atdb/taskdatabase/services/algorithms.py
index 1bec39fa..42536608 100644
--- a/atdb/taskdatabase/services/algorithms.py
+++ b/atdb/taskdatabase/services/algorithms.py
@@ -938,8 +938,6 @@ def construct_default_summary(task):
return results
-
-
def construct_imaging_summary(task):
totals = ""
@@ -1072,6 +1070,63 @@ def construct_imaging_summary(task):
return results
+def construct_linc_summary(task):
+ totals = ""
+ results = ""
+ total_size_to_process = 0
+ total_size_processed = 0
+ total_total_processing_time = 0
+
+ quality_values = {'poor': 0, 'moderate': 0, 'good': 0}
+
+ sas_id = task.sas_id
+ title = "<h4>Summary File for SAS_ID " + task.sas_id + "</h4> "
+
+ tasks = Task.objects.filter(sas_id=sas_id)
+
+ for task in tasks:
+
+ # skip 'suspended' and 'discarded' tasks
+ if task.status in ['suspended', 'discarded']:
+ continue
+
+ results += '<tr style="background-color:#7EB1C4"><td colspan="3"><b>Task ' + str(task.id) + '</b></td></tr>'
+
+ total_size_to_process += task.size_to_process
+ total_size_processed += task.size_processed
+ total_total_processing_time += task.total_processing_time
+
+ # find the summary in the quality json structure
+ try:
+ summary = task.quality_json["summary"]
+ results += '<tr><td><b>size_to_process</b></td><td>' + str(task.size_to_process) + '</td></tr>'
+ results += '<tr><td><b>size_processed</b></td><td>' + str(task.size_processed) + '</td></tr>'
+
+ results += '<tr><td><b>basename</b></td><td>' + str(summary['basename']) + '</td></tr>'
+ results += '<tr><td><b>checksum</b></td><td>' + str(summary['checksum']) + '</td></tr>'
+ results += '<tr><td><b>class</b></td><td>' + str(summary['class']) + '</td></tr>'
+ results += '<tr><td><b>location</b></td><td>' + str(summary['location']) + '</td></tr>'
+ results += '<tr><td><b>nameext</b></td><td>' + str(summary['nameext']) + '</td></tr>'
+ results += '<tr><td><b>nameroot</b></td><td>' + str(summary['nameroot']) + '</td></tr>'
+ results += '<tr><td><b>size</b></td><td>' + str(summary['size']) + '</td></tr>'
+ results += '<tr><td><b>surl</b></td><td>' + str(summary['surl']) + '</td></tr>'
+
+ except:
+ pass
+
+ totals += '<th>Totals</th><th></th><th width="35%"></th>'
+ try:
+
+ totals += '<tr><td><b>Size to process</b></td colspan="2"><td>' + str(total_size_to_process) + '</td></tr>'
+ totals += '<tr><td><b>Size processed</b><td colspan="2">' + str(total_size_processed) + '</td></tr>'
+
+ except:
+ pass
+
+ results = title + totals + results
+ return results
+
+
def construct_summary(task):
summary_flavour = get_summary_flavour(task)
@@ -1080,7 +1135,13 @@ def construct_summary(task):
if summary_flavour == SummaryFlavour.DEFAULT.value:
html = construct_default_summary(task)
- elif summary_flavour == SummaryFlavour.IMAGING.value:
+ elif summary_flavour == SummaryFlavour.IMAGING_COMPRESSION.value:
html = construct_imaging_summary(task)
+ elif summary_flavour == SummaryFlavour.LINC_CALIBRATOR.value:
+ html = construct_linc_summary(task)
+
+ elif summary_flavour == SummaryFlavour.LINC_TARGET.value:
+ html = construct_linc_summary(task)
+
return html
\ No newline at end of file
diff --git a/atdb/taskdatabase/services/calculated_qualities.py b/atdb/taskdatabase/services/calculated_qualities.py
index 164a0dcc..cad619b0 100644
--- a/atdb/taskdatabase/services/calculated_qualities.py
+++ b/atdb/taskdatabase/services/calculated_qualities.py
@@ -26,7 +26,7 @@ def calculate_qualities(task, tasks_for_this_sasid, quality_thresholds):
summary_flavour = get_summary_flavour(task)
- if summary_flavour == SummaryFlavour.IMAGING.value:
+ if summary_flavour == SummaryFlavour.IMAGING_COMPRESSION.value:
rfi_percent = float(summary['details']['rfi_percentage'])
if summary_flavour == SummaryFlavour.DEFAULT.value:
@@ -35,6 +35,7 @@ def calculate_qualities(task, tasks_for_this_sasid, quality_thresholds):
record = summary[key]
rfi_percent = float(record['rfi_percent'])
+
if rfi_percent > 0:
quality = "good"
if rfi_percent >= quality_thresholds['moderate']:
@@ -44,6 +45,8 @@ def calculate_qualities(task, tasks_for_this_sasid, quality_thresholds):
return quality
except Exception as error:
+ # when rfi_percentage is missing, then the quality cannot be calculated.
+ # Just continue without it
pass
def calculate_quality_sasid(unsaved_task, tasks_for_this_sasid):
diff --git a/atdb/taskdatabase/services/common.py b/atdb/taskdatabase/services/common.py
index 21a13e5c..eae0d8dd 100644
--- a/atdb/taskdatabase/services/common.py
+++ b/atdb/taskdatabase/services/common.py
@@ -10,7 +10,9 @@ logger = logging.getLogger(__name__)
class SummaryFlavour(Enum):
DEFAULT = "default"
- IMAGING = "imaging"
+ LINC_CALIBRATOR = "linc_calibrator"
+ LINC_TARGET = "linc_target"
+ IMAGING_COMPRESSION = "imaging_compression"
# this is a decorator that can be put in front (around) a function all to measure its execution time
def timeit(method):
@@ -33,7 +35,7 @@ def get_summary_flavour(task):
# determine the flavour based on the selected task, and construct the html accordingly
# this could be made implicit in the future by adding a setting to the Workflow, but currently it is derived
"""
- summary_flavour = "default"
+ summary_flavour = SummaryFlavour.DEFAULT.value
try:
summary = task.quality_json["summary"]
@@ -41,9 +43,18 @@ def get_summary_flavour(task):
# no summary found
return None
+ # so... yikes... what distinguishes the summary flavours?
+ workflow_uri = task.workflow.workflow_uri
+
+ # for linc, look at the workflow_uri
+ if "linc_calibrator" in workflow_uri:
+ return SummaryFlavour.LINC_CALIBRATOR.value
+ if "linc_target" in workflow_uri:
+ return SummaryFlavour.LINC_TARGET.value
+
try:
d = summary["details"]
- summary_flavour = "imaging"
+ summary_flavour = SummaryFlavour.IMAGING_COMPRESSION.value
except:
# this is not an imaging summary, continue with the default
pass
diff --git a/atdb/taskdatabase/templates/taskdatabase/index.html b/atdb/taskdatabase/templates/taskdatabase/index.html
index 01a7d013..968c2aa5 100644
--- a/atdb/taskdatabase/templates/taskdatabase/index.html
+++ b/atdb/taskdatabase/templates/taskdatabase/index.html
@@ -31,7 +31,7 @@
{% include 'taskdatabase/pagination.html' %}
</div>
</div>
- <p class="footer"> Version 28 Aug 2023
+ <p class="footer"> Version 31 Aug 2023
</div>
{% include 'taskdatabase/refresh.html' %}
diff --git a/atdb/taskdatabase/templates/taskdatabase/validation/tasks.html b/atdb/taskdatabase/templates/taskdatabase/validation/tasks.html
index 5077a5d5..50a645d4 100644
--- a/atdb/taskdatabase/templates/taskdatabase/validation/tasks.html
+++ b/atdb/taskdatabase/templates/taskdatabase/validation/tasks.html
@@ -29,10 +29,10 @@
<img src="{% static 'taskdatabase/surfsara.jpg' %}" height="20" alt="inspection plots">
</a>
- <a class="open-modal btn btn-warning btn-sm"
+<!-- <a class="open-modal btn btn-warning btn-sm"
data-popup-url="{% url 'inspection-plots-sasid' task.id True %}">
<i class="fas fa-images"></i>
- </a>
+ </a> -->
</td>
{% else %}
--
GitLab