diff --git a/atdb/taskdatabase/services/algorithms.py b/atdb/taskdatabase/services/algorithms.py
index b4bd4905dd6439603a8d91d15c38e21164b2e85d..1bec39fa35e77fbf4f6fb483cc2a1a1870cc733c 100644
--- a/atdb/taskdatabase/services/algorithms.py
+++ b/atdb/taskdatabase/services/algorithms.py
@@ -9,7 +9,7 @@ import base64
from datetime import datetime
from django.db.models import Q, Sum
import logging
-from .common import timeit
+from .common import timeit, get_summary_flavour, SummaryFlavour
from django.urls import reverse
from ..models import Task, LogEntry, Workflow, Configuration
from django.conf import settings
@@ -83,7 +83,7 @@ def convert_logentries_to_html(log_entries):
results = ""
try:
- results += '<th>service</th><th>step</th><th>status</th><th width="200pixels">timestamp</th><th>cpu_cycles</th><th>wall_clock_time</th><th>log</th>'
+ results += '<th>service</th><th>step</th><th>status</th><th width="200px">timestamp</th><th>cpu_cycles</th><th>wall_clock_time</th><th>log</th>'
results += "<tbody>"
for log in log_entries:
line = "<tr><td><b>" + str(log.service) + '</b></td>'
@@ -799,7 +799,7 @@ def construct_inspectionplots(task, expand_image="False", source='task_id'):
return results
-def construct_summary(task):
+def construct_default_summary(task):
totals = ""
results = ""
@@ -937,3 +937,150 @@ def construct_summary(task):
results = title + totals + results
return results
+
+
+
+def construct_imaging_summary(task):
+
+ totals = ""
+ results = ""
+ total_size_to_process = 0
+ total_size_processed = 0
+ total_total_processing_time = 0
+
+ quality_values = {'poor': 0, 'moderate': 0, 'good': 0}
+
+ sas_id = task.sas_id
+ title = "<h4>Summary File for SAS_ID " + task.sas_id + "</h4> "
+
+ tasks = Task.objects.filter(sas_id=sas_id)
+
+ for task in tasks:
+
+ # skip 'suspended' and 'discarded' tasks
+ if task.status in ['suspended', 'discarded']:
+ continue
+
+ results += '<tr style="background-color:#7EB1C4"><td colspan="3"><b>Task ' + str(task.id) + '</b></td></tr>'
+
+ total_size_to_process += task.size_to_process
+ total_size_processed += task.size_processed
+ total_total_processing_time += task.total_processing_time
+
+ # find the summary in the quality json structure
+ try:
+ summary = task.quality_json["summary"]
+ results += '<tr><td><b>size_to_process</b></td><td>' + str(task.size_to_process) + '</td></tr>'
+ results += '<tr><td><b>size_processed</b></td><td>' + str(task.size_processed) + '</td></tr>'
+
+ results += '<tr><td><b>applied_fixes</b></td><td>' + str(summary['applied_fixes']) + '</td></tr>'
+ results += '<tr><td><b>rfi_perc_total</b></td><td>' + str(summary['rfi_perc_total']) + '</td></tr>'
+ results += '<tr><td><b>elevation_score</b></td><td>' + str(summary['elevation_score']) + '</td></tr>'
+ results += '<tr><td><b>sun_interference</b></td><td>' + str(summary['sun_interference']) + '</td></tr>'
+ results += '<tr><td><b>unfixable_issues</b></td><td>' + str(summary['unfixable_issues']) + '</td></tr>'
+ results += '<tr><td><b>moon_interference</b></td><td>' + str(summary['moon_interference']) + '</td></tr>'
+ results += '<tr><td><b>jupiter_interference</b></td><td>' + str(summary['jupiter_interference']) + '</td></tr>'
+ results += '<tr><td><b>degree_incompleteness_array</b></td><td>' + str(summary['degree_incompleteness_array']) + '</td></tr>'
+ results += '<tr><td><b>array_missing_important_pairs_is</b></td><td>' + str(summary['array_missing_important_pairs_is']) + '</td></tr>'
+ results += '<tr><td><b>array_missing_important_pairs_dutch</b></td><td>' + str(summary['array_missing_important_pairs_dutch']) + '</td></tr>'
+ results += '<tr><td><b>aggregated_array_data_losses_percentage</b></td><td>' + str(summary['aggregated_array_data_losses_percentage']) + '</td></tr>'
+ results += '<tr><td><b>array_high_data_loss_on_is_important_pair</b></td><td>' + str(summary['array_high_data_loss_on_is_important_pair']) + '</td></tr>'
+ results += '<tr><td><b>array_high_data_loss_on_dutch_important_pair</b></td><td>' + str(summary['array_high_data_loss_on_dutch_important_pair']) + '</td></tr>'
+
+ results += '<tr style="background-color:#7EB100"><td colspan="3"><b>Details</b></td></tr>'
+ details = summary["details"]
+
+ results += '<tr><td><b>Antenna configuration</b></td><td>' + str(details['antenna_configuration']) + '</td></tr>'
+ results += '<tr><td><b>Antennas not available</b></td><td>' + str(details['antennas_not_available']) + '</td></tr>'
+
+
+ if 'rfi_percentage' in details:
+ # add RFI percentage (if present)
+ rfi = details['rfi_percentage']
+ results += '<tr><td><b>RFI percentage</b></td>'
+ results += '<td colspan="2">' + str(rfi) + '</td>'
+ results += '</tr>'
+
+ try:
+ # add calculated quality (if present)
+ calculated_qualities = task.calculated_qualities
+ if calculated_qualities:
+ task_quality = calculated_qualities['per_task']
+
+ results += '<tr><td><b>Calculated Quality</b></td>'
+ results += '<td class="' + task_quality + '">' + str(task_quality) + '</td>'
+ results += '</tr>'
+
+ except:
+ pass
+
+ try:
+ key = task.calculated_qualities['per_task']
+ quality_values[key] = quality_values[key] + 1
+ except:
+ # ignore the tasks that have no calculated quality.
+ pass
+
+ results += '<tr style="background-color:#7EB100"><td colspan="3"><b>Target</b></td></tr>'
+ results += f'<tr><td>{details["target"]}</td></tr>'
+
+ results += '<tr style="background-color:#7EB100"><td colspan="3"><b>Pointing</b></td></tr>'
+ results += f'<tr><td colspan="3">{details["pointing"]}</td></tr>'
+
+ stations = details["DStDev"]
+ results += '<tr style="background-color:#7EB100"><td colspan="3"><b>Stations</b></td></tr>'
+ results += f'<tr><td colspan="3">{stations}</td></tr>'
+
+ antennas = details["antennas"]
+ results += '<tr style="background-color:#7EB100"><td colspan="3"><b>Antennas</b></td></tr>'
+ results += f'<tr><td colspan="3">{antennas}</td></tr>'
+
+
+ except:
+ pass
+
+ totals += '<th>Totals</th><th></th><th width="35%"></th>'
+ try:
+
+ totals += '<tr><td><b>Size to process</b></td colspan="2"><td>' + str(total_size_to_process) + '</td></tr>'
+ totals += '<tr><td><b>Size processed</b><td colspan="2">' + str(total_size_processed) + '</td></tr>'
+
+ try:
+ # add calculated quality per sasid (if present)
+ if calculated_qualities:
+ sasid_quality = calculated_qualities['per_sasid']
+ totals += '<tr><td><b>Calculated Quality</b></td>'
+ totals += '<td colspan="2" class="' + sasid_quality + '">' + str(sasid_quality) + '</td></tr>'
+
+ try:
+ quality_thresholds = json.loads(Configuration.objects.get(key='quality_thresholds').value)
+
+ totals += '<tr>'
+ totals += '<td><b>RFI thresholds</b></td>'
+ totals += '<td colspan="2">M, rfi>'+ str(quality_thresholds['poor']) + '% = P, rfi<=' + str(quality_thresholds['moderate']) + '% = G</td>'
+ totals += '</tr>'
+
+ except:
+ pass
+ except:
+ pass
+
+ except:
+ pass
+
+ results = title + totals + results
+ return results
+
+
+def construct_summary(task):
+
+ summary_flavour = get_summary_flavour(task)
+
+ # construct the appropriate summary html
+ if summary_flavour == SummaryFlavour.DEFAULT.value:
+ html = construct_default_summary(task)
+
+ elif summary_flavour == SummaryFlavour.IMAGING.value:
+ html = construct_imaging_summary(task)
+
+ return html
\ No newline at end of file
diff --git a/atdb/taskdatabase/services/calculated_qualities.py b/atdb/taskdatabase/services/calculated_qualities.py
index e7d518f46052c294c19677603d7ddb964defb967..164a0dcc7881205efc722da6eb5100f6e6fc3ec5 100644
--- a/atdb/taskdatabase/services/calculated_qualities.py
+++ b/atdb/taskdatabase/services/calculated_qualities.py
@@ -1,6 +1,8 @@
import logging
+from .common import get_summary_flavour, SummaryFlavour
logger = logging.getLogger(__name__)
+
def calculate_qualities(task, tasks_for_this_sasid, quality_thresholds):
""""
calculate the quality for this task, but also the quality for all the combined tasks of this sas_id
@@ -22,15 +24,23 @@ def calculate_qualities(task, tasks_for_this_sasid, quality_thresholds):
summary = task.quality_json["summary"]
quality = None
- for key in summary:
- record = summary[key]
- rfi_percent = int(record['rfi_percent'])
- if rfi_percent > 0:
- quality = "good"
- if rfi_percent >= quality_thresholds['moderate']:
- quality = "moderate"
- if rfi_percent > quality_thresholds['poor']:
- quality = "poor"
+ summary_flavour = get_summary_flavour(task)
+
+ if summary_flavour == SummaryFlavour.IMAGING.value:
+ rfi_percent = float(summary['details']['rfi_percentage'])
+
+ if summary_flavour == SummaryFlavour.DEFAULT.value:
+ # there is 1 key, but it is a filename which not known
+ for key in summary:
+ record = summary[key]
+ rfi_percent = float(record['rfi_percent'])
+
+ if rfi_percent > 0:
+ quality = "good"
+ if rfi_percent >= quality_thresholds['moderate']:
+ quality = "moderate"
+ if rfi_percent > quality_thresholds['poor']:
+ quality = "poor"
return quality
except Exception as error:
diff --git a/atdb/taskdatabase/services/common.py b/atdb/taskdatabase/services/common.py
index 7aa0aa258040959cff4e562ff906a53877d11ff0..21a13e5c34a1b57aec8f0ca52c24f52c0685e8ee 100644
--- a/atdb/taskdatabase/services/common.py
+++ b/atdb/taskdatabase/services/common.py
@@ -4,9 +4,14 @@ common helper functions
import logging;
from datetime import *
import time
+from enum import Enum
logger = logging.getLogger(__name__)
+class SummaryFlavour(Enum):
+ DEFAULT = "default"
+ IMAGING = "imaging"
+
# this is a decorator that can be put in front (around) a function all to measure its execution time
def timeit(method):
def timed(*args, **kw):
@@ -20,4 +25,27 @@ def timeit(method):
print('execution time: %r %2.2f ms' % \
(method.__name__, (te - ts) * 1000))
return result
- return timed
\ No newline at end of file
+ return timed
+
+def get_summary_flavour(task):
+ """
+ # not every workflow has the same summary structure
+ # determine the flavour based on the selected task, and construct the html accordingly
+ # this could be made implicit in the future by adding a setting to the Workflow, but currently it is derived
+ """
+ summary_flavour = "default"
+
+ try:
+ summary = task.quality_json["summary"]
+ except:
+ # no summary found
+ return None
+
+ try:
+ d = summary["details"]
+ summary_flavour = "imaging"
+ except:
+ # this is not an imaging summary, continue with the default
+ pass
+
+ return summary_flavour
\ No newline at end of file
diff --git a/atdb/taskdatabase/templates/taskdatabase/index.html b/atdb/taskdatabase/templates/taskdatabase/index.html
index 0c25d8e8e45694026eb2d357ad1d7f7c02fc95f8..01a7d0135691539899809a5b7ea838612e3800c5 100644
--- a/atdb/taskdatabase/templates/taskdatabase/index.html
+++ b/atdb/taskdatabase/templates/taskdatabase/index.html
@@ -31,7 +31,7 @@
{% include 'taskdatabase/pagination.html' %}
</div>
</div>
- <p class="footer"> Version 22 Aug 2023
+ <p class="footer"> Version 28 Aug 2023
</div>
{% include 'taskdatabase/refresh.html' %}