Merge branch 'SDC-1005-multiple-summary-structures' into 'master'

SDC 1005 multiple summary structures

See merge request !320

atdb/taskdatabase/services/algorithms.py

@@ -9,7 +9,7 @@ import base64
 from datetime import datetime
 from django.db.models import Q, Sum
 import logging
-from .common import timeit
+from .common import timeit, get_summary_flavour, SummaryFlavour
 from django.urls import reverse
 from ..models import Task, LogEntry, Workflow, Configuration
 from django.conf import settings
@@ -83,7 +83,7 @@ def convert_logentries_to_html(log_entries):
     results = ""
 
     try:
-        results += '<th>service</th><th>step</th><th>status</th><th width="200pixels">timestamp</th><th>cpu_cycles</th><th>wall_clock_time</th><th>log</th>'
+        results += '<th>service</th><th>step</th><th>status</th><th width="200px">timestamp</th><th>cpu_cycles</th><th>wall_clock_time</th><th>log</th>'
         results += "<tbody>"
         for log in log_entries:
             line = "<tr><td><b>" + str(log.service) + '</b></td>'
@@ -799,7 +799,7 @@ def construct_inspectionplots(task, expand_image="False", source='task_id'):
     return results
 
 
-def construct_summary(task):
+def construct_default_summary(task):
 
     totals = ""
     results = ""
@@ -937,3 +937,150 @@ def construct_summary(task):
     results = title + totals + results
     return results
 
+
+
+
+def construct_imaging_summary(task):
+
+    totals = ""
+    results = ""
+    total_size_to_process = 0
+    total_size_processed = 0
+    total_total_processing_time = 0
+
+    quality_values = {'poor': 0, 'moderate': 0, 'good': 0}
+
+    sas_id = task.sas_id
+    title = "<h4>Summary File for SAS_ID " + task.sas_id + "</h4> "
+
+    tasks = Task.objects.filter(sas_id=sas_id)
+
+    for task in tasks:
+
+        # skip 'suspended' and 'discarded' tasks
+        if task.status in ['suspended', 'discarded']:
+            continue
+
+        results += '<tr style="background-color:#7EB1C4"><td colspan="3"><b>Task ' + str(task.id) + '</b></td></tr>'
+
+        total_size_to_process += task.size_to_process
+        total_size_processed += task.size_processed
+        total_total_processing_time += task.total_processing_time
+
+        # find the summary in the quality json structure
+        try:
+            summary = task.quality_json["summary"]
+            results += '<tr><td><b>size_to_process</b></td><td>' + str(task.size_to_process) + '</td></tr>'
+            results += '<tr><td><b>size_processed</b></td><td>' + str(task.size_processed) + '</td></tr>'
+
+            results += '<tr><td><b>applied_fixes</b></td><td>' + str(summary['applied_fixes']) + '</td></tr>'
+            results += '<tr><td><b>rfi_perc_total</b></td><td>' + str(summary['rfi_perc_total']) + '</td></tr>'
+            results += '<tr><td><b>elevation_score</b></td><td>' + str(summary['elevation_score']) + '</td></tr>'
+            results += '<tr><td><b>sun_interference</b></td><td>' + str(summary['sun_interference']) + '</td></tr>'
+            results += '<tr><td><b>unfixable_issues</b></td><td>' + str(summary['unfixable_issues']) + '</td></tr>'
+            results += '<tr><td><b>moon_interference</b></td><td>' + str(summary['moon_interference']) + '</td></tr>'
+            results += '<tr><td><b>jupiter_interference</b></td><td>' + str(summary['jupiter_interference']) + '</td></tr>'
+            results += '<tr><td><b>degree_incompleteness_array</b></td><td>' + str(summary['degree_incompleteness_array']) + '</td></tr>'
+            results += '<tr><td><b>array_missing_important_pairs_is</b></td><td>' + str(summary['array_missing_important_pairs_is']) + '</td></tr>'
+            results += '<tr><td><b>array_missing_important_pairs_dutch</b></td><td>' + str(summary['array_missing_important_pairs_dutch']) + '</td></tr>'
+            results += '<tr><td><b>aggregated_array_data_losses_percentage</b></td><td>' + str(summary['aggregated_array_data_losses_percentage']) + '</td></tr>'
+            results += '<tr><td><b>array_high_data_loss_on_is_important_pair</b></td><td>' + str(summary['array_high_data_loss_on_is_important_pair']) + '</td></tr>'
+            results += '<tr><td><b>array_high_data_loss_on_dutch_important_pair</b></td><td>' + str(summary['array_high_data_loss_on_dutch_important_pair']) + '</td></tr>'
+
+            results += '<tr style="background-color:#7EB100"><td colspan="3"><b>Details</b></td></tr>'
+            details = summary["details"]
+
+            results += '<tr><td><b>Antenna configuration</b></td><td>' + str(details['antenna_configuration']) + '</td></tr>'
+            results += '<tr><td><b>Antennas not available</b></td><td>' + str(details['antennas_not_available']) + '</td></tr>'
+
+
+            if 'rfi_percentage' in details:
+                # add RFI percentage (if present)
+                rfi = details['rfi_percentage']
+                results += '<tr><td><b>RFI percentage</b></td>'
+                results += '<td colspan="2">' + str(rfi) + '</td>'
+                results += '</tr>'
+
+            try:
+                # add calculated quality (if present)
+                calculated_qualities = task.calculated_qualities
+                if calculated_qualities:
+                    task_quality = calculated_qualities['per_task']
+
+                    results += '<tr><td><b>Calculated Quality</b></td>'
+                    results += '<td class="' + task_quality + '">' + str(task_quality) + '</td>'
+                    results += '</tr>'
+
+            except:
+                pass
+
+            try:
+                key = task.calculated_qualities['per_task']
+                quality_values[key] = quality_values[key] + 1
+            except:
+                # ignore the tasks that have no calculated quality.
+                pass
+
+            results += '<tr style="background-color:#7EB100"><td colspan="3"><b>Target</b></td></tr>'
+            results += f'<tr><td>{details["target"]}</td></tr>'
+
+            results += '<tr style="background-color:#7EB100"><td colspan="3"><b>Pointing</b></td></tr>'
+            results += f'<tr><td colspan="3">{details["pointing"]}</td></tr>'
+
+            stations = details["DStDev"]
+            results += '<tr style="background-color:#7EB100"><td colspan="3"><b>Stations</b></td></tr>'
+            results += f'<tr><td colspan="3">{stations}</td></tr>'
+
+            antennas = details["antennas"]
+            results += '<tr style="background-color:#7EB100"><td colspan="3"><b>Antennas</b></td></tr>'
+            results += f'<tr><td colspan="3">{antennas}</td></tr>'
+
+
+        except:
+            pass
+
+    totals += '<th>Totals</th><th></th><th width="35%"></th>'
+    try:
+
+        totals += '<tr><td><b>Size to process</b></td  colspan="2"><td>' + str(total_size_to_process) + '</td></tr>'
+        totals += '<tr><td><b>Size processed</b><td  colspan="2">' + str(total_size_processed) + '</td></tr>'
+
+        try:
+            # add calculated quality per sasid (if present)
+            if calculated_qualities:
+                sasid_quality = calculated_qualities['per_sasid']
+                totals += '<tr><td><b>Calculated Quality</b></td>'
+                totals += '<td colspan="2" class="' + sasid_quality + '">' + str(sasid_quality) + '</td></tr>'
+
+                try:
+                    quality_thresholds = json.loads(Configuration.objects.get(key='quality_thresholds').value)
+
+                    totals += '<tr>'
+                    totals += '<td><b>RFI thresholds</b></td>'
+                    totals += '<td colspan="2">M, rfi>'+ str(quality_thresholds['poor']) + '% = P, rfi<=' + str(quality_thresholds['moderate']) + '% = G</td>'
+                    totals += '</tr>'
+
+                except:
+                    pass
+        except:
+            pass
+
+    except:
+        pass
+
+    results = title + totals + results
+    return results
+
+
+def construct_summary(task):
+
+    summary_flavour = get_summary_flavour(task)
+
+    # construct the appropriate summary html
+    if summary_flavour == SummaryFlavour.DEFAULT.value:
+        html = construct_default_summary(task)
+
+    elif summary_flavour == SummaryFlavour.IMAGING.value:
+        html = construct_imaging_summary(task)
+
+    return html
\ No newline at end of file

atdb/taskdatabase/services/calculated_qualities.py

 import logging
+from .common import get_summary_flavour, SummaryFlavour
 logger = logging.getLogger(__name__)
 
+
 def calculate_qualities(task, tasks_for_this_sasid, quality_thresholds):
     """"
     calculate the quality for this task, but also the quality for all the combined tasks of this sas_id
@@ -22,9 +24,17 @@ def calculate_qualities(task, tasks_for_this_sasid, quality_thresholds):
             summary = task.quality_json["summary"]
             quality = None
 
+            summary_flavour = get_summary_flavour(task)
+
+            if summary_flavour == SummaryFlavour.IMAGING.value:
+                rfi_percent = float(summary['details']['rfi_percentage'])
+
+            if summary_flavour == SummaryFlavour.DEFAULT.value:
+                # there is 1 key, but it is a filename which not known
                 for key in summary:
                     record = summary[key]
-                rfi_percent = int(record['rfi_percent'])
+                    rfi_percent = float(record['rfi_percent'])
+
             if rfi_percent > 0:
                 quality = "good"
                 if rfi_percent >= quality_thresholds['moderate']:

atdb/taskdatabase/services/common.py

@@ -4,9 +4,14 @@ common helper functions
 import logging;
 from datetime import *
 import time
+from enum import Enum
 
 logger = logging.getLogger(__name__)
 
+class SummaryFlavour(Enum):
+    DEFAULT = "default"
+    IMAGING = "imaging"
+
 # this is a decorator that can be put in front (around) a function all to measure its execution time
 def timeit(method):
     def timed(*args, **kw):
@@ -21,3 +26,26 @@ def timeit(method):
                   (method.__name__, (te - ts) * 1000))
         return result
     return timed
+
+def get_summary_flavour(task):
+    """
+    # not every workflow has the same summary structure
+    # determine the flavour based on the selected task, and construct the html accordingly
+    # this could be made implicit in the future by adding a setting to the Workflow, but currently it is derived
+    """
+    summary_flavour = "default"
+
+    try:
+        summary = task.quality_json["summary"]
+    except:
+        # no summary found
+        return None
+
+    try:
+        d = summary["details"]
+        summary_flavour = "imaging"
+    except:
+        # this is not an imaging summary, continue with the default
+        pass
+
+    return summary_flavour
\ No newline at end of file

atdb/taskdatabase/templates/taskdatabase/index.html

@@ -31,7 +31,7 @@
             {% include 'taskdatabase/pagination.html' %}
         </div>
     </div>
-    <p class="footer"> Version 22 Aug 2023
+    <p class="footer"> Version 28 Aug 2023
 </div>
 
 {% include 'taskdatabase/refresh.html' %}