Skip to content
Snippets Groups Projects

SDC 1005 multiple summary structures

Merged SDC 1005 multiple summary structures
SDC-1005-multiple-summary-structures into master
Merged Nico Vermaas requested to merge SDC-1005-multiple-summary-structures into master
Compare
and
4 files
+ 199
− 14
Compare changes
  • Side-by-side
  • Inline
Files
4
atdb/taskdatabase/services/algorithms.py
+ 150
− 3
@@ -9,7 +9,7 @@ import base64
@@ -9,7 +9,7 @@ import base64
from datetime import datetime
from datetime import datetime
from django.db.models import Q, Sum
from django.db.models import Q, Sum
import logging
import logging
from .common import timeit
from .common import timeit, get_summary_flavour, SummaryFlavour
from django.urls import reverse
from django.urls import reverse
from ..models import Task, LogEntry, Workflow, Configuration
from ..models import Task, LogEntry, Workflow, Configuration
from django.conf import settings
from django.conf import settings
@@ -83,7 +83,7 @@ def convert_logentries_to_html(log_entries):
@@ -83,7 +83,7 @@ def convert_logentries_to_html(log_entries):
results = ""
results = ""
try:
try:
results += '<th>service</th><th>step</th><th>status</th><th width="200pixels">timestamp</th><th>cpu_cycles</th><th>wall_clock_time</th><th>log</th>'
results += '<th>service</th><th>step</th><th>status</th><th width="200px">timestamp</th><th>cpu_cycles</th><th>wall_clock_time</th><th>log</th>'
results += "<tbody>"
results += "<tbody>"
for log in log_entries:
for log in log_entries:
line = "<tr><td><b>" + str(log.service) + '</b></td>'
line = "<tr><td><b>" + str(log.service) + '</b></td>'
@@ -799,7 +799,7 @@ def construct_inspectionplots(task, expand_image="False", source='task_id'):
@@ -799,7 +799,7 @@ def construct_inspectionplots(task, expand_image="False", source='task_id'):
return results
return results
def construct_summary(task):
def construct_default_summary(task):
totals = ""
totals = ""
results = ""
results = ""
@@ -937,3 +937,150 @@ def construct_summary(task):
@@ -937,3 +937,150 @@ def construct_summary(task):
results = title + totals + results
results = title + totals + results
return results
return results
 
 
 
 
def construct_imaging_summary(task):
 
 
totals = ""
 
results = ""
 
total_size_to_process = 0
 
total_size_processed = 0
 
total_total_processing_time = 0
 
 
quality_values = {'poor': 0, 'moderate': 0, 'good': 0}
 
 
sas_id = task.sas_id
 
title = "<h4>Summary File for SAS_ID " + task.sas_id + "</h4> "
 
 
tasks = Task.objects.filter(sas_id=sas_id)
 
 
for task in tasks:
 
 
# skip 'suspended' and 'discarded' tasks
 
if task.status in ['suspended', 'discarded']:
 
continue
 
 
results += '<tr style="background-color:#7EB1C4"><td colspan="3"><b>Task ' + str(task.id) + '</b></td></tr>'
 
 
total_size_to_process += task.size_to_process
 
total_size_processed += task.size_processed
 
total_total_processing_time += task.total_processing_time
 
 
# find the summary in the quality json structure
 
try:
 
summary = task.quality_json["summary"]
 
results += '<tr><td><b>size_to_process</b></td><td>' + str(task.size_to_process) + '</td></tr>'
 
results += '<tr><td><b>size_processed</b></td><td>' + str(task.size_processed) + '</td></tr>'
 
 
results += '<tr><td><b>applied_fixes</b></td><td>' + str(summary['applied_fixes']) + '</td></tr>'
 
results += '<tr><td><b>rfi_perc_total</b></td><td>' + str(summary['rfi_perc_total']) + '</td></tr>'
 
results += '<tr><td><b>elevation_score</b></td><td>' + str(summary['elevation_score']) + '</td></tr>'
 
results += '<tr><td><b>sun_interference</b></td><td>' + str(summary['sun_interference']) + '</td></tr>'
 
results += '<tr><td><b>unfixable_issues</b></td><td>' + str(summary['unfixable_issues']) + '</td></tr>'
 
results += '<tr><td><b>moon_interference</b></td><td>' + str(summary['moon_interference']) + '</td></tr>'
 
results += '<tr><td><b>jupiter_interference</b></td><td>' + str(summary['jupiter_interference']) + '</td></tr>'
 
results += '<tr><td><b>degree_incompleteness_array</b></td><td>' + str(summary['degree_incompleteness_array']) + '</td></tr>'
 
results += '<tr><td><b>array_missing_important_pairs_is</b></td><td>' + str(summary['array_missing_important_pairs_is']) + '</td></tr>'
 
results += '<tr><td><b>array_missing_important_pairs_dutch</b></td><td>' + str(summary['array_missing_important_pairs_dutch']) + '</td></tr>'
 
results += '<tr><td><b>aggregated_array_data_losses_percentage</b></td><td>' + str(summary['aggregated_array_data_losses_percentage']) + '</td></tr>'
 
results += '<tr><td><b>array_high_data_loss_on_is_important_pair</b></td><td>' + str(summary['array_high_data_loss_on_is_important_pair']) + '</td></tr>'
 
results += '<tr><td><b>array_high_data_loss_on_dutch_important_pair</b></td><td>' + str(summary['array_high_data_loss_on_dutch_important_pair']) + '</td></tr>'
 
 
results += '<tr style="background-color:#7EB100"><td colspan="3"><b>Details</b></td></tr>'
 
details = summary["details"]
 
 
results += '<tr><td><b>Antenna configuration</b></td><td>' + str(details['antenna_configuration']) + '</td></tr>'
 
results += '<tr><td><b>Antennas not available</b></td><td>' + str(details['antennas_not_available']) + '</td></tr>'
 
 
 
if 'rfi_percentage' in details:
 
# add RFI percentage (if present)
 
rfi = details['rfi_percentage']
 
results += '<tr><td><b>RFI percentage</b></td>'
 
results += '<td colspan="2">' + str(rfi) + '</td>'
 
results += '</tr>'
 
 
try:
 
# add calculated quality (if present)
 
calculated_qualities = task.calculated_qualities
 
if calculated_qualities:
 
task_quality = calculated_qualities['per_task']
 
 
results += '<tr><td><b>Calculated Quality</b></td>'
 
results += '<td class="' + task_quality + '">' + str(task_quality) + '</td>'
 
results += '</tr>'
 
 
except:
 
pass
 
 
try:
 
key = task.calculated_qualities['per_task']
 
quality_values[key] = quality_values[key] + 1
 
except:
 
# ignore the tasks that have no calculated quality.
 
pass
 
 
results += '<tr style="background-color:#7EB100"><td colspan="3"><b>Target</b></td></tr>'
 
results += f'<tr><td>{details["target"]}</td></tr>'
 
 
results += '<tr style="background-color:#7EB100"><td colspan="3"><b>Pointing</b></td></tr>'
 
results += f'<tr><td colspan="3">{details["pointing"]}</td></tr>'
 
 
stations = details["DStDev"]
 
results += '<tr style="background-color:#7EB100"><td colspan="3"><b>Stations</b></td></tr>'
 
results += f'<tr><td colspan="3">{stations}</td></tr>'
 
 
antennas = details["antennas"]
 
results += '<tr style="background-color:#7EB100"><td colspan="3"><b>Antennas</b></td></tr>'
 
results += f'<tr><td colspan="3">{antennas}</td></tr>'
 
 
 
except:
 
pass
 
 
totals += '<th>Totals</th><th></th><th width="35%"></th>'
 
try:
 
 
totals += '<tr><td><b>Size to process</b></td colspan="2"><td>' + str(total_size_to_process) + '</td></tr>'
 
totals += '<tr><td><b>Size processed</b><td colspan="2">' + str(total_size_processed) + '</td></tr>'
 
 
try:
 
# add calculated quality per sasid (if present)
 
if calculated_qualities:
 
sasid_quality = calculated_qualities['per_sasid']
 
totals += '<tr><td><b>Calculated Quality</b></td>'
 
totals += '<td colspan="2" class="' + sasid_quality + '">' + str(sasid_quality) + '</td></tr>'
 
 
try:
 
quality_thresholds = json.loads(Configuration.objects.get(key='quality_thresholds').value)
 
 
totals += '<tr>'
 
totals += '<td><b>RFI thresholds</b></td>'
 
totals += '<td colspan="2">M, rfi>'+ str(quality_thresholds['poor']) + '% = P, rfi<=' + str(quality_thresholds['moderate']) + '% = G</td>'
 
totals += '</tr>'
 
 
except:
 
pass
 
except:
 
pass
 
 
except:
 
pass
 
 
results = title + totals + results
 
return results
 
 
 
def construct_summary(task):
 
 
summary_flavour = get_summary_flavour(task)
 
 
# construct the appropriate summary html
 
if summary_flavour == SummaryFlavour.DEFAULT.value:
 
html = construct_default_summary(task)
 
 
elif summary_flavour == SummaryFlavour.IMAGING.value:
 
html = construct_imaging_summary(task)
 
 
return html
 
\ No newline at end of file