diff --git a/atdb/taskdatabase/services/algorithms.py b/atdb/taskdatabase/services/algorithms.py
index cb761767f6a3fe47736ac4db54837ded4d18631c..5ccddf0d90366a11849be3c031c10ab5eb720f97 100644
--- a/atdb/taskdatabase/services/algorithms.py
+++ b/atdb/taskdatabase/services/algorithms.py
@@ -949,6 +949,13 @@ def construct_imaging_summary(task):
title = "<h4>Summary File for SAS_ID " + task.sas_id + "</h4> "
tasks = Task.objects.filter(sas_id=sas_id).order_by('task_type','id')
+
+ # isolate the aggregation task, which is to be used to read the 'quality_indicators from
+ try:
+ aggregation_task = tasks.filter(task_type='aggregation')[0]
+ except:
+ aggregation_task = None
+
for task in tasks:
try:
@@ -1044,11 +1051,18 @@ def construct_imaging_summary(task):
if task_quality:
results += '<tr><td><b>Calculated Quality</b></td>'
- results += '<td class="' + task_quality + '">' + str(task_quality) + '</td>'
+
+ # task_quality can also be a list of qualities... if so, use the first element as class
+ quality_class = task_quality
+ if isinstance(task_quality, list):
+ quality_class = task_quality[0]
+
+ results += '<td class="' + quality_class + '">' + str(task_quality) + '</td>'
results += '</tr>'
- except:
- pass
+ except Exception as error:
+ logger.info(error)
+ #pass
try:
key = task.calculated_qualities['per_task']
@@ -1091,19 +1105,12 @@ def construct_imaging_summary(task):
totals += '<tr><td><b>Calculated Quality</b></td>'
totals += '<td colspan="2" class="' + sasid_quality + '">' + str(sasid_quality) + '</td></tr>'
- try:
- quality_thresholds = json.loads(Configuration.objects.get(key='quality_thresholds').value)
- except:
- quality_thresholds = {
- "moderate": 20,
- "poor": 50,
- "overall_poor": 50,
- "overall_good": 90,
- }
- totals += '<tr>'
- totals += '<td><b>RFI thresholds</b></td>'
- totals += '<td colspan="2">M, rfi>'+ str(quality_thresholds['poor']) + '% = P, rfi<=' + str(quality_thresholds['moderate']) + '% = G</td>'
- totals += '</tr>'
+ if aggregation_task:
+ quality_indicators = aggregation_task.quality_json["summary"]['details']['quality_indicators']
+ totals += '<tr>'
+ totals += '<td><b>Quality Indicators</b></td>'
+ totals += f'<td colspan="2">{quality_indicators}</td>'
+ totals += '</tr>'
except:
pass
diff --git a/atdb/taskdatabase/services/calculated_qualities.py b/atdb/taskdatabase/services/calculated_qualities.py
index f07fbd813514ff68294a9c871efe004a16555b70..178494e9bed8dda87e45078e4f6835e2c0c85d51 100644
--- a/atdb/taskdatabase/services/calculated_qualities.py
+++ b/atdb/taskdatabase/services/calculated_qualities.py
@@ -68,7 +68,7 @@ def calculate_qualities(task, tasks_for_this_sasid, quality_thresholds):
if summary_flavour == SummaryFlavour.IMAGING_COMPRESSION.value:
quality_per_file = None
- # shortcut, if quality is already calculated by the workflow itself, then no need to recalculate
+ # override, if quality is already calculated by the workflow itself, then do not recalculate
try:
quality_from_summary = summary['details']['quality']
if quality_from_summary in ['poor', 'moderate', 'good']:
@@ -109,59 +109,76 @@ def calculate_qualities(task, tasks_for_this_sasid, quality_thresholds):
def calculate_quality_sasid(unsaved_task, tasks_for_this_sasid):
"""
calculate the overall quality per sas_id, based on other tasks with the same sas_id
- The threshold values are written from a configuration json blob
+
+ For the imaging compression pipeline (IMAGING_COMPRESSION flavour) this value is read from the aggregation task
+ For other pipelines it is calculated based on threshold values stored in ATDB
+
+ The threshold values are read from a configuration json blob
Using this algorithm from SDCO:
if more then 90 % of all files have a good quality then the dataset has good condition.
If more then 50 % of all files have a poor quality then the dataset is poor
otherwise is moderate.
"""
- try:
- # gather the results of all the calculated_quality values for this sas_id
- qualities = {'poor': 0, 'moderate': 0, 'good': 0}
- # also add the currently unsaved task to the list for the quality calculation per sas_id
- tasks = list(tasks_for_this_sasid)
- tasks.append(unsaved_task)
+ summary_flavour = get_summary_flavour(unsaved_task)
- for task in tasks:
+ if summary_flavour == SummaryFlavour.IMAGING_COMPRESSION.value:
+ # because the quality is read from the aggregation task, only the aggregation task needs to be checked
+ if unsaved_task.task_type == 'aggregation':
+ summary = unsaved_task.quality_json["summary"]
+ quality_from_summary = summary['details']['quality']
+ return quality_from_summary
+ else:
+ return None
- # skip 'suspended' and 'discarded' tasks
- if task.status in ['suspended', 'discarded']:
- continue
+ if summary_flavour == SummaryFlavour.DEFAULT.value:
+ try:
+ # gather the results of all the calculated_quality values for this sas_id
+ qualities = {'poor': 0, 'moderate': 0, 'good': 0}
- # because this all happens in the overridden 'Task.save', the actual saving has not yet occurred
- # So use the calculated quality from the unsaved task instead.
- if task.id == unsaved_task.id:
- t = unsaved_task
- else:
- t = task
+ # also add the currently unsaved task to the list for the quality calculation per sas_id
+ tasks = list(tasks_for_this_sasid)
+ tasks.append(unsaved_task)
- try:
- qualities = unpack_qualities_per_task(t, qualities)
+ for task in tasks:
- except:
- # ignore the tasks that have no calculated quality.
- pass
+ # skip 'suspended' and 'discarded' tasks
+ if task.status in ['suspended', 'discarded']:
+ continue
+ # because this all happens in the overridden 'Task.save', the actual saving has not yet occurred
+ # So use the calculated quality from the unsaved task instead.
+ if task.id == unsaved_task.id:
+ t = unsaved_task
+ else:
+ t = task
- total = qualities['poor'] + qualities['moderate'] + qualities['good']
- quality_sasid = None
- if total > 0:
- percentage_poor = (qualities['poor'] / total) * 100
- percentage_good = (qualities['good'] / total) * 100
- quality_sasid = "moderate"
+ try:
+ qualities = unpack_qualities_per_task(t, qualities)
- if percentage_poor >= quality_thresholds['overall_poor']:
- quality_sasid = 'poor'
+ except:
+ # ignore the tasks that have no calculated quality.
+ pass
- if percentage_good >= quality_thresholds['overall_good']:
- quality_sasid = 'good'
- return quality_sasid
+ total = qualities['poor'] + qualities['moderate'] + qualities['good']
+ quality_sasid = None
+ if total > 0:
+ percentage_poor = (qualities['poor'] / total) * 100
+ percentage_good = (qualities['good'] / total) * 100
+ quality_sasid = "moderate"
- except Exception as error:
- logger.error(error)
+ if percentage_poor >= quality_thresholds['overall_poor']:
+ quality_sasid = 'poor'
+
+ if percentage_good >= quality_thresholds['overall_good']:
+ quality_sasid = 'good'
+
+ return quality_sasid
+
+ except Exception as error:
+ logger.error(error)
# --- main function body ---
@@ -181,12 +198,13 @@ def calculate_qualities(task, tasks_for_this_sasid, quality_thresholds):
# update the overall quality of all tasks for this sas_id
calculated_quality_sasid = calculate_quality_sasid(task, tasks_for_this_sasid)
- # store the result in task.calculated_qualities (not yet saved in the database)
- qualities['per_sasid'] = calculated_quality_sasid
+ if calculated_quality_sasid:
+ # store the result in task.calculated_qualities (not yet saved in the database)
+ qualities['per_sasid'] = calculated_quality_sasid
- # store the result in the activity, and save it
- task.activity.calculated_quality = calculated_quality_sasid
- task.activity.save()
+ # store the result in the activity, and save it
+ task.activity.calculated_quality = calculated_quality_sasid
+ task.activity.save()
except Exception as error:
logger.error(error)
diff --git a/atdb/taskdatabase/tests/test_calculated_qualities.py b/atdb/taskdatabase/tests/test_calculated_qualities.py
index b06b8a5036da89e5e77b273e4dab36158fce8eea..319f22c291fa9a5a3c2ec499f2568a589d2c9630 100644
--- a/atdb/taskdatabase/tests/test_calculated_qualities.py
+++ b/atdb/taskdatabase/tests/test_calculated_qualities.py
@@ -51,6 +51,8 @@ class TestCalculatedQualities(TestCase):
workflow_imaging_compression = Workflow(workflow_uri="imaging_compress_pipeline_v011")
workflow_imaging_compression.save()
Task.objects.get_or_create(sas_id=55555, status='processed', outputs=outputs.imaging_compression_summary_flavor_with_rfi_1, workflow=workflow_imaging_compression)
+ Task.objects.get_or_create(sas_id=55555, task_type='aggregation', status='processed', outputs=outputs.imaging_compression_aggregation_task, workflow=workflow_imaging_compression)
+
# LINK pipelines (no rfi_percent onboard yet)
workflow_link_calibrator = Workflow(workflow_uri="linc_calibrator_v4_2")
@@ -75,7 +77,7 @@ class TestCalculatedQualities(TestCase):
if task.calculated_qualities['per_task']:
count += 1
- self.assertEqual(count,6)
+ self.assertEqual(count,7)
def test_calculated_qualities(self):
@@ -120,6 +122,41 @@ class TestCalculatedQualities(TestCase):
self.assertEqual(quality_per_sasid,'moderate')
+ def test_calculated_qualities_imaging_compression(self):
+ """
+ for imaging compression the qualities are read from the aggregation task, instead of calculated by ATDB
+
+ """
+
+ # read the quality thresholds from the test database
+ quality_thresholds = json.loads(Configuration.objects.get(key="quality_thresholds").value)
+ quality_indicators = None
+
+ # get the tasks for sas_id 54321
+ tasks_for_this_sasid = Task.objects.filter(sas_id=55555)
+
+ # run the algorithms and gather the values
+ quality_values = {'poor': 0, 'moderate': 0, 'good': 0}
+
+ for task in tasks_for_this_sasid:
+ q = qualities.calculate_qualities(task, tasks_for_this_sasid, quality_thresholds)
+ quality_values = qualities.unpack_qualities_per_task(task,quality_values)
+
+ if task.task_type=='aggregation':
+ quality_indicators = task.quality_json["summary"]['details']['quality_indicators']
+
+ try:
+ quality_per_sasid = task.calculated_qualities['per_sasid']
+ except:
+ # ignore the tasks that have no calculated quality.
+ pass
+
+ self.assertEqual(quality_values,{'poor': 0, 'moderate': 1, 'good': 1})
+
+ # assert values read from the aggregation task
+ self.assertEqual(quality_per_sasid,'moderate')
+ self.assertEqual(quality_indicators, 'rfi_limit:0.3; median_dataloss_limit:1.5; sun_angle_limit:20; moon_angle_limit:10; jupiter_angle_limit:5')
+
def test_calculated_qualities_with_optimistic_thresholds(self):
"""
calculate the quality per task and per sas_id based on rfi_percent values
diff --git a/atdb/taskdatabase/tests/test_calculated_qualities_outputs.py b/atdb/taskdatabase/tests/test_calculated_qualities_outputs.py
index 37332d077e9198eec986bdbda40620463ae61fdc..d3725e5ee893d4604c303f1743c40f4b68d1f901 100644
--- a/atdb/taskdatabase/tests/test_calculated_qualities_outputs.py
+++ b/atdb/taskdatabase/tests/test_calculated_qualities_outputs.py
@@ -370,6 +370,183 @@ imaging_compression_summary_flavor_with_rfi_1 = {
},
}
+
+imaging_compression_aggregation_task = {
+ "quality": {
+ "summary": {
+ "details": {
+ "DStDev": {
+ "CS001": 15372035.9671943,
+ "CS002": 14858111.10350275,
+ "CS003": 10147611.046423668,
+ "CS004": 18980165.035334244,
+ "CS005": 9209186.769605417,
+ "CS006": 15544054.561004732,
+ "CS007": 15737019.571027506,
+ "CS011": 14245094.062691605,
+ "CS013": 10705936.655357886,
+ "CS017": 14099126.5756219,
+ "CS021": 13172990.03150767,
+ "CS024": 14696724.018343825,
+ "CS026": 18501377.981954917,
+ "CS028": 14326771.584380083,
+ "CS030": 16033335.687642261,
+ "CS031": 20901704.500670947,
+ "CS032": 18795952.493532542,
+ "CS101": 67594399.69123329,
+ "CS103": 14555006.230974862,
+ "CS201": 11491082.207871344,
+ "CS301": 40468265.70497692,
+ "CS302": 17781663.389931183,
+ "CS401": 19556709.0685369,
+ "CS501": 27261643.796409346,
+ "DE601": 17777132.55854045,
+ "DE602": 19748901.556048356,
+ "DE603": 26819783.45521549,
+ "DE604": 14385497.046839358,
+ "DE605": 12729490.454671673,
+ "DE609": 11548756.244492985,
+ "FR606": 13169448.64903064,
+ "IE613": 13395597.406249378,
+ "LV614": 11668296.995990513,
+ "PL610": 14960883.74047425,
+ "PL611": 17196733.845408365,
+ "PL612": 10283464.55136512,
+ "RS106": 12128338.820957774,
+ "RS205": 42916272.60510826,
+ "RS208": 9365468.17970955,
+ "RS210": 47000312.251054004,
+ "RS305": 15538054.639135055,
+ "RS306": 14206058.107420009,
+ "RS307": 14757246.239034232,
+ "RS310": 14171170.538164835,
+ "RS406": 15226166.937623236,
+ "RS407": 14530681.276822567,
+ "RS409": 14725610.814889988,
+ "RS503": 11508097.846546676,
+ "RS508": 28514459.964421105,
+ "RS509": 19256534.542812303,
+ "SE607": 30430197.90790976,
+ "UK608": 22423233.01862699
+ },
+ "target": [
+ "3C295"
+ ],
+ "quality": "moderate",
+ "median_dataloss": 1.23456789,
+ "quality_indicators": "rfi_limit:0.3; median_dataloss_limit:1.5; sun_angle_limit:20; moon_angle_limit:10; jupiter_angle_limit:5",
+
+ "antennas": [
+ "CS001HBA0",
+ "CS001HBA1",
+ "CS002HBA0",
+ "CS002HBA1",
+ "CS003HBA0",
+ "CS003HBA1",
+ "CS004HBA0",
+ "CS004HBA1",
+ "CS005HBA0",
+ "CS005HBA1",
+ "CS006HBA0",
+ "CS006HBA1",
+ "CS007HBA0",
+ "CS007HBA1",
+ "CS011HBA0",
+ "CS011HBA1",
+ "CS013HBA0",
+ "CS013HBA1",
+ "CS017HBA0",
+ "CS017HBA1",
+ "CS021HBA0",
+ "CS021HBA1",
+ "CS024HBA0",
+ "CS024HBA1",
+ "CS026HBA0",
+ "CS026HBA1",
+ "CS028HBA0",
+ "CS028HBA1",
+ "CS030HBA0",
+ "CS030HBA1",
+ "CS031HBA0",
+ "CS031HBA1",
+ "CS032HBA0",
+ "CS032HBA1",
+ "CS101HBA0",
+ "CS101HBA1",
+ "CS103HBA0",
+ "CS103HBA1",
+ "CS201HBA0",
+ "CS201HBA1",
+ "CS301HBA0",
+ "CS301HBA1",
+ "CS302HBA0",
+ "CS302HBA1",
+ "CS401HBA0",
+ "CS401HBA1",
+ "CS501HBA0",
+ "CS501HBA1",
+ "RS106HBA",
+ "RS205HBA",
+ "RS208HBA",
+ "RS210HBA",
+ "RS305HBA",
+ "RS306HBA",
+ "RS307HBA",
+ "RS310HBA",
+ "RS406HBA",
+ "RS407HBA",
+ "RS409HBA",
+ "RS503HBA",
+ "RS508HBA",
+ "RS509HBA",
+ "DE601HBA",
+ "DE602HBA",
+ "DE603HBA",
+ "DE604HBA",
+ "DE605HBA",
+ "FR606HBA",
+ "SE607HBA",
+ "UK608HBA",
+ "DE609HBA",
+ "PL610HBA",
+ "PL611HBA",
+ "PL612HBA",
+ "IE613HBA"
+ ],
+ "pointing": {
+ "Sun": 98.62325727494583,
+ "CasA": 63.8887478639975,
+ "CygA": 57.33860706164162,
+ "HerA": 57.53230892059052,
+ "Moon": 82.10124202600636,
+ "TauA": 93.60818880478796,
+ "VirA": 44.64319497995252,
+ "Jupiter": 65.56149628509407,
+ "elevation_fraction": 1
+ },
+ "rfi_percent": 1.7186448587105623,
+ "antenna_configuration": "FULL",
+ "antennas_not_available": [
+ "LV614"
+ ]
+ },
+ "applied_fixes": [],
+ "rfi_perc_total": "good",
+ "elevation_score": "good",
+ "sun_interference": "good",
+ "unfixable_issues": [],
+ "moon_interference": "good",
+ "jupiter_interference": "good",
+ "degree_incompleteness_array": [],
+ "array_missing_important_pairs_is": "good",
+ "array_missing_important_pairs_dutch": "good",
+ "aggregated_array_data_losses_percentage": "poor",
+ "array_high_data_loss_on_is_important_pair": "good",
+ "array_high_data_loss_on_dutch_important_pair": "good"
+ }
+ },
+}
+
link_calibrator_summary_without_rfi = {
"quality": {
"details": {},