Skip to content
Snippets Groups Projects
Commit d4e5a21d authored by Nico Vermaas's avatar Nico Vermaas
Browse files

Merge branch 'SDC-1649-compression-pipeline-quality-indicators' into 'master' 

Sdc 1649 compression pipeline quality indicators

See merge request !385
parents 8dfdd388 772ceb98
No related branches found
No related tags found
1 merge request!385Sdc 1649 compression pipeline quality indicators
Pipeline #105768 passed
Stage: test
Stage: build
Stage: deploy_to_test
Stage: deploy_to_production
Show whitespace changes
Inline Side-by-side
atdb/taskdatabase/services/algorithms.py
+ 23
− 16
View file @ d4e5a21d
......@@ -949,6 +949,13 @@ def construct_imaging_summary(task):
title = "<h4>Summary File for SAS_ID " + task.sas_id + "</h4> "
tasks = Task.objects.filter(sas_id=sas_id).order_by('task_type','id')
# isolate the aggregation task, which is to be used to read the 'quality_indicators from
try:
aggregation_task = tasks.filter(task_type='aggregation')[0]
except:
aggregation_task = None
for task in tasks:
try:
......@@ -1044,11 +1051,18 @@ def construct_imaging_summary(task):
if task_quality:
results += '<tr><td><b>Calculated Quality</b></td>'
results += '<td class="' + task_quality + '">' + str(task_quality) + '</td>'
# task_quality can also be a list of qualities... if so, use the first element as class
quality_class = task_quality
if isinstance(task_quality, list):
quality_class = task_quality[0]
results += '<td class="' + quality_class + '">' + str(task_quality) + '</td>'
results += '</tr>'
except:
pass
except Exception as error:
logger.info(error)
#pass
try:
key = task.calculated_qualities['per_task']
......@@ -1091,18 +1105,11 @@ def construct_imaging_summary(task):
totals += '<tr><td><b>Calculated Quality</b></td>'
totals += '<td colspan="2" class="' + sasid_quality + '">' + str(sasid_quality) + '</td></tr>'
try:
quality_thresholds = json.loads(Configuration.objects.get(key='quality_thresholds').value)
except:
quality_thresholds = {
"moderate": 20,
"poor": 50,
"overall_poor": 50,
"overall_good": 90,
}
if aggregation_task:
quality_indicators = aggregation_task.quality_json["summary"]['details']['quality_indicators']
totals += '<tr>'
totals += '<td><b>RFI thresholds</b></td>'
totals += '<td colspan="2">M, rfi>'+ str(quality_thresholds['poor']) + '% = P, rfi<=' + str(quality_thresholds['moderate']) + '% = G</td>'
totals += '<td><b>Quality Indicators</b></td>'
totals += f'<td colspan="2">{quality_indicators}</td>'
totals += '</tr>'
except:
......
atdb/taskdatabase/services/calculated_qualities.py
+ 59
− 41
View file @ d4e5a21d
......@@ -68,7 +68,7 @@ def calculate_qualities(task, tasks_for_this_sasid, quality_thresholds):
if summary_flavour == SummaryFlavour.IMAGING_COMPRESSION.value:
quality_per_file = None
# shortcut, if quality is already calculated by the workflow itself, then no need to recalculate
# override, if quality is already calculated by the workflow itself, then do not recalculate
try:
quality_from_summary = summary['details']['quality']
if quality_from_summary in ['poor', 'moderate', 'good']:
......@@ -109,13 +109,30 @@ def calculate_qualities(task, tasks_for_this_sasid, quality_thresholds):
def calculate_quality_sasid(unsaved_task, tasks_for_this_sasid):
"""
calculate the overall quality per sas_id, based on other tasks with the same sas_id
The threshold values are written from a configuration json blob
For the imaging compression pipeline (IMAGING_COMPRESSION flavour) this value is read from the aggregation task
For other pipelines it is calculated based on threshold values stored in ATDB
The threshold values are read from a configuration json blob
Using this algorithm from SDCO:
if more then 90 % of all files have a good quality then the dataset has good condition.
If more then 50 % of all files have a poor quality then the dataset is poor
otherwise is moderate.
"""
summary_flavour = get_summary_flavour(unsaved_task)
if summary_flavour == SummaryFlavour.IMAGING_COMPRESSION.value:
# because the quality is read from the aggregation task, only the aggregation task needs to be checked
if unsaved_task.task_type == 'aggregation':
summary = unsaved_task.quality_json["summary"]
quality_from_summary = summary['details']['quality']
return quality_from_summary
else:
return None
if summary_flavour == SummaryFlavour.DEFAULT.value:
try:
# gather the results of all the calculated_quality values for this sas_id
qualities = {'poor': 0, 'moderate': 0, 'good': 0}
......@@ -181,6 +198,7 @@ def calculate_qualities(task, tasks_for_this_sasid, quality_thresholds):
# update the overall quality of all tasks for this sas_id
calculated_quality_sasid = calculate_quality_sasid(task, tasks_for_this_sasid)
if calculated_quality_sasid:
# store the result in task.calculated_qualities (not yet saved in the database)
qualities['per_sasid'] = calculated_quality_sasid
......
atdb/taskdatabase/tests/test_calculated_qualities.py
+ 38
− 1
View file @ d4e5a21d
......@@ -51,6 +51,8 @@ class TestCalculatedQualities(TestCase):
workflow_imaging_compression = Workflow(workflow_uri="imaging_compress_pipeline_v011")
workflow_imaging_compression.save()
Task.objects.get_or_create(sas_id=55555, status='processed', outputs=outputs.imaging_compression_summary_flavor_with_rfi_1, workflow=workflow_imaging_compression)
Task.objects.get_or_create(sas_id=55555, task_type='aggregation', status='processed', outputs=outputs.imaging_compression_aggregation_task, workflow=workflow_imaging_compression)
# LINK pipelines (no rfi_percent onboard yet)
workflow_link_calibrator = Workflow(workflow_uri="linc_calibrator_v4_2")
......@@ -75,7 +77,7 @@ class TestCalculatedQualities(TestCase):
if task.calculated_qualities['per_task']:
count += 1
self.assertEqual(count,6)
self.assertEqual(count,7)
def test_calculated_qualities(self):
......@@ -120,6 +122,41 @@ class TestCalculatedQualities(TestCase):
self.assertEqual(quality_per_sasid,'moderate')
def test_calculated_qualities_imaging_compression(self):
"""
for imaging compression the qualities are read from the aggregation task, instead of calculated by ATDB
"""
# read the quality thresholds from the test database
quality_thresholds = json.loads(Configuration.objects.get(key="quality_thresholds").value)
quality_indicators = None
# get the tasks for sas_id 54321
tasks_for_this_sasid = Task.objects.filter(sas_id=55555)
# run the algorithms and gather the values
quality_values = {'poor': 0, 'moderate': 0, 'good': 0}
for task in tasks_for_this_sasid:
q = qualities.calculate_qualities(task, tasks_for_this_sasid, quality_thresholds)
quality_values = qualities.unpack_qualities_per_task(task,quality_values)
if task.task_type=='aggregation':
quality_indicators = task.quality_json["summary"]['details']['quality_indicators']
try:
quality_per_sasid = task.calculated_qualities['per_sasid']
except:
# ignore the tasks that have no calculated quality.
pass
self.assertEqual(quality_values,{'poor': 0, 'moderate': 1, 'good': 1})
# assert values read from the aggregation task
self.assertEqual(quality_per_sasid,'moderate')
self.assertEqual(quality_indicators, 'rfi_limit:0.3; median_dataloss_limit:1.5; sun_angle_limit:20; moon_angle_limit:10; jupiter_angle_limit:5')
def test_calculated_qualities_with_optimistic_thresholds(self):
"""
calculate the quality per task and per sas_id based on rfi_percent values
......
atdb/taskdatabase/tests/test_calculated_qualities_outputs.py
+ 177
− 0
View file @ d4e5a21d
......@@ -370,6 +370,183 @@ imaging_compression_summary_flavor_with_rfi_1 = {
},
}
imaging_compression_aggregation_task = {
"quality": {
"summary": {
"details": {
"DStDev": {
"CS001": 15372035.9671943,
"CS002": 14858111.10350275,
"CS003": 10147611.046423668,
"CS004": 18980165.035334244,
"CS005": 9209186.769605417,
"CS006": 15544054.561004732,
"CS007": 15737019.571027506,
"CS011": 14245094.062691605,
"CS013": 10705936.655357886,
"CS017": 14099126.5756219,
"CS021": 13172990.03150767,
"CS024": 14696724.018343825,
"CS026": 18501377.981954917,
"CS028": 14326771.584380083,
"CS030": 16033335.687642261,
"CS031": 20901704.500670947,
"CS032": 18795952.493532542,
"CS101": 67594399.69123329,
"CS103": 14555006.230974862,
"CS201": 11491082.207871344,
"CS301": 40468265.70497692,
"CS302": 17781663.389931183,
"CS401": 19556709.0685369,
"CS501": 27261643.796409346,
"DE601": 17777132.55854045,
"DE602": 19748901.556048356,
"DE603": 26819783.45521549,
"DE604": 14385497.046839358,
"DE605": 12729490.454671673,
"DE609": 11548756.244492985,
"FR606": 13169448.64903064,
"IE613": 13395597.406249378,
"LV614": 11668296.995990513,
"PL610": 14960883.74047425,
"PL611": 17196733.845408365,
"PL612": 10283464.55136512,
"RS106": 12128338.820957774,
"RS205": 42916272.60510826,
"RS208": 9365468.17970955,
"RS210": 47000312.251054004,
"RS305": 15538054.639135055,
"RS306": 14206058.107420009,
"RS307": 14757246.239034232,
"RS310": 14171170.538164835,
"RS406": 15226166.937623236,
"RS407": 14530681.276822567,
"RS409": 14725610.814889988,
"RS503": 11508097.846546676,
"RS508": 28514459.964421105,
"RS509": 19256534.542812303,
"SE607": 30430197.90790976,
"UK608": 22423233.01862699
},
"target": [
"3C295"
],
"quality": "moderate",
"median_dataloss": 1.23456789,
"quality_indicators": "rfi_limit:0.3; median_dataloss_limit:1.5; sun_angle_limit:20; moon_angle_limit:10; jupiter_angle_limit:5",
"antennas": [
"CS001HBA0",
"CS001HBA1",
"CS002HBA0",
"CS002HBA1",
"CS003HBA0",
"CS003HBA1",
"CS004HBA0",
"CS004HBA1",
"CS005HBA0",
"CS005HBA1",
"CS006HBA0",
"CS006HBA1",
"CS007HBA0",
"CS007HBA1",
"CS011HBA0",
"CS011HBA1",
"CS013HBA0",
"CS013HBA1",
"CS017HBA0",
"CS017HBA1",
"CS021HBA0",
"CS021HBA1",
"CS024HBA0",
"CS024HBA1",
"CS026HBA0",
"CS026HBA1",
"CS028HBA0",
"CS028HBA1",
"CS030HBA0",
"CS030HBA1",
"CS031HBA0",
"CS031HBA1",
"CS032HBA0",
"CS032HBA1",
"CS101HBA0",
"CS101HBA1",
"CS103HBA0",
"CS103HBA1",
"CS201HBA0",
"CS201HBA1",
"CS301HBA0",
"CS301HBA1",
"CS302HBA0",
"CS302HBA1",
"CS401HBA0",
"CS401HBA1",
"CS501HBA0",
"CS501HBA1",
"RS106HBA",
"RS205HBA",
"RS208HBA",
"RS210HBA",
"RS305HBA",
"RS306HBA",
"RS307HBA",
"RS310HBA",
"RS406HBA",
"RS407HBA",
"RS409HBA",
"RS503HBA",
"RS508HBA",
"RS509HBA",
"DE601HBA",
"DE602HBA",
"DE603HBA",
"DE604HBA",
"DE605HBA",
"FR606HBA",
"SE607HBA",
"UK608HBA",
"DE609HBA",
"PL610HBA",
"PL611HBA",
"PL612HBA",
"IE613HBA"
],
"pointing": {
"Sun": 98.62325727494583,
"CasA": 63.8887478639975,
"CygA": 57.33860706164162,
"HerA": 57.53230892059052,
"Moon": 82.10124202600636,
"TauA": 93.60818880478796,
"VirA": 44.64319497995252,
"Jupiter": 65.56149628509407,
"elevation_fraction": 1
},
"rfi_percent": 1.7186448587105623,
"antenna_configuration": "FULL",
"antennas_not_available": [
"LV614"
]
},
"applied_fixes": [],
"rfi_perc_total": "good",
"elevation_score": "good",
"sun_interference": "good",
"unfixable_issues": [],
"moon_interference": "good",
"jupiter_interference": "good",
"degree_incompleteness_array": [],
"array_missing_important_pairs_is": "good",
"array_missing_important_pairs_dutch": "good",
"aggregated_array_data_losses_percentage": "poor",
"array_high_data_loss_on_is_important_pair": "good",
"array_high_data_loss_on_dutch_important_pair": "good"
}
},
}
link_calibrator_summary_without_rfi = {
"quality": {
"details": {},
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment