diff --git a/atdb/taskdatabase/services/algorithms.py b/atdb/taskdatabase/services/algorithms.py
index 9326e6cdcf52f7cb859c63f33070b3fe11334280..b29eafc3512e7ecfdd41ddbb504423edf3a3b3cc 100644
--- a/atdb/taskdatabase/services/algorithms.py
+++ b/atdb/taskdatabase/services/algorithms.py
@@ -1077,8 +1077,6 @@ def construct_linc_summary(task):
total_size_processed = 0
total_total_processing_time = 0
- quality_values = {'poor': 0, 'moderate': 0, 'good': 0}
-
sas_id = task.sas_id
title = "<h4>Summary File for SAS_ID " + task.sas_id + "</h4> "
@@ -1098,7 +1096,12 @@ def construct_linc_summary(task):
# find the summary in the quality json structure
try:
- summary = task.quality_json["summary"]
+ #NV_25mar2023:
+ # it is not yet clear if LINC will have its 'summary' metadata directly under outputs or under outputs.quality
+ # for now assuming directly under outputs
+ #summary = task.quality_json["summary"]
+ summary = task.outputs["summary"]
+
results += '<tr><td><b>size_to_process</b></td><td>' + str(task.size_to_process) + '</td></tr>'
results += '<tr><td><b>size_processed</b></td><td>' + str(task.size_processed) + '</td></tr>'
@@ -1129,6 +1132,7 @@ def construct_linc_summary(task):
def construct_summary(task, format='html'):
+ # summary_flavour will be DEFAULT, unless proof is found that it is something else
summary_flavour = get_summary_flavour(task)
logger.info(f'summary_flavour = {summary_flavour}')
@@ -1147,9 +1151,16 @@ def construct_summary(task, format='html'):
return construct_linc_summary(task)
elif format=='json':
+
if summary_flavour == SummaryFlavour.DEFAULT.value:
return construct_default_summary_json(task)
+ elif summary_flavour == SummaryFlavour.LINC_CALIBRATOR.value:
+ return construct_linc_summary_json(task)
+
+ elif summary_flavour == SummaryFlavour.LINC_TARGET.value:
+ return construct_linc_summary_json(task)
+
return None
@@ -1259,3 +1270,68 @@ def construct_default_summary_json(task):
return summary_json
+
+def construct_linc_summary_json(task):
+
+ total_size_to_process = 0
+ total_size_processed = 0
+ total_total_processing_time = 0
+
+ sas_id = task.sas_id
+ title = f'Summary File for SAS_ID {task.sas_id}'
+ summary_json = {}
+ summary_json['title'] = title
+
+ tasks = Task.objects.filter(sas_id=sas_id)
+ tasks_records = []
+
+ for task in tasks:
+
+ task_record = {}
+
+ # skip 'suspended' and 'discarded' tasks
+ if task.status in ['suspended', 'discarded']:
+ continue
+
+ task_record['task'] = task.id
+
+ total_size_to_process += task.size_to_process
+ total_size_processed += task.size_processed
+ total_total_processing_time += task.total_processing_time
+
+ # find the summary in the quality json structure
+ try:
+ #NV_25mar2023:
+ # it is not yet clear if LINC will have its 'summary' metadata directly under outputs or under outputs.quality
+ # for now assuming directly under outputs. If that turns out differently then change the line below accordingly.
+ # summary = task.quality_json["summary"]
+ summary = task.outputs["summary"]
+
+ task_record['size_to_process'] = task.size_to_process
+ task_record['size_processed'] = task.size_processed
+ task_record['basename'] = summary['basename']
+ task_record['checksum'] = summary['checksum']
+ task_record['class'] = summary['class']
+ task_record['location'] = summary['location']
+ task_record['nameext'] = summary['nameext']
+ task_record['nameroot'] = summary['nameroot']
+ task_record['size'] = summary['size']
+ task_record['surl'] = summary['surl']
+
+ tasks_records.append(task_record)
+ except:
+ pass
+
+ totals_record = {}
+ try:
+ totals_record['total_size_to_process'] = total_size_to_process
+ totals_record['total_size_processed'] = total_size_processed
+
+ except:
+ pass
+
+ summary_json['totals'] = totals_record
+ summary_json['tasks'] = tasks_records
+
+ return summary_json
+
diff --git a/atdb/taskdatabase/tests/test_views_get_summary.py b/atdb/taskdatabase/tests/test_views_get_summary.py
index ee64db5c1ac7e3a00f3b6552c820331037d4a3c1..df31ba49e1c194eed48ef8ef91ab073551735e94 100644
--- a/atdb/taskdatabase/tests/test_views_get_summary.py
+++ b/atdb/taskdatabase/tests/test_views_get_summary.py
@@ -18,7 +18,7 @@ class GetSummaryTestCase(TestCase):
outputs=outputs.default_summary_flavour_with_rfi_percent_zero_1,
workflow=workflow_requantisation)
- # rfi_percent 11,22,31,52
+ # default summary flavour
Task.objects.get_or_create(sas_id=54321, status='processed', outputs=outputs.default_summary_flavour_with_rfi_1,
workflow=workflow_requantisation)
Task.objects.get_or_create(sas_id=54321, status='processed', outputs=outputs.default_summary_flavour_with_rfi_2,
@@ -28,6 +28,21 @@ class GetSummaryTestCase(TestCase):
Task.objects.get_or_create(sas_id=54321, status='processed', outputs=outputs.default_summary_flavour_with_rfi_4,
workflow=workflow_requantisation)
+ # test image compression, rfi_percentage=1.7186448587105623
+ workflow_imaging_compression = Workflow(workflow_uri="imaging_compress_pipeline_v011")
+ workflow_imaging_compression.save()
+ Task.objects.get_or_create(sas_id=55555, status='processed', outputs=outputs.imaging_compression_summary_flavor_with_rfi_1, workflow=workflow_imaging_compression)
+
+ # LINC pipelines (no rfi_percent onboard yet)
+ workflow_linc_calibrator = Workflow(workflow_uri="linc_calibrator_v4_2")
+ workflow_linc_calibrator.save()
+ Task.objects.get_or_create(sas_id=666666, status='processed', outputs=outputs.link_calibrator_summary_without_rfi, workflow=workflow_linc_calibrator)
+
+ workflow_linc_target = Workflow(workflow_uri="linc_target_v4_2")
+ workflow_linc_target.save()
+ Task.objects.get_or_create(sas_id=666667, status='processed', outputs=outputs.link_target_summary_without_rfi, workflow=workflow_linc_target)
+
+
def test_summary_json_response(self):
# Mock request
response = self.client.get(reverse('get-summary', args=['54321', 'json']))
@@ -63,6 +78,33 @@ class GetSummaryTestCase(TestCase):
self.assertEqual(t['output_size'], 283791360)
self.assertEqual(t['size_ratio'], '0.572')
+ def test_summary_json_contents_linc(self):
+ response = self.client.get(reverse('get-summary', args=['666666', 'json']))
+
+ # Check if response is JsonResponse
+ self.assertIsInstance(response, JsonResponse)
+
+ # Add more assertions as needed
+ json_data = json.loads(response.content.decode('utf-8'))
+
+ # is this json generated for the expected SAS_ID?
+ expected = "Summary File for SAS_ID 666666"
+ actual = json_data['summary']['title']
+ self.assertEqual(expected, actual)
+
+ # are all the tasks in the json?
+ tasks = json_data['summary']['tasks']
+ actual = len(tasks)
+ expected = 1
+ self.assertEqual(expected, actual)
+
+ # check 1 task for correct contents
+ t = tasks[0]
+ self.assertEqual(t['basename'], '3c48_LINC_calibrator_summary.json')
+ self.assertEqual(t['class'], 'File')
+ self.assertEqual(t['checksum'], 'sha1$531646ff527d76f4facdabf72d939bac302eaf1f')
+ self.assertEqual(t['location'], 'file:///project/ldv/Share/run/2023/6/16/1352_35011/3c48_LINC_calibrator_summary.json')
+ self.assertEqual(t['surl'], 'srm://srm.grid.sara.nl/pnfs/grid.sara.nl/data/lofar/ops/disk/ldv/lt10_010/689478/35011/3c48_LINC_calibrator_summary.json')
def test_summary_html_response(self):
# Mock request