add linc pipelines json output and unittests for it.

atdb/taskdatabase/services/algorithms.py

@@ -1077,8 +1077,6 @@ def construct_linc_summary(task):
     total_size_processed = 0
     total_total_processing_time = 0
 
-    quality_values = {'poor': 0, 'moderate': 0, 'good': 0}
-
     sas_id = task.sas_id
     title = "<h4>Summary File for SAS_ID " + task.sas_id + "</h4> "
 
@@ -1098,7 +1096,12 @@ def construct_linc_summary(task):
 
         # find the summary in the quality json structure
         try:
-            summary = task.quality_json["summary"]
+            #NV_25mar2023:
+            # it is not yet clear if LINC will have its 'summary' metadata directly under outputs or under outputs.quality
+            # for now assuming directly under outputs
+            #summary = task.quality_json["summary"]
+            summary = task.outputs["summary"]
+
             results += '<tr><td><b>size_to_process</b></td><td>' + str(task.size_to_process) + '</td></tr>'
             results += '<tr><td><b>size_processed</b></td><td>' + str(task.size_processed) + '</td></tr>'
 
@@ -1129,6 +1132,7 @@ def construct_linc_summary(task):
 
 def construct_summary(task, format='html'):
 
+    # summary_flavour will be DEFAULT, unless proof is found that it is something else
     summary_flavour = get_summary_flavour(task)
     logger.info(f'summary_flavour = {summary_flavour}')
 
@@ -1147,9 +1151,16 @@ def construct_summary(task, format='html'):
             return construct_linc_summary(task)
 
     elif format=='json':
+
         if summary_flavour == SummaryFlavour.DEFAULT.value:
             return construct_default_summary_json(task)
 
+        elif summary_flavour == SummaryFlavour.LINC_CALIBRATOR.value:
+            return construct_linc_summary_json(task)
+
+        elif summary_flavour == SummaryFlavour.LINC_TARGET.value:
+            return construct_linc_summary_json(task)
+
     return None
 
 
@@ -1259,3 +1270,68 @@ def construct_default_summary_json(task):
 
     return summary_json
 
+
+def construct_linc_summary_json(task):
+
+    total_size_to_process = 0
+    total_size_processed = 0
+    total_total_processing_time = 0
+
+    sas_id = task.sas_id
+    title = f'Summary File for SAS_ID {task.sas_id}'
+    summary_json = {}
+    summary_json['title'] = title
+
+    tasks = Task.objects.filter(sas_id=sas_id)
+    tasks_records = []
+
+    for task in tasks:
+
+        task_record = {}
+
+        # skip 'suspended' and 'discarded' tasks
+        if task.status in ['suspended', 'discarded']:
+            continue
+
+        task_record['task'] = task.id
+
+        total_size_to_process += task.size_to_process
+        total_size_processed += task.size_processed
+        total_total_processing_time += task.total_processing_time
+
+        # find the summary in the quality json structure
+        try:
+            #NV_25mar2023:
+            # it is not yet clear if LINC will have its 'summary' metadata directly under outputs or under outputs.quality
+            # for now assuming directly under outputs. If that turns out differently then change the line below accordingly.
+            # summary = task.quality_json["summary"]
+            summary = task.outputs["summary"]
+
+            task_record['size_to_process'] = task.size_to_process
+            task_record['size_processed'] = task.size_processed
+            task_record['basename'] = summary['basename']
+            task_record['checksum'] = summary['checksum']
+            task_record['class'] = summary['class']
+            task_record['location'] = summary['location']
+            task_record['nameext'] = summary['nameext']
+            task_record['nameroot'] = summary['nameroot']
+            task_record['size'] = summary['size']
+            task_record['surl'] = summary['surl']
+
+            tasks_records.append(task_record)
+        except:
+            pass
+
+    totals_record = {}
+    try:
+        totals_record['total_size_to_process'] = total_size_to_process
+        totals_record['total_size_processed'] = total_size_processed
+
+    except:
+        pass
+
+    summary_json['totals'] = totals_record
+    summary_json['tasks'] = tasks_records
+
+    return summary_json
+

atdb/taskdatabase/tests/test_views_get_summary.py

@@ -18,7 +18,7 @@ class GetSummaryTestCase(TestCase):
                                    outputs=outputs.default_summary_flavour_with_rfi_percent_zero_1,
                                    workflow=workflow_requantisation)
 
-        # rfi_percent 11,22,31,52
+        # default summary flavour
         Task.objects.get_or_create(sas_id=54321, status='processed', outputs=outputs.default_summary_flavour_with_rfi_1,
                                    workflow=workflow_requantisation)
         Task.objects.get_or_create(sas_id=54321, status='processed', outputs=outputs.default_summary_flavour_with_rfi_2,
@@ -28,6 +28,21 @@ class GetSummaryTestCase(TestCase):
         Task.objects.get_or_create(sas_id=54321, status='processed', outputs=outputs.default_summary_flavour_with_rfi_4,
                                    workflow=workflow_requantisation)
 
+        # test image compression, rfi_percentage=1.7186448587105623
+        workflow_imaging_compression = Workflow(workflow_uri="imaging_compress_pipeline_v011")
+        workflow_imaging_compression.save()
+        Task.objects.get_or_create(sas_id=55555, status='processed', outputs=outputs.imaging_compression_summary_flavor_with_rfi_1, workflow=workflow_imaging_compression)
+
+        # LINC pipelines (no rfi_percent onboard yet)
+        workflow_linc_calibrator = Workflow(workflow_uri="linc_calibrator_v4_2")
+        workflow_linc_calibrator.save()
+        Task.objects.get_or_create(sas_id=666666, status='processed', outputs=outputs.link_calibrator_summary_without_rfi, workflow=workflow_linc_calibrator)
+
+        workflow_linc_target = Workflow(workflow_uri="linc_target_v4_2")
+        workflow_linc_target.save()
+        Task.objects.get_or_create(sas_id=666667, status='processed', outputs=outputs.link_target_summary_without_rfi, workflow=workflow_linc_target)
+
+
     def test_summary_json_response(self):
         # Mock request
         response = self.client.get(reverse('get-summary', args=['54321', 'json']))
@@ -63,6 +78,33 @@ class GetSummaryTestCase(TestCase):
         self.assertEqual(t['output_size'], 283791360)
         self.assertEqual(t['size_ratio'], '0.572')
 
+    def test_summary_json_contents_linc(self):
+        response = self.client.get(reverse('get-summary', args=['666666', 'json']))
+
+        # Check if response is JsonResponse
+        self.assertIsInstance(response, JsonResponse)
+
+        # Add more assertions as needed
+        json_data = json.loads(response.content.decode('utf-8'))
+
+        # is this json generated for the expected SAS_ID?
+        expected = "Summary File for SAS_ID 666666"
+        actual = json_data['summary']['title']
+        self.assertEqual(expected, actual)
+
+        # are all the tasks in the json?
+        tasks = json_data['summary']['tasks']
+        actual = len(tasks)
+        expected = 1
+        self.assertEqual(expected, actual)
+
+        # check 1 task for correct contents
+        t = tasks[0]
+        self.assertEqual(t['basename'], '3c48_LINC_calibrator_summary.json')
+        self.assertEqual(t['class'], 'File')
+        self.assertEqual(t['checksum'], 'sha1$531646ff527d76f4facdabf72d939bac302eaf1f')
+        self.assertEqual(t['location'], 'file:///project/ldv/Share/run/2023/6/16/1352_35011/3c48_LINC_calibrator_summary.json')
+        self.assertEqual(t['surl'], 'srm://srm.grid.sara.nl/pnfs/grid.sara.nl/data/lofar/ops/disk/ldv/lt10_010/689478/35011/3c48_LINC_calibrator_summary.json')
 
     def test_summary_html_response(self):
         # Mock request