diff --git a/atdb/requirements/base.txt b/atdb/requirements/base.txt index 38935ee2a06d85b69c5d5bafc3a26dfeb44da837..eb63b89a13274589c3866ed061fdba550b169cb4 100644 --- a/atdb/requirements/base.txt +++ b/atdb/requirements/base.txt @@ -18,3 +18,4 @@ requests-oauthlib==1.3.1 six==1.15.0 whitenoise==5.0.1 pytz==2022.6 +xhtml2pdf==0.2.15 \ No newline at end of file diff --git a/atdb/taskdatabase/serializers.py b/atdb/taskdatabase/serializers.py index f80c4eb13fcfc3e9c0d7ddf93963f69114b2da89..55484a36048d1b2872f560b55f2615de05f1c2f5 100644 --- a/atdb/taskdatabase/serializers.py +++ b/atdb/taskdatabase/serializers.py @@ -132,7 +132,7 @@ class TaskReadSerializerFast(serializers.ModelSerializer): """ class Meta: model = Task - fields = ['id','task_type','creationTime','filter','predecessor','predecessor_status', + fields = ['id','task_type','is_summary','creationTime','filter','predecessor','predecessor_status', #'joined_input_tasks', 'joined_output_task', 'joined_status', 'project','sas_id','priority','purge_policy','cleanup_policy','resume', 'workflow', diff --git a/atdb/taskdatabase/services/algorithms.py b/atdb/taskdatabase/services/algorithms.py index 7651909e711f59dbab809ca77b4bcfc8ce55ec5a..fda6b71f52f25dc8cbf79787b6866722c3a8110b 100644 --- a/atdb/taskdatabase/services/algorithms.py +++ b/atdb/taskdatabase/services/algorithms.py @@ -833,7 +833,7 @@ def construct_default_summary(task): line = '' line += '<tr style="background-color:#7EB100"><td colspan="3"><b>' + key + '</b></td></tr>' - line += '<th></th><th>Name</th><th>Size</th>' + line += '<td></td><td><b>Name</b></td><td><b>Size</b></td>' line += '<tr><td><b>Input</b></td>' line += '<td>' + record['input_name'] + '</td>' line += '<td>' + str(record['input_size']) + ' (' + record['input_size_str'] + ')</td>' @@ -861,7 +861,7 @@ def construct_default_summary(task): task_quality = calculated_qualities['per_task'] line += '<tr><td><b>Calculated Quality</b></td>' - line += '<td class="' + task_quality + '">' + str(task_quality) + '</td>' + line += '<td colspan="2" class="' + task_quality + '">' + str(task_quality) + '</td>' line += '</tr>' except: @@ -870,18 +870,18 @@ def construct_default_summary(task): try: added = record['added'] if added: - line += '<th>Added</th>' + line += '<td><b>Added</b></td><td></td><td></td>' for filename in added: - line += '<tr><td colspan="3">' + filename + '<td></tr>' + line += '<tr><td colspan="2">' + filename + '<td></tr>' except: pass try: deleted = record['deleted'] if deleted: - line += '<th>Deleted</th>' + line += '<td><b>Deleted</b></td><td></td><td></td>' for filename in deleted: - line += '<tr><td colspan="3">' +filename + '<td></tr>' + line += '<tr><td colspan="2">' +filename + '<td></tr>' except: pass @@ -898,7 +898,7 @@ def construct_default_summary(task): except: pass - totals += '<th>Totals</th><th></th><th width="35%"></th>' + totals += '<td><b>Totals</b></td><td></td><td width="35%"></td>' try: totals += '<tr><td colspan="2"><b>Input size</b></td><td>' + str(total_size_input) + '</td></tr>' totals += '<tr><td colspan="2"><b>Output size</b><td>' + str(total_size_output) + '</td></tr>' @@ -1077,8 +1077,6 @@ def construct_linc_summary(task): total_size_processed = 0 total_total_processing_time = 0 - quality_values = {'poor': 0, 'moderate': 0, 'good': 0} - sas_id = task.sas_id title = "<h4>Summary File for SAS_ID " + task.sas_id + "</h4> " @@ -1098,7 +1096,12 @@ def construct_linc_summary(task): # find the summary in the quality json structure try: - summary = task.quality_json["summary"] + #NV_25mar2024: + # it is not yet clear if LINC will have its 'summary' metadata directly under outputs or under outputs.quality + # for now assuming directly under outputs + #summary = task.quality_json["summary"] + summary = task.outputs["summary"] + results += '<tr><td><b>size_to_process</b></td><td>' + str(task.size_to_process) + '</td></tr>' results += '<tr><td><b>size_processed</b></td><td>' + str(task.size_processed) + '</td></tr>' @@ -1127,21 +1130,208 @@ def construct_linc_summary(task): return results -def construct_summary(task): +def construct_summary(task, format='html'): + # summary_flavour will be DEFAULT, unless proof is found that it is something else summary_flavour = get_summary_flavour(task) + logger.info(f'summary_flavour = {summary_flavour}') + + if format=='html': + # construct the appropriate summary html + if summary_flavour == SummaryFlavour.IMAGING_COMPRESSION.value: + return construct_imaging_summary(task) + + elif summary_flavour == SummaryFlavour.LINC_CALIBRATOR.value: + return construct_linc_summary(task) + + elif summary_flavour == SummaryFlavour.LINC_TARGET.value: + return construct_linc_summary(task) + + return construct_default_summary(task) + + elif format=='json': + # construct the appropriate summary json + if summary_flavour == SummaryFlavour.IMAGING_COMPRESSION.value: + return construct_imaging_summary_json(task) + + if summary_flavour == SummaryFlavour.LINC_CALIBRATOR.value: + return construct_linc_summary_json(task) + + elif summary_flavour == SummaryFlavour.LINC_TARGET.value: + return construct_linc_summary_json(task) + + return construct_default_summary_json(task) + + return None + + +def construct_default_summary_json(task): + + total_size_input = 0 + total_size_output = 0 + quality_values = {'poor': 0, 'moderate': 0, 'good': 0} + + sas_id = task.sas_id + title = f'Summary File for SAS_ID {task.sas_id}' + summary_json = {'title':title} + + tasks = Task.objects.filter(sas_id=sas_id) + tasks_records = [] + + for task in tasks: + + # skip 'suspended' and 'discarded' tasks + if task.status in ['suspended', 'discarded']: + continue - # construct the appropriate summary html - if summary_flavour == SummaryFlavour.DEFAULT.value: - html = construct_default_summary(task) + task_record = {'task': task.id} - elif summary_flavour == SummaryFlavour.IMAGING_COMPRESSION.value: - html = construct_imaging_summary(task) + # find the summary in the quality json structure + try: + summary = task.quality_json["summary"] + + for key in summary: + record = summary[key] + total_size_input += record['input_size'] + total_size_output+= record['output_size'] + + task_record['file'] = key + task_record['input_name'] = record['input_name'] + task_record['input_size'] = record['input_size'] + task_record['output_name'] = record['output_name'] + task_record['output_size'] = record['output_size'] + task_record['size_ratio'] = str(round(record['size_ratio'],3)) + + if 'rfi_percent' in record: + # add RFI percentage (if present) + task_record['rfi_percent'] = str(record['rfi_percent']) + + try: + # add calculated quality (if present) + calculated_qualities = task.calculated_qualities + if calculated_qualities: + task_quality = calculated_qualities['per_task'] + task_record['task_quality'] = str(task_quality) + + except: + pass + + if 'added' in record: + task_record['added'] = record['added'] + + if 'deleted' in record: + task_record['deleted'] = record['deleted'] + + try: + key = task.calculated_qualities['per_task'] + quality_values[key] = quality_values[key] + 1 + except: + # ignore the tasks that have no calculated quality. + pass + + tasks_records.append(task_record) + except: + pass + + # calculate totals + totals_record = {} + + try: + totals_record['input_size'] = total_size_input + totals_record['output_size'] = total_size_output + totals_record['ratio'] = round(total_size_output / total_size_input, 3) + + try: + # add calculated quality per sasid (if present) + if calculated_qualities: + sasid_quality = calculated_qualities['per_sasid'] + totals_record['sasid_quality'] = str(sasid_quality) + totals_record['quality_values'] = str(quality_values) + + try: + quality_thresholds = json.loads(Configuration.objects.get(key='quality_thresholds').value) + totals_record['rfi_tresholds'] = quality_thresholds + + except: + pass + except: + pass + + except: + pass + + summary_json['totals'] = totals_record + summary_json['tasks'] = tasks_records + + return summary_json + + +def construct_imaging_summary_json(task): + # example: http://localhost:8000/atdb/get_summary/658584/json + results = {'result': 'summary not yet implemented for imaging'} + return results + + +def construct_linc_summary_json(task): + + total_size_to_process = 0 + total_size_processed = 0 + total_total_processing_time = 0 + + sas_id = task.sas_id + title = f'Summary File for SAS_ID {task.sas_id}' + summary_json = {'title': title} + + tasks = Task.objects.filter(sas_id=sas_id) + tasks_records = [] + + for task in tasks: + + task_record = {} + + # skip 'suspended' and 'discarded' tasks + if task.status in ['suspended', 'discarded']: + continue + + task_record['task'] = task.id + + total_size_to_process += task.size_to_process + total_size_processed += task.size_processed + total_total_processing_time += task.total_processing_time + + # find the summary in the quality json structure + try: + #NV_25mar2024: + # it is not yet clear if LINC will have its 'summary' metadata directly under outputs or under outputs.quality + # for now assuming directly under outputs. If that turns out differently then change the line below accordingly. + # summary = task.quality_json["summary"] + summary = task.outputs["summary"] + + task_record['size_to_process'] = task.size_to_process + task_record['size_processed'] = task.size_processed + task_record['basename'] = summary['basename'] + task_record['checksum'] = summary['checksum'] + task_record['class'] = summary['class'] + task_record['location'] = summary['location'] + task_record['nameext'] = summary['nameext'] + task_record['nameroot'] = summary['nameroot'] + task_record['size'] = summary['size'] + task_record['surl'] = summary['surl'] + + tasks_records.append(task_record) + except: + pass + + totals_record = {} + try: + totals_record['total_size_to_process'] = total_size_to_process + totals_record['total_size_processed'] = total_size_processed + + except: + pass - elif summary_flavour == SummaryFlavour.LINC_CALIBRATOR.value: - html = construct_linc_summary(task) + summary_json['totals'] = totals_record + summary_json['tasks'] = tasks_records - elif summary_flavour == SummaryFlavour.LINC_TARGET.value: - html = construct_linc_summary(task) + return summary_json - return html diff --git a/atdb/taskdatabase/templates/taskdatabase/index.html b/atdb/taskdatabase/templates/taskdatabase/index.html index 7e1037563da30e372565efbed9170cb5201ae88c..5d0d60f5e193ac7b21c31e989f212912b5ec5507 100644 --- a/atdb/taskdatabase/templates/taskdatabase/index.html +++ b/atdb/taskdatabase/templates/taskdatabase/index.html @@ -31,7 +31,7 @@ {% include 'taskdatabase/pagination.html' %} </div> </div> - <p class="footer"> Version 21 Mar 2024 + <p class="footer"> Version 26 Mar 2024 </div> {% include 'taskdatabase/refresh.html' %} diff --git a/atdb/taskdatabase/templates/taskdatabase/tasks/set_status_buttons.html b/atdb/taskdatabase/templates/taskdatabase/tasks/set_status_buttons.html index dfb555e457daa0e70d1f047983ac79908c9a0b0a..f4aab591726dd4ab4ce621df7dbab2cc04d5b84f 100644 --- a/atdb/taskdatabase/templates/taskdatabase/tasks/set_status_buttons.html +++ b/atdb/taskdatabase/templates/taskdatabase/tasks/set_status_buttons.html @@ -6,6 +6,7 @@ <a href="{% url 'task-details-setstatus' task.pk 'stored' %}" class="btn btn-warning btn-sm" role="button"><i class="fas fa-sync-alt"></i> stored</a></td> <a href="{% url 'task-details-setstatus' task.pk 'validated' %}" class="btn btn-warning btn-sm" role="button"><i class="fas fa-sync-alt"></i> validated</a> <a href="{% url 'task-details-setstatus' task.pk 'scrubbed' %}" class="btn btn-warning btn-sm" role="button"><i class="fas fa-sync-alt"></i> scrubbed</a> +<a href="{% url 'task-details-setstatus' task.pk 'pre_archived' %}" class="btn btn-warning btn-sm" role="button"><i class="fas fa-sync-alt"></i> pre_archived</a> <a href="{% url 'task-details-setstatus' task.pk 'archived' %}" class="btn btn-warning btn-sm" role="button"><i class="fas fa-sync-alt"></i> archived</a> <a href="{% url 'task-details-setstatus' task.pk 'finished' %}" class="btn btn-warning btn-sm" role="button"><i class="fas fa-sync-alt"></i> finished</a> <a href="{% url 'task-details-setstatus' task.pk 'suspended' %}" class="btn btn-warning btn-sm" role="button"><i class="fas fa-sync-alt"></i> suspended</a> diff --git a/atdb/taskdatabase/tests/test_views_diagram_page.py b/atdb/taskdatabase/tests/test_views_diagram_page.py index aadaf8300eb0b278bc98a6a1a464359cd3a99fda..49c1175e26d2be05c87723b28eb027fb41dffdba 100644 --- a/atdb/taskdatabase/tests/test_views_diagram_page.py +++ b/atdb/taskdatabase/tests/test_views_diagram_page.py @@ -4,18 +4,6 @@ from django.urls import reverse from taskdatabase.models import Task, Workflow class DiagramPageViewTest(TestCase): - @classmethod - def setUpTestData(cls): - - # Set up non-modified objects used by all test methods - workflow = Workflow() - workflow.save() - - # create a list of Tasks - Task.objects.get_or_create(sas_id=12345, status='finished', workflow = workflow) - Task.objects.get_or_create(sas_id=12345, status='finished', workflow = workflow) - Task.objects.get_or_create(sas_id=12345, status='stored', workflow = workflow) - def test_url_exists_at_desired_location(self): response = self.client.get('/atdb/diagram/') self.assertEqual(response.status_code, 200) diff --git a/atdb/taskdatabase/tests/test_views_discarded_page.py b/atdb/taskdatabase/tests/test_views_discarded_page.py index 9e79f2a6dca3ece16bec5db6fa0e6d877422bee8..da0400cb7a7d95e159171b20d5828f4b97789931 100644 --- a/atdb/taskdatabase/tests/test_views_discarded_page.py +++ b/atdb/taskdatabase/tests/test_views_discarded_page.py @@ -4,18 +4,6 @@ from django.urls import reverse from taskdatabase.models import Task, Workflow class DiscardedPageViewTest(TestCase): - @classmethod - def setUpTestData(cls): - - # Set up non-modified objects used by all test methods - workflow = Workflow() - workflow.save() - - # create a list of Tasks - Task.objects.get_or_create(sas_id=12345, status='discarded', workflow = workflow) - Task.objects.get_or_create(sas_id=12345, status='discarded', workflow = workflow) - Task.objects.get_or_create(sas_id=12345, status='stored', workflow = workflow) - def test_url_exists_at_desired_location(self): response = self.client.get('/atdb/discarded') self.assertEqual(response.status_code, 200) diff --git a/atdb/taskdatabase/tests/test_views_failures_page.py b/atdb/taskdatabase/tests/test_views_failures_page.py index a2342724238963eef495229587c9d62c0e5a1851..662a43191fac451d23540af6cc8635b8558116b0 100644 --- a/atdb/taskdatabase/tests/test_views_failures_page.py +++ b/atdb/taskdatabase/tests/test_views_failures_page.py @@ -4,17 +4,6 @@ from django.urls import reverse from taskdatabase.models import Task, Workflow class FailuresPageViewTest(TestCase): - @classmethod - def setUpTestData(cls): - - # Set up non-modified objects used by all test methods - workflow = Workflow() - workflow.save() - - # create a list of Tasks - Task.objects.get_or_create(sas_id=12345, status='processed_failed', workflow = workflow) - Task.objects.get_or_create(sas_id=12345, status='processed_failed', workflow = workflow) - Task.objects.get_or_create(sas_id=12345, status='stored', workflow = workflow) def test_url_exists_at_desired_location(self): response = self.client.get('/atdb/failures') diff --git a/atdb/taskdatabase/tests/test_views_filter_page.py b/atdb/taskdatabase/tests/test_views_filter_page.py index 545c5448bf48dba8aa964c517ee02f896fb0cead..18a81731aa3922de2fcf1cd3a398b2c8fa5f76a9 100644 --- a/atdb/taskdatabase/tests/test_views_filter_page.py +++ b/atdb/taskdatabase/tests/test_views_filter_page.py @@ -4,18 +4,6 @@ from django.urls import reverse from taskdatabase.models import Task, Workflow class FilterPageViewTest(TestCase): - @classmethod - def setUpTestData(cls): - - # Set up non-modified objects used by all test methods - workflow = Workflow() - workflow.save() - - # create a list of Tasks - Task.objects.get_or_create(sas_id=12345, status='finished', workflow = workflow) - Task.objects.get_or_create(sas_id=12345, status='finished', workflow = workflow) - Task.objects.get_or_create(sas_id=12345, status='stored', workflow = workflow) - def test_url_exists_at_desired_location(self): response = self.client.get('/atdb/query/') self.assertEqual(response.status_code, 200) diff --git a/atdb/taskdatabase/tests/test_views_finished_page.py b/atdb/taskdatabase/tests/test_views_finished_page.py index 3de45fa27df79bd994a2763b845408de1d62a4d7..df8ba4138f44277ebd961f50e1b40793c3803d37 100644 --- a/atdb/taskdatabase/tests/test_views_finished_page.py +++ b/atdb/taskdatabase/tests/test_views_finished_page.py @@ -4,17 +4,6 @@ from django.urls import reverse from taskdatabase.models import Task, Workflow class FinishedPageViewTest(TestCase): - @classmethod - def setUpTestData(cls): - - # Set up non-modified objects used by all test methods - workflow = Workflow() - workflow.save() - - # create a list of Tasks - Task.objects.get_or_create(sas_id=12345, status='finished', workflow = workflow) - Task.objects.get_or_create(sas_id=12345, status='finished', workflow = workflow) - Task.objects.get_or_create(sas_id=12345, status='stored', workflow = workflow) def test_url_exists_at_desired_location(self): response = self.client.get('/atdb/finished') diff --git a/atdb/taskdatabase/tests/test_views_get_summary.py b/atdb/taskdatabase/tests/test_views_get_summary.py new file mode 100644 index 0000000000000000000000000000000000000000..22deb354511a295e1989d46dd74575ae15d5a68c --- /dev/null +++ b/atdb/taskdatabase/tests/test_views_get_summary.py @@ -0,0 +1,145 @@ +from django.test import TestCase +from django.urls import reverse +from django.http import JsonResponse, HttpResponse + +from taskdatabase.models import Task, Workflow +import taskdatabase.tests.test_calculated_qualities_outputs as outputs +import json + +class GetSummaryTestCase(TestCase): + def setUp(self): + + workflow_requantisation = Workflow(workflow_uri="psrfits_requantisation") + workflow_requantisation.save() + + # rfi_percent=0 + Task.objects.get_or_create(sas_id=54321, status='processed', + outputs=outputs.default_summary_flavour_with_rfi_percent_zero_1, + workflow=workflow_requantisation) + + # default summary flavour + Task.objects.get_or_create(sas_id=54321, status='processed', outputs=outputs.default_summary_flavour_with_rfi_1, + workflow=workflow_requantisation) + Task.objects.get_or_create(sas_id=54321, status='processed', outputs=outputs.default_summary_flavour_with_rfi_2, + workflow=workflow_requantisation) + Task.objects.get_or_create(sas_id=54321, status='processed', outputs=outputs.default_summary_flavour_with_rfi_3, + workflow=workflow_requantisation) + Task.objects.get_or_create(sas_id=54321, status='processed', outputs=outputs.default_summary_flavour_with_rfi_4, + workflow=workflow_requantisation) + + # test image compression, rfi_percentage=1.7186448587105623 + workflow_imaging_compression = Workflow(workflow_uri="imaging_compress_pipeline_v011") + workflow_imaging_compression.save() + Task.objects.get_or_create(sas_id=55555, status='processed', outputs=outputs.imaging_compression_summary_flavor_with_rfi_1, workflow=workflow_imaging_compression) + + # LINC pipelines (no rfi_percent onboard yet) + workflow_linc_calibrator = Workflow(workflow_uri="linc_calibrator_v4_2") + workflow_linc_calibrator.save() + Task.objects.get_or_create(sas_id=666666, status='processed', outputs=outputs.link_calibrator_summary_without_rfi, workflow=workflow_linc_calibrator) + + workflow_linc_target = Workflow(workflow_uri="linc_target_v4_2") + workflow_linc_target.save() + Task.objects.get_or_create(sas_id=666667, status='processed', outputs=outputs.link_target_summary_without_rfi, workflow=workflow_linc_target) + + + def test_summary_json_response(self): + # Mock request + response = self.client.get(reverse('get-summary', args=['54321', 'json'])) + + # Check if response is JsonResponse + self.assertIsInstance(response, JsonResponse) + + def test_summary_json_contents(self): + response = self.client.get(reverse('get-summary', args=['54321', 'json'])) + + # Check if response is JsonResponse + self.assertIsInstance(response, JsonResponse) + + # Add more assertions as needed + json_data = json.loads(response.content.decode('utf-8')) + + # is this json generated for the expected SAS_ID? + expected = "Summary File for SAS_ID 54321" + actual = json_data['summary']['title'] + self.assertEqual(expected, actual) + + # are all the tasks in the json? + tasks = json_data['summary']['tasks'] + actual = len(tasks) + expected = 5 + self.assertEqual(expected, actual) + + # check 1 task for correct contents + t = tasks[0] + self.assertEqual(t['file'], 'L526107_summaryIS.tar') + self.assertEqual(t['input_name'], 'L526107_summaryIS.tar') + self.assertEqual(t['input_size'], 495749120) + self.assertEqual(t['output_size'], 283791360) + self.assertEqual(t['size_ratio'], '0.572') + + def test_summary_json_contents_linc(self): + response = self.client.get(reverse('get-summary', args=['666666', 'json'])) + + # Check if response is JsonResponse + self.assertIsInstance(response, JsonResponse) + + # Add more assertions as needed + json_data = json.loads(response.content.decode('utf-8')) + + # is this json generated for the expected SAS_ID? + expected = "Summary File for SAS_ID 666666" + actual = json_data['summary']['title'] + self.assertEqual(expected, actual) + + # are all the tasks in the json? + tasks = json_data['summary']['tasks'] + actual = len(tasks) + expected = 1 + self.assertEqual(expected, actual) + + # check 1 task for correct contents + t = tasks[0] + self.assertEqual(t['basename'], '3c48_LINC_calibrator_summary.json') + self.assertEqual(t['class'], 'File') + self.assertEqual(t['checksum'], 'sha1$531646ff527d76f4facdabf72d939bac302eaf1f') + self.assertEqual(t['location'], 'file:///project/ldv/Share/run/2023/6/16/1352_35011/3c48_LINC_calibrator_summary.json') + self.assertEqual(t['surl'], 'srm://srm.grid.sara.nl/pnfs/grid.sara.nl/data/lofar/ops/disk/ldv/lt10_010/689478/35011/3c48_LINC_calibrator_summary.json') + + def test_summary_html_response(self): + # Mock request + response = self.client.get(reverse('get-summary', args=['your_sas_id', 'html'])) + + # Check if response is HttpResponse + self.assertIsInstance(response, HttpResponse) + + + def test_summary_html_contents(self): + response = self.client.get(reverse('get-summary', args=['54321', 'html'])) + + # Check if response is JsonResponse + self.assertIsInstance(response, HttpResponse) + + # Add more assertions as needed + html_data = response.content.decode('utf-8') + + # is this html generated for the expected SAS_ID? + title = "Summary File for SAS_ID 54321" + found = False + if title in html_data: + found = True + self.assertEqual(found, True) + + # does this filename exist in the html? + input_name = "L526107_summaryIS.tar" + found = False + if input_name in html_data: + found = True + self.assertEqual(found, True) + + + def test_summary_pdf_response(self): + # Mock request + response = self.client.get(reverse('get-summary', args=['your_sas_id', 'pdf'])) + + # Check if response is HttpResponse + self.assertIsInstance(response, HttpResponse) \ No newline at end of file diff --git a/atdb/taskdatabase/urls.py b/atdb/taskdatabase/urls.py index b846c15dcf892c1cfd9ba1dbc854ea4b46acec19..03577e508ebd3c351653728ddb1c36689d101c0d 100644 --- a/atdb/taskdatabase/urls.py +++ b/atdb/taskdatabase/urls.py @@ -101,6 +101,10 @@ urlpatterns = [ path('get_unique_values_for_key/<str:aggregation_key>/', views.GetUniqueValuesForKey.as_view(), name='get-unique-values-for-key-view'), + # ancillary dataproducts endpoints (retrieved by archiver service for copy to LTA dcache). + # /atdb/get_summary/606942/html + path('get_summary/<sas_id>/<format>', views.GetSummary, name='get-summary'), # format can be 'json', 'html' or 'pdf' + # --- controller resources --- path('tasks/<int:pk>/setstatus/<new_status>/<page>', views.TaskSetStatus, name='task-setstatus-view'), path('tasks/<int:pk>/setstatus/<new_status>', views.TaskSetStatus, name='task-details-setstatus'), diff --git a/atdb/taskdatabase/views.py b/atdb/taskdatabase/views.py index ffb7a79b57aac6ecc8eada10b7ca026428e6da4d..e3469e622ebb30439a37691d76cdcd5b1140a1fa 100644 --- a/atdb/taskdatabase/views.py +++ b/atdb/taskdatabase/views.py @@ -15,9 +15,10 @@ from django.contrib.auth.decorators import login_required from django.views.generic import ListView from django.contrib import messages + from rest_framework import generics from rest_framework.response import Response -from django.http import JsonResponse +from django.http import JsonResponse, HttpResponse from django_filters import rest_framework as filters from django_filters.views import FilterView @@ -26,9 +27,10 @@ from django_tables2.views import SingleTableMixin from django.shortcuts import render, redirect, reverse from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger from django.contrib.admin.views.decorators import staff_member_required -from django.db.models import Count, F, Func, ExpressionWrapper, DurationField -from django.db.models.functions import TruncHour, TruncDay -from django.db.models import IntegerField, DateTimeField + +from django.template.loader import get_template +from xhtml2pdf import pisa +from io import BytesIO from rest_framework.request import Request #from silk.profiling.profiler import silk_profile @@ -1701,6 +1703,71 @@ class GetUniqueValuesForKey(generics.ListAPIView): 'error': str(error) }) + + +def GetSummary(request, sas_id, format): + """ + Construct and return a summary structure for given sas_id in the requested format (json, html or pdf) + This is the same informtion and algorithm as used when the user clicks the SUM button on the Validation page. + See documentation: https://drive.google.com/file/d/16R8L06OFiKHFHBUA6FhrNVZVAaQBC2tU/view?usp=sharing + + The return is either a JsonResponse (for format='json') or a HttpResponse for format is 'html' or 'pdf' + example: /atdb/get_summary/606942/json + """ + try: + + # use a trick to be able to use the existing task based code + queryset = Task.objects.filter(sas_id=sas_id) + task = queryset[0] + + if format == 'json': + summary_json = algorithms.construct_summary(task,format='json') + + return JsonResponse({ + 'summary': summary_json + }) + + # for both the formats 'html' and 'pdf' the html must be constructed first + # add some basic layout without using the summary.html template + head_html=""" + <head> + <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css"> + <link href='https://fonts.googleapis.com/css?family=Arial' rel='stylesheet' type='text/css'> + </head> + """ + + template = get_template("taskdatabase/validation/summary.html") + summary_html = algorithms.construct_summary(task) + context = {'task': task, 'my_summary': summary_html} + html = head_html + template.render(context) + + if format == 'html': + # for 'html' the operation is ready, return the html + return HttpResponse(html) + + if format == 'pdf': + # for pdf, convert the html to pdf + + # Create a BytesIO object to receive the PDF data + result = BytesIO() + + # Convert HTML to PDF + # TODO: fonts are wrong, can that be fixed somehow? + pdf = pisa.pisaDocument(BytesIO(html.encode("UTF-8")), result) + pdf_name = sas_id + '_summary.pdf' + if not pdf.err: + # Return the PDF as a response + response = HttpResponse(result.getvalue(), content_type='application/pdf') + response['Content-Disposition'] = f'attachment; filename={pdf_name}' + return response + + except Exception as error: + logger.error(error) + return JsonResponse({ + 'error': str(error) + }) + + @staff_member_required def AssociateActivities(request): # disconnect the signals to avoid unneccesary updates