Merge branch 'master' into TMSS-637

SAS/TMSS/backend/src/tmss/tmssapp/adapters/CMakeLists.txt

@@ -5,6 +5,7 @@ set(_py_files
     parset.py
     sip.py
     feedback.py
+    reports.py
     )
 
 python_install(${_py_files}

SAS/TMSS/backend/src/tmss/tmssapp/adapters/reports.py

+from django.db.models import Sum
+from lofar.sas.tmss.tmss.tmssapp import models
+from lofar.sas.tmss.tmss.tmssapp import serializers
+
+from rest_framework.request import Request
+from datetime import timedelta
+
+
+def create_project_report(request: Request, project: models.Project) -> {}:
+    """
+    Create a project report as a JSON object.
+    """
+    result = {'project': project.pk}
+    result['quota'] = _get_quotas_from_project(request, project.pk)
+    result['durations'] = _get_subs_and_durations_from_project(project.pk)
+    result['LTA dataproducts'] = _get_lta_dataproducts(project.name)
+    result['SAPs'] = _get_saps(project.pk)
+
+    return result
+
+
+def _get_quotas_from_project(request: Request, project_pk: int) -> []:
+    """
+    Help function to retrieve quotas.
+    """
+    project_quotas = models.ProjectQuota.objects.filter(project=project_pk)
+    project_quotas_data = [serializers.ProjectQuotaSerializer(pq, context={'request': request}).data for pq in project_quotas]
+    quotas = [{k: pqd[k] for k in ('id', 'resource_type_id', 'value')} for pqd in project_quotas_data]
+    return quotas
+
+
+def _get_subs_and_durations_from_project(project_pk: int) -> {}:
+    """
+    Help function to retrieve durations and scheduling_units distinguished by success/fail.
+    """
+    # Get SUBs related to the project
+    scheduling_unit_blueprints = models.SchedulingUnitBlueprint.objects.filter(draft__scheduling_set__project__pk=project_pk)
+    # TODO: Split into total, prio A, prio B? See TMSS-592.
+    total_duration, total_succeeded_duration, total_failed_duration = timedelta(), timedelta(), timedelta()
+    subs_succeeded, subs_failed = [], []
+
+    # NOTE: This might be optimised later with the use of Django's ORM as done for LTA dataproducts.
+    for sub in scheduling_unit_blueprints:  # Distinguish between succeeded and failed observations
+        # TODO: Use QA workflow flag instead of the finished status? See TMSS-592.
+        if sub.status == 'finished':        # Succeeded observations
+            total_succeeded_duration += sub.duration
+            subs_succeeded.append({'id': sub.pk, 'name': sub.name, 'duration': sub.duration.total_seconds()})
+        elif sub.status == 'cancelled':     # Failed observations
+            total_failed_duration += sub.duration
+            subs_failed.append({'id': sub.pk, 'name': sub.name, 'duration': sub.duration.total_seconds()})
+        total_duration += sub.duration      # Total duration without considering the status of the obs.
+
+    total_not_cancelled = total_duration - total_failed_duration  # Calculate not_cancelled duration
+    durations = {'total': total_duration.total_seconds(), 'total_succeeded': total_succeeded_duration.total_seconds(),
+                 'total_not_cancelled': total_not_cancelled.total_seconds(), 'total_failed': total_failed_duration.total_seconds(),
+                 'scheduling_unit_blueprints_finished': subs_succeeded, 'scheduling_unit_blueprints_failed': subs_failed}
+    return durations
+
+
+def _get_lta_dataproducts(project_name: str) -> int:
+    """
+    Help function to retrieve the sum of the LTA dataproducts sizes.
+    """
+    # Query dataproducts from Subtasks of type 'ingest' within 'finished' status
+    return models.Dataproduct.objects.filter(producer__subtask__specifications_template__type='ingest') \
+        .filter(producer__subtask__state__value='finished') \
+        .filter(producer__subtask__task_blueprint__draft__scheduling_unit_draft__scheduling_set__project__name=project_name) \
+        .aggregate(Sum('size'))
+
+
+def _get_saps(project_pk: int) -> []:
+    """
+    Help function to retrieve SAPs.
+    """
+    # TODO: For each unique target (SAP name) get the sum of target observation durations from the tasks.
+    return [{'sap_name': 'placeholder', 'total_exposure': 0}, ]

SAS/TMSS/backend/src/tmss/tmssapp/viewsets/specification.py

@@ -22,6 +22,7 @@ from drf_yasg.openapi import Parameter
 from lofar.sas.tmss.tmss.tmssapp.viewsets.lofar_viewset import LOFARViewSet, LOFARNestedViewSet, AbstractTemplateViewSet, LOFARCopyViewSet, LOFARFilterBackend
 from lofar.sas.tmss.tmss.tmssapp import models
 from lofar.sas.tmss.tmss.tmssapp import serializers
+from lofar.sas.tmss.tmss.tmssapp.adapters.reports import create_project_report
 from django.http import JsonResponse
 
 from datetime import datetime
@@ -347,6 +348,15 @@ class ProjectViewSet(LOFARViewSet):
 
         return queryset
 
+    @swagger_auto_schema(responses={200: 'The Report information',
+                                    403: 'forbidden'},
+                         operation_description="Get Report information for the project.")
+    @action(methods=['get'], detail=True, url_name="report", name="Get Report")
+    def report(self, request, pk=None):
+        project = get_object_or_404(models.Project, pk=pk)
+        result = create_project_report(request, project)
+        return Response(result, status=status.HTTP_200_OK)
+
 
 class ProjectNestedViewSet(LOFARNestedViewSet):
     queryset = models.Project.objects.all()

SAS/TMSS/backend/test/t_adapter.py

@@ -508,6 +508,88 @@ _isCobalt=T
         self.assertEqual(self.feedback_pipe_complete.strip(), subtask_pipe.raw_feedback.strip())
 
 
+class ProjectReportTest(unittest.TestCase):
+    def setUp(self):
+        # Create requirements
+        self.project = models.Project.objects.create(**Project_test_data(name='test_for_report'))
+        self.project_quota = models.ProjectQuota.objects.create(
+            **ProjectQuota_test_data(project=self.project, resource_type=models.ResourceType.objects.create(
+                **ResourceType_test_data(quantity=models.Quantity.objects.get(value=models.Quantity.Choices.NUMBER.value)))))
+        self.scheduling_set = models.SchedulingSet.objects.create(**SchedulingSet_test_data(project=self.project))
+        self.scheduling_unit_draft = models.SchedulingUnitDraft.objects.create(
+            **SchedulingUnitDraft_test_data(scheduling_set=self.scheduling_set))
+        self.task_draft = models.TaskDraft.objects.create(
+            **TaskDraft_test_data(scheduling_unit_draft=self.scheduling_unit_draft))
+
+        # Create test_data_creator as superuser
+        self.test_data_creator = TMSSRESTTestDataCreator(BASE_URL, AUTH)
+        response = requests.get(self.test_data_creator.django_api_url + '/', auth=self.test_data_creator.auth)
+
+    def _get_SUB_with_subtask_and_set_status(self, status=None):
+        """
+        Help method to create SUB, TaskBlueprint, Subtask and (optionally) set the latter's status.
+        """
+        sub = models.SchedulingUnitBlueprint.objects.create(**SchedulingUnitBlueprint_test_data(draft=self.scheduling_unit_draft))
+        tb = models.TaskBlueprint.objects.create(**TaskBlueprint_test_data(task_draft=self.task_draft, scheduling_unit_blueprint=sub))
+        # Create Subtask of type 'ingest'
+        subtask_template = models.SubtaskTemplate.objects.create(**SubtaskTemplate_test_data(subtask_type_value='ingest'))
+        subtask = models.Subtask.objects.create(**Subtask_test_data(task_blueprint=tb, subtask_template=subtask_template))
+
+        if status:  # Set Subtask status to 'cancelled'
+            with tmss_test_env.create_tmss_client() as client:
+                client.set_subtask_status(subtask.pk, status)
+            # Refreshing Subtask from cache
+            subtask = models.Subtask.objects.get(pk=subtask.pk)
+            while subtask.state.value != status:
+                subtask = models.Subtask.objects.get(pk=subtask.pk)
+
+        return sub, tb, subtask
+
+    def test_create_project_report(self):
+        """
+        Test create project extra action.
+        """
+        # Create and set three SUBs and respectively set the following states: 'finished', 'cancelled', 'defined' (not cancelled)
+        succeeded_sub, _, succeeded_subtask = self._get_SUB_with_subtask_and_set_status('finished')
+        cancelled_sub, _, cancelled_subtask = self._get_SUB_with_subtask_and_set_status('cancelled')
+        not_cancelled_sub, _, not_cancelled_subtask = self._get_SUB_with_subtask_and_set_status('defined')
+
+        # Create SubtaskOutput and Dataproducts from subtask_output
+        subtask_output = models.SubtaskOutput.objects.create(**SubtaskOutput_test_data(subtask=succeeded_subtask))
+        dataproduct1 = models.Dataproduct.objects.create(**Dataproduct_test_data(producer=subtask_output))
+        dataproduct2 = models.Dataproduct.objects.create(**Dataproduct_test_data(producer=subtask_output))
+
+        # Calculate expected durations
+        total = succeeded_subtask.duration.total_seconds() + cancelled_subtask.duration.total_seconds() + \
+                not_cancelled_subtask.duration.total_seconds()
+        total_succeeded = succeeded_subtask.duration.total_seconds()
+        total_not_cancelled = succeeded_subtask.duration.total_seconds() + not_cancelled_subtask.duration.total_seconds()
+        total_failed = cancelled_subtask.duration.total_seconds()
+
+        # Assert we get the expected object
+        response = requests.get(BASE_URL + '/project/%s/report' % self.project.pk, auth=self.test_data_creator.auth)
+        result = response.json()
+
+        # Assert Project and ProjectQuota ids
+        self.assertEqual(result['project'], self.project.pk)
+        self.assertEqual(result['quota'][0]['id'], self.project_quota.pk)
+
+        # Assert durations are well calculated
+        self.assertAlmostEqual(result['durations']['total'], total)
+        self.assertAlmostEqual(result['durations']['total_succeeded'], total_succeeded)
+        self.assertAlmostEqual(result['durations']['total_not_cancelled'], total_not_cancelled)
+        self.assertAlmostEqual(result['durations']['total_failed'], total_failed)
+
+        # There is only one finished SUB
+        self.assertEqual(result['durations']['scheduling_unit_blueprints_finished'][0]['id'], succeeded_sub.pk)
+        # There is only one cancelled SUB
+        self.assertEqual(result['durations']['scheduling_unit_blueprints_failed'][0]['id'], cancelled_sub.pk)
+
+        # There are just two dataproducts
+        self.assertEqual(result['LTA dataproducts']['size__sum'], dataproduct1.size + dataproduct2.size)
+        # Just to check if the placeholder was added
+        self.assertIsNotNone(result['SAPs'])
+
 
 if __name__ == "__main__":
     os.environ['TZ'] = 'UTC'