TMSS-857: Fix successful/failed SUBs filter retrieve, cleanup

SAS/TMSS/backend/src/tmss/tmssapp/adapters/reports.py

@@ -47,11 +47,11 @@ def _get_telescope_time_distribution(cycle: models.Cycle) -> {}:
             # Get durations for single project
             subs = models.SchedulingUnitBlueprint.objects.filter(draft__scheduling_set__project=p.pk)
             for sub in subs:
-                accepted = SchedulingUnitProcess.objects.filter(su=sub.pk, results_accepted__isnull=False).values('results_accepted')
+                sup = SchedulingUnitProcess.objects.filter(su=sub).first()
                 # Aggregate total, successful and failed durations
                 total += sub.observed_duration.total_seconds()
-                succeeded += sub.observed_duration.total_seconds() if accepted else 0
-                failed += sub.observed_duration.total_seconds() if accepted is False else 0
+                succeeded += sub.observed_duration.total_seconds() if sup and sup.results_accepted else 0
+                failed += sub.observed_duration.total_seconds() if sup and sup.results_accepted is False else 0
         idle = total - succeeded - failed
         result[c if c == 'UNASSIGNED' or c == 'FILLER' else c.name] = {'durations': {'total': total, 'succeeded': succeeded,
                                                                        'failed': failed, 'idle': idle}}
@@ -82,9 +82,9 @@ def _get_average_efficiency(cycle: models.Cycle) -> {}:
         for sub in subs:
             # Aggregate total and successful durations
             if sub.observed_duration and d <= sub.observed_start_time < d + step and sub.observed_end_time < d + step:
-                accepted = SchedulingUnitProcess.objects.filter(su=sub.pk, results_accepted__isnull=False).values('results_accepted')
+                sup = SchedulingUnitProcess.objects.filter(su=sub).first()
                 total_per_day += sub.observed_duration.total_seconds()
-                total_succeeded_per_day += sub.observed_duration.total_seconds() if accepted else 0
+                total_succeeded_per_day += sub.observed_duration.total_seconds() if sup and sup.results_accepted else 0
         efficiency_per_day += total_succeeded_per_day / total_per_day if total_per_day > 0 else 0
         i += 1
         d += step
@@ -111,9 +111,9 @@ def _get_completion_level(cycle: models.Cycle) -> {}:
     for sub in subs:
         # Aggregate total and successful durations
         if sub.observed_duration:
-            accepted = SchedulingUnitProcess.objects.filter(su=sub.pk, results_accepted__isnull=False).values('results_accepted')
+            sup = SchedulingUnitProcess.objects.filter(su=sub).first()
             total += sub.observed_duration.total_seconds()
-            total_succeeded += sub.observed_duration.total_seconds() if accepted else 0
+            total_succeeded += sub.observed_duration.total_seconds() if sup and sup.results_accepted else 0
 
     result['total'], result['succeeded'] = total, total_succeeded
 
@@ -136,9 +136,9 @@ def _get_observation_hours_per_category(cycle: models.Cycle) -> {}:
         for sub in subs:
             result['total_duration_idle'] += sub.observed_duration.total_seconds() if sub.observed_duration else 0
             if sub.observed_duration:
-                accepted = SchedulingUnitProcess.objects.filter(su=sub.pk, results_accepted__isnull=False).values('results_accepted')
-                result[f'total_duration_{prio.name}'] += sub.observed_duration.total_seconds() if accepted else 0
-                result['total_duration_failed'] += sub.observed_duration.total_seconds() if accepted is False else 0
+                sup = SchedulingUnitProcess.objects.filter(su=sub).first()
+                result[f'total_duration_{prio.name}'] += sub.observed_duration.total_seconds() if sup and sup.results_accepted else 0
+                result['total_duration_failed'] += sub.observed_duration.total_seconds() if sup and sup.results_accepted is False else 0
         # Subtract prio states from total to get partial idle
         result['total_duration_idle'] -= result[f'total_duration_{prio.name}']
     # Subtract total failed to get total idle eventually
@@ -169,9 +169,9 @@ def _get_weekly_efficiency(cycle: models.Cycle) -> {}:
         for sub in subs:
             # Aggregate total and successful durations
             if sub.observed_duration and d <= sub.observed_start_time < d + step and sub.observed_end_time < d + step:
-                accepted = SchedulingUnitProcess.objects.filter(su=sub.pk, results_accepted__isnull=False).values('results_accepted')
+                sup = SchedulingUnitProcess.objects.filter(su=sub).first()
                 total_per_week += sub.observed_duration.total_seconds()
-                total_succeeded_per_week += sub.observed_duration.total_seconds() if accepted else 0
+                total_succeeded_per_week += sub.observed_duration.total_seconds() if sup and sup.results_accepted else 0
         result['weeks'].append(
             {'week': d.date().isoformat(), 'efficiency': total_succeeded_per_week / total_per_week if total_per_week > 0 else None})
         d += step
@@ -287,9 +287,9 @@ def _get_failures(cycle: models.Cycle) -> {}:
         for sub in subs:
             # Aggregate total and successful durations
             if sub.observed_duration and d <= sub.observed_start_time < d + step and sub.observed_end_time < d + step:
-                accepted = SchedulingUnitProcess.objects.filter(su=sub.pk, results_accepted__isnull=False).values('results_accepted')
+                sup = SchedulingUnitProcess.objects.filter(su=sub).first()
                 total_per_month += sub.observed_duration.total_seconds()
-                total_failed_per_month += sub.observed_duration.total_seconds() if accepted is False else 0
+                total_failed_per_month += sub.observed_duration.total_seconds() if sup and sup.results_accepted is False else 0
         result['months'].append({'month': d.date().isoformat(), 'total': total_per_month, 'total_failed': total_failed_per_month})
         d += step
 
@@ -329,24 +329,25 @@ def _get_subs_and_durations_from_project(project_pk: int) -> ({}, {}):
     # Get SUBs related to the project
     scheduling_unit_blueprints = models.SchedulingUnitBlueprint.objects.filter(draft__scheduling_set__project__pk=project_pk)
     # TODO: Split into total, prio A, prio B. See TMSS-592.
-    total_duration, total_succeeded_duration, total_failed_duration = timedelta(), timedelta(), timedelta()
+    total_duration, total_succeeded_duration, total_failed_duration, total_not_cancelled = timedelta(), timedelta(), timedelta(), timedelta()
     subs_succeeded, subs_failed = [], []
 
     # NOTE: This might be optimised later with the use of Django's ORM as done for LTA dataproducts.
     for sub in scheduling_unit_blueprints:  # Distinguish between succeeded and failed observations
         # TODO: Clarify if this info should be related only to obs or all SUBs in general. The latter are considered for now.
         #       We can filter observations by considering observed_duration in the SUB, for now. See TMSS-610 comments.
-        accepted = SchedulingUnitProcess.objects.filter(su=sub.pk, results_accepted__isnull=False).values('results_accepted')
-        if accepted:  # Succeeded SUBs
+        sup = SchedulingUnitProcess.objects.filter(su=sub).first()
+        if sup and sup.results_accepted:  # Succeeded SUBs
             total_succeeded_duration += sub.duration
             subs_succeeded.append({'id': sub.pk, 'name': sub.name, 'duration': sub.duration.total_seconds()})
-        elif accepted is False:  # Failed SUBs
+        elif sup and sup.results_accepted is False:  # Failed SUBs
             total_failed_duration += sub.duration
             subs_failed.append({'id': sub.pk, 'name': sub.name, 'duration': sub.duration.total_seconds()})
         # Total duration without considering the status of the SUBs.
         total_duration += sub.duration
+        if sub.status != 'cancelled':   # Calculate not_cancelled duration
+            total_not_cancelled += sub.duration
 
-    total_not_cancelled = total_duration - total_failed_duration  # Calculate not_cancelled duration
     durations = {'total': total_duration.total_seconds(), 'total_succeeded': total_succeeded_duration.total_seconds(),
                  'total_not_cancelled': total_not_cancelled.total_seconds(), 'total_failed': total_failed_duration.total_seconds()}
     subs = {'finished': subs_succeeded, 'failed': subs_failed}

SAS/TMSS/backend/test/t_adapter.py

@@ -33,9 +33,24 @@ exit_with_skipped_code_if_skip_integration_tests()
 # Do Mandatory setup step:
 # use setup/teardown magic for tmss test database, ldap server and django server
 # (ignore pycharm unused import statement, python unittests does use at RunTime the tmss_test_environment_unittest_setup module)
-from lofar.sas.tmss.test.tmss_test_environment_unittest_setup import *
-tmss_test_env.populate_schemas()
-# TODO: Start the workflow service as well as viewflow.
+
+from lofar.sas.tmss.test.test_environment import TMSSTestEnvironment
+tmss_test_env = TMSSTestEnvironment(start_workflow_service=True, enable_viewflow=True)
+try:
+    tmss_test_env.start()
+    tmss_test_env.populate_schemas()
+except Exception as e:
+    logger.exception(str(e))
+    tmss_test_env.stop()
+    exit(1)
+
+# tell unittest to stop (and automagically cleanup) the test database once all testing is done.
+def tearDownModule():
+    tmss_test_env.stop()
+
+
+AUTH = requests.auth.HTTPBasicAuth(tmss_test_env.ldap_server.dbcreds.user, tmss_test_env.ldap_server.dbcreds.password)
+BASE_URL = tmss_test_env.django_server.url[:-1] if tmss_test_env.django_server.url.endswith('/')  else tmss_test_env.django_server.url
 
 from lofar.sas.tmss.test.tmss_test_data_django_models import *
 
@@ -44,6 +59,7 @@ from lofar.sas.tmss.test.tmss_test_data_rest import TMSSRESTTestDataCreator
 rest_data_creator = TMSSRESTTestDataCreator(BASE_URL, AUTH)
 
 from lofar.sas.tmss.tmss.tmssapp import models
+from lofar.sas.tmss.tmss.workflowapp.models.schedulingunitflow import SchedulingUnitProcess
 from lofar.sas.tmss.tmss.exceptions import SubtaskInvalidStateException
 from lofar.sas.tmss.tmss.tmssapp.adapters.parset import convert_to_parset, convert_to_parset_dict
 from lofar.common.json_utils import get_default_json_object_for_schema, add_defaults_to_json_object_for_schema
@@ -476,10 +492,11 @@ class ProjectReportTest(unittest.TestCase):
         """
         Test create project extra action.
         """
-        # FIXME: Distinguish SUBs according to the workflow flag (and consider observed_duration instead of duration?).
         # Create and set three SUBs and respectively set the following states: 'finished', 'cancelled', 'defined' (not cancelled)
         succeeded_sub, _, succeeded_subtask = self._get_SUB_with_subtask_and_set_status('finished')
-        cancelled_sub, _, cancelled_subtask = self._get_SUB_with_subtask_and_set_status('cancelled')
+        SchedulingUnitProcess.objects.create(su=succeeded_sub, results_accepted=True)
+        failed_sub, _, failed_subtask = self._get_SUB_with_subtask_and_set_status('cancelled')
+        SchedulingUnitProcess.objects.create(su=failed_sub, results_accepted=False)
         not_cancelled_sub, _, not_cancelled_subtask = self._get_SUB_with_subtask_and_set_status('defined')
 
         # Create SubtaskOutput and Dataproducts from subtask_output
@@ -487,13 +504,6 @@ class ProjectReportTest(unittest.TestCase):
         dataproduct1 = models.Dataproduct.objects.create(**Dataproduct_test_data(producer=subtask_output))
         dataproduct2 = models.Dataproduct.objects.create(**Dataproduct_test_data(producer=subtask_output))
 
-        # Calculate expected durations
-        total = succeeded_subtask.duration.total_seconds() + cancelled_subtask.duration.total_seconds() + \
-                not_cancelled_subtask.duration.total_seconds()
-        total_succeeded = succeeded_subtask.duration.total_seconds()
-        total_not_cancelled = succeeded_subtask.duration.total_seconds() + not_cancelled_subtask.duration.total_seconds()
-        total_failed = cancelled_subtask.duration.total_seconds()
-
         # Assert we get the expected object
         response = requests.get(BASE_URL + '/project/%s/report' % self.project.pk, auth=self.test_data_creator.auth)
         self.assertEqual(response.status_code, 200)
@@ -504,15 +514,16 @@ class ProjectReportTest(unittest.TestCase):
         self.assertEqual(result['quota'][0]['id'], self.project_quota.pk)
 
         # Assert durations are well calculated
-        self.assertAlmostEqual(result['durations']['total'], total)
-        self.assertAlmostEqual(result['durations']['total_succeeded'], total_succeeded)
-        self.assertAlmostEqual(result['durations']['total_not_cancelled'], total_not_cancelled)
-        self.assertAlmostEqual(result['durations']['total_failed'], total_failed)
+        # Note: there are three SUBs: one successful, one failed and one not cancelled. Each has a duration=600.
+        self.assertAlmostEqual(result['durations']['total'], 1800.0, places=4)
+        self.assertAlmostEqual(result['durations']['total_succeeded'], 600.0, places=4)
+        self.assertAlmostEqual(result['durations']['total_not_cancelled'], 1200.0, places=4)
+        self.assertAlmostEqual(result['durations']['total_failed'], 600.0, places=4)
 
         # There is only one finished SUB
         self.assertEqual(result['SUBs']['finished'][0]['id'], succeeded_sub.pk)
         # There is only one cancelled SUB
-        self.assertEqual(result['SUBs']['failed'][0]['id'], cancelled_sub.pk)
+        self.assertEqual(result['SUBs']['failed'][0]['id'], failed_sub.pk)
 
         # There are just two dataproducts
         self.assertEqual(result['LTA dataproducts']['size__sum'], dataproduct1.size + dataproduct2.size)