TMSS-662: fix bug with duplicate entries in SU table data and add flag to...

TMSS-662: fix bug with duplicate entries in SU table data and add flag to restrict histogram timespan

SAS/TMSS/backend/src/tmss/tmssapp/adapters/reports.py

@@ -698,14 +698,17 @@ class FailureReport():
             aggregates[f'by_{field_name}'] = group_aggregates
         return aggregates
 
-    def _get_lost_observing_time_histogram(self) -> {}:
+    def _get_lost_observing_time_histogram(self, ignore_time_range=True) -> {}:
         """
         Help method to get lost observing time over entire lifetime (not restricted to report period)
         Note: This is using on sky times of scheduling unit that are effected by system events of severity 'failure'.
         """
         histogram = {}
-        # do not use self.system_events here because we want to ignore the reporting period here
-        system_events = models.SystemEvent.objects.filter(severity__value=models.SystemEventSeverity.Choices.FAILURE.value)
+        if ignore_time_range:
+            # do not use self.system_events here because we want to ignore the reporting period here
+            system_events = models.SystemEvent.objects.filter(severity__value=models.SystemEventSeverity.Choices.FAILURE.value)
+        else:
+            system_events = self.system_events
         affected_tasks = models.TaskBlueprint.objects.filter(system_events__in=system_events)
         affected_units = (models.SchedulingUnitBlueprint.objects.filter(task_blueprints__in=affected_tasks))
 
@@ -717,14 +720,17 @@ class FailureReport():
         histogram = {k.date().isoformat():v for k,v in histogram.items()}
         return histogram
 
-    def _get_system_event_time_histogram(self) -> {}:
+    def _get_system_event_time_histogram(self, ignore_time_range=True) -> {}:
         """
         Help method to get lost observing time over entire lifetime (not restricted to report period)
         Note: This is based on reported system event start and stop times, not based on the affected observations.
         """
         histogram = {}
-        # do not use self.system_events here because we want to ignore the reporting period here
-        system_events = models.SystemEvent.objects.filter(severity__value=models.SystemEventSeverity.Choices.FAILURE.value).order_by('start')
+        if ignore_time_range:
+            # do not use self.system_events here because we want to ignore the reporting period here
+            system_events = models.SystemEvent.objects.filter(severity__value=models.SystemEventSeverity.Choices.FAILURE.value).order_by('start')
+        else:
+            system_events = self.system_events.order_by('start')
 
         for event in system_events.all():
             event_stop = event.stop if event.stop else datetime.utcnow()
@@ -759,7 +765,7 @@ class FailureReport():
         """
         unit_list = []
         affected_tasks = models.TaskBlueprint.objects.filter(system_events__in=self.system_events)
-        affected_units = (models.SchedulingUnitBlueprint.objects.filter(task_blueprints__in=affected_tasks))
+        affected_units = models.SchedulingUnitBlueprint.objects.filter(task_blueprints__in=affected_tasks).distinct()
         for unit in affected_units.all():
             observation_subtasks = (models.Subtask.independent_subtasks().filter(task_blueprint__scheduling_unit_blueprint_id=unit.id)
                                                                          .filter(specifications_template__type__value=models.SubtaskType.Choices.OBSERVATION.value))
@@ -776,13 +782,13 @@ class FailureReport():
             })
         return unit_list
 
-    def create_failure_report(self) -> {}:
+    def create_failure_report(self, ignore_time_range_for_histogram=True) -> {}:
         """
         Create a failure report as a JSON object.
         """
         result = {'system_event_summary': self._get_system_event_summary(),
-                  'lost_time_histogram_on_sky': self._get_lost_observing_time_histogram(),
-                  'lost_time_histogram_event': self._get_system_event_time_histogram(),
+                  'lost_time_histogram_on_sky': self._get_lost_observing_time_histogram(ignore_time_range=ignore_time_range_for_histogram),
+                  'lost_time_histogram_event': self._get_system_event_time_histogram(ignore_time_range=ignore_time_range_for_histogram),
                   'failed_scheduling_units': self._get_failed_scheduling_units()}
         result['total_duration_lost_event'] = sum([d['duration_lost_event'] for d in result['system_event_summary']['by_issue_type']])
         result['total_duration_lost_on_sky'] = sum([d['duration_lost_on_sky'] for d in result['system_event_summary']['by_issue_type']])
@@ -795,9 +801,9 @@ class FailureReport():
         return result
 
 
-def create_failure_report(start: datetime, stop: datetime) -> {}:
+def create_failure_report(start: datetime, stop: datetime, ignore_time_range_for_histogram=True) -> {}:
     """
     Create a failure report as a JSON object.
     """
-    return FailureReport(start, stop).create_failure_report()
+    return FailureReport(start, stop).create_failure_report(ignore_time_range_for_histogram)
 

SAS/TMSS/backend/src/tmss/tmssapp/views.py

@@ -31,6 +31,7 @@ from lofar.sas.tmss.tmss.tmssapp.conversions import local_sidereal_time_for_utc_
 from lofar.sas.tmss.tmss.tmssapp.adapters.keycloak import get_names_by_role_in_project
 from lofar.sas.tmss.tmss.tmssapp.viewsets.permissions import get_project_roles_for_user
 from lofar.sas.tmss.tmss.tmssapp.adapters.reports import create_failure_report
+from distutils.util import strtobool
 
 from django.views.decorators.cache import cache_page
 
@@ -724,12 +725,16 @@ def lst_target_pressure_plot(request, from_date: str=None, until_date: str=None)
                      manual_parameters=[Parameter(name='start', required=True, type='string', in_='query',
                                                   description="Datetime in isoformat for start of reporting period"),
                                         Parameter(name='stop', required=True, type='string', in_='query',
-                                                  description="Datetime in isoformat for stop of reporting period")])
+                                                  description="Datetime in isoformat for stop of reporting period"),
+                                        Parameter(name='ignore_time_range_for_histogram', required=False, type='string', in_='query',
+                                                  description="for the lost time histogram, aggregate over all events known to TMSS"),
+                                        ])
 @api_view(['GET'])
 def failure_report(request):
 
     # Check start and stop times
     start, stop = request.query_params.get('start', None), request.query_params.get('stop', None)
+    ignore_time_range_for_histogram = strtobool(request.query_params.get('ignore_time_range_for_histogram', 'True'))
     if start:
         try:
             start = dateutil.parser.parse(start, ignoretz=True)
@@ -741,5 +746,5 @@ def failure_report(request):
         except Exception:
             return RestResponse('Error: please specify an isoformat timestamp for stop_time', status=400)
 
-    result = create_failure_report(start, stop)
+    result = create_failure_report(start, stop, ignore_time_range_for_histogram)
     return JsonResponse(result, status=status.HTTP_200_OK)
\ No newline at end of file