diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/adapters/reports.py b/SAS/TMSS/backend/src/tmss/tmssapp/adapters/reports.py
index 90336cab44e9e41b4bad8605d2fdc1f61723d318..b371d9f179688876f0a69a681d620825d30f417a 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/adapters/reports.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/adapters/reports.py
@@ -3,11 +3,12 @@ from lofar.sas.tmss.tmss.tmssapp import models
from lofar.sas.tmss.tmss.tmssapp import serializers
from rest_framework.request import Request
-from datetime import datetime, timedelta
-
+from datetime import timedelta
+from dateutil.relativedelta import relativedelta
# Cycle Report
+
def create_cycle_report(request: Request, cycle: models.Cycle) -> {}:
"""
Create a cycle report as a JSON object.
@@ -31,9 +32,10 @@ def _get_telescope_time_distribution(cycle: models.Cycle) -> {}:
"""
Help function to retrieve telescope time distribution info.
"""
+ from lofar.sas.tmss.tmss.workflowapp.models.schedulingunitflow import SchedulingUnitProcess
+
result = {}
- # TODO: Add SYSTEM/IDLE
# Consider UNASSIGNED and FILLER as categories for the purposes of reporting
categories = ['UNASSIGNED', 'FILLER', ] + [c for c in models.ProjectCategory.Choices]
for c in categories:
@@ -41,15 +43,15 @@ def _get_telescope_time_distribution(cycle: models.Cycle) -> {}:
projects = models.Project.objects.filter(cycles=cycle, project_category=c.value) if (c != 'UNASSIGNED' and c != 'FILLER') \
else models.Project.objects.filter(cycles=cycle, filler=True) if c == 'FILLER' \
else models.Project.objects.filter(cycles=cycle, project_category__isnull=True)
- # TODO: Use QA workflow flag to get successful or failed SUBs, instead of SUBs' states.
- # At the moment just return some 0 placeholder values.
- # for p in projects:
- # # Get durations for single project and aggregate to get the totals
- # # Note: We can filter observations by considering observed_duration in the SUB, for now. See TMSS-610.
- # _, durations = _get_subs_and_durations_from_project(p)
- # total += durations['total']
- # succeeded += durations['total_succeeded']
- # failed += durations['total_failed']
+ for p in projects:
+ # Get durations for single project
+ subs = models.SchedulingUnitBlueprint.objects.filter(draft__scheduling_set__project=p.pk)
+ for sub in subs:
+ accepted = SchedulingUnitProcess.objects.filter(su=sub.pk, results_accepted__is_null=False).results_accepted
+ # Aggregate total, successful and failed durations
+ total += sub.observed_duration.total_seconds()
+ succeeded += sub.observed_duration.total_seconds() if accepted else 0
+ failed += sub.observed_duration.total_seconds() if accepted is False else 0
idle = total - succeeded - failed
result[c if c == 'UNASSIGNED' or c == 'FILLER' else c.name] = {'durations': {'total': total, 'succeeded': succeeded,
'failed': failed, 'idle': idle}}
@@ -61,6 +63,8 @@ def _get_average_efficiency(cycle: models.Cycle) -> {}:
"""
Help function to retrieve the average efficiency with total and total successful obs durations per day.
"""
+ from lofar.sas.tmss.tmss.workflowapp.models.schedulingunitflow import SchedulingUnitProcess
+
result = {'target': '0.65'} # TODO: Default efficiency is 65%. Change it properly when it will be implemented.
efficiency_per_day = 0
@@ -76,15 +80,11 @@ def _get_average_efficiency(cycle: models.Cycle) -> {}:
total_per_day = 0
total_succeeded_per_day = 0
for sub in subs:
- # TODO: This loop takes too much time.
- total_per_day += sub.observed_duration.total_seconds() if sub.observed_duration and \
- d <= sub.observed_start_time < d + step and \
- sub.observed_end_time < d + step else 0
- # TODO: Use QA workflow flag to get successful or failed SUBs, instead of SUBs' states.
- # At the moment just return some 0 placeholder values.
- # total_succeeded_per_day += sub.observed_duration.total_seconds() \
- # if sub.observed_duration and sub.status == 'finished' and \
- # d <= sub.observed_start_time < d + step and sub.observed_end_time < d + step else 0
+ # Aggregate total and successful durations
+ if sub.observed_duration and d <= sub.observed_start_time < d + step and sub.observed_end_time < d + step:
+ accepted = SchedulingUnitProcess.objects.filter(su=sub.pk, results_accepted__is_null=False).results_accepted
+ total_per_day += sub.observed_duration.total_seconds()
+ total_succeeded_per_day += sub.observed_duration.total_seconds() if accepted else 0
efficiency_per_day += total_succeeded_per_day / total_per_day if total_per_day > 0 else 0
i += 1
d += step
@@ -98,7 +98,10 @@ def _get_completion_level(cycle: models.Cycle) -> {}:
"""
Help function to retrieve the completion level info.
"""
- result = {'target': '0.0'} # TODO: Change it properly when it will be implemented.
+ from lofar.sas.tmss.tmss.workflowapp.models.schedulingunitflow import SchedulingUnitProcess
+
+ # TODO: Change it properly when it will be implemented.
+ result = {'target': '0.0'}
# Get SchedulingUnitBlueprints related to the cycle
subs = models.SchedulingUnitBlueprint.objects.filter(draft__scheduling_set__project__cycles=cycle.pk)
@@ -106,10 +109,12 @@ def _get_completion_level(cycle: models.Cycle) -> {}:
total = 0
total_succeeded = 0
for sub in subs:
- total += sub.observed_duration.total_seconds() if sub.observed_duration else 0
- # TODO: Use QA workflow flag to get successful SUBs, instead of SUBs' states.
- # At the moment just return some 0 placeholder values.
- # total_succeeded += sub.observed_duration.total_seconds() if sub.observed_duration and sub.status == 'finished' else 0
+ # Aggregate total and successful durations
+ if sub.observed_duration:
+ accepted = SchedulingUnitProcess.objects.filter(su=sub.pk, results_accepted__is_null=False).results_accepted
+ total += sub.observed_duration.total_seconds()
+ total_succeeded += sub.observed_duration.total_seconds() if accepted else 0
+
result['total'], result['succeeded'] = total, total_succeeded
return result
@@ -119,6 +124,8 @@ def _get_observation_hours_per_category(cycle: models.Cycle) -> {}:
"""
Help function to retrieve observation hours per category info.
"""
+ from lofar.sas.tmss.tmss.workflowapp.models.schedulingunitflow import SchedulingUnitProcess
+
result = {'total_duration_failed': 0, 'total_duration_idle': 0}
# TODO: Filter also according to "DDT Com Rep", and "System Unavailability".
@@ -128,12 +135,10 @@ def _get_observation_hours_per_category(cycle: models.Cycle) -> {}:
subs = models.SchedulingUnitBlueprint.objects.filter(draft__scheduling_set__project__cycles=cycle.pk).filter(priority_queue=prio.value)
for sub in subs:
result['total_duration_idle'] += sub.observed_duration.total_seconds() if sub.observed_duration else 0
- # TODO: Use QA workflow flag to get successful or failed SUBs, instead of SUBs' states.
- # At the moment just return some 0 placeholder values.
- # if sub.status == 'finished':
- # result[f'total_duration_{prio.name}'] += sub.observed_duration.total_seconds() if sub.observed_duration else 0
- # if sub.status == 'error':
- # result['total_duration_failed'] += sub.observed_duration.total_seconds() if sub.observed_duration else 0
+ if sub.observed_duration:
+ accepted = SchedulingUnitProcess.objects.filter(su=sub.pk, results_accepted__is_null=False).results_accepted
+ result[f'total_duration_{prio.name}'] += sub.observed_duration.total_seconds() if accepted else 0
+ result['total_duration_failed'] += sub.observed_duration.total_seconds() if accepted is False else 0
# Subtract prio states from total to get partial idle
result['total_duration_idle'] -= result[f'total_duration_{prio.name}']
# Subtract total failed to get total idle eventually
@@ -142,10 +147,12 @@ def _get_observation_hours_per_category(cycle: models.Cycle) -> {}:
return result
-def _get_weekly_efficiency(cycle: models.Cycle):
+def _get_weekly_efficiency(cycle: models.Cycle) -> {}:
"""
Help function to retrieve the weekly efficiency with total successful obs durations per week.
"""
+ from lofar.sas.tmss.tmss.workflowapp.models.schedulingunitflow import SchedulingUnitProcess
+
result = {'weeks': []}
# Get SchedulingUnitBlueprints related to the cycle
@@ -160,14 +167,11 @@ def _get_weekly_efficiency(cycle: models.Cycle):
total_per_week = 0
total_succeeded_per_week = 0
for sub in subs:
- total_per_week += sub.observed_duration.total_seconds() if sub.observed_duration and \
- d <= sub.observed_start_time < d + step and \
- sub.observed_end_time < d + step else 0
- # TODO: Use QA workflow flag to get successful or failed SUBs, instead of SUBs' states.
- # At the moment just return some 0 placeholder values.
- # total_succeeded_per_week += sub.observed_duration.total_seconds() \
- # if sub.observed_duration and sub.status == 'finished' and \
- # d <= sub.observed_start_time < d + step and sub.observed_end_time < d + step else 0
+ # Aggregate total and successful durations
+ if sub.observed_duration and d <= sub.observed_start_time < d + step and sub.observed_end_time < d + step:
+ accepted = SchedulingUnitProcess.objects.filter(su=sub.pk, results_accepted__is_null=False).results_accepted
+ total_per_week += sub.observed_duration.total_seconds()
+ total_succeeded_per_week += sub.observed_duration.total_seconds() if accepted else 0
result['weeks'].append(
{'week': d.date().isoformat(), 'efficiency': total_succeeded_per_week / total_per_week if total_per_week > 0 else None})
d += step
@@ -260,12 +264,37 @@ def _get_usage_mode(cycle: models.Cycle) -> {}:
return result
-def _get_failures(cycle: models.Cycle):
+def _get_failures(cycle: models.Cycle) -> {}:
"""
Help function to retrieve failures rate info as a function of time.
"""
+ from lofar.sas.tmss.tmss.workflowapp.models.schedulingunitflow import SchedulingUnitProcess
+
# TODO: See TMSS-662 for details.
- return None
+ result = {'months': []}
+
+ # Get SchedulingUnitBlueprints related to the cycle
+ subs = models.SchedulingUnitBlueprint.objects.filter(
+ draft__scheduling_set__project__cycles=cycle.pk)
+
+ # Get start and stop
+ start, stop = cycle.start, cycle.stop
+
+ # Iterate through weeks and sum durations per week
+ step, d = relativedelta(months=1), start
+ while d < stop:
+ total_per_month = 0
+ total_failed_per_month = 0
+ for sub in subs:
+ # Aggregate total and successful durations
+ if sub.observed_duration and d <= sub.observed_start_time < d + step and sub.observed_end_time < d + step:
+ accepted = SchedulingUnitProcess.objects.filter(su=sub.pk, results_accepted__is_null=False).results_accepted
+ total_per_month += sub.observed_duration.total_seconds()
+ total_failed_per_month += sub.observed_duration.total_seconds() if accepted == False else 0
+ result['months'].append({'month': d.date().isoformat(), 'total': total_per_month, 'total_failed': total_failed_per_month})
+ d += step
+
+ return result
# Project Report
@@ -297,6 +326,8 @@ def _get_subs_and_durations_from_project(project_pk: int) -> ({}, {}):
"""
Help function to retrieve durations and scheduling_units distinguished by success/fail.
"""
+ from lofar.sas.tmss.tmss.workflowapp.models.schedulingunitflow import SchedulingUnitProcess
+
# Get SUBs related to the project
scheduling_unit_blueprints = models.SchedulingUnitBlueprint.objects.filter(draft__scheduling_set__project__pk=project_pk)
# TODO: Split into total, prio A, prio B. See TMSS-592.
@@ -305,17 +336,17 @@ def _get_subs_and_durations_from_project(project_pk: int) -> ({}, {}):
# NOTE: This might be optimised later with the use of Django's ORM as done for LTA dataproducts.
for sub in scheduling_unit_blueprints: # Distinguish between succeeded and failed observations
- # TODO: Use QA workflow flag to get successful or failed SUBs, instead of SUBs' states.
- # Cancelled SUBs are not failed SUBs. We need to adjust this once the QA workflow flag will be defined.
- # Also clarify if this info should be related only to obs or all SUBs in general. The latter are considered for now.
+ # TODO: Clarify if this info should be related only to obs or all SUBs in general. The latter are considered for now.
# We can filter observations by considering observed_duration in the SUB, for now. See TMSS-610 comments.
- if sub.status == 'finished': # Succeeded SUBs
+ accepted = SchedulingUnitProcess.objects.filter(su=sub.pk, results_accepted__is_null=False).results_accepted
+ if accepted: # Succeeded SUBs
total_succeeded_duration += sub.duration
subs_succeeded.append({'id': sub.pk, 'name': sub.name, 'duration': sub.duration.total_seconds()})
- elif sub.status == 'cancelled': # Failed SUBs
+ elif accepted is False: # Failed SUBs
total_failed_duration += sub.duration
subs_failed.append({'id': sub.pk, 'name': sub.name, 'duration': sub.duration.total_seconds()})
- total_duration += sub.duration # Total duration without considering the status of the SUBs.
+ # Total duration without considering the status of the SUBs.
+ total_duration += sub.duration
total_not_cancelled = total_duration - total_failed_duration # Calculate not_cancelled duration
durations = {'total': total_duration.total_seconds(), 'total_succeeded': total_succeeded_duration.total_seconds(),
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/views.py b/SAS/TMSS/backend/src/tmss/tmssapp/views.py
index 7620179e0a574d3b5cd51d0f2be6f0379c8364a8..14bcd8cd3ea4f5da42ff36748aff9006afc08441 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/views.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/views.py
@@ -348,7 +348,10 @@ def get_cycles_report(request):
results = {}
for c_pk in cycles:
c = get_object_or_404(models.Cycle, pk=c_pk)
- results[c_pk] = create_cycle_report(request, c)
+ try:
+ results[c_pk] = create_cycle_report(request, c)
+ except RuntimeError:
+ return HttpResponse('Error: workflowapp is not running. It is needed to retrieve some reporting information.', status=503)
return JsonResponse(results)
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/specification.py b/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/specification.py
index 752ea356aaa6ec0ba81abe864314030ccbefb7ea..3d85e66c8d5cca2d9adeaf4400c4b7be1b9fd7e0 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/specification.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/specification.py
@@ -351,7 +351,10 @@ class CycleViewSet(LOFARViewSet):
@action(methods=['get'], detail=True, url_name="report", name="Get Report")
def report(self, request, pk=None):
cycle = get_object_or_404(models.Cycle, pk=pk)
- result = create_cycle_report(request, cycle)
+ try:
+ result = create_cycle_report(request, cycle)
+ except RuntimeError:
+ return Response('Error: workflowapp is not running. It is needed to retrieve some reporting information.', status=status.HTTP_503_SERVICE_UNAVAILABLE)
return Response(result, status=status.HTTP_200_OK)
@@ -391,7 +394,10 @@ class ProjectViewSet(LOFARViewSet):
@action(methods=['get'], detail=True, url_name="report", name="Get Report")
def report(self, request, pk=None):
project = get_object_or_404(models.Project, pk=pk)
- result = create_project_report(request, project)
+ try:
+ result = create_project_report(request, project)
+ except RuntimeError:
+ return Response('Error: workflowapp is not running. It is needed to retrieve some reporting information.', status=status.HTTP_503_SERVICE_UNAVAILABLE)
return Response(result, status=status.HTTP_200_OK)