From 5baa800f9e07aaa4839a5eddd79461bd7a6c4266 Mon Sep 17 00:00:00 2001
From: Mattia Mancini <mancini@astron.nl>
Date: Tue, 23 Oct 2018 14:20:52 +0000
Subject: [PATCH] OSB-28: increase speed in quering the latestObservations
---
.../monitoringdb/views/controllers.py | 126 ++++++++----------
1 file changed, 58 insertions(+), 68 deletions(-)
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py b/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py
index d80c7346aa1..06b580f380c 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py
@@ -263,14 +263,13 @@ class ControllerLatestObservations(APIView):
DEFAULT_STATION_GROUP = 'A'
DEFAULT_ONLY_ERRORS = True
- queryset = StationTest.objects.all()
schema = ManualSchema(fields=[
coreapi.Field(
"station_group",
required=False,
location='query',
schema=coreschema.Enum(['C', 'R', 'I', 'A'], description=
- 'Station group to select for choices are [C|R|I|ALL]',
+ 'Station group to select for choices are [C|R|I|A]',
)
),
coreapi.Field(
@@ -299,19 +298,6 @@ class ControllerLatestObservations(APIView):
except Exception as e:
raise ValueError('cannot parse %s with format %s - %s' % (date, expected_format, e))
- def compute_rtsm_observation_summary(self, rtsm_errors):
- errors_summary = OrderedDict()
-
- errors_summary_query = rtsm_errors.annotate(total=
- Window(expression=Count('rcu'),
- partition_by=[F(
- 'error_type')])).values(
- 'error_type', 'total').distinct()
-
- for error_summary in errors_summary_query:
- errors_summary[error_summary['error_type']] = error_summary['total']
- return errors_summary
-
def validate_query_parameters(self, request):
self.errors_only = request.query_params.get('errors_only', self.DEFAULT_ONLY_ERRORS)
self.station_group = request.query_params.get('station_group', self.DEFAULT_STATION_GROUP)
@@ -328,62 +314,66 @@ class ControllerLatestObservations(APIView):
except KeyError as e:
return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
data='Please specify both the start and the end date: %s' % (e,))
-
- rtsm_observation_entities = RTSMObservation.objects.defer('errors').filter(
- start_datetime__gte=self.from_date)
- for group in self.station_group:
- if group is not 'A':
- rtsm_observation_entities = rtsm_observation_entities.filter(station__type=group)
-
- # Since django preferes a ordered dict over a dict we make it happy... for now
- response_payload = list()
- for rtsm_observation_entity in rtsm_observation_entities.values('observation_id',
- 'start_datetime',
- 'end_datetime'). \
- distinct().order_by('-start_datetime'):
- observation_payload = OrderedDict()
-
- observation_payload['observation_id'] = rtsm_observation_entity['observation_id']
- observation_payload['start_datetime'] = rtsm_observation_entity['start_datetime']
- observation_payload['end_datetime'] = rtsm_observation_entity['end_datetime']
-
- rtsm_list = RTSMObservation.objects.defer('errors').filter(
- start_datetime__gte=self.from_date,
- observation_id=rtsm_observation_entity['observation_id'])
- unique_modes = [item['errors_summary__mode'] for item in
- rtsm_list.values('errors_summary__mode').distinct()]
-
- observation_payload['mode'] = unique_modes
- observation_payload['total_component_errors'] = 0
-
- station_involved_list = list()
-
- for rtsm_entry_per_station in rtsm_list.order_by('station__name'):
- station_summary = OrderedDict()
- station_summary['station_name'] = rtsm_entry_per_station.station.name
-
- station_summary['n_errors'] = \
- rtsm_entry_per_station.errors.values('rcu').distinct().count()
- station_summary['component_error_summary'] = self.compute_rtsm_observation_summary(
- rtsm_entry_per_station.errors_summary)
-
- station_involved_list.append(station_summary)
-
- observation_payload['total_component_errors'] += station_summary['n_errors']
-
- observation_payload['station_involved'] = station_involved_list
-
- response_payload.append(observation_payload)
- if self.errors_only and self.errors_only != 'false':
-
- response_payload = filter(
- lambda station_entry:
- len(station_entry['total_component_errors']) > 0,
- response_payload)
+ filtered_entities = RTSMObservation.objects\
+ .filter(start_datetime__gte=self.from_date)
+ if self.station_group != 'A':
+ filtered_entities = filtered_entities\
+ .filter(station__type=self.station_group)
+ if self.errors_only:
+ filtered_entities = filtered_entities.exclude(errors_summary__isnull=True)
+
+ errors_summary = filtered_entities\
+ .values('observation_id',
+ 'station__name',
+ 'start_datetime',
+ 'end_datetime',
+ 'errors_summary__error_type',
+ 'errors_summary__mode')\
+ .annotate(total=Count('errors_summary__error_type'))\
+ .order_by('observation_id', 'station__name')
+ response = dict()
+
+ for error_summary in errors_summary:
+ observation_id = error_summary['observation_id']
+ station_name = error_summary['station__name']
+ start_datetime = error_summary['start_datetime']
+ end_datetime = error_summary['end_datetime']
+ mode = error_summary['errors_summary__mode']
+ error_type = error_summary['errors_summary__error_type']
+ total = error_summary['total']
+
+ if observation_id not in response:
+ response[observation_id] = OrderedDict()
+ response[observation_id]['observation_id'] = observation_id
+ response[observation_id]['start_datetime'] = start_datetime
+ response[observation_id]['end_datetime'] = end_datetime
+
+ response[observation_id]['total_component_errors'] = 0
+ response[observation_id]['mode'] = list()
+ response[observation_id]['station_involved'] = dict()
+
+ if total == 0:
+ continue
+
+ response[observation_id]['total_component_errors'] += total
+ station_involved_summary = response[observation_id]['station_involved']
+ response[observation_id]['mode'] += [mode]\
+ if mode not in response[observation_id]['mode'] else []
+ if station_name not in station_involved_summary:
+ station_involved_summary[station_name] = OrderedDict()
+ station_involved_summary[station_name]['station_name'] = station_name
+ station_involved_summary[station_name]['n_errors'] = 0
+ station_involved_summary[station_name]['component_error_summary'] = OrderedDict()
+
+ station_involved_summary[station_name]['n_errors'] += total
+ station_involved_summary[station_name]['component_error_summary'][error_type] = total
+
+ response_payload = sorted(response.values(),
+ key= lambda item: item['start_datetime'],
+ reverse=True)
return Response(status=status.HTTP_200_OK, data=response_payload)
-
class ControllerStationTestStatistics(APIView):
"""
--
GitLab