OSB-28: increase speed in quering the latestObservations

LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py

@@ -263,14 +263,13 @@ class ControllerLatestObservations(APIView):
     DEFAULT_STATION_GROUP = 'A'
     DEFAULT_ONLY_ERRORS = True
 
-    queryset = StationTest.objects.all()
     schema = ManualSchema(fields=[
         coreapi.Field(
             "station_group",
             required=False,
             location='query',
             schema=coreschema.Enum(['C', 'R', 'I', 'A'], description=
-            'Station group to select for choices are [C|R|I|ALL]',
+            'Station group to select for choices are [C|R|I|A]',
                                    )
         ),
         coreapi.Field(
@@ -299,19 +298,6 @@ class ControllerLatestObservations(APIView):
         except Exception as e:
             raise ValueError('cannot parse %s with format %s - %s' % (date, expected_format, e))
 
-    def compute_rtsm_observation_summary(self, rtsm_errors):
-        errors_summary = OrderedDict()
-
-        errors_summary_query = rtsm_errors.annotate(total=
-                                                    Window(expression=Count('rcu'),
-                                                           partition_by=[F(
-                                                               'error_type')])).values(
-            'error_type', 'total').distinct()
-
-        for error_summary in errors_summary_query:
-            errors_summary[error_summary['error_type']] = error_summary['total']
-        return errors_summary
-
     def validate_query_parameters(self, request):
         self.errors_only = request.query_params.get('errors_only', self.DEFAULT_ONLY_ERRORS)
         self.station_group = request.query_params.get('station_group', self.DEFAULT_STATION_GROUP)
@@ -328,62 +314,66 @@ class ControllerLatestObservations(APIView):
         except KeyError as e:
             return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
                             data='Please specify both the start and the end date: %s' % (e,))
-
-        rtsm_observation_entities = RTSMObservation.objects.defer('errors').filter(
-            start_datetime__gte=self.from_date)
-        for group in self.station_group:
-            if group is not 'A':
-                rtsm_observation_entities = rtsm_observation_entities.filter(station__type=group)
-
-        # Since django preferes a ordered dict over a dict we make it happy... for now
-        response_payload = list()
-        for rtsm_observation_entity in rtsm_observation_entities.values('observation_id',
-                                                                        'start_datetime',
-                                                                        'end_datetime'). \
-                distinct().order_by('-start_datetime'):
-            observation_payload = OrderedDict()
-
-            observation_payload['observation_id'] = rtsm_observation_entity['observation_id']
-            observation_payload['start_datetime'] = rtsm_observation_entity['start_datetime']
-            observation_payload['end_datetime'] = rtsm_observation_entity['end_datetime']
-
-            rtsm_list = RTSMObservation.objects.defer('errors').filter(
-                start_datetime__gte=self.from_date,
-                observation_id=rtsm_observation_entity['observation_id'])
-            unique_modes = [item['errors_summary__mode'] for item in
-                            rtsm_list.values('errors_summary__mode').distinct()]
-
-            observation_payload['mode'] = unique_modes
-            observation_payload['total_component_errors'] = 0
-
-            station_involved_list = list()
-
-            for rtsm_entry_per_station in rtsm_list.order_by('station__name'):
-                station_summary = OrderedDict()
-                station_summary['station_name'] = rtsm_entry_per_station.station.name
-
-                station_summary['n_errors'] = \
-                    rtsm_entry_per_station.errors.values('rcu').distinct().count()
-                station_summary['component_error_summary'] = self.compute_rtsm_observation_summary(
-                    rtsm_entry_per_station.errors_summary)
-
-                station_involved_list.append(station_summary)
-
-                observation_payload['total_component_errors'] += station_summary['n_errors']
-
-            observation_payload['station_involved'] = station_involved_list
-
-            response_payload.append(observation_payload)
-        if self.errors_only and self.errors_only != 'false':
-
-            response_payload = filter(
-                lambda station_entry:
-                len(station_entry['total_component_errors']) > 0,
-                response_payload)
+        filtered_entities = RTSMObservation.objects\
+            .filter(start_datetime__gte=self.from_date)
+        if self.station_group != 'A':
+            filtered_entities = filtered_entities\
+                .filter(station__type=self.station_group)
+        if self.errors_only:
+            filtered_entities = filtered_entities.exclude(errors_summary__isnull=True)
+
+        errors_summary = filtered_entities\
+            .values('observation_id',
+                    'station__name',
+                    'start_datetime',
+                    'end_datetime',
+                    'errors_summary__error_type',
+                    'errors_summary__mode')\
+            .annotate(total=Count('errors_summary__error_type'))\
+            .order_by('observation_id', 'station__name')
+        response = dict()
+
+        for error_summary in errors_summary:
+            observation_id = error_summary['observation_id']
+            station_name = error_summary['station__name']
+            start_datetime = error_summary['start_datetime']
+            end_datetime = error_summary['end_datetime']
+            mode = error_summary['errors_summary__mode']
+            error_type = error_summary['errors_summary__error_type']
+            total = error_summary['total']
+
+            if observation_id not in response:
+                response[observation_id] = OrderedDict()
+                response[observation_id]['observation_id'] = observation_id
+                response[observation_id]['start_datetime'] = start_datetime
+                response[observation_id]['end_datetime'] = end_datetime
+
+                response[observation_id]['total_component_errors'] = 0
+                response[observation_id]['mode'] = list()
+                response[observation_id]['station_involved'] = dict()
+
+            if total == 0:
+                continue
+
+            response[observation_id]['total_component_errors'] += total
+            station_involved_summary = response[observation_id]['station_involved']
+            response[observation_id]['mode'] += [mode]\
+                if mode not in response[observation_id]['mode'] else []
+            if station_name not in station_involved_summary:
+                station_involved_summary[station_name] = OrderedDict()
+                station_involved_summary[station_name]['station_name'] = station_name
+                station_involved_summary[station_name]['n_errors'] = 0
+                station_involved_summary[station_name]['component_error_summary'] = OrderedDict()
+
+            station_involved_summary[station_name]['n_errors'] += total
+            station_involved_summary[station_name]['component_error_summary'][error_type] = total
+
+        response_payload = sorted(response.values(),
+                                  key= lambda item: item['start_datetime'],
+                                  reverse=True)
 
         return Response(status=status.HTTP_200_OK, data=response_payload)
 
-
 class ControllerStationTestStatistics(APIView):
     """