OSB-28: small fixes

LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py

@@ -179,7 +179,7 @@ def dict_from_component_error(content):
 
     component_error = dict(component=component,
                            details=error_details,
-                           type=error_type)
+                           type=error_type.strip())
 
     for element_error in element_errors:
         element_error['component_error'] = dict(component_error)

LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py

 import datetime
 from collections import OrderedDict
+from math import ceil
 
 import coreapi
 import coreschema
 from django.db.models import Count
-
-from math import ceil
 from django.db.models import Window, F
-
 from rest_framework import status
 from rest_framework.response import Response
 from rest_framework.schemas import ManualSchema
 from rest_framework.views import APIView
 
+from lofar.maintenance.monitoringdb.models.component_error import ComponentError
+from lofar.maintenance.monitoringdb.models.rtsm import RTSMErrorSummary
 from lofar.maintenance.monitoringdb.models.rtsm import RTSMObservation
 from lofar.maintenance.monitoringdb.models.station import Station
 from lofar.maintenance.monitoringdb.models.station_test import StationTest
-from lofar.maintenance.monitoringdb.models.component_error import ComponentError
-from lofar.maintenance.monitoringdb.models.rtsm import RTSMErrorSummary
 
 
 class ControllerStationOverview(APIView):
@@ -123,14 +121,14 @@ class ControllerStationOverview(APIView):
                 rtsm_payload['start_datetime'] = rtsm.start_datetime
                 rtsm_payload['end_datetime'] = rtsm.end_datetime
 
-                unique_modes = [item['mode'] for item in rtsm.errors.values('mode').distinct()]
+                unique_modes = [item['mode'] for item in rtsm.errors_summary.values('mode').distinct()]
                 rtsm_payload['mode'] = unique_modes
                 rtsm_payload['total_component_errors'] = rtsm.errors_summary.count()
 
                 errors_summary = OrderedDict()
 
-                errors_summary_query = rtsm.errors_summary.annotate(total=Count('error_type')).values(
-                    'error_type', 'total').distinct()
+                errors_summary_query = rtsm.errors_summary.values('error_type').annotate(
+                    total=Count('error_type'))
 
                 for error_summary in errors_summary_query:
                     errors_summary[error_summary['error_type']] = error_summary['total']
@@ -197,7 +195,7 @@ class ControllerStationTestsSummary(APIView):
         self.station_group = request.query_params.get('station_group', self.DEFAULT_STATION_GROUP)
 
         self.lookback_time = datetime.timedelta(int(request.query_params.get('lookback_time',
-                                                            self.DEFAULT_LOOKBACK_TIME_IN_DAYS)))
+                                                                             self.DEFAULT_LOOKBACK_TIME_IN_DAYS)))
 
     def get(self, request, format=None):
         try:
@@ -209,12 +207,12 @@ class ControllerStationTestsSummary(APIView):
             return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
                             data='Please specify all the required parameters: %s' % (e,))
 
-        station_test_list = StationTest.objects\
-            .filter(start_datetime__gte=datetime.date.today()-self.lookback_time)\
+        station_test_list = StationTest.objects \
+            .filter(start_datetime__gte=datetime.date.today() - self.lookback_time) \
             .order_by('-start_datetime', 'station__name')
         for group in self.station_group:
             if group is not 'A':
-                station_entities = station_test_list.filter(type=group)
+                station_test_list = station_test_list.filter(station__type=group)
 
         # Since django preferes a ordered dict over a dict we make it happy... for now
         response_payload = list()
@@ -331,7 +329,7 @@ class ControllerLatestObservations(APIView):
             return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
                             data='Please specify both the start and the end date: %s' % (e,))
 
-        rtsm_observation_entities = RTSMObservation.objects.filter(
+        rtsm_observation_entities = RTSMObservation.objects.defer('errors').filter(
             start_datetime__gte=self.from_date)
         for group in self.station_group:
             if group is not 'A':
@@ -342,26 +340,25 @@ class ControllerLatestObservations(APIView):
         for rtsm_observation_entity in rtsm_observation_entities.values('observation_id',
                                                                         'start_datetime',
                                                                         'end_datetime'). \
-                distinct().order_by('-observation_id'):
+                distinct().order_by('-start_datetime'):
             observation_payload = OrderedDict()
 
             observation_payload['observation_id'] = rtsm_observation_entity['observation_id']
             observation_payload['start_datetime'] = rtsm_observation_entity['start_datetime']
             observation_payload['end_datetime'] = rtsm_observation_entity['end_datetime']
 
-            rtsm_list = RTSMObservation.objects.filter(
+            rtsm_list = RTSMObservation.objects.defer('errors').filter(
                 start_datetime__gte=self.from_date,
                 observation_id=rtsm_observation_entity['observation_id'])
-            unique_modes = [item['errors__mode'] for item in rtsm_list.values('errors__mode').distinct()]
+            unique_modes = [item['errors_summary__mode'] for item in
+                            rtsm_list.values('errors_summary__mode').distinct()]
 
             observation_payload['mode'] = unique_modes
             observation_payload['total_component_errors'] = 0
 
-
             station_involved_list = list()
 
             for rtsm_entry_per_station in rtsm_list.order_by('station__name'):
-
                 station_summary = OrderedDict()
                 station_summary['station_name'] = rtsm_entry_per_station.station.name
 
@@ -378,7 +375,7 @@ class ControllerLatestObservations(APIView):
 
             response_payload.append(observation_payload)
         if self.errors_only and self.errors_only != 'false':
-            print(self.errors_only)
+
             response_payload = filter(
                 lambda station_entry:
                 len(station_entry['total_component_errors']) > 0,
@@ -488,27 +485,32 @@ result:
         if self.test_type not in ['R', 'S', 'B']:
             raise ValueError('test_type is not one of [R,S,B]')
 
-        self.averaging_interval = datetime.timedelta(int(request.query_params.get('averaging_interval')))
+        self.averaging_interval = datetime.timedelta(
+            int(request.query_params.get('averaging_interval')))
 
-    def compute_errors_per_station(self, from_date, to_date, central_time, station_group, test_type):
+    def compute_errors_per_station(self, from_date, to_date, central_time, station_group,
+                                   test_type):
 
         component_errors = ComponentError.objects.all()
         rtsm_summary_errors = RTSMErrorSummary.objects.all()
 
         if station_group:
             component_errors = component_errors.filter(station_test__station__type=station_group)
-            rtsm_summary_errors = rtsm_summary_errors.filter(observation__station__type=station_group)
+            rtsm_summary_errors = rtsm_summary_errors.filter(
+                observation__station__type=station_group)
 
         station_test_results = []
         rtsm_results = []
         if test_type in ['S', 'B']:
             station_test_results = component_errors. \
-                filter(station_test__start_datetime__gt=from_date, station_test__start_datetime__lt=to_date). \
+                filter(station_test__start_datetime__gt=from_date,
+                       station_test__start_datetime__lt=to_date). \
                 values('station_test__station__name'). \
                 annotate(n_errors=Count('station_test__station__name'))
         if test_type in ['R', 'B']:
             rtsm_results = rtsm_summary_errors. \
-                filter(observation__start_datetime__gt=from_date, observation__start_datetime__lt=to_date). \
+                filter(observation__start_datetime__gt=from_date,
+                       observation__start_datetime__lt=to_date). \
                 values('observation__station__name'). \
                 annotate(n_errors=Count('observation__station__name'))
 
@@ -541,18 +543,22 @@ result:
         station_test_results = []
         rtsm_results = []
 
+        central_time_str = central_time.strftime('%Y-%m-%d')
         if station_group:
             component_errors = component_errors.filter(station_test__station__type=station_group)
-            rtsm_summary_errors = rtsm_summary_errors.filter(observation__station__type=station_group)
+            rtsm_summary_errors = rtsm_summary_errors.filter(
+                observation__station__type=station_group)
 
         if test_type in ['S', 'B']:
             station_test_results = component_errors. \
-                filter(station_test__start_datetime__gt=from_date, station_test__start_datetime__lt=to_date). \
+                filter(station_test__start_datetime__gt=from_date,
+                       station_test__start_datetime__lt=to_date). \
                 values('type'). \
                 annotate(n_errors=Count('type'))
         if test_type in ['R', 'B']:
             rtsm_results = rtsm_summary_errors. \
-                filter(observation__start_datetime__gt=from_date, observation__start_datetime__lt=to_date). \
+                filter(observation__start_datetime__gt=from_date,
+                       observation__start_datetime__lt=to_date). \
                 values('error_type'). \
                 annotate(n_errors=Count('error_type'))
 
@@ -563,14 +569,14 @@ result:
                 error_type = result['type']
                 errors_per_error_type_in_bin[error_type] = dict(error_type=error_type,
                                                                 n_errors=result['n_errors'],
-                                                                time=central_time)
+                                                                time=central_time_str)
         if test_type in ['R', 'B']:
             for result in rtsm_results:
                 error_type = result['error_type']
                 if error_type not in errors_per_error_type_in_bin:
                     errors_per_error_type_in_bin[error_type] = dict(error_type=error_type,
                                                                     n_errors=result['n_errors'],
-                                                                    time=central_time)
+                                                                    time=central_time_str)
                 else:
                     errors_per_error_type_in_bin[error_type]['n_errors'] += result['n_errors']
 
@@ -601,18 +607,19 @@ result:
                 station_group = None
             else:
                 station_group = self.station_group
-            errors_per_station.append(
-                self.compute_errors_per_station(from_date=self.from_date + i * self.averaging_interval,
-                                                to_date=self.from_date + (i + 1) * self.averaging_interval,
-                                                central_time=self.from_date + (i + .5) * self.averaging_interval,
-                                                station_group=station_group,
-                                                test_type=self.test_type))
-            errors_per_type.append(
-                self.compute_errors_per_type(from_date=self.from_date + i * self.averaging_interval,
-                                             to_date=self.from_date + (i + 1) * self.averaging_interval,
-                                             central_time=self.from_date + (i + .5) * self.averaging_interval,
-                                             station_group=station_group,
-                                             test_type=self.test_type))
+            errors_per_station += self.compute_errors_per_station(
+                from_date=self.from_date + i * self.averaging_interval,
+                to_date=self.from_date + (i + 1) * self.averaging_interval,
+                central_time=self.from_date + (i + .5) * self.averaging_interval,
+                station_group=station_group,
+                test_type=self.test_type)
+
+            errors_per_type += self.compute_errors_per_type(
+                from_date=self.from_date + i * self.averaging_interval,
+                to_date=self.from_date + (i + 1) * self.averaging_interval,
+                central_time=self.from_date + (i + .5) * self.averaging_interval,
+                station_group=station_group,
+                test_type=self.test_type)
 
         response_payload['errors_per_station'] = errors_per_station
         response_payload['errors_per_type'] = errors_per_type
@@ -627,4 +634,4 @@ class ControllerAllComponentErrorTypes(APIView):
 
     def get(self, request, format=None):
         data = [item['type'] for item in ComponentError.objects.values('type').distinct()]
-        return Response(status=status.HTTP_200_OK, data=data)
\ No newline at end of file
+        return Response(status=status.HTTP_200_OK, data=data)

LCU/Maintenance/MDB_tools/cli/station_tests_watchdog.py

@@ -172,7 +172,7 @@ def handle_event_file_created(mdb_address, event):
 
 def main():
     setup_logging_framework()
-    station_tests_watchdog(sys.argv)
+    station_tests_watchdog(sys.argv[1:])
 
 
 if __name__ == '__main__':