OSB-28: end of work day commit

LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py

@@ -64,7 +64,8 @@ class RTSMObservationSerializer(serializers.ModelSerializer):
 
         station_entry = Station.objects.filter(name=station_name).first()
         if station_entry is None:
-            station_entry = Station(name=station_name, type=station_type).save()
+            station_entry = Station(name=station_name, type=station_type)
+            station_entry.save()
         # searches if the RTSM is already present
         observation_id = validated_data['observation_id']
         RTSMObservation_instance = RTSMObservation.objects.filter(observation_id=observation_id,

LCU/Maintenance/DBInterface/monitoringdb/urls.py

@@ -44,6 +44,8 @@ urlpatterns = [
     url(r'^api/stationtests/raw/insert', insert_raw_station_test),
     url(r'^api/rtsm/raw/insert', insert_raw_rtsm_test),
     url(r'^api/view/ctrl_stationoverview', ControllerStationOverview.as_view()),
+    url(r'^api/view/ctrl_stationtestsummary', ControllerStationTestsSummary.as_view()),
+    url(r'^api/view/ctrl_latest_observation', ControllerLatestObservations.as_view()),
 
     url(r'^api/docs', include_docs_urls(title='Monitoring DB API'))
 ]

LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py

+import datetime
 from collections import OrderedDict
 
 import coreapi
 import coreschema
-from django.db.models import Count
-from rest_framework.schemas import ManualSchema
-from rest_framework.views import APIView
+from django.db.models import Count, Sum
+from django.db.models import Window, F
 from rest_framework import status
-
-from lofar.maintenance.monitoringdb.models.station_test import StationTest
-from lofar.maintenance.monitoringdb.models.station import Station
 from rest_framework.response import Response
+from rest_framework.schemas import ManualSchema
+from rest_framework.views import APIView
 
 from lofar.maintenance.monitoringdb.models.rtsm import RTSMObservationSummary
+from lofar.maintenance.monitoringdb.models.station import Station
+from lofar.maintenance.monitoringdb.models.station_test import StationTest
 
 
 class ControllerStationOverview(APIView):
@@ -20,7 +21,7 @@ class ControllerStationOverview(APIView):
     """
 
     DEFAULT_STATION_GROUP = 'A'
-    DEFAULT_ONLY_ERRORS = False
+    DEFAULT_ONLY_ERRORS = True
     DEFAULT_N_STATION_TESTS = 4
     DEFAULT_N_RTSM = 4
 
@@ -80,7 +81,7 @@ class ControllerStationOverview(APIView):
             station_test_list = StationTest.objects.filter(
                 station__name=station_entity.name).order_by('-end_datetime')[:n_station_tests - 1]
             rtsm_list = RTSMObservationSummary.objects.filter(
-                station__name=station_entity.name).order_by('-end_time')[:n_rtsm - 1]
+                station__name=station_entity.name).order_by('-end_datetime')[:n_rtsm - 1]
 
             station_payload['station_tests'] = list()
             for station_test in station_test_list:
@@ -88,7 +89,7 @@ class ControllerStationOverview(APIView):
                 component_errors = station_test.component_errors
 
                 station_test_payload[
-                    'total_component_errors'] = station_test.component_errors.count()
+                    'total_component_errorss'] = station_test.component_errors.count()
                 station_test_payload['start_datetime'] = station_test.start_datetime
                 station_test_payload['end_datetime'] = station_test.end_datetime
                 station_test_payload['checks'] = station_test.checks
@@ -114,13 +115,13 @@ class ControllerStationOverview(APIView):
             for rtsm in rtsm_list:
                 rtsm_payload = OrderedDict()
                 rtsm_payload['observation_id'] = rtsm.observation_id
-                rtsm_payload['start_datetime'] = rtsm.start_time
-                rtsm_payload['end_datetime'] = rtsm.end_time
+                rtsm_payload['start_datetime'] = rtsm.start_datetime
+                rtsm_payload['end_datetime'] = rtsm.end_datetime
 
                 unique_modes = [item['mode'] for item in rtsm.errors.values('mode').distinct()]
                 rtsm_payload['mode'] = unique_modes
-                rtsm_payload['total_component_error'] = rtsm.errors_found.count()
-                from django.db.models import Window, F
+                rtsm_payload['total_component_errors'] = rtsm.errors_found.count()
+
                 errors_summary = OrderedDict()
 
                 errors_summary_query = rtsm.errors_found.annotate(total=
@@ -136,10 +137,272 @@ class ControllerStationOverview(APIView):
                 station_payload['rtsm'].append(rtsm_payload)
 
             response_payload.append(station_payload)
-        if errors_only:
+        if self.errors_only and self.errors_only is not 'false':
             response_payload = filter(
                 lambda station_entry:
                 len(station_entry['station_tests']) + len(station_entry['rtsm']) > 0,
                 response_payload)
         response_payload = sorted(response_payload, key=lambda item: item['station_name'])
         return Response(status=status.HTTP_200_OK, data=response_payload)
+
+
+class ControllerStationTestsSummary(APIView):
+    """
+    Overview of the latest station tests performed on the stations
+    """
+
+    DEFAULT_STATION_GROUP = 'A'
+    DEFAULT_ONLY_ERRORS = True
+
+    queryset = StationTest.objects.all()
+    schema = ManualSchema(fields=[
+        coreapi.Field(
+            "station_group",
+            required=False,
+            location='query',
+            schema=coreschema.Enum(['C', 'R', 'I', 'A'], description=
+            'Station group to select for choices are [C|R|I|ALL]',
+                                   )
+        ),
+        coreapi.Field(
+            "errors_only",
+            required=False,
+            location='query',
+            schema=coreschema.Boolean(
+                description='displays or not only the station with more than one error')
+        ),
+        coreapi.Field(
+            "start_date",
+            required=True,
+            location='query',
+            schema=coreschema.String(
+                description='select station tests from date (ex. YYYY-MM-DD)')
+        ),
+        coreapi.Field(
+            "end_date",
+            required=True,
+            location='query',
+            schema=coreschema.String(
+                description='select station tests to date (ex. YYYY-MM-DD)')
+        )
+    ]
+    )
+
+    @staticmethod
+    def parse_date(date):
+        expected_format = '%Y-%m-%d'
+        try:
+            parsed_date = datetime.datetime.strptime(date, expected_format)
+            return parsed_date
+        except Exception as e:
+            raise ValueError('cannot parse %s with format %s - %s' % (date, expected_format, e))
+
+    def validate_query_parameters(self, request):
+        self.errors_only = request.query_params.get('errors_only', self.DEFAULT_ONLY_ERRORS)
+        self.station_group = request.query_params.get('station_group', self.DEFAULT_STATION_GROUP)
+
+        start_date = request.query_params.get('start_date')
+        self.start_date = ControllerStationTestsSummary.parse_date(start_date)
+
+        end_date = request.query_params.get('end_date')
+        self.end_date = ControllerStationTestsSummary.parse_date(end_date)
+
+    def get(self, request, format=None):
+        try:
+            self.validate_query_parameters(request)
+        except ValueError as e:
+            return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
+                            data='Please specify the date in the format YYYY-MM-DD: %s' % (e,))
+        except KeyError as e:
+            return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
+                            data='Please specify both the start and the end date: %s' % (e,))
+
+        station_entities = Station.objects.all()
+        for group in self.station_group:
+            if group is not 'A':
+                station_entities = station_entities.filter(type=group)
+
+        # Since django preferes a ordered dict over a dict we make it happy... for now
+        response_payload = list()
+        for station_entity in station_entities:
+            station_payload = OrderedDict()
+
+            station_payload['station_name'] = station_entity.name
+
+            station_test_list = StationTest.objects.filter(
+                start_datetime__gte=self.start_date,
+                end_datetime__lte=self.end_date,
+                station__name=station_entity.name).order_by('-end_datetime')
+            station_payload['station_tests'] = list()
+            for station_test in station_test_list:
+
+                station_test_payload = OrderedDict()
+                component_errors = station_test.component_errors
+
+                station_test_payload[
+                    'total_component_errors'] = station_test.component_errors.count()
+                station_test_payload['start_datetime'] = station_test.start_datetime
+                station_test_payload['end_datetime'] = station_test.end_datetime
+                station_test_payload['checks'] = station_test.checks
+                component_errors_summary = component_errors. \
+                    values('component__type', 'type').annotate(
+                    total=Count('type')).order_by('-total')
+                component_errors_summary_dict = OrderedDict()
+                for item in component_errors_summary:
+                    item_component_type = item['component__type']
+                    item_error_type = item['type']
+                    item_error_total = item['total']
+
+                    if item_component_type not in component_errors_summary_dict:
+                        component_errors_summary_dict[item_component_type] = OrderedDict()
+
+                    component_errors_summary_dict[item_component_type][item_error_type] = \
+                        item_error_total
+                station_test_payload['component_error_summary'] = component_errors_summary_dict
+
+                station_payload['station_tests'].append(station_test_payload)
+
+            response_payload.append(station_payload)
+        if self.errors_only and self.errors_only is not 'false':
+            response_payload = filter(
+                lambda station_entry:
+                len(station_entry['station_tests']) > 0,
+                response_payload)
+        response_payload = sorted(response_payload, key=lambda item: item['station_name'])
+        return Response(status=status.HTTP_200_OK, data=response_payload)
+
+
+class ControllerLatestObservations(APIView):
+    """
+    Overview of the latest observations performed on the stations
+    """
+
+    DEFAULT_STATION_GROUP = 'A'
+    DEFAULT_ONLY_ERRORS = True
+
+    queryset = StationTest.objects.all()
+    schema = ManualSchema(fields=[
+        coreapi.Field(
+            "station_group",
+            required=False,
+            location='query',
+            schema=coreschema.Enum(['C', 'R', 'I', 'A'], description=
+            'Station group to select for choices are [C|R|I|ALL]',
+                                   )
+        ),
+        coreapi.Field(
+            "errors_only",
+            required=False,
+            location='query',
+            schema=coreschema.Boolean(
+                description='displays or not only the station with more than one error')
+        ),
+        coreapi.Field(
+            "from_date",
+            required=True,
+            location='query',
+            schema=coreschema.String(
+                description='select rtsm from date (ex. YYYY-MM-DD)')
+        )
+    ]
+    )
+
+    @staticmethod
+    def parse_date(date):
+        expected_format = '%Y-%m-%d'
+        try:
+            parsed_date = datetime.datetime.strptime(date, expected_format)
+            return parsed_date
+        except Exception as e:
+            raise ValueError('cannot parse %s with format %s - %s' % (date, expected_format, e))
+
+    def compute_rtsm_observation_summary(self, rtsm_errors):
+        errors_summary = OrderedDict()
+
+        errors_summary_query = rtsm_errors.annotate(total=
+                                             Window(expression=Count('rcu'),
+                                                    partition_by=[F(
+                                                        'error_type')])).values(
+            'error_type', 'total').distinct()
+
+        for error_summary in errors_summary_query:
+            errors_summary[error_summary['error_type']] = error_summary['total']
+        return errors_summary
+
+    def validate_query_parameters(self, request):
+        self.errors_only = request.query_params.get('errors_only', self.DEFAULT_ONLY_ERRORS)
+        self.station_group = request.query_params.get('station_group', self.DEFAULT_STATION_GROUP)
+
+        start_date = request.query_params.get('from_date')
+        self.from_date = ControllerLatestObservations.parse_date(start_date)
+
+    def get(self, request, format=None):
+        try:
+            self.validate_query_parameters(request)
+        except ValueError as e:
+            return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
+                            data='Please specify the date in the format YYYY-MM-DD: %s' % (e,))
+        except KeyError as e:
+            return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
+                            data='Please specify both the start and the end date: %s' % (e,))
+
+        rtsm_observation_entities = RTSMObservationSummary.objects.filter(
+                start_datetime__gte=self.from_date)
+        for group in self.station_group:
+            if group is not 'A':
+                rtsm_observation_entities = rtsm_observation_entities.filter(station__type=group)
+
+        # Since django preferes a ordered dict over a dict we make it happy... for now
+        response_payload = list()
+        for rtsm_observation_entity in rtsm_observation_entities.values('observation_id',
+                                                                        'start_datetime',
+                                                                        'end_datetime'). \
+                distinct():
+            observation_payload = OrderedDict()
+
+            observation_payload['observation_id'] = rtsm_observation_entity['observation_id']
+            observation_payload['start_datetime'] = rtsm_observation_entity['start_datetime']
+            observation_payload['end_datetime'] = rtsm_observation_entity['end_datetime']
+
+            rtsm_list = RTSMObservationSummary.objects.filter(
+                start_datetime__gte=self.from_date,
+                observation_id=rtsm_observation_entity['observation_id']). \
+                order_by('-end_datetime')
+            unique_modes = [item['errors__mode'] for item in rtsm_list.values('errors__mode').distinct()]
+
+            observation_payload['mode'] = unique_modes
+            observation_payload['total_component_errors'] = 0
+
+            station_list = rtsm_list.values('station').distinct()
+            station_involved_list = list()
+
+            for station in station_list:
+                rtsm_entry_per_station = rtsm_list.filter(station__pk=station['station']).first()
+
+                station_summary = OrderedDict()
+                station_summary['station_name'] = rtsm_entry_per_station.station.name
+
+                station_summary['n_errors'] =\
+                    rtsm_entry_per_station.errors.values('rcu').distinct().count()
+                station_summary['component_error_summary'] = self.compute_rtsm_observation_summary(
+                    rtsm_entry_per_station.errors_found)
+
+                station_involved_list.append(station_summary)
+
+                observation_payload['total_component_errors'] += station_summary['n_errors']
+
+            station_involved_list = sorted(station_involved_list,
+                                           key=lambda rtsm_per_station: rtsm_per_station['n_errors'],
+                                           reverse=True)
+            observation_payload['station_involved'] = station_involved_list
+
+            response_payload.append(observation_payload)
+        if self.errors_only and self.errors_only != 'false':
+            print(self.errors_only)
+            response_payload = filter(
+                lambda station_entry:
+                len(station_entry['total_component_errors']) > 0,
+                response_payload)
+        response_payload = sorted(response_payload, key=lambda item: item['total_component_errors'],
+                                  reverse=True)
+        return Response(status=status.HTTP_200_OK, data=response_payload)

LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py

 from .common import *
-from ..models.rtsm import RTSMObservation, RTSMError, RTSMSpectrum
+from ..models.rtsm import RTSMObservation, RTSMError, RTSMSpectrum, RTSMObservationSummary
 from ..serializers.rtsm import RTSMObservationSerializer, RTSMErrorSerializer,\
                                RTSMObservationSummarySerializer, RTSMSpectrumSerializer
 from ..rtsm_test_raw_parser import parse_rtsm_test
@@ -33,7 +33,7 @@ class RTSMObservationViewSet(viewsets.ReadOnlyModelViewSet):
 
 
 class RTSMObservationSummaryViewSet(viewsets.ReadOnlyModelViewSet):
-    queryset = RTSMObservation.objects.all()
+    queryset = RTSMObservationSummary.objects.all()
     serializer_class = RTSMObservationSummarySerializer
     filter_fields = '__all__'