diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py
index 035857bec458cfc126d59cce224528a9e2b884cb..a08a30f76b5666efc37383dba7e3d4a3792767a5 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py
@@ -64,7 +64,8 @@ class RTSMObservationSerializer(serializers.ModelSerializer):
station_entry = Station.objects.filter(name=station_name).first()
if station_entry is None:
- station_entry = Station(name=station_name, type=station_type).save()
+ station_entry = Station(name=station_name, type=station_type)
+ station_entry.save()
# searches if the RTSM is already present
observation_id = validated_data['observation_id']
RTSMObservation_instance = RTSMObservation.objects.filter(observation_id=observation_id,
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/urls.py b/LCU/Maintenance/DBInterface/monitoringdb/urls.py
index 229568882f3ed42b69e7a7f4727a39b6ba337d06..7733500227debed91440e72ebd5bf6b77e74f30d 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/urls.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/urls.py
@@ -44,6 +44,8 @@ urlpatterns = [
url(r'^api/stationtests/raw/insert', insert_raw_station_test),
url(r'^api/rtsm/raw/insert', insert_raw_rtsm_test),
url(r'^api/view/ctrl_stationoverview', ControllerStationOverview.as_view()),
+ url(r'^api/view/ctrl_stationtestsummary', ControllerStationTestsSummary.as_view()),
+ url(r'^api/view/ctrl_latest_observation', ControllerLatestObservations.as_view()),
url(r'^api/docs', include_docs_urls(title='Monitoring DB API'))
]
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py b/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py
index 2426e49c4ca5ed2a1ddc2bf26b590096b2ae1a94..46a9c4cb07b81b543e4f73be415bb81f31edba10 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py
@@ -1,17 +1,18 @@
+import datetime
from collections import OrderedDict
import coreapi
import coreschema
-from django.db.models import Count
-from rest_framework.schemas import ManualSchema
-from rest_framework.views import APIView
+from django.db.models import Count, Sum
+from django.db.models import Window, F
from rest_framework import status
-
-from lofar.maintenance.monitoringdb.models.station_test import StationTest
-from lofar.maintenance.monitoringdb.models.station import Station
from rest_framework.response import Response
+from rest_framework.schemas import ManualSchema
+from rest_framework.views import APIView
from lofar.maintenance.monitoringdb.models.rtsm import RTSMObservationSummary
+from lofar.maintenance.monitoringdb.models.station import Station
+from lofar.maintenance.monitoringdb.models.station_test import StationTest
class ControllerStationOverview(APIView):
@@ -20,7 +21,7 @@ class ControllerStationOverview(APIView):
"""
DEFAULT_STATION_GROUP = 'A'
- DEFAULT_ONLY_ERRORS = False
+ DEFAULT_ONLY_ERRORS = True
DEFAULT_N_STATION_TESTS = 4
DEFAULT_N_RTSM = 4
@@ -80,7 +81,7 @@ class ControllerStationOverview(APIView):
station_test_list = StationTest.objects.filter(
station__name=station_entity.name).order_by('-end_datetime')[:n_station_tests - 1]
rtsm_list = RTSMObservationSummary.objects.filter(
- station__name=station_entity.name).order_by('-end_time')[:n_rtsm - 1]
+ station__name=station_entity.name).order_by('-end_datetime')[:n_rtsm - 1]
station_payload['station_tests'] = list()
for station_test in station_test_list:
@@ -88,7 +89,7 @@ class ControllerStationOverview(APIView):
component_errors = station_test.component_errors
station_test_payload[
- 'total_component_errors'] = station_test.component_errors.count()
+ 'total_component_errorss'] = station_test.component_errors.count()
station_test_payload['start_datetime'] = station_test.start_datetime
station_test_payload['end_datetime'] = station_test.end_datetime
station_test_payload['checks'] = station_test.checks
@@ -114,13 +115,13 @@ class ControllerStationOverview(APIView):
for rtsm in rtsm_list:
rtsm_payload = OrderedDict()
rtsm_payload['observation_id'] = rtsm.observation_id
- rtsm_payload['start_datetime'] = rtsm.start_time
- rtsm_payload['end_datetime'] = rtsm.end_time
+ rtsm_payload['start_datetime'] = rtsm.start_datetime
+ rtsm_payload['end_datetime'] = rtsm.end_datetime
unique_modes = [item['mode'] for item in rtsm.errors.values('mode').distinct()]
rtsm_payload['mode'] = unique_modes
- rtsm_payload['total_component_error'] = rtsm.errors_found.count()
- from django.db.models import Window, F
+ rtsm_payload['total_component_errors'] = rtsm.errors_found.count()
+
errors_summary = OrderedDict()
errors_summary_query = rtsm.errors_found.annotate(total=
@@ -136,10 +137,272 @@ class ControllerStationOverview(APIView):
station_payload['rtsm'].append(rtsm_payload)
response_payload.append(station_payload)
- if errors_only:
+ if self.errors_only and self.errors_only is not 'false':
response_payload = filter(
lambda station_entry:
len(station_entry['station_tests']) + len(station_entry['rtsm']) > 0,
response_payload)
response_payload = sorted(response_payload, key=lambda item: item['station_name'])
return Response(status=status.HTTP_200_OK, data=response_payload)
+
+
+class ControllerStationTestsSummary(APIView):
+ """
+ Overview of the latest station tests performed on the stations
+ """
+
+ DEFAULT_STATION_GROUP = 'A'
+ DEFAULT_ONLY_ERRORS = True
+
+ queryset = StationTest.objects.all()
+ schema = ManualSchema(fields=[
+ coreapi.Field(
+ "station_group",
+ required=False,
+ location='query',
+ schema=coreschema.Enum(['C', 'R', 'I', 'A'], description=
+ 'Station group to select for choices are [C|R|I|ALL]',
+ )
+ ),
+ coreapi.Field(
+ "errors_only",
+ required=False,
+ location='query',
+ schema=coreschema.Boolean(
+ description='displays or not only the station with more than one error')
+ ),
+ coreapi.Field(
+ "start_date",
+ required=True,
+ location='query',
+ schema=coreschema.String(
+ description='select station tests from date (ex. YYYY-MM-DD)')
+ ),
+ coreapi.Field(
+ "end_date",
+ required=True,
+ location='query',
+ schema=coreschema.String(
+ description='select station tests to date (ex. YYYY-MM-DD)')
+ )
+ ]
+ )
+
+ @staticmethod
+ def parse_date(date):
+ expected_format = '%Y-%m-%d'
+ try:
+ parsed_date = datetime.datetime.strptime(date, expected_format)
+ return parsed_date
+ except Exception as e:
+ raise ValueError('cannot parse %s with format %s - %s' % (date, expected_format, e))
+
+ def validate_query_parameters(self, request):
+ self.errors_only = request.query_params.get('errors_only', self.DEFAULT_ONLY_ERRORS)
+ self.station_group = request.query_params.get('station_group', self.DEFAULT_STATION_GROUP)
+
+ start_date = request.query_params.get('start_date')
+ self.start_date = ControllerStationTestsSummary.parse_date(start_date)
+
+ end_date = request.query_params.get('end_date')
+ self.end_date = ControllerStationTestsSummary.parse_date(end_date)
+
+ def get(self, request, format=None):
+ try:
+ self.validate_query_parameters(request)
+ except ValueError as e:
+ return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
+ data='Please specify the date in the format YYYY-MM-DD: %s' % (e,))
+ except KeyError as e:
+ return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
+ data='Please specify both the start and the end date: %s' % (e,))
+
+ station_entities = Station.objects.all()
+ for group in self.station_group:
+ if group is not 'A':
+ station_entities = station_entities.filter(type=group)
+
+ # Since django preferes a ordered dict over a dict we make it happy... for now
+ response_payload = list()
+ for station_entity in station_entities:
+ station_payload = OrderedDict()
+
+ station_payload['station_name'] = station_entity.name
+
+ station_test_list = StationTest.objects.filter(
+ start_datetime__gte=self.start_date,
+ end_datetime__lte=self.end_date,
+ station__name=station_entity.name).order_by('-end_datetime')
+ station_payload['station_tests'] = list()
+ for station_test in station_test_list:
+
+ station_test_payload = OrderedDict()
+ component_errors = station_test.component_errors
+
+ station_test_payload[
+ 'total_component_errors'] = station_test.component_errors.count()
+ station_test_payload['start_datetime'] = station_test.start_datetime
+ station_test_payload['end_datetime'] = station_test.end_datetime
+ station_test_payload['checks'] = station_test.checks
+ component_errors_summary = component_errors. \
+ values('component__type', 'type').annotate(
+ total=Count('type')).order_by('-total')
+ component_errors_summary_dict = OrderedDict()
+ for item in component_errors_summary:
+ item_component_type = item['component__type']
+ item_error_type = item['type']
+ item_error_total = item['total']
+
+ if item_component_type not in component_errors_summary_dict:
+ component_errors_summary_dict[item_component_type] = OrderedDict()
+
+ component_errors_summary_dict[item_component_type][item_error_type] = \
+ item_error_total
+ station_test_payload['component_error_summary'] = component_errors_summary_dict
+
+ station_payload['station_tests'].append(station_test_payload)
+
+ response_payload.append(station_payload)
+ if self.errors_only and self.errors_only is not 'false':
+ response_payload = filter(
+ lambda station_entry:
+ len(station_entry['station_tests']) > 0,
+ response_payload)
+ response_payload = sorted(response_payload, key=lambda item: item['station_name'])
+ return Response(status=status.HTTP_200_OK, data=response_payload)
+
+
+class ControllerLatestObservations(APIView):
+ """
+ Overview of the latest observations performed on the stations
+ """
+
+ DEFAULT_STATION_GROUP = 'A'
+ DEFAULT_ONLY_ERRORS = True
+
+ queryset = StationTest.objects.all()
+ schema = ManualSchema(fields=[
+ coreapi.Field(
+ "station_group",
+ required=False,
+ location='query',
+ schema=coreschema.Enum(['C', 'R', 'I', 'A'], description=
+ 'Station group to select for choices are [C|R|I|ALL]',
+ )
+ ),
+ coreapi.Field(
+ "errors_only",
+ required=False,
+ location='query',
+ schema=coreschema.Boolean(
+ description='displays or not only the station with more than one error')
+ ),
+ coreapi.Field(
+ "from_date",
+ required=True,
+ location='query',
+ schema=coreschema.String(
+ description='select rtsm from date (ex. YYYY-MM-DD)')
+ )
+ ]
+ )
+
+ @staticmethod
+ def parse_date(date):
+ expected_format = '%Y-%m-%d'
+ try:
+ parsed_date = datetime.datetime.strptime(date, expected_format)
+ return parsed_date
+ except Exception as e:
+ raise ValueError('cannot parse %s with format %s - %s' % (date, expected_format, e))
+
+ def compute_rtsm_observation_summary(self, rtsm_errors):
+ errors_summary = OrderedDict()
+
+ errors_summary_query = rtsm_errors.annotate(total=
+ Window(expression=Count('rcu'),
+ partition_by=[F(
+ 'error_type')])).values(
+ 'error_type', 'total').distinct()
+
+ for error_summary in errors_summary_query:
+ errors_summary[error_summary['error_type']] = error_summary['total']
+ return errors_summary
+
+ def validate_query_parameters(self, request):
+ self.errors_only = request.query_params.get('errors_only', self.DEFAULT_ONLY_ERRORS)
+ self.station_group = request.query_params.get('station_group', self.DEFAULT_STATION_GROUP)
+
+ start_date = request.query_params.get('from_date')
+ self.from_date = ControllerLatestObservations.parse_date(start_date)
+
+ def get(self, request, format=None):
+ try:
+ self.validate_query_parameters(request)
+ except ValueError as e:
+ return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
+ data='Please specify the date in the format YYYY-MM-DD: %s' % (e,))
+ except KeyError as e:
+ return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
+ data='Please specify both the start and the end date: %s' % (e,))
+
+ rtsm_observation_entities = RTSMObservationSummary.objects.filter(
+ start_datetime__gte=self.from_date)
+ for group in self.station_group:
+ if group is not 'A':
+ rtsm_observation_entities = rtsm_observation_entities.filter(station__type=group)
+
+ # Since django preferes a ordered dict over a dict we make it happy... for now
+ response_payload = list()
+ for rtsm_observation_entity in rtsm_observation_entities.values('observation_id',
+ 'start_datetime',
+ 'end_datetime'). \
+ distinct():
+ observation_payload = OrderedDict()
+
+ observation_payload['observation_id'] = rtsm_observation_entity['observation_id']
+ observation_payload['start_datetime'] = rtsm_observation_entity['start_datetime']
+ observation_payload['end_datetime'] = rtsm_observation_entity['end_datetime']
+
+ rtsm_list = RTSMObservationSummary.objects.filter(
+ start_datetime__gte=self.from_date,
+ observation_id=rtsm_observation_entity['observation_id']). \
+ order_by('-end_datetime')
+ unique_modes = [item['errors__mode'] for item in rtsm_list.values('errors__mode').distinct()]
+
+ observation_payload['mode'] = unique_modes
+ observation_payload['total_component_errors'] = 0
+
+ station_list = rtsm_list.values('station').distinct()
+ station_involved_list = list()
+
+ for station in station_list:
+ rtsm_entry_per_station = rtsm_list.filter(station__pk=station['station']).first()
+
+ station_summary = OrderedDict()
+ station_summary['station_name'] = rtsm_entry_per_station.station.name
+
+ station_summary['n_errors'] =\
+ rtsm_entry_per_station.errors.values('rcu').distinct().count()
+ station_summary['component_error_summary'] = self.compute_rtsm_observation_summary(
+ rtsm_entry_per_station.errors_found)
+
+ station_involved_list.append(station_summary)
+
+ observation_payload['total_component_errors'] += station_summary['n_errors']
+
+ station_involved_list = sorted(station_involved_list,
+ key=lambda rtsm_per_station: rtsm_per_station['n_errors'],
+ reverse=True)
+ observation_payload['station_involved'] = station_involved_list
+
+ response_payload.append(observation_payload)
+ if self.errors_only and self.errors_only != 'false':
+ print(self.errors_only)
+ response_payload = filter(
+ lambda station_entry:
+ len(station_entry['total_component_errors']) > 0,
+ response_payload)
+ response_payload = sorted(response_payload, key=lambda item: item['total_component_errors'],
+ reverse=True)
+ return Response(status=status.HTTP_200_OK, data=response_payload)
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py b/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py
index 660b38d0f04f3ff6b1a3f4f400b254101192a165..8459d523e71e9c9105b7274b90d683d748a59d9c 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py
@@ -1,5 +1,5 @@
from .common import *
-from ..models.rtsm import RTSMObservation, RTSMError, RTSMSpectrum
+from ..models.rtsm import RTSMObservation, RTSMError, RTSMSpectrum, RTSMObservationSummary
from ..serializers.rtsm import RTSMObservationSerializer, RTSMErrorSerializer,\
RTSMObservationSummarySerializer, RTSMSpectrumSerializer
from ..rtsm_test_raw_parser import parse_rtsm_test
@@ -33,7 +33,7 @@ class RTSMObservationViewSet(viewsets.ReadOnlyModelViewSet):
class RTSMObservationSummaryViewSet(viewsets.ReadOnlyModelViewSet):
- queryset = RTSMObservation.objects.all()
+ queryset = RTSMObservationSummary.objects.all()
serializer_class = RTSMObservationSummarySerializer
filter_fields = '__all__'