From 4c63e4e712124bc70e01b6c4fa42695ab7ff40ec Mon Sep 17 00:00:00 2001
From: Mattia Mancini <mancini@astron.nl>
Date: Fri, 26 Oct 2018 10:16:39 +0000
Subject: [PATCH] OSB-28: improved fields validation
---
.../DBInterface/django_postgresql/settings.py | 4 +
.../monitoringdb/views/controllers.py | 292 +++++++++---------
2 files changed, 147 insertions(+), 149 deletions(-)
diff --git a/LCU/Maintenance/DBInterface/django_postgresql/settings.py b/LCU/Maintenance/DBInterface/django_postgresql/settings.py
index ba6750a9c70..be5844ac85b 100644
--- a/LCU/Maintenance/DBInterface/django_postgresql/settings.py
+++ b/LCU/Maintenance/DBInterface/django_postgresql/settings.py
@@ -106,6 +106,10 @@ LOGGING = {
'serializers': {
'handlers': ['console'],
'level': 'DEBUG',
+ },
+ 'controllers': {
+ 'handlers': ['console'],
+ 'level': 'DEBUG'
}
},
}
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py b/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py
index 06b580f380c..7f1c3478f57 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py
@@ -1,11 +1,11 @@
import datetime
+import logging
from collections import OrderedDict
from math import ceil
import coreapi
import coreschema
from django.db.models import Count
-from django.db.models import Window, F
from rest_framework import status
from rest_framework.response import Response
from rest_framework.schemas import ManualSchema
@@ -17,19 +17,96 @@ from lofar.maintenance.monitoringdb.models.rtsm import RTSMObservation
from lofar.maintenance.monitoringdb.models.station import Station
from lofar.maintenance.monitoringdb.models.station_test import StationTest
+logger = logging.getLogger(__name__)
-class ControllerStationOverview(APIView):
+
+def parse_date(date):
+ expected_format = '%Y-%m-%d'
+ try:
+ parsed_date = datetime.datetime.strptime(date, expected_format)
+ return parsed_date
+ except Exception as e:
+ raise ValueError('cannot parse %s with format %s - %s' % (date, expected_format, e))
+
+
+def parse_bool(boolean_value_str):
+ boolean_value_str = boolean_value_str.lower()
+ if boolean_value_str in ['t', 'true', '1']:
+ return True
+ elif boolean_value_str in ['f', 'false', '0']:
+ return False
+ else:
+ raise ValueError('%s is neither true or false' % boolean_value_str)
+
+
+class ValidableReadOnlyView(APIView):
+ """
+ Convenience APIView class to have the validation of the query parameters on a get http request
+ """
+
+ # Override this to make the schema validation work
+ fields = []
+
+ def compute_response(self):
+ raise NotImplementedError()
+
+ @property
+ def schema(self):
+ return ManualSchema(fields=self.fields)
+
+ def validate_query_parameters(self, request):
+ """
+ Validated the request parameters and stores them as fields
+ :param request: the http request to the api call
+ :type request: rest_framework.request.Request
+ :raises ValueError: if the parameter is not valid
+ :raises KeyError: if the requested parameter is missing
+ """
+ for field in self.fields:
+ if field.required and field.name not in request.query_params:
+ raise KeyError('%s parameter is missing' % field.name)
+ elif field.name not in request.query_params:
+ continue
+ else:
+ value = self.request.query_params.get(field.name)
+ if field.type:
+ self.__setattr__(field.name, field.type(value))
+ else:
+ self.__setattr__(field.name, value)
+ errors = field.schema.validate(self.__getattribute__(field.name))
+ for error in errors:
+ raise ValueError(error.text)
+
+ def get(self, request):
+ try:
+ self.validate_query_parameters(request)
+ except ValueError as e:
+ return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
+ data='Please specify the correct parameters: %s' % (e,))
+ except KeyError as e:
+ return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
+ data='Please specify all the required parameters: %s' % (e,))
+
+ try:
+ response = self.compute_response()
+ except Exception as e:
+ logger.exception(e)
+ return Response(status=status.HTTP_500_INTERNAL_SERVER_ERROR,
+ data='exception occurred: %s' % e)
+ return response
+
+
+class ControllerStationOverview(ValidableReadOnlyView):
"""
Overview of the latest tests performed on the stations
"""
- DEFAULT_STATION_GROUP = 'A'
- DEFAULT_ONLY_ERRORS = True
- DEFAULT_N_STATION_TESTS = 4
- DEFAULT_N_RTSM = 4
+ station_group = 'A'
+ errors_only = 'true'
+ n_station_tests = 4
+ n_rtsm = 4
- queryset = StationTest.objects.all()
- schema = ManualSchema(fields=[
+ fields = [
coreapi.Field(
"station_group",
required=False,
@@ -42,6 +119,7 @@ class ControllerStationOverview(APIView):
"n_station_tests",
required=False,
location='query',
+ type=int,
schema=coreschema.Integer(description='number of station tests to select',
minimum=1)
),
@@ -49,6 +127,7 @@ class ControllerStationOverview(APIView):
"n_rtsm",
required=False,
location='query',
+ type=int,
schema=coreschema.Integer(description='number of station tests to select',
minimum=1)
),
@@ -56,21 +135,16 @@ class ControllerStationOverview(APIView):
"errors_only",
required=False,
location='query',
+ type=parse_bool,
schema=coreschema.Boolean(
description='displays or not only the station with more than one error')
)
]
- )
- def get(self, request, format=None):
- errors_only = request.query_params.get('errors_only', self.DEFAULT_ONLY_ERRORS)
- station_group = request.query_params.get('station_group', self.DEFAULT_STATION_GROUP)
- n_station_tests = int(
- request.query_params.get('n_station_tests', self.DEFAULT_N_STATION_TESTS))
- n_rtsm = int(request.query_params.get('n_rtsm', self.DEFAULT_N_RTSM))
+ def compute_response(self):
station_entities = Station.objects.all()
- for group in station_group:
+ for group in self.station_group:
if group is not 'A':
station_entities = station_entities.filter(type=group)
@@ -82,9 +156,9 @@ class ControllerStationOverview(APIView):
station_payload['station_name'] = station_entity.name
station_test_list = StationTest.objects.filter(
- station__name=station_entity.name).order_by('-end_datetime')[:n_station_tests]
+ station__name=station_entity.name).order_by('-end_datetime')[:self.n_station_tests]
rtsm_list = RTSMObservation.objects.filter(
- station__name=station_entity.name).order_by('-end_datetime')[:n_rtsm]
+ station__name=station_entity.name).order_by('-end_datetime')[:self.n_rtsm]
station_payload['station_tests'] = list()
for station_test in station_test_list:
@@ -121,7 +195,8 @@ class ControllerStationOverview(APIView):
rtsm_payload['start_datetime'] = rtsm.start_datetime
rtsm_payload['end_datetime'] = rtsm.end_datetime
- unique_modes = [item['mode'] for item in rtsm.errors_summary.values('mode').distinct()]
+ unique_modes = [item['mode'] for item in
+ rtsm.errors_summary.values('mode').distinct()]
rtsm_payload['mode'] = unique_modes
rtsm_payload['total_component_errors'] = rtsm.errors_summary.count()
@@ -137,7 +212,7 @@ class ControllerStationOverview(APIView):
station_payload['rtsm'].append(rtsm_payload)
response_payload.append(station_payload)
- if errors_only and errors_only is not 'false':
+ if self.errors_only:
response_payload = filter(
lambda station_entry:
len(station_entry['station_tests']) + len(station_entry['rtsm']) > 0,
@@ -146,17 +221,15 @@ class ControllerStationOverview(APIView):
return Response(status=status.HTTP_200_OK, data=response_payload)
-class ControllerStationTestsSummary(APIView):
+class ControllerStationTestsSummary(ValidableReadOnlyView):
"""
Overview of the latest station tests performed on the stations # lookback days before now
"""
+ station_group = 'A'
+ errors_only = 'true'
+ lookback_time = 7
- DEFAULT_STATION_GROUP = 'A'
- DEFAULT_ONLY_ERRORS = True
- DEFAULT_LOOKBACK_TIME_IN_DAYS = 7
-
- queryset = StationTest.objects.all()
- schema = ManualSchema(fields=[
+ fields = [
coreapi.Field(
"station_group",
required=False,
@@ -168,44 +241,22 @@ class ControllerStationTestsSummary(APIView):
"errors_only",
required=False,
location='query',
+ type=parse_bool,
schema=coreschema.Boolean(
description='displays or not only the station with more than one error')
),
coreapi.Field(
"lookback_time",
required=False,
+ type=int,
location='query',
schema=coreschema.Integer(description='number of days from now (default 7)',
minimum=1)
)
]
- )
-
- @staticmethod
- def parse_date(date):
- expected_format = '%Y-%m-%d'
- try:
- parsed_date = datetime.datetime.strptime(date, expected_format)
- return parsed_date
- except Exception as e:
- raise ValueError('cannot parse %s with format %s - %s' % (date, expected_format, e))
- def validate_query_parameters(self, request):
- self.errors_only = request.query_params.get('errors_only', self.DEFAULT_ONLY_ERRORS)
- self.station_group = request.query_params.get('station_group', self.DEFAULT_STATION_GROUP)
-
- self.lookback_time = datetime.timedelta(int(request.query_params.get('lookback_time',
- self.DEFAULT_LOOKBACK_TIME_IN_DAYS)))
-
- def get(self, request, format=None):
- try:
- self.validate_query_parameters(request)
- except ValueError as e:
- return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
- data='Please specify the correct parameters: %s' % (e,))
- except KeyError as e:
- return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
- data='Please specify all the required parameters: %s' % (e,))
+ def compute_response(self):
+ self.lookback_time = datetime.timedelta(days=self.lookback_time)
station_test_list = StationTest.objects \
.filter(start_datetime__gte=datetime.date.today() - self.lookback_time) \
@@ -246,7 +297,8 @@ class ControllerStationTestsSummary(APIView):
station_test_payload['component_error_summary'] = component_errors_summary_dict
response_payload.append(station_test_payload)
- if self.errors_only and self.errors_only is not 'false':
+
+ if self.errors_only:
response_payload = filter(
lambda station_test_entry:
station_test_entry['total_component_errors'] > 0,
@@ -255,15 +307,15 @@ class ControllerStationTestsSummary(APIView):
return Response(status=status.HTTP_200_OK, data=response_payload)
-class ControllerLatestObservations(APIView):
+class ControllerLatestObservations(ValidableReadOnlyView):
"""
Overview of the latest observations performed on the stations
"""
- DEFAULT_STATION_GROUP = 'A'
- DEFAULT_ONLY_ERRORS = True
+ station_group = 'A'
+ errors_only = 'true'
- schema = ManualSchema(fields=[
+ fields = [
coreapi.Field(
"station_group",
required=False,
@@ -276,6 +328,7 @@ class ControllerLatestObservations(APIView):
"errors_only",
required=False,
location='query',
+ type=parse_bool,
schema=coreschema.Boolean(
description='displays or not only the station with more than one error')
),
@@ -287,49 +340,26 @@ class ControllerLatestObservations(APIView):
description='select rtsm from date (ex. YYYY-MM-DD)')
)
]
- )
-
- @staticmethod
- def parse_date(date):
- expected_format = '%Y-%m-%d'
- try:
- parsed_date = datetime.datetime.strptime(date, expected_format)
- return parsed_date
- except Exception as e:
- raise ValueError('cannot parse %s with format %s - %s' % (date, expected_format, e))
-
- def validate_query_parameters(self, request):
- self.errors_only = request.query_params.get('errors_only', self.DEFAULT_ONLY_ERRORS)
- self.station_group = request.query_params.get('station_group', self.DEFAULT_STATION_GROUP)
- start_date = request.query_params.get('from_date')
- self.from_date = ControllerLatestObservations.parse_date(start_date)
+ def compute_response(self):
+ self.from_date = parse_date(self.from_date)
- def get(self, request, format=None):
- try:
- self.validate_query_parameters(request)
- except ValueError as e:
- return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
- data='Please specify the date in the format YYYY-MM-DD: %s' % (e,))
- except KeyError as e:
- return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
- data='Please specify both the start and the end date: %s' % (e,))
- filtered_entities = RTSMObservation.objects\
+ filtered_entities = RTSMObservation.objects \
.filter(start_datetime__gte=self.from_date)
if self.station_group != 'A':
- filtered_entities = filtered_entities\
+ filtered_entities = filtered_entities \
.filter(station__type=self.station_group)
if self.errors_only:
filtered_entities = filtered_entities.exclude(errors_summary__isnull=True)
- errors_summary = filtered_entities\
+ errors_summary = filtered_entities \
.values('observation_id',
'station__name',
'start_datetime',
'end_datetime',
'errors_summary__error_type',
- 'errors_summary__mode')\
- .annotate(total=Count('errors_summary__error_type'))\
+ 'errors_summary__mode') \
+ .annotate(total=Count('errors_summary__error_type')) \
.order_by('observation_id', 'station__name')
response = dict()
@@ -357,7 +387,7 @@ class ControllerLatestObservations(APIView):
response[observation_id]['total_component_errors'] += total
station_involved_summary = response[observation_id]['station_involved']
- response[observation_id]['mode'] += [mode]\
+ response[observation_id]['mode'] += [mode] \
if mode not in response[observation_id]['mode'] else []
if station_name not in station_involved_summary:
station_involved_summary[station_name] = OrderedDict()
@@ -369,12 +399,13 @@ class ControllerLatestObservations(APIView):
station_involved_summary[station_name]['component_error_summary'][error_type] = total
response_payload = sorted(response.values(),
- key= lambda item: item['start_datetime'],
+ key=lambda item: item['start_datetime'],
reverse=True)
return Response(status=status.HTTP_200_OK, data=response_payload)
-class ControllerStationTestStatistics(APIView):
+
+class ControllerStationTestStatistics(ValidableReadOnlyView):
"""
/views/ctrl_stationtest_statistics:
@@ -404,11 +435,10 @@ result:
....
]
"""
- DEFAULT_STATION_GROUP = 'A'
- DEFAULT_TEST_TYPE = 'B'
+ station_group = 'A'
+ test_type = 'B'
- queryset = StationTest.objects.all()
- schema = ManualSchema(fields=[
+ fields = [
coreapi.Field(
"test_type",
required=False,
@@ -444,39 +474,11 @@ result:
"averaging_interval",
required=True,
location='query',
+ type=int,
schema=coreschema.Integer(
description='averaging interval in days')
)
]
- )
-
- @staticmethod
- def parse_date(date):
- expected_format = '%Y-%m-%d'
- try:
- parsed_date = datetime.datetime.strptime(date, expected_format)
- return parsed_date
- except Exception as e:
- raise ValueError('cannot parse %s with format %s - %s' % (date, expected_format, e))
-
- def validate_query_parameters(self, request):
- self.station_group = request.query_params.get('station_group', self.DEFAULT_STATION_GROUP)
- if self.station_group not in ['C', 'R', 'I', 'A']:
- raise ValueError('station_group is not one of [C,R,I,A]')
-
- from_date = request.query_params.get('from_date')
- self.from_date = ControllerLatestObservations.parse_date(from_date)
-
- to_date = request.query_params.get('to_date')
- self.to_date = ControllerLatestObservations.parse_date(to_date)
-
- self.test_type = request.query_params.get('test_type', self.DEFAULT_TEST_TYPE)
-
- if self.test_type not in ['R', 'S', 'B']:
- raise ValueError('test_type is not one of [R,S,B]')
-
- self.averaging_interval = datetime.timedelta(
- int(request.query_params.get('averaging_interval')))
def compute_errors_per_station(self, from_date, to_date, central_time, station_group,
test_type):
@@ -572,25 +574,19 @@ result:
return errors_per_error_type_in_bin.values()
- def get(self, request, format=None):
- try:
- self.validate_query_parameters(request)
- except ValueError as e:
- return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
- data='Error wrong format: %s' % (e,))
- except KeyError as e:
- return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
- data='Please specify all the correct parameters: %s' % (e,))
-
+ def compute_response(self):
+ from_date = parse_date(self.from_date)
+ to_date = parse_date(self.to_date)
+ averaging_interval = datetime.timedelta(days=self.averaging_interval)
response_payload = OrderedDict()
- response_payload['start_date'] = self.from_date
- response_payload['end_date'] = self.to_date
- response_payload['averaging_interval'] = self.averaging_interval
+ response_payload['start_date'] = from_date
+ response_payload['end_date'] = to_date
+ response_payload['averaging_interval'] = averaging_interval
errors_per_station = []
errors_per_type = []
- n_bins = int(ceil((self.to_date - self.from_date) / self.averaging_interval))
+ n_bins = int(ceil((to_date - from_date) / averaging_interval))
for i in range(n_bins):
if self.station_group is 'A':
@@ -598,16 +594,16 @@ result:
else:
station_group = self.station_group
errors_per_station += self.compute_errors_per_station(
- from_date=self.from_date + i * self.averaging_interval,
- to_date=self.from_date + (i + 1) * self.averaging_interval,
- central_time=self.from_date + (i + .5) * self.averaging_interval,
+ from_date=from_date + i * averaging_interval,
+ to_date=from_date + (i + 1) * averaging_interval,
+ central_time=from_date + (i + .5) * averaging_interval,
station_group=station_group,
test_type=self.test_type)
errors_per_type += self.compute_errors_per_type(
- from_date=self.from_date + i * self.averaging_interval,
- to_date=self.from_date + (i + 1) * self.averaging_interval,
- central_time=self.from_date + (i + .5) * self.averaging_interval,
+ from_date=from_date + i * averaging_interval,
+ to_date=from_date + (i + 1) * averaging_interval,
+ central_time=from_date + (i + .5) * averaging_interval,
station_group=station_group,
test_type=self.test_type)
@@ -617,11 +613,9 @@ result:
return Response(status=status.HTTP_200_OK, data=response_payload)
-class ControllerAllComponentErrorTypes(APIView):
- """
- Returns all distinct component errors
- """
+class ControllerAllComponentErrorTypes(ValidableReadOnlyView):
+ schema = ManualSchema(description="Returns all distinct component errors", fields=[])
- def get(self, request, format=None):
+ def compute_response(self):
data = [item['type'] for item in ComponentError.objects.values('type').distinct()]
return Response(status=status.HTTP_200_OK, data=data)
--
GitLab