Newer
Older
from collections import OrderedDict
import coreapi
import coreschema
from django.db.models import Count
from lofar.maintenance.monitoringdb.models.component_error import ComponentError
from lofar.maintenance.monitoringdb.models.rtsm import RTSMErrorSummary, MODE_TO_COMPONENT
from lofar.maintenance.monitoringdb.models.rtsm import RTSMObservation
from lofar.maintenance.monitoringdb.models.station import Station
from lofar.maintenance.monitoringdb.models.station_test import StationTest
from rest_framework import status
from rest_framework.response import Response
from rest_framework.schemas import ManualSchema
from rest_framework.views import APIView
def parse_date(date):
expected_format = '%Y-%m-%d'
try:
parsed_date = datetime.datetime.strptime(date, expected_format)
return parsed_date
except Exception as e:
raise ValueError('cannot parse %s with format %s - %s' % (date, expected_format, e))
def parse_bool(boolean_value_str):
boolean_value_str = boolean_value_str.lower()
if boolean_value_str in ['t', 'true', '1']:
return True
elif boolean_value_str in ['f', 'false', '0']:
return False
else:
raise ValueError('%s is neither true or false' % boolean_value_str)
class ValidableReadOnlyView(APIView):
"""
Convenience APIView class to have the validation of the query parameters on a get http request
"""
# Override this to make the schema validation work
fields = []
description = ''
def compute_response(self):
raise NotImplementedError()
@property
def schema(self):
return ManualSchema(fields=self.fields, description=self.description)
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
def validate_query_parameters(self, request):
"""
Validated the request parameters and stores them as fields
:param request: the http request to the api call
:type request: rest_framework.request.Request
:raises ValueError: if the parameter is not valid
:raises KeyError: if the requested parameter is missing
"""
for field in self.fields:
if field.required and field.name not in request.query_params:
raise KeyError('%s parameter is missing' % field.name)
elif field.name not in request.query_params:
continue
else:
value = self.request.query_params.get(field.name)
if field.type:
self.__setattr__(field.name, field.type(value))
else:
self.__setattr__(field.name, value)
errors = field.schema.validate(self.__getattribute__(field.name))
for error in errors:
raise ValueError(error.text)
def get(self, request):
try:
self.validate_query_parameters(request)
except ValueError as e:
return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
data='Please specify the correct parameters: %s' % (e,))
except KeyError as e:
return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
data='Please specify all the required parameters: %s' % (e,))
try:
response = self.compute_response()
except ValueError as e:
return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
data='Please specify the correct parameters: %s' % (e,))
except Exception as e:
logger.exception(e)
return Response(status=status.HTTP_500_INTERNAL_SERVER_ERROR,
data='exception occurred: %s' % e)
return response
class ControllerStationOverview(ValidableReadOnlyView):
description = "Overview of the latest tests performed on the stations"
station_group = 'A'
errors_only = 'true'
n_station_tests = 4
n_rtsm = 4
coreapi.Field(
"station_group",
required=False,
location='query',
schema=coreschema.Enum(['C', 'R', 'I', 'A'], description=
'Station group to select for choices are [C|R|I|ALL]',
)
),
coreapi.Field(
"n_station_tests",
required=False,
location='query',
schema=coreschema.Integer(description='number of station tests to select',
minimum=1)
),
coreapi.Field(
"n_rtsm",
required=False,
location='query',
schema=coreschema.Integer(description='number of station tests to select',
minimum=1)
),
coreapi.Field(
"errors_only",
required=False,
location='query',
schema=coreschema.Boolean(
description='displays or not only the station with more than one error')
)
]
station_entities = Station.objects.all()
if group is not 'A':
station_entities = station_entities.filter(type=group)
# Since django preferes a ordered dict over a dict we make it happy... for now
response_payload = list()
for station_entity in station_entities:
station_payload = OrderedDict()
station_payload['station_name'] = station_entity.name
station_test_list = StationTest.objects.filter(
station__name=station_entity.name).order_by('-end_datetime')[:self.n_station_tests]
rtsm_list = RTSMObservation.objects.filter(
station__name=station_entity.name).order_by('-end_datetime')[:self.n_rtsm]
station_payload['station_tests'] = list()
for station_test in station_test_list:
station_test_payload = OrderedDict()
component_errors = station_test.component_errors
station_test_payload[
'total_component_errors'] = station_test.component_errors.count()
station_test_payload['start_datetime'] = station_test.start_datetime
station_test_payload['end_datetime'] = station_test.end_datetime
station_test_payload['checks'] = station_test.checks
component_errors_summary = component_errors. \
values('component__type', 'type').annotate(
total=Count('type')).order_by('-total')
component_errors_summary_dict = OrderedDict()
for item in component_errors_summary:
item_component_type = item['component__type']
item_error_type = item['type']
item_error_total = item['total']
if item_component_type not in component_errors_summary_dict:
component_errors_summary_dict[item_component_type] = OrderedDict()
component_errors_summary_dict[item_component_type][item_error_type] = \
item_error_total
station_test_payload['component_error_summary'] = component_errors_summary_dict
station_payload['station_tests'].append(station_test_payload)
station_payload['rtsm'] = list()
for rtsm in rtsm_list:
rtsm_payload = OrderedDict()
rtsm_payload['observation_id'] = rtsm.observation_id
rtsm_payload['start_datetime'] = rtsm.start_datetime
rtsm_payload['end_datetime'] = rtsm.end_datetime
unique_modes = [item['mode'] for item in
rtsm.errors_summary.values('mode').distinct()]
rtsm_payload['mode'] = unique_modes
rtsm_payload['total_component_errors'] = rtsm.errors_summary.count()
errors_summary = OrderedDict()
errors_summary_query = rtsm.errors_summary.values('error_type').annotate(
total=Count('error_type'))
for error_summary in errors_summary_query:
errors_summary[error_summary['error_type']] = error_summary['total']
rtsm_payload['error_summary'] = errors_summary
station_payload['rtsm'].append(rtsm_payload)
response_payload.append(station_payload)
response_payload = filter(
lambda station_entry:
len(station_entry['station_tests']) + len(station_entry['rtsm']) > 0,
response_payload)
response_payload = sorted(response_payload, key=lambda item: item['station_name'])
return Response(status=status.HTTP_200_OK, data=response_payload)
class ControllerStationTestsSummary(ValidableReadOnlyView):
description = "Overview of the latest station tests performed on" \
" the stations a [loopback_time] days ago from now"
station_group = 'A'
errors_only = 'true'
lookback_time = 7
coreapi.Field(
"station_group",
required=False,
location='query',
Mattia Mancini
committed
schema=coreschema.Enum(['C', 'R', 'I', 'A'],
description='Station group to select for choices are [C|R|I|ALL]')
),
coreapi.Field(
"errors_only",
required=False,
location='query',
schema=coreschema.Boolean(
description='displays or not only the station with more than one error')
),
coreapi.Field(
Mattia Mancini
committed
"lookback_time",
required=False,
Mattia Mancini
committed
schema=coreschema.Integer(description='number of days from now (default 7)',
minimum=1)
def compute_response(self):
self.lookback_time = datetime.timedelta(days=self.lookback_time)
station_test_list = StationTest.objects \
.filter(start_datetime__gte=datetime.date.today() - self.lookback_time) \
Mattia Mancini
committed
.order_by('-start_datetime', 'station__name')
for group in self.station_group:
if group is not 'A':
station_test_list = station_test_list.filter(station__type=group)
# Since django preferes a ordered dict over a dict we make it happy... for now
response_payload = list()
Mattia Mancini
committed
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
for station_test in station_test_list:
station_test_payload = OrderedDict()
component_errors = station_test.component_errors
station_test_payload['station_name'] = station_test.station.name
station_test_payload[
'total_component_errors'] = station_test.component_errors.count()
station_test_payload['date'] = station_test.start_datetime.strftime('%Y-%m-%d')
station_test_payload['start_datetime'] = station_test.start_datetime
station_test_payload['end_datetime'] = station_test.end_datetime
station_test_payload['checks'] = station_test.checks
component_errors_summary = component_errors. \
values('component__type', 'type').annotate(
total=Count('type')).order_by('-total')
component_errors_summary_dict = OrderedDict()
for item in component_errors_summary:
item_component_type = item['component__type']
item_error_type = item['type']
item_error_total = item['total']
if item_component_type not in component_errors_summary_dict:
component_errors_summary_dict[item_component_type] = OrderedDict()
component_errors_summary_dict[item_component_type][item_error_type] = \
item_error_total
station_test_payload['component_error_summary'] = component_errors_summary_dict
response_payload.append(station_test_payload)
lambda station_test_entry:
station_test_entry['total_component_errors'] > 0,
return Response(status=status.HTTP_200_OK, data=response_payload)
class ControllerLatestObservations(ValidableReadOnlyView):
description = "Overview of the latest observations performed on the stations"
station_group = 'A'
errors_only = 'true'
coreapi.Field(
"station_group",
required=False,
location='query',
schema=coreschema.Enum(['C', 'R', 'I', 'A'], description=
'Station group to select for choices are [C|R|I|A]',
)
),
coreapi.Field(
"errors_only",
required=False,
location='query',
schema=coreschema.Boolean(
description='displays or not only the station with more than one error')
),
coreapi.Field(
"from_date",
required=True,
location='query',
schema=coreschema.String(
description='select rtsm from date (ex. YYYY-MM-DD)')
)
]
def compute_response(self):
self.from_date = parse_date(self.from_date)
filtered_entities = RTSMObservation.objects \
.filter(start_datetime__gte=self.from_date)
if self.station_group != 'A':
filtered_entities = filtered_entities \
.filter(station__type=self.station_group)
if self.errors_only:
filtered_entities = filtered_entities.exclude(errors_summary__isnull=True)
errors_summary = filtered_entities \
.values('observation_id',
'station__name',
'start_datetime',
'end_datetime',
'errors_summary__error_type',
'errors_summary__mode') \
.annotate(total=Count('errors_summary__error_type')) \
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
.order_by('observation_id', 'station__name')
response = dict()
for error_summary in errors_summary:
observation_id = error_summary['observation_id']
station_name = error_summary['station__name']
start_datetime = error_summary['start_datetime']
end_datetime = error_summary['end_datetime']
mode = error_summary['errors_summary__mode']
error_type = error_summary['errors_summary__error_type']
total = error_summary['total']
if observation_id not in response:
response[observation_id] = OrderedDict()
response[observation_id]['observation_id'] = observation_id
response[observation_id]['start_datetime'] = start_datetime
response[observation_id]['end_datetime'] = end_datetime
response[observation_id]['total_component_errors'] = 0
response[observation_id]['mode'] = list()
response[observation_id]['station_involved'] = dict()
if total == 0:
continue
response[observation_id]['total_component_errors'] += total
station_involved_summary = response[observation_id]['station_involved']
response[observation_id]['mode'] += [mode] \
if mode not in response[observation_id]['mode'] else []
if station_name not in station_involved_summary:
station_involved_summary[station_name] = OrderedDict()
station_involved_summary[station_name]['station_name'] = station_name
station_involved_summary[station_name]['n_errors'] = 0
station_involved_summary[station_name]['component_error_summary'] = OrderedDict()
station_involved_summary[station_name]['n_errors'] += total
station_involved_summary[station_name]['component_error_summary'][error_type] = total
response_payload = sorted(response.values(),
key=lambda item: item['start_datetime'],
Mattia Mancini
committed
return Response(status=status.HTTP_200_OK, data=response_payload)
class ControllerStationTestStatistics(ValidableReadOnlyView):
description = "Statistical summary of both or either the station test and RTSM"
station_group = 'A'
test_type = 'B'
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
coreapi.Field(
"test_type",
required=False,
location='query',
schema=coreschema.Enum(['R', 'S', 'B'],
description='select the type of test possible values are (R, RTSM),'
' (S, Station test), (B, both)[DEFAULT=B]',
)
),
coreapi.Field(
"station_group",
required=False,
location='query',
schema=coreschema.Enum(['C', 'R', 'I', 'A'], description=
'Station group to select for choices are [C|R|I|ALL]',
)
),
coreapi.Field(
"from_date",
required=True,
location='query',
schema=coreschema.String(
description='select tests from date (ex. YYYY-MM-DD)')
),
coreapi.Field(
"to_date",
required=True,
location='query',
schema=coreschema.String(
description='select tests to date (ex. YYYY-MM-DD)')
),
coreapi.Field(
"averaging_interval",
required=True,
location='query',
schema=coreschema.Integer(
description='averaging interval in days')
)
]
def compute_errors_per_station(self, from_date, to_date, central_time, station_group,
test_type):
component_errors = ComponentError.objects.all()
rtsm_summary_errors = RTSMErrorSummary.objects.all()
if station_group:
component_errors = component_errors.filter(station_test__station__type=station_group)
rtsm_summary_errors = rtsm_summary_errors.filter(
observation__station__type=station_group)
station_test_results = []
rtsm_results = []
if test_type in ['S', 'B']:
station_test_results = component_errors. \
filter(station_test__start_datetime__gt=from_date,
station_test__start_datetime__lt=to_date). \
values('station_test__station__name'). \
annotate(n_errors=Count('station_test__station__name'))
if test_type in ['R', 'B']:
rtsm_results = rtsm_summary_errors. \
filter(observation__start_datetime__gt=from_date,
observation__start_datetime__lt=to_date). \
values('observation__station__name'). \
annotate(n_errors=Count('observation__station__name'))
errors_per_station_in_bin = dict()
central_time_str = central_time.strftime('%Y-%m-%d')
if test_type in ['S', 'B']:
for result in station_test_results:
station_name = result['station_test__station__name']
errors_per_station_in_bin[station_name] = dict(station_name=station_name,
n_errors=result['n_errors'],
if test_type in ['R', 'B']:
for result in rtsm_results:
station_name = result['observation__station__name']
if station_name not in errors_per_station_in_bin:
errors_per_station_in_bin[station_name] = dict(station_name=station_name,
n_errors=result['n_errors'],
else:
errors_per_station_in_bin[station_name]['n_errors'] += result['n_errors']
return errors_per_station_in_bin.values()
def compute_errors_per_type(self, from_date, to_date, central_time, station_group, test_type):
component_errors = ComponentError.objects.all()
rtsm_summary_errors = RTSMErrorSummary.objects.all()
station_test_results = []
rtsm_results = []
central_time_str = central_time.strftime('%Y-%m-%d')
if station_group:
component_errors = component_errors.filter(station_test__station__type=station_group)
rtsm_summary_errors = rtsm_summary_errors.filter(
observation__station__type=station_group)
if test_type in ['S', 'B']:
station_test_results = component_errors. \
filter(station_test__start_datetime__gt=from_date,
station_test__start_datetime__lt=to_date). \
values('type'). \
annotate(n_errors=Count('type'))
if test_type in ['R', 'B']:
rtsm_results = rtsm_summary_errors. \
filter(observation__start_datetime__gt=from_date,
observation__start_datetime__lt=to_date). \
values('error_type'). \
annotate(n_errors=Count('error_type'))
errors_per_error_type_in_bin = dict()
if test_type in ['S', 'B']:
for result in station_test_results:
error_type = result['type']
errors_per_error_type_in_bin[error_type] = dict(error_type=error_type,
n_errors=result['n_errors'],
if test_type in ['R', 'B']:
for result in rtsm_results:
error_type = result['error_type']
if error_type not in errors_per_error_type_in_bin:
errors_per_error_type_in_bin[error_type] = dict(error_type=error_type,
n_errors=result['n_errors'],
else:
errors_per_error_type_in_bin[error_type]['n_errors'] += result['n_errors']
return errors_per_error_type_in_bin.values()
def compute_response(self):
from_date = parse_date(self.from_date)
to_date = parse_date(self.to_date)
averaging_interval = datetime.timedelta(days=self.averaging_interval)
response_payload = OrderedDict()
response_payload['start_date'] = from_date
response_payload['end_date'] = to_date
response_payload['averaging_interval'] = averaging_interval
errors_per_station = []
errors_per_type = []
n_bins = int(ceil((to_date - from_date) / averaging_interval))
for i in range(n_bins):
if self.station_group is 'A':
station_group = None
else:
station_group = self.station_group
errors_per_station += self.compute_errors_per_station(
from_date=from_date + i * averaging_interval,
to_date=from_date + (i + 1) * averaging_interval,
central_time=from_date + (i + .5) * averaging_interval,
station_group=station_group,
test_type=self.test_type)
errors_per_type += self.compute_errors_per_type(
from_date=from_date + i * averaging_interval,
to_date=from_date + (i + 1) * averaging_interval,
central_time=from_date + (i + .5) * averaging_interval,
station_group=station_group,
test_type=self.test_type)
response_payload['errors_per_station'] = errors_per_station
response_payload['errors_per_type'] = errors_per_type
return Response(status=status.HTTP_200_OK, data=response_payload)
class ControllerAllComponentErrorTypes(ValidableReadOnlyView):
description = "Lists all the presents component error types"
data = [item['type'] for item in ComponentError.objects.values('type').distinct()]
return Response(status=status.HTTP_200_OK, data=data)
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
class ControllerStationComponentErrors(ValidableReadOnlyView):
description = "Provides a summary per station of the component errors"
# required parameters
station_name = None
from_date = None
to_date = None
# optional parameters
test_type = 'B'
fields = [
coreapi.Field(
'station_name',
required=True,
location='query',
schema=coreschema.String(description='name of the station to select')
),
coreapi.Field(
'from_date',
required=True,
location='query',
schema=coreschema.String(description='select tests from date (ex. YYYY-MM-DD)')
),
coreapi.Field(
'to_date',
required=True,
location='query',
schema=coreschema.String(description='select tests from date (ex. YYYY-MM-DD)')
),
coreapi.Field(
'test_type',
required=False,
location='query',
schema=coreschema.Enum(
['R', 'S', 'B'],
description='select the type of test possible values are (R, RTSM),'
' (S, Station test), (B, both)[DEFAULT=B]',
)
)
]
def collect_station_test_errors(self):
station_entry = Station.objects.filter(name=self.station_name).first()
response_payload = OrderedDict()
station_tests = station_entry.stationtest_set \
.filter(start_datetime__gte=self.from_date,
end_datetime__lte=self.to_date)
failing_component_types = station_tests.distinct('component_errors__component__type').values_list('component_errors__component__type')
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
for failing_component_type in failing_component_types:
failing_component_type = failing_component_type[0]
component_type_errors_list = list()
response_payload[failing_component_type] = component_type_errors_list
for station_test in station_tests.order_by('-start_datetime'):
test_summary = OrderedDict()
test_summary['test_type'] = 'S'
test_summary['start_date'] = station_test.start_datetime
test_summary['end_date'] = station_test.end_datetime
component_errors_dict = OrderedDict()
test_summary['component_errors'] = component_errors_dict
component_errors = station_test.component_errors\
.filter(component__type=failing_component_type)
for component_error in component_errors:
component_id = component_error.component.component_id
error_type = component_error.type
details = component_error.details
if component_id not in component_errors:
component_errors_dict[str(component_id)] = list()
component_errors_dict[str(component_id)] += [dict(error_type=error_type,
details=details)]
component_type_errors_list.append(test_summary)
return response_payload
def collect_rtsm_errors(self):
station_entry = Station.objects.filter(name=self.station_name).first()
response_payload = OrderedDict()
rtsm_observations = station_entry.rtsmobservation_set \
.filter(start_datetime__gte=self.from_date,
end_datetime__lte=self.to_date)
failing_component_modes = rtsm_observations.exclude(errors_summary__isnull=True).distinct(
'errors_summary__mode').values_list('errors_summary__mode')
for observing_mode in failing_component_modes:
observing_mode = observing_mode[0]
rtsm_errors_per_component_type = list()
response_payload[MODE_TO_COMPONENT[observing_mode]] = rtsm_errors_per_component_type
for rtsm_observation in rtsm_observations.order_by('-start_datetime'):
rtsm_summary = OrderedDict()
rtsm_errors_per_component_type.append(rtsm_summary)
rtsm_summary['test_type'] = 'R'
rtsm_summary['start_date'] = rtsm_observation.start_datetime
rtsm_summary['end_date'] = rtsm_observation.end_datetime
component_errors_dict = OrderedDict()
rtsm_summary['component_errors'] = component_errors_dict
component_errors = rtsm_observation.errors_summary\
.filter(mode=observing_mode)\
.values('error_type', 'start_frequency',
'stop_frequency', 'percentage',
'error_type', 'count', 'rcu')
for component_error in component_errors:
component_id = component_error['rcu']
details = dict(percentage = component_error['percentage'],
start_frequency = component_error['start_frequency'],
stop_frequency = component_error['stop_frequency'],
count = component_error['count'])
error_type = component_error['error_type']
if component_id not in component_errors:
component_errors_dict[str(component_id)] = list()
component_errors_dict[str(component_id)] += [dict(error_type=error_type, details=details)]
return response_payload
def compute_response(self):
self.from_date = parse_date(self.from_date)
self.to_date = parse_date(self.to_date)
station_test_errors = {}
rtsm_errors = {}
if self.test_type in ['S', 'B']:
station_test_errors = self.collect_station_test_errors()
if self.test_type in ['R', 'B']:
rtsm_errors = self.collect_rtsm_errors()
payload = OrderedDict()
for component_type in set(rtsm_errors.keys() | station_test_errors.keys()):
station_test_errors_per_type = station_test_errors.get(component_type, [])
rtsm_errors_per_type = rtsm_errors.get(component_type, [])
payload[component_type] = sorted(station_test_errors_per_type + rtsm_errors_per_type, key=lambda item: item['start_date'])
return Response(status=status.HTTP_200_OK, data=payload)