Newer
Older
from collections import OrderedDict
import coreapi
import coreschema
from django.db.models import Count, Sum
from django.db.models import Window, F
from rest_framework import status
from rest_framework.response import Response
from rest_framework.schemas import ManualSchema
from rest_framework.views import APIView
from lofar.maintenance.monitoringdb.models.rtsm import RTSMObservation
from lofar.maintenance.monitoringdb.models.station import Station
from lofar.maintenance.monitoringdb.models.station_test import StationTest
class ControllerStationOverview(APIView):
"""
Overview of the latest tests performed on the stations
"""
DEFAULT_STATION_GROUP = 'A'
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
DEFAULT_N_STATION_TESTS = 4
DEFAULT_N_RTSM = 4
queryset = StationTest.objects.all()
schema = ManualSchema(fields=[
coreapi.Field(
"station_group",
required=False,
location='query',
schema=coreschema.Enum(['C', 'R', 'I', 'A'], description=
'Station group to select for choices are [C|R|I|ALL]',
)
),
coreapi.Field(
"n_station_tests",
required=False,
location='query',
schema=coreschema.Integer(description='number of station tests to select',
minimum=1)
),
coreapi.Field(
"n_rtsm",
required=False,
location='query',
schema=coreschema.Integer(description='number of station tests to select',
minimum=1)
),
coreapi.Field(
"errors_only",
required=False,
location='query',
schema=coreschema.Boolean(
description='displays or not only the station with more than one error')
)
]
)
def get(self, request, format=None):
errors_only = request.query_params.get('errors_only', self.DEFAULT_ONLY_ERRORS)
station_group = request.query_params.get('station_group', self.DEFAULT_STATION_GROUP)
n_station_tests = int(
request.query_params.get('n_station_tests', self.DEFAULT_N_STATION_TESTS))
n_rtsm = int(request.query_params.get('n_rtsm', self.DEFAULT_N_RTSM))
station_entities = Station.objects.all()
for group in station_group:
if group is not 'A':
station_entities = station_entities.filter(type=group)
# Since django preferes a ordered dict over a dict we make it happy... for now
response_payload = list()
for station_entity in station_entities:
station_payload = OrderedDict()
station_payload['station_name'] = station_entity.name
station_test_list = StationTest.objects.filter(
station__name=station_entity.name).order_by('-end_datetime')[:n_station_tests - 1]
rtsm_list = RTSMObservation.objects.filter(
station__name=station_entity.name).order_by('-end_datetime')[:n_rtsm - 1]
station_payload['station_tests'] = list()
for station_test in station_test_list:
station_test_payload = OrderedDict()
component_errors = station_test.component_errors
station_test_payload[
'total_component_errors'] = station_test.component_errors.count()
station_test_payload['start_datetime'] = station_test.start_datetime
station_test_payload['end_datetime'] = station_test.end_datetime
station_test_payload['checks'] = station_test.checks
component_errors_summary = component_errors. \
values('component__type', 'type').annotate(
total=Count('type')).order_by('-total')
component_errors_summary_dict = OrderedDict()
for item in component_errors_summary:
item_component_type = item['component__type']
item_error_type = item['type']
item_error_total = item['total']
if item_component_type not in component_errors_summary_dict:
component_errors_summary_dict[item_component_type] = OrderedDict()
component_errors_summary_dict[item_component_type][item_error_type] = \
item_error_total
station_test_payload['component_error_summary'] = component_errors_summary_dict
station_payload['station_tests'].append(station_test_payload)
station_payload['rtsm'] = list()
for rtsm in rtsm_list:
rtsm_payload = OrderedDict()
rtsm_payload['observation_id'] = rtsm.observation_id
rtsm_payload['start_datetime'] = rtsm.start_datetime
rtsm_payload['end_datetime'] = rtsm.end_datetime
unique_modes = [item['mode'] for item in rtsm.errors.values('mode').distinct()]
rtsm_payload['mode'] = unique_modes
rtsm_payload['total_component_errors'] = rtsm.errors_summary.count()
errors_summary = OrderedDict()
errors_summary_query = rtsm.errors_summary.annotate(total=Count('error_type')).values(
'error_type', 'total').distinct()
for error_summary in errors_summary_query:
errors_summary[error_summary['error_type']] = error_summary['total']
rtsm_payload['error_summary'] = errors_summary
station_payload['rtsm'].append(rtsm_payload)
response_payload.append(station_payload)
if errors_only and errors_only is not 'false':
response_payload = filter(
lambda station_entry:
len(station_entry['station_tests']) + len(station_entry['rtsm']) > 0,
response_payload)
response_payload = sorted(response_payload, key=lambda item: item['station_name'])
return Response(status=status.HTTP_200_OK, data=response_payload)
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
class ControllerStationTestsSummary(APIView):
"""
Overview of the latest station tests performed on the stations
"""
DEFAULT_STATION_GROUP = 'A'
DEFAULT_ONLY_ERRORS = True
queryset = StationTest.objects.all()
schema = ManualSchema(fields=[
coreapi.Field(
"station_group",
required=False,
location='query',
schema=coreschema.Enum(['C', 'R', 'I', 'A'], description=
'Station group to select for choices are [C|R|I|ALL]',
)
),
coreapi.Field(
"errors_only",
required=False,
location='query',
schema=coreschema.Boolean(
description='displays or not only the station with more than one error')
),
coreapi.Field(
"start_date",
required=True,
location='query',
schema=coreschema.String(
description='select station tests from date (ex. YYYY-MM-DD)')
),
coreapi.Field(
"end_date",
required=True,
location='query',
schema=coreschema.String(
description='select station tests to date (ex. YYYY-MM-DD)')
)
]
)
@staticmethod
def parse_date(date):
expected_format = '%Y-%m-%d'
try:
parsed_date = datetime.datetime.strptime(date, expected_format)
return parsed_date
except Exception as e:
raise ValueError('cannot parse %s with format %s - %s' % (date, expected_format, e))
def validate_query_parameters(self, request):
self.errors_only = request.query_params.get('errors_only', self.DEFAULT_ONLY_ERRORS)
self.station_group = request.query_params.get('station_group', self.DEFAULT_STATION_GROUP)
start_date = request.query_params.get('start_date')
self.start_date = ControllerStationTestsSummary.parse_date(start_date)
end_date = request.query_params.get('end_date')
self.end_date = ControllerStationTestsSummary.parse_date(end_date)
def get(self, request, format=None):
try:
self.validate_query_parameters(request)
except ValueError as e:
return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
data='Please specify the date in the format YYYY-MM-DD: %s' % (e,))
except KeyError as e:
return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
data='Please specify both the start and the end date: %s' % (e,))
station_entities = Station.objects.all()
for group in self.station_group:
if group is not 'A':
station_entities = station_entities.filter(type=group)
# Since django preferes a ordered dict over a dict we make it happy... for now
response_payload = list()
for station_entity in station_entities:
station_payload = OrderedDict()
station_payload['station_name'] = station_entity.name
station_test_list = StationTest.objects.filter(
start_datetime__gte=self.start_date,
end_datetime__lte=self.end_date,
station__name=station_entity.name).order_by('-end_datetime')
station_payload['station_tests'] = list()
for station_test in station_test_list:
station_test_payload = OrderedDict()
component_errors = station_test.component_errors
station_test_payload[
'total_component_errors'] = station_test.component_errors.count()
station_test_payload['start_datetime'] = station_test.start_datetime
station_test_payload['end_datetime'] = station_test.end_datetime
station_test_payload['checks'] = station_test.checks
component_errors_summary = component_errors. \
values('component__type', 'type').annotate(
total=Count('type')).order_by('-total')
component_errors_summary_dict = OrderedDict()
for item in component_errors_summary:
item_component_type = item['component__type']
item_error_type = item['type']
item_error_total = item['total']
if item_component_type not in component_errors_summary_dict:
component_errors_summary_dict[item_component_type] = OrderedDict()
component_errors_summary_dict[item_component_type][item_error_type] = \
item_error_total
station_test_payload['component_error_summary'] = component_errors_summary_dict
station_payload['station_tests'].append(station_test_payload)
response_payload.append(station_payload)
if self.errors_only and self.errors_only is not 'false':
response_payload = filter(
lambda station_entry:
len(station_entry['station_tests']) > 0,
response_payload)
response_payload = sorted(response_payload, key=lambda item: item['station_name'])
return Response(status=status.HTTP_200_OK, data=response_payload)
class ControllerLatestObservations(APIView):
"""
Overview of the latest observations performed on the stations
"""
DEFAULT_STATION_GROUP = 'A'
DEFAULT_ONLY_ERRORS = True
queryset = StationTest.objects.all()
schema = ManualSchema(fields=[
coreapi.Field(
"station_group",
required=False,
location='query',
schema=coreschema.Enum(['C', 'R', 'I', 'A'], description=
'Station group to select for choices are [C|R|I|ALL]',
)
),
coreapi.Field(
"errors_only",
required=False,
location='query',
schema=coreschema.Boolean(
description='displays or not only the station with more than one error')
),
coreapi.Field(
"from_date",
required=True,
location='query',
schema=coreschema.String(
description='select rtsm from date (ex. YYYY-MM-DD)')
)
]
)
@staticmethod
def parse_date(date):
expected_format = '%Y-%m-%d'
try:
parsed_date = datetime.datetime.strptime(date, expected_format)
return parsed_date
except Exception as e:
raise ValueError('cannot parse %s with format %s - %s' % (date, expected_format, e))
def compute_rtsm_observation_summary(self, rtsm_errors):
errors_summary = OrderedDict()
errors_summary_query = rtsm_errors.annotate(total=
Window(expression=Count('rcu'),
partition_by=[F(
'error_type')])).values(
'error_type', 'total').distinct()
for error_summary in errors_summary_query:
errors_summary[error_summary['error_type']] = error_summary['total']
return errors_summary
def validate_query_parameters(self, request):
self.errors_only = request.query_params.get('errors_only', self.DEFAULT_ONLY_ERRORS)
self.station_group = request.query_params.get('station_group', self.DEFAULT_STATION_GROUP)
start_date = request.query_params.get('from_date')
self.from_date = ControllerLatestObservations.parse_date(start_date)
def get(self, request, format=None):
try:
self.validate_query_parameters(request)
except ValueError as e:
return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
data='Please specify the date in the format YYYY-MM-DD: %s' % (e,))
except KeyError as e:
return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
data='Please specify both the start and the end date: %s' % (e,))
rtsm_observation_entities = RTSMObservation.objects.filter(
start_datetime__gte=self.from_date)
for group in self.station_group:
if group is not 'A':
rtsm_observation_entities = rtsm_observation_entities.filter(station__type=group)
# Since django preferes a ordered dict over a dict we make it happy... for now
response_payload = list()
for rtsm_observation_entity in rtsm_observation_entities.values('observation_id',
'start_datetime',
'end_datetime'). \
distinct():
observation_payload = OrderedDict()
observation_payload['observation_id'] = rtsm_observation_entity['observation_id']
observation_payload['start_datetime'] = rtsm_observation_entity['start_datetime']
observation_payload['end_datetime'] = rtsm_observation_entity['end_datetime']
rtsm_list = RTSMObservation.objects.filter(
start_datetime__gte=self.from_date,
observation_id=rtsm_observation_entity['observation_id']). \
order_by('-end_datetime')
unique_modes = [item['errors__mode'] for item in rtsm_list.values('errors__mode').distinct()]
observation_payload['mode'] = unique_modes
observation_payload['total_component_errors'] = 0
station_list = rtsm_list.values('station').distinct()
station_involved_list = list()
for station in station_list:
rtsm_entry_per_station = rtsm_list.filter(station__pk=station['station']).first()
station_summary = OrderedDict()
station_summary['station_name'] = rtsm_entry_per_station.station.name
rtsm_entry_per_station.errors.values('rcu').distinct().count()
station_summary['component_error_summary'] = self.compute_rtsm_observation_summary(
rtsm_entry_per_station.errors_summary)
station_involved_list.append(station_summary)
observation_payload['total_component_errors'] += station_summary['n_errors']
station_involved_list = sorted(station_involved_list,
key=lambda rtsm_per_station: rtsm_per_station['n_errors'],
reverse=True)
observation_payload['station_involved'] = station_involved_list
response_payload.append(observation_payload)
if self.errors_only and self.errors_only != 'false':
print(self.errors_only)
response_payload = filter(
lambda station_entry:
len(station_entry['total_component_errors']) > 0,
response_payload)
response_payload = sorted(response_payload, key=lambda item: item['total_component_errors'],
reverse=True)
return Response(status=status.HTTP_200_OK, data=response_payload)