Skip to content
Snippets Groups Projects
Commit 04c48e56 authored by Mattia Mancini's avatar Mattia Mancini
Browse files

OSB-28: small fixes

parent 36ccddd1
No related branches found
No related tags found
2 merge requests!89Monitoring maintenance Epic branch merge,!1Resolve OSB-13 "Monitoringmaintenance "
Hide whitespace changes
Inline Side-by-side
LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py
+ 1
− 1
View file @ 04c48e56
...@@ -179,7 +179,7 @@ def dict_from_component_error(content): ...@@ -179,7 +179,7 @@ def dict_from_component_error(content):
component_error = dict(component=component, component_error = dict(component=component,
details=error_details, details=error_details,
type=error_type) type=error_type.strip())
for element_error in element_errors: for element_error in element_errors:
element_error['component_error'] = dict(component_error) element_error['component_error'] = dict(component_error)
......
LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py
+ 49
− 42
View file @ 04c48e56
import datetime import datetime
from collections import OrderedDict from collections import OrderedDict
from math import ceil
import coreapi import coreapi
import coreschema import coreschema
from django.db.models import Count from django.db.models import Count
from math import ceil
from django.db.models import Window, F from django.db.models import Window, F
from rest_framework import status from rest_framework import status
from rest_framework.response import Response from rest_framework.response import Response
from rest_framework.schemas import ManualSchema from rest_framework.schemas import ManualSchema
from rest_framework.views import APIView from rest_framework.views import APIView
from lofar.maintenance.monitoringdb.models.component_error import ComponentError
from lofar.maintenance.monitoringdb.models.rtsm import RTSMErrorSummary
from lofar.maintenance.monitoringdb.models.rtsm import RTSMObservation from lofar.maintenance.monitoringdb.models.rtsm import RTSMObservation
from lofar.maintenance.monitoringdb.models.station import Station from lofar.maintenance.monitoringdb.models.station import Station
from lofar.maintenance.monitoringdb.models.station_test import StationTest from lofar.maintenance.monitoringdb.models.station_test import StationTest
from lofar.maintenance.monitoringdb.models.component_error import ComponentError
from lofar.maintenance.monitoringdb.models.rtsm import RTSMErrorSummary
class ControllerStationOverview(APIView): class ControllerStationOverview(APIView):
...@@ -123,14 +121,14 @@ class ControllerStationOverview(APIView): ...@@ -123,14 +121,14 @@ class ControllerStationOverview(APIView):
rtsm_payload['start_datetime'] = rtsm.start_datetime rtsm_payload['start_datetime'] = rtsm.start_datetime
rtsm_payload['end_datetime'] = rtsm.end_datetime rtsm_payload['end_datetime'] = rtsm.end_datetime
unique_modes = [item['mode'] for item in rtsm.errors.values('mode').distinct()] unique_modes = [item['mode'] for item in rtsm.errors_summary.values('mode').distinct()]
rtsm_payload['mode'] = unique_modes rtsm_payload['mode'] = unique_modes
rtsm_payload['total_component_errors'] = rtsm.errors_summary.count() rtsm_payload['total_component_errors'] = rtsm.errors_summary.count()
errors_summary = OrderedDict() errors_summary = OrderedDict()
errors_summary_query = rtsm.errors_summary.annotate(total=Count('error_type')).values( errors_summary_query = rtsm.errors_summary.values('error_type').annotate(
'error_type', 'total').distinct() total=Count('error_type'))
for error_summary in errors_summary_query: for error_summary in errors_summary_query:
errors_summary[error_summary['error_type']] = error_summary['total'] errors_summary[error_summary['error_type']] = error_summary['total']
...@@ -197,7 +195,7 @@ class ControllerStationTestsSummary(APIView): ...@@ -197,7 +195,7 @@ class ControllerStationTestsSummary(APIView):
self.station_group = request.query_params.get('station_group', self.DEFAULT_STATION_GROUP) self.station_group = request.query_params.get('station_group', self.DEFAULT_STATION_GROUP)
self.lookback_time = datetime.timedelta(int(request.query_params.get('lookback_time', self.lookback_time = datetime.timedelta(int(request.query_params.get('lookback_time',
self.DEFAULT_LOOKBACK_TIME_IN_DAYS))) self.DEFAULT_LOOKBACK_TIME_IN_DAYS)))
def get(self, request, format=None): def get(self, request, format=None):
try: try:
...@@ -209,12 +207,12 @@ class ControllerStationTestsSummary(APIView): ...@@ -209,12 +207,12 @@ class ControllerStationTestsSummary(APIView):
return Response(status=status.HTTP_406_NOT_ACCEPTABLE, return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
data='Please specify all the required parameters: %s' % (e,)) data='Please specify all the required parameters: %s' % (e,))
station_test_list = StationTest.objects\ station_test_list = StationTest.objects \
.filter(start_datetime__gte=datetime.date.today()-self.lookback_time)\ .filter(start_datetime__gte=datetime.date.today() - self.lookback_time) \
.order_by('-start_datetime', 'station__name') .order_by('-start_datetime', 'station__name')
for group in self.station_group: for group in self.station_group:
if group is not 'A': if group is not 'A':
station_entities = station_test_list.filter(type=group) station_test_list = station_test_list.filter(station__type=group)
# Since django preferes a ordered dict over a dict we make it happy... for now # Since django preferes a ordered dict over a dict we make it happy... for now
response_payload = list() response_payload = list()
...@@ -331,7 +329,7 @@ class ControllerLatestObservations(APIView): ...@@ -331,7 +329,7 @@ class ControllerLatestObservations(APIView):
return Response(status=status.HTTP_406_NOT_ACCEPTABLE, return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
data='Please specify both the start and the end date: %s' % (e,)) data='Please specify both the start and the end date: %s' % (e,))
rtsm_observation_entities = RTSMObservation.objects.filter( rtsm_observation_entities = RTSMObservation.objects.defer('errors').filter(
start_datetime__gte=self.from_date) start_datetime__gte=self.from_date)
for group in self.station_group: for group in self.station_group:
if group is not 'A': if group is not 'A':
...@@ -342,26 +340,25 @@ class ControllerLatestObservations(APIView): ...@@ -342,26 +340,25 @@ class ControllerLatestObservations(APIView):
for rtsm_observation_entity in rtsm_observation_entities.values('observation_id', for rtsm_observation_entity in rtsm_observation_entities.values('observation_id',
'start_datetime', 'start_datetime',
'end_datetime'). \ 'end_datetime'). \
distinct().order_by('-observation_id'): distinct().order_by('-start_datetime'):
observation_payload = OrderedDict() observation_payload = OrderedDict()
observation_payload['observation_id'] = rtsm_observation_entity['observation_id'] observation_payload['observation_id'] = rtsm_observation_entity['observation_id']
observation_payload['start_datetime'] = rtsm_observation_entity['start_datetime'] observation_payload['start_datetime'] = rtsm_observation_entity['start_datetime']
observation_payload['end_datetime'] = rtsm_observation_entity['end_datetime'] observation_payload['end_datetime'] = rtsm_observation_entity['end_datetime']
rtsm_list = RTSMObservation.objects.filter( rtsm_list = RTSMObservation.objects.defer('errors').filter(
start_datetime__gte=self.from_date, start_datetime__gte=self.from_date,
observation_id=rtsm_observation_entity['observation_id']) observation_id=rtsm_observation_entity['observation_id'])
unique_modes = [item['errors__mode'] for item in rtsm_list.values('errors__mode').distinct()] unique_modes = [item['errors_summary__mode'] for item in
rtsm_list.values('errors_summary__mode').distinct()]
observation_payload['mode'] = unique_modes observation_payload['mode'] = unique_modes
observation_payload['total_component_errors'] = 0 observation_payload['total_component_errors'] = 0
station_involved_list = list() station_involved_list = list()
for rtsm_entry_per_station in rtsm_list.order_by('station__name'): for rtsm_entry_per_station in rtsm_list.order_by('station__name'):
station_summary = OrderedDict() station_summary = OrderedDict()
station_summary['station_name'] = rtsm_entry_per_station.station.name station_summary['station_name'] = rtsm_entry_per_station.station.name
...@@ -378,7 +375,7 @@ class ControllerLatestObservations(APIView): ...@@ -378,7 +375,7 @@ class ControllerLatestObservations(APIView):
response_payload.append(observation_payload) response_payload.append(observation_payload)
if self.errors_only and self.errors_only != 'false': if self.errors_only and self.errors_only != 'false':
print(self.errors_only)
response_payload = filter( response_payload = filter(
lambda station_entry: lambda station_entry:
len(station_entry['total_component_errors']) > 0, len(station_entry['total_component_errors']) > 0,
...@@ -488,27 +485,32 @@ result: ...@@ -488,27 +485,32 @@ result:
if self.test_type not in ['R', 'S', 'B']: if self.test_type not in ['R', 'S', 'B']:
raise ValueError('test_type is not one of [R,S,B]') raise ValueError('test_type is not one of [R,S,B]')
self.averaging_interval = datetime.timedelta(int(request.query_params.get('averaging_interval'))) self.averaging_interval = datetime.timedelta(
int(request.query_params.get('averaging_interval')))
def compute_errors_per_station(self, from_date, to_date, central_time, station_group, test_type): def compute_errors_per_station(self, from_date, to_date, central_time, station_group,
test_type):
component_errors = ComponentError.objects.all() component_errors = ComponentError.objects.all()
rtsm_summary_errors = RTSMErrorSummary.objects.all() rtsm_summary_errors = RTSMErrorSummary.objects.all()
if station_group: if station_group:
component_errors = component_errors.filter(station_test__station__type=station_group) component_errors = component_errors.filter(station_test__station__type=station_group)
rtsm_summary_errors = rtsm_summary_errors.filter(observation__station__type=station_group) rtsm_summary_errors = rtsm_summary_errors.filter(
observation__station__type=station_group)
station_test_results = [] station_test_results = []
rtsm_results = [] rtsm_results = []
if test_type in ['S', 'B']: if test_type in ['S', 'B']:
station_test_results = component_errors. \ station_test_results = component_errors. \
filter(station_test__start_datetime__gt=from_date, station_test__start_datetime__lt=to_date). \ filter(station_test__start_datetime__gt=from_date,
station_test__start_datetime__lt=to_date). \
values('station_test__station__name'). \ values('station_test__station__name'). \
annotate(n_errors=Count('station_test__station__name')) annotate(n_errors=Count('station_test__station__name'))
if test_type in ['R', 'B']: if test_type in ['R', 'B']:
rtsm_results = rtsm_summary_errors. \ rtsm_results = rtsm_summary_errors. \
filter(observation__start_datetime__gt=from_date, observation__start_datetime__lt=to_date). \ filter(observation__start_datetime__gt=from_date,
observation__start_datetime__lt=to_date). \
values('observation__station__name'). \ values('observation__station__name'). \
annotate(n_errors=Count('observation__station__name')) annotate(n_errors=Count('observation__station__name'))
...@@ -541,18 +543,22 @@ result: ...@@ -541,18 +543,22 @@ result:
station_test_results = [] station_test_results = []
rtsm_results = [] rtsm_results = []
central_time_str = central_time.strftime('%Y-%m-%d')
if station_group: if station_group:
component_errors = component_errors.filter(station_test__station__type=station_group) component_errors = component_errors.filter(station_test__station__type=station_group)
rtsm_summary_errors = rtsm_summary_errors.filter(observation__station__type=station_group) rtsm_summary_errors = rtsm_summary_errors.filter(
observation__station__type=station_group)
if test_type in ['S', 'B']: if test_type in ['S', 'B']:
station_test_results = component_errors. \ station_test_results = component_errors. \
filter(station_test__start_datetime__gt=from_date, station_test__start_datetime__lt=to_date). \ filter(station_test__start_datetime__gt=from_date,
station_test__start_datetime__lt=to_date). \
values('type'). \ values('type'). \
annotate(n_errors=Count('type')) annotate(n_errors=Count('type'))
if test_type in ['R', 'B']: if test_type in ['R', 'B']:
rtsm_results = rtsm_summary_errors. \ rtsm_results = rtsm_summary_errors. \
filter(observation__start_datetime__gt=from_date, observation__start_datetime__lt=to_date). \ filter(observation__start_datetime__gt=from_date,
observation__start_datetime__lt=to_date). \
values('error_type'). \ values('error_type'). \
annotate(n_errors=Count('error_type')) annotate(n_errors=Count('error_type'))
...@@ -563,14 +569,14 @@ result: ...@@ -563,14 +569,14 @@ result:
error_type = result['type'] error_type = result['type']
errors_per_error_type_in_bin[error_type] = dict(error_type=error_type, errors_per_error_type_in_bin[error_type] = dict(error_type=error_type,
n_errors=result['n_errors'], n_errors=result['n_errors'],
time=central_time) time=central_time_str)
if test_type in ['R', 'B']: if test_type in ['R', 'B']:
for result in rtsm_results: for result in rtsm_results:
error_type = result['error_type'] error_type = result['error_type']
if error_type not in errors_per_error_type_in_bin: if error_type not in errors_per_error_type_in_bin:
errors_per_error_type_in_bin[error_type] = dict(error_type=error_type, errors_per_error_type_in_bin[error_type] = dict(error_type=error_type,
n_errors=result['n_errors'], n_errors=result['n_errors'],
time=central_time) time=central_time_str)
else: else:
errors_per_error_type_in_bin[error_type]['n_errors'] += result['n_errors'] errors_per_error_type_in_bin[error_type]['n_errors'] += result['n_errors']
...@@ -601,18 +607,19 @@ result: ...@@ -601,18 +607,19 @@ result:
station_group = None station_group = None
else: else:
station_group = self.station_group station_group = self.station_group
errors_per_station.append( errors_per_station += self.compute_errors_per_station(
self.compute_errors_per_station(from_date=self.from_date + i * self.averaging_interval, from_date=self.from_date + i * self.averaging_interval,
to_date=self.from_date + (i + 1) * self.averaging_interval, to_date=self.from_date + (i + 1) * self.averaging_interval,
central_time=self.from_date + (i + .5) * self.averaging_interval, central_time=self.from_date + (i + .5) * self.averaging_interval,
station_group=station_group, station_group=station_group,
test_type=self.test_type)) test_type=self.test_type)
errors_per_type.append(
self.compute_errors_per_type(from_date=self.from_date + i * self.averaging_interval, errors_per_type += self.compute_errors_per_type(
to_date=self.from_date + (i + 1) * self.averaging_interval, from_date=self.from_date + i * self.averaging_interval,
central_time=self.from_date + (i + .5) * self.averaging_interval, to_date=self.from_date + (i + 1) * self.averaging_interval,
station_group=station_group, central_time=self.from_date + (i + .5) * self.averaging_interval,
test_type=self.test_type)) station_group=station_group,
test_type=self.test_type)
response_payload['errors_per_station'] = errors_per_station response_payload['errors_per_station'] = errors_per_station
response_payload['errors_per_type'] = errors_per_type response_payload['errors_per_type'] = errors_per_type
...@@ -627,4 +634,4 @@ class ControllerAllComponentErrorTypes(APIView): ...@@ -627,4 +634,4 @@ class ControllerAllComponentErrorTypes(APIView):
def get(self, request, format=None): def get(self, request, format=None):
data = [item['type'] for item in ComponentError.objects.values('type').distinct()] data = [item['type'] for item in ComponentError.objects.values('type').distinct()]
return Response(status=status.HTTP_200_OK, data=data) return Response(status=status.HTTP_200_OK, data=data)
\ No newline at end of file
LCU/Maintenance/MDB_tools/cli/station_tests_watchdog.py
+ 1
− 1
View file @ 04c48e56
...@@ -172,7 +172,7 @@ def handle_event_file_created(mdb_address, event): ...@@ -172,7 +172,7 @@ def handle_event_file_created(mdb_address, event):
def main(): def main():
setup_logging_framework() setup_logging_framework()
station_tests_watchdog(sys.argv) station_tests_watchdog(sys.argv[1:])
if __name__ == '__main__': if __name__ == '__main__':
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment