From 04c48e56979092ce0b6c2f5655456319b7b19fab Mon Sep 17 00:00:00 2001
From: Mattia Mancini <mancini@astron.nl>
Date: Mon, 22 Oct 2018 12:56:46 +0000
Subject: [PATCH] OSB-28: small fixes
---
.../monitoringdb/station_test_raw_parser.py | 2 +-
.../monitoringdb/views/controllers.py | 91 ++++++++++---------
.../MDB_tools/cli/station_tests_watchdog.py | 2 +-
3 files changed, 51 insertions(+), 44 deletions(-)
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py b/LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py
index 3f4ff081edd..f47b1ae7c47 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py
@@ -179,7 +179,7 @@ def dict_from_component_error(content):
component_error = dict(component=component,
details=error_details,
- type=error_type)
+ type=error_type.strip())
for element_error in element_errors:
element_error['component_error'] = dict(component_error)
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py b/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py
index 08008cc6689..21bc826a45d 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py
@@ -1,23 +1,21 @@
import datetime
from collections import OrderedDict
+from math import ceil
import coreapi
import coreschema
from django.db.models import Count
-
-from math import ceil
from django.db.models import Window, F
-
from rest_framework import status
from rest_framework.response import Response
from rest_framework.schemas import ManualSchema
from rest_framework.views import APIView
+from lofar.maintenance.monitoringdb.models.component_error import ComponentError
+from lofar.maintenance.monitoringdb.models.rtsm import RTSMErrorSummary
from lofar.maintenance.monitoringdb.models.rtsm import RTSMObservation
from lofar.maintenance.monitoringdb.models.station import Station
from lofar.maintenance.monitoringdb.models.station_test import StationTest
-from lofar.maintenance.monitoringdb.models.component_error import ComponentError
-from lofar.maintenance.monitoringdb.models.rtsm import RTSMErrorSummary
class ControllerStationOverview(APIView):
@@ -123,14 +121,14 @@ class ControllerStationOverview(APIView):
rtsm_payload['start_datetime'] = rtsm.start_datetime
rtsm_payload['end_datetime'] = rtsm.end_datetime
- unique_modes = [item['mode'] for item in rtsm.errors.values('mode').distinct()]
+ unique_modes = [item['mode'] for item in rtsm.errors_summary.values('mode').distinct()]
rtsm_payload['mode'] = unique_modes
rtsm_payload['total_component_errors'] = rtsm.errors_summary.count()
errors_summary = OrderedDict()
- errors_summary_query = rtsm.errors_summary.annotate(total=Count('error_type')).values(
- 'error_type', 'total').distinct()
+ errors_summary_query = rtsm.errors_summary.values('error_type').annotate(
+ total=Count('error_type'))
for error_summary in errors_summary_query:
errors_summary[error_summary['error_type']] = error_summary['total']
@@ -197,7 +195,7 @@ class ControllerStationTestsSummary(APIView):
self.station_group = request.query_params.get('station_group', self.DEFAULT_STATION_GROUP)
self.lookback_time = datetime.timedelta(int(request.query_params.get('lookback_time',
- self.DEFAULT_LOOKBACK_TIME_IN_DAYS)))
+ self.DEFAULT_LOOKBACK_TIME_IN_DAYS)))
def get(self, request, format=None):
try:
@@ -209,12 +207,12 @@ class ControllerStationTestsSummary(APIView):
return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
data='Please specify all the required parameters: %s' % (e,))
- station_test_list = StationTest.objects\
- .filter(start_datetime__gte=datetime.date.today()-self.lookback_time)\
+ station_test_list = StationTest.objects \
+ .filter(start_datetime__gte=datetime.date.today() - self.lookback_time) \
.order_by('-start_datetime', 'station__name')
for group in self.station_group:
if group is not 'A':
- station_entities = station_test_list.filter(type=group)
+ station_test_list = station_test_list.filter(station__type=group)
# Since django preferes a ordered dict over a dict we make it happy... for now
response_payload = list()
@@ -331,7 +329,7 @@ class ControllerLatestObservations(APIView):
return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
data='Please specify both the start and the end date: %s' % (e,))
- rtsm_observation_entities = RTSMObservation.objects.filter(
+ rtsm_observation_entities = RTSMObservation.objects.defer('errors').filter(
start_datetime__gte=self.from_date)
for group in self.station_group:
if group is not 'A':
@@ -342,26 +340,25 @@ class ControllerLatestObservations(APIView):
for rtsm_observation_entity in rtsm_observation_entities.values('observation_id',
'start_datetime',
'end_datetime'). \
- distinct().order_by('-observation_id'):
+ distinct().order_by('-start_datetime'):
observation_payload = OrderedDict()
observation_payload['observation_id'] = rtsm_observation_entity['observation_id']
observation_payload['start_datetime'] = rtsm_observation_entity['start_datetime']
observation_payload['end_datetime'] = rtsm_observation_entity['end_datetime']
- rtsm_list = RTSMObservation.objects.filter(
+ rtsm_list = RTSMObservation.objects.defer('errors').filter(
start_datetime__gte=self.from_date,
observation_id=rtsm_observation_entity['observation_id'])
- unique_modes = [item['errors__mode'] for item in rtsm_list.values('errors__mode').distinct()]
+ unique_modes = [item['errors_summary__mode'] for item in
+ rtsm_list.values('errors_summary__mode').distinct()]
observation_payload['mode'] = unique_modes
observation_payload['total_component_errors'] = 0
-
station_involved_list = list()
for rtsm_entry_per_station in rtsm_list.order_by('station__name'):
-
station_summary = OrderedDict()
station_summary['station_name'] = rtsm_entry_per_station.station.name
@@ -378,7 +375,7 @@ class ControllerLatestObservations(APIView):
response_payload.append(observation_payload)
if self.errors_only and self.errors_only != 'false':
- print(self.errors_only)
+
response_payload = filter(
lambda station_entry:
len(station_entry['total_component_errors']) > 0,
@@ -488,27 +485,32 @@ result:
if self.test_type not in ['R', 'S', 'B']:
raise ValueError('test_type is not one of [R,S,B]')
- self.averaging_interval = datetime.timedelta(int(request.query_params.get('averaging_interval')))
+ self.averaging_interval = datetime.timedelta(
+ int(request.query_params.get('averaging_interval')))
- def compute_errors_per_station(self, from_date, to_date, central_time, station_group, test_type):
+ def compute_errors_per_station(self, from_date, to_date, central_time, station_group,
+ test_type):
component_errors = ComponentError.objects.all()
rtsm_summary_errors = RTSMErrorSummary.objects.all()
if station_group:
component_errors = component_errors.filter(station_test__station__type=station_group)
- rtsm_summary_errors = rtsm_summary_errors.filter(observation__station__type=station_group)
+ rtsm_summary_errors = rtsm_summary_errors.filter(
+ observation__station__type=station_group)
station_test_results = []
rtsm_results = []
if test_type in ['S', 'B']:
station_test_results = component_errors. \
- filter(station_test__start_datetime__gt=from_date, station_test__start_datetime__lt=to_date). \
+ filter(station_test__start_datetime__gt=from_date,
+ station_test__start_datetime__lt=to_date). \
values('station_test__station__name'). \
annotate(n_errors=Count('station_test__station__name'))
if test_type in ['R', 'B']:
rtsm_results = rtsm_summary_errors. \
- filter(observation__start_datetime__gt=from_date, observation__start_datetime__lt=to_date). \
+ filter(observation__start_datetime__gt=from_date,
+ observation__start_datetime__lt=to_date). \
values('observation__station__name'). \
annotate(n_errors=Count('observation__station__name'))
@@ -541,18 +543,22 @@ result:
station_test_results = []
rtsm_results = []
+ central_time_str = central_time.strftime('%Y-%m-%d')
if station_group:
component_errors = component_errors.filter(station_test__station__type=station_group)
- rtsm_summary_errors = rtsm_summary_errors.filter(observation__station__type=station_group)
+ rtsm_summary_errors = rtsm_summary_errors.filter(
+ observation__station__type=station_group)
if test_type in ['S', 'B']:
station_test_results = component_errors. \
- filter(station_test__start_datetime__gt=from_date, station_test__start_datetime__lt=to_date). \
+ filter(station_test__start_datetime__gt=from_date,
+ station_test__start_datetime__lt=to_date). \
values('type'). \
annotate(n_errors=Count('type'))
if test_type in ['R', 'B']:
rtsm_results = rtsm_summary_errors. \
- filter(observation__start_datetime__gt=from_date, observation__start_datetime__lt=to_date). \
+ filter(observation__start_datetime__gt=from_date,
+ observation__start_datetime__lt=to_date). \
values('error_type'). \
annotate(n_errors=Count('error_type'))
@@ -563,14 +569,14 @@ result:
error_type = result['type']
errors_per_error_type_in_bin[error_type] = dict(error_type=error_type,
n_errors=result['n_errors'],
- time=central_time)
+ time=central_time_str)
if test_type in ['R', 'B']:
for result in rtsm_results:
error_type = result['error_type']
if error_type not in errors_per_error_type_in_bin:
errors_per_error_type_in_bin[error_type] = dict(error_type=error_type,
n_errors=result['n_errors'],
- time=central_time)
+ time=central_time_str)
else:
errors_per_error_type_in_bin[error_type]['n_errors'] += result['n_errors']
@@ -601,18 +607,19 @@ result:
station_group = None
else:
station_group = self.station_group
- errors_per_station.append(
- self.compute_errors_per_station(from_date=self.from_date + i * self.averaging_interval,
- to_date=self.from_date + (i + 1) * self.averaging_interval,
- central_time=self.from_date + (i + .5) * self.averaging_interval,
- station_group=station_group,
- test_type=self.test_type))
- errors_per_type.append(
- self.compute_errors_per_type(from_date=self.from_date + i * self.averaging_interval,
- to_date=self.from_date + (i + 1) * self.averaging_interval,
- central_time=self.from_date + (i + .5) * self.averaging_interval,
- station_group=station_group,
- test_type=self.test_type))
+ errors_per_station += self.compute_errors_per_station(
+ from_date=self.from_date + i * self.averaging_interval,
+ to_date=self.from_date + (i + 1) * self.averaging_interval,
+ central_time=self.from_date + (i + .5) * self.averaging_interval,
+ station_group=station_group,
+ test_type=self.test_type)
+
+ errors_per_type += self.compute_errors_per_type(
+ from_date=self.from_date + i * self.averaging_interval,
+ to_date=self.from_date + (i + 1) * self.averaging_interval,
+ central_time=self.from_date + (i + .5) * self.averaging_interval,
+ station_group=station_group,
+ test_type=self.test_type)
response_payload['errors_per_station'] = errors_per_station
response_payload['errors_per_type'] = errors_per_type
@@ -627,4 +634,4 @@ class ControllerAllComponentErrorTypes(APIView):
def get(self, request, format=None):
data = [item['type'] for item in ComponentError.objects.values('type').distinct()]
- return Response(status=status.HTTP_200_OK, data=data)
\ No newline at end of file
+ return Response(status=status.HTTP_200_OK, data=data)
diff --git a/LCU/Maintenance/MDB_tools/cli/station_tests_watchdog.py b/LCU/Maintenance/MDB_tools/cli/station_tests_watchdog.py
index 4b4ae3fff66..d4e7d675776 100644
--- a/LCU/Maintenance/MDB_tools/cli/station_tests_watchdog.py
+++ b/LCU/Maintenance/MDB_tools/cli/station_tests_watchdog.py
@@ -172,7 +172,7 @@ def handle_event_file_created(mdb_address, event):
def main():
setup_logging_framework()
- station_tests_watchdog(sys.argv)
+ station_tests_watchdog(sys.argv[1:])
if __name__ == '__main__':
--
GitLab