From 1f440dcca66281356100337a8faf9ec0e1ddce3f Mon Sep 17 00:00:00 2001
From: Mattia Mancini <mancini@astron.nl>
Date: Mon, 15 Oct 2018 12:29:05 +0000
Subject: [PATCH] OSB-28: minor fixes
---
.../monitoringdb/models/component_error.py | 2 +-
.../monitoringdb/models/element_error.py | 2 +-
.../DBInterface/monitoringdb/models/rtsm.py | 13 +-
.../monitoringdb/rtsm_test_raw_parser.py | 15 ++-
.../monitoringdb/serializers/rtsm.py | 48 ++++----
.../monitoringdb/views/rtsm_views.py | 4 +-
.../monitoringdb/views/station_test_views.py | 112 +++++++++++++-----
7 files changed, 125 insertions(+), 71 deletions(-)
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/component_error.py b/LCU/Maintenance/DBInterface/monitoringdb/models/component_error.py
index 23d2bd3cf9c..de50e7bce8a 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/models/component_error.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/models/component_error.py
@@ -9,4 +9,4 @@ class ComponentError(models.Model):
component = models.ForeignKey(Component, on_delete=models.DO_NOTHING)
station_test = models.ForeignKey(StationTest, on_delete=models.DO_NOTHING,
related_name='component_errors')
- details = JSONField(default={})
\ No newline at end of file
+ details = JSONField(default=dict)
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/element_error.py b/LCU/Maintenance/DBInterface/monitoringdb/models/element_error.py
index fb2e6683363..a7a14dcf547 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/models/element_error.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/models/element_error.py
@@ -12,4 +12,4 @@ class ElementError(models.Model):
related_name='failing_elements')
type = models.CharField(max_length=50)
- details = JSONField(default={})
\ No newline at end of file
+ details = JSONField(default=dict)
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py b/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py
index 1a63be8f95f..19c2693950f 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py
@@ -3,7 +3,7 @@ In this file all the model regarding the real time station monitor output are co
"""
from django.db import models
from django.contrib.postgres.fields import ArrayField
-
+from .station import Station
from .fixed_types import STATION_TYPES
@@ -11,11 +11,10 @@ from .fixed_types import STATION_TYPES
class RTSMObservation(models.Model):
observation_id = models.PositiveIntegerField(default=0)
- station_name = models.CharField(max_length=6)
- station_type = models.CharField(max_length=1, choices=STATION_TYPES)
+ station = models.ForeignKey(Station, on_delete=models.SET_NULL, null=True)
- start_time = models.DateTimeField()
- end_time = models.DateTimeField()
+ start_datetime = models.DateTimeField()
+ end_datetime = models.DateTimeField()
samples = models.IntegerField(default=None, null=True)
@@ -24,7 +23,7 @@ class RTSMObservationSummary(RTSMObservation):
@property
def errors_found(self):
return RTSMError.objects.defer('spectra'). \
- filter(observation=self).values('error_type', 'rcu'). \
+ filter(observation=self).values('error_type', 'mode', 'rcu'). \
annotate(count=models.Count('error_type', unique=True, output_field=models.FloatField())). \
annotate(percentage=models.ExpressionWrapper(models.F('count') * 100. / models.F('observation__samples'),
output_field=models.FloatField()))
@@ -39,7 +38,7 @@ class RTSMError(models.Model):
rcu = models.SmallIntegerField(default=None, null=True)
mode = models.SmallIntegerField(default=None, null=True)
- error_type = models.CharField(max_length=10)
+ error_type = models.CharField(max_length=50)
start_frequency = models.FloatField(default=None, null=True)
stop_frequency = models.FloatField(default=None, null=True)
time = models.DateTimeField()
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/rtsm_test_raw_parser.py b/LCU/Maintenance/DBInterface/monitoringdb/rtsm_test_raw_parser.py
index 132b355b3a7..a6c7bf90793 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/rtsm_test_raw_parser.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/rtsm_test_raw_parser.py
@@ -1,6 +1,15 @@
from datetime import datetime
import pytz
+rtsm_error_types_to_error_types = dict(HN='HIGH_NOISE',
+ SN='SUMMATOR_NOISE',
+ CR='CABLE_REFLECTION',
+ LN='LOW_NOISE',
+ J='JITTER',
+ OSC='OSCILLATION',
+ FLAT='FLAT',
+ DOWN='DOWN',
+ SHORT='SHORT')
def split_raw_rtsm_test(content):
"""
@@ -44,15 +53,15 @@ def preparse_rtsm_test_file(content):
rcu, mode, observation_id, error_type, start_frequency, stop_frequency, time = value.split(',')
result.update(rcu=int(rcu),
mode=int(mode),
- error_type=error_type,
+ error_type=rtsm_error_types_to_error_types[error_type],
start_frequency=float(start_frequency),
stop_frequency=float(stop_frequency),
time=parse_data(time))
elif 'INFO' in key:
observation_id, start_time, stop_time, samples = value.split(',')
observation.update(observation_id=int(observation_id),
- start_time=parse_data(start_time),
- end_time=parse_data(stop_time),
+ start_datetime=parse_data(start_time),
+ end_datetime=parse_data(stop_time),
samples=int(samples))
elif 'BAD' in key:
values = [float(item) for item in value.lstrip('[').rstrip(']').strip().split(' ')]
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py
index e1617cf099f..035857bec45 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py
@@ -3,8 +3,8 @@ from rest_framework import serializers
import logging
from django.db.transaction import atomic
from ..exceptions import ItemAlreadyExists
-from django.db.models import ObjectDoesNotExist
from .log import ActionLogSerializer
+from .station import Station
logger = logging.getLogger('serializers')
@@ -56,34 +56,28 @@ class RTSMObservationSerializer(serializers.ModelSerializer):
@atomic
def create(self, validated_data):
rtsm_errors = validated_data.pop('errors')
- try:
-
-
- # derive the station type from the station name
- station_name = validated_data['station_name']
- station_type = station_name[0].capitalize()
- station_type = station_type if station_type in ['C', 'R'] else 'I'
- validated_data.update(station_type=station_type)
-
- # searches if the RTSM is already present
- observation_id = validated_data['observation_id']
- element = RTSMObservation.objects.filter(observation_id=
- observation_id,
- station_name=station_name).first()
-
- if element is None:
- logger.info('rtsm not found inserting %s', validated_data)
- else:
- logger.info('RTSM test for sas id %s started at station %s is present already. Skipping insertion',
- observation_id, station_name)
- raise ItemAlreadyExists(element)
- except ObjectDoesNotExist as e:
- logger.info('rtsm not found inserting %s', validated_data)
- RTSMObservation_instance = RTSMObservation.objects.create(**validated_data)
- RTSMObservation_instance.save()
- ActionLogSerializer.log_model_insert(RTSMObservation_instance)
+ # derive the station type from the station name
+ station_name = validated_data.pop('station_name')
+ station_type = station_name[0].capitalize()
+ station_type = station_type if station_type in ['C', 'R'] else 'I'
+
+ station_entry = Station.objects.filter(name=station_name).first()
+ if station_entry is None:
+ station_entry = Station(name=station_name, type=station_type).save()
+ # searches if the RTSM is already present
+ observation_id = validated_data['observation_id']
+ RTSMObservation_instance = RTSMObservation.objects.filter(observation_id=observation_id,
+ station=station_entry).first()
+ if RTSMObservation_instance is None:
+ logger.info('rtsm not found inserting %s', validated_data)
+ RTSMObservation_instance = RTSMObservation.objects.create(station=station_entry,
+ **validated_data)
+ RTSMObservation_instance.save()
+ ActionLogSerializer.log_model_insert(RTSMObservation_instance)
+ else:
+ raise ItemAlreadyExists(RTSMObservation_instance)
for rtsm_error in rtsm_errors:
rtsm_error.update(observation=RTSMObservation_instance)
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py b/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py
index ae05720ce6c..660b38d0f04 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py
@@ -65,11 +65,11 @@ def insert_raw_rtsm_test(request):
entry['station_name'])
try:
- rtsm_item = RTSMObservationSerializer().create(entry)
+ rtsm_item = RTSMObservationSerializer().create(dict(entry))
rtsm_id = rtsm_item.id
except ItemAlreadyExists as e:
return Response(status=status.HTTP_200_OK,
- data='RTSM test was already inserted with id {}\n'.format(e.instance_id))
+ data='RTSM test was already present with id {}\n'.format(e.instance_id))
logger.info('RTSM inserted successfully for obsid %d and station %s',
entry['observation_id'],
entry['station_name'])
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/views/station_test_views.py b/LCU/Maintenance/DBInterface/monitoringdb/views/station_test_views.py
index 511bbbd7e5d..95d17cc8cda 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/views/station_test_views.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/views/station_test_views.py
@@ -1,26 +1,29 @@
+from collections import OrderedDict
+
+import coreapi
+import coreschema
+from django.db.models import Count
from django_filters import rest_framework as filters
-from rest_framework.viewsets import ReadOnlyModelViewSet
-from rest_framework.views import APIView
-import coreapi, coreschema
from rest_framework.schemas import ManualSchema
-
+from rest_framework.views import APIView
+from rest_framework.viewsets import ReadOnlyModelViewSet
from .common import *
from ..models.component import Component
from ..models.component_error import ComponentError
from ..models.element import Element
from ..models.element_error import ElementError
+from ..models.rtsm import RTSMObservationSummary
from ..models.station import Station
from ..models.station_test import StationTest
from ..serializers.component import ComponentSerializer
-from collections import OrderedDict
from ..serializers.component_error import ComponentErrorSerializer
from ..serializers.element import ElementSerializer
from ..serializers.element_error import ElementErrorSerializer
from ..serializers.station import StationSerializer
from ..serializers.station_tests import StationTestSerializer
from ..station_test_raw_parser import parse_raw_station_test
-from django.db.models import Count
+
logger = logging.getLogger('views')
@@ -73,10 +76,12 @@ class ElementErrorFilterSet(filters.FilterSet):
lookup_expr='type__contains',
help_text='station type')
- from_date = filters.DateFilter(field_name='component_error__station_test', lookup_expr='start_datetime__gt',
+ from_date = filters.DateFilter(field_name='component_error__station_test',
+ lookup_expr='start_datetime__gt',
help_text='select element errors from date time')
- to_date = filters.DateFilter(field_name='component_error__station_test', lookup_expr='end_datetime__lt',
- help_text='select element errors until date time')
+ to_date = filters.DateFilter(field_name='component_error__station_test',
+ lookup_expr='end_datetime__lt',
+ help_text='select element errors until date time')
type = filters.CharFilter(field_name='type',
lookup_expr='contains', help_text='element error type')
@@ -96,7 +101,6 @@ class StationViewSet(ReadOnlyModelViewSet):
class ComponentViewSet(ReadOnlyModelViewSet):
-
serializer_class = ComponentSerializer
queryset = Component.objects.all()
filter_fields = '__all__'
@@ -164,7 +168,7 @@ def insert_raw_station_test(request):
return Response(exception=True,
data="the post message is not correct." +
" It has to be of the form \{'content':[RAWSTRING]\}: %s. Request provided %s" % (
- e, request),
+ e, request),
status=status.HTTP_400_BAD_REQUEST)
else:
return Response(exception=True,
@@ -181,7 +185,7 @@ class ControllerStationOverview(APIView):
Overview of the latest tests performed on the stations
"""
- DEFAULT_STATION_GROUP = 'ALL'
+ DEFAULT_STATION_GROUP = 'A'
DEFAULT_ONLY_ERRORS = False
DEFAULT_N_STATION_TESTS = 4
DEFAULT_N_RTSM = 4
@@ -193,7 +197,7 @@ class ControllerStationOverview(APIView):
required=False,
location='query',
schema=coreschema.Enum(['C', 'R', 'I', 'A'], description=
- 'Station group to select for choices are [C|R|I|ALL]',
+ 'Station group to select for choices are [C|R|I|ALL]',
)
),
coreapi.Field(
@@ -214,20 +218,18 @@ class ControllerStationOverview(APIView):
"errors_only",
required=False,
location='query',
- schema=coreschema.Boolean(description=
- 'displays or not only the station with more than one error')
+ schema=coreschema.Boolean(
+ description='displays or not only the station with more than one error')
)
]
)
-
def get(self, request, format=None):
-
errors_only = request.query_params.get('errors_only', self.DEFAULT_ONLY_ERRORS)
-
station_group = request.query_params.get('station_group', self.DEFAULT_STATION_GROUP)
- n_station_tests = int(request.query_params.get('n_station_tests', self.DEFAULT_N_STATION_TESTS))
- n_rtsm = request.query_params.get('n_rtsm', self.DEFAULT_N_RTSM)
+ n_station_tests = int(
+ request.query_params.get('n_station_tests', self.DEFAULT_N_STATION_TESTS))
+ n_rtsm = int(request.query_params.get('n_rtsm', self.DEFAULT_N_RTSM))
station_entities = Station.objects.all()
for group in station_group:
@@ -235,25 +237,75 @@ class ControllerStationOverview(APIView):
station_entities = station_entities.filter(type=group)
# Since django preferes a ordered dict over a dict we make it happy... for now
- response_payload = OrderedDict()
+ response_payload = list()
for station_entity in station_entities:
+ station_payload = OrderedDict()
+
+ station_payload['station_name'] = station_entity.name
station_test_list = StationTest.objects.filter(
- station__name=station_entity.name).order_by('-end_datetime')[:n_station_tests-1]
- response_payload[station_entity.name] = list()
+ station__name=station_entity.name).order_by('-end_datetime')[:n_station_tests - 1]
+ rtsm_list = RTSMObservationSummary.objects.filter(
+ station__name=station_entity.name).order_by('-end_time')[:n_rtsm - 1]
+
+ station_payload['station_tests'] = list()
for station_test in station_test_list:
station_test_payload = OrderedDict()
component_errors = station_test.component_errors
- station_test_payload['total_component_errors'] = station_test.component_errors.count()
+ station_test_payload[
+ 'total_component_errors'] = station_test.component_errors.count()
station_test_payload['start_datetime'] = station_test.start_datetime
station_test_payload['end_datetime'] = station_test.end_datetime
station_test_payload['checks'] = station_test.checks
- component_errors_summary = component_errors.\
+ component_errors_summary = component_errors. \
values('component__type', 'type').annotate(
total=Count('type')).order_by('-total')
- station_test_payload['STUFF'] = [(item['component__type'], item['type'], item['total']) for item in component_errors_summary]
-
- response_payload[station_entity.name].append(station_test_payload)
-
- return Response(status=status.HTTP_200_OK, data=response_payload)
\ No newline at end of file
+ component_errors_summary_dict = OrderedDict()
+ for item in component_errors_summary:
+ item_component_type = item['component__type']
+ item_error_type = item['type']
+ item_error_total = item['total']
+
+ if item_component_type not in component_errors_summary_dict:
+ component_errors_summary_dict[item_component_type] = OrderedDict()
+
+ component_errors_summary_dict[item_component_type][item_error_type] = \
+ item_error_total
+ station_test_payload['component_error_summary'] = component_errors_summary_dict
+
+ station_payload['station_tests'].append(station_test_payload)
+
+ station_payload['rtsm'] = list()
+ for rtsm in rtsm_list:
+ rtsm_payload = OrderedDict()
+ rtsm_payload['observation_id'] = rtsm.observation_id
+ rtsm_payload['start_datetime'] = rtsm.start_time
+ rtsm_payload['end_datetime'] = rtsm.end_time
+
+ unique_modes = [item['mode'] for item in rtsm.errors.values('mode').distinct()]
+ rtsm_payload['mode'] = unique_modes
+ rtsm_payload['total_component_error'] = rtsm.errors_found.count()
+ from django.db.models import Window, F
+ errors_summary = OrderedDict()
+
+ errors_summary_query = rtsm.errors_found.annotate(total=
+ Window(expression=Count('rcu'),
+ partition_by=[F(
+ 'error_type')])).values(
+ 'error_type', 'total').distinct()
+
+ for error_summary in errors_summary_query:
+ errors_summary[error_summary['error_type']] = error_summary['total']
+
+ rtsm_payload['error_summary'] = errors_summary
+ station_payload['rtsm'].append(rtsm_payload)
+
+ response_payload.append(station_payload)
+ if errors_only:
+ response_payload = filter(
+ lambda station_entry:
+ len(station_entry['station_tests']) + len(station_entry['rtsm']) > 0,
+ response_payload)
+ response_payload = sorted(response_payload, key=lambda item: item['station_name'])
+ return Response(status=status.HTTP_200_OK, data=response_payload)
--
GitLab