From 57cecc086d943e694a01bca56e1029cd62a8abad Mon Sep 17 00:00:00 2001
From: Mattia Mancini <mancini@astron.nl>
Date: Tue, 5 Jun 2018 16:40:45 +0000
Subject: [PATCH] Story SW-300: fixes in raw test parser when it founds
corresponding error in tile and rcu
---
.gitattributes | 3 ++-
.../DBInterface/monitoringdb/models/rtsm.py | 9 +++++++++
.../monitoringdb/serializers/rtsm.py | 16 +++++++++++++++
.../monitoringdb/station_test_raw_parser.py | 5 ++++-
.../DBInterface/monitoringdb/tests/common.py | 6 ++++--
.../tests/t_stationtest_parser.py | 16 +++++++++++++--
...a.csv => test_generic_stationtestdata.csv} | 0
.../test_multiple_tileerror_and_rcuerror.csv | 20 +++++++++++++++++++
.../DBInterface/monitoringdb/urls.py | 2 ++
.../monitoringdb/views/rtsm_views.py | 18 +++++++++++++++--
10 files changed, 87 insertions(+), 8 deletions(-)
rename LCU/Maintenance/DBInterface/monitoringdb/tests/{test_stationtestdata.csv => test_generic_stationtestdata.csv} (100%)
create mode 100644 LCU/Maintenance/DBInterface/monitoringdb/tests/test_multiple_tileerror_and_rcuerror.csv
diff --git a/.gitattributes b/.gitattributes
index f37db444339..d4bd8f4099f 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1813,7 +1813,8 @@ LCU/Maintenance/DBInterface/monitoringdb/tests/common.py -text
LCU/Maintenance/DBInterface/monitoringdb/tests/old_tests.py -text
LCU/Maintenance/DBInterface/monitoringdb/tests/t_stationtest_model_creation.py -text
LCU/Maintenance/DBInterface/monitoringdb/tests/t_stationtest_parser.py -text
-LCU/Maintenance/DBInterface/monitoringdb/tests/test_stationtestdata.csv -text
+LCU/Maintenance/DBInterface/monitoringdb/tests/test_generic_stationtestdata.csv -text
+LCU/Maintenance/DBInterface/monitoringdb/tests/test_multiple_tileerror_and_rcuerror.csv -text
LCU/Maintenance/DBInterface/monitoringdb/urls.py -text
LCU/Maintenance/DBInterface/monitoringdb/views/__init__.py -text
LCU/Maintenance/DBInterface/monitoringdb/views/common.py -text
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py b/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py
index 8632e83b704..ddc39b87cbd 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py
@@ -4,6 +4,7 @@ In this file all the model regarding the real time station monitor output are co
from django.db import models
from django.contrib.postgres.fields import ArrayField
+
from .fixed_types import STATION_TYPES
@@ -18,6 +19,14 @@ class RTSMObservation(models.Model):
samples = models.IntegerField(default=None, null=True)
+ @property
+ def errors_found(self):
+ return RTSMErrorSummary.objects.defer('bad_spectrum', 'average_spectrum').\
+ filter(observation=self).values('error_type', 'rcu').\
+ annotate(count=models.Count('error_type', unique=True, output_field=models.FloatField())).\
+ annotate(percentage=models.ExpressionWrapper(models.F('count') * 100. / models.F('observation__samples'),
+ output_field=models.FloatField()))
+
class RTSMErrorSummary(models.Model):
rcu = models.SmallIntegerField(default=None, null=True)
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py
index 749d9b03c20..9b738ca2b07 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py
@@ -8,6 +8,13 @@ from django.db.models import ObjectDoesNotExist
logger = logging.getLogger('serializers')
+class RTSMErrorsAggregateSerializer(serializers.Serializer):
+ count = serializers.IntegerField()
+ percentage = serializers.FloatField()
+ error_type = serializers.CharField()
+ rcu = serializers.IntegerField()
+
+
class RTSMErrorSerializer(serializers.ModelSerializer):
bad_spectrum = serializers.ListField(child=serializers.FloatField())
average_spectrum = serializers.ListField(child=serializers.FloatField())
@@ -29,8 +36,17 @@ class RTSMErrorSummarySerializer(serializers.ModelSerializer):
fields = '__all__'
+class RTSMObservationSummarySerializer(serializers.ModelSerializer):
+ errors_found = RTSMErrorsAggregateSerializer(many=True)
+
+ class Meta:
+ model = RTSMObservation
+ fields = '__all__'
+
+
class RTSMObservationSerializer(serializers.ModelSerializer):
errors_summary = RTSMErrorSummarySerializer(many=True)
+ errors_found = RTSMErrorsAggregateSerializer(many=True)
class Meta:
model = RTSMObservation
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py b/LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py
index fc8105ba51b..b2612dc0d36 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py
@@ -407,6 +407,7 @@ def dict_from_raw_station_test(content):
for tile_error in tile_errors:
linked_type = translate_tile_error_to_component_error_type[tile_error['resourcetype']]
component_id = int(row[2])
+
if (component_id, linked_type) not in result['components_error']:
error_type = translate_tile_error_to_component_error[tile_error['resourcetype']]
if error_type == 'NOISE':
@@ -423,9 +424,11 @@ def dict_from_raw_station_test(content):
if row[2] == '---':
logger.error('row type %s unrecognized: %s', row_type, row)
else:
- result['components_error'][(int(row[2]), row_type)] = dict_from_component_error(row)
+ component_error_dict = dict_from_component_error(row)
+ result['components_error'][(int(row[2]), component_error_dict['resourcetype'])] = component_error_dict
result['components_error'] = result['components_error'].values()
+
return result
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/tests/common.py b/LCU/Maintenance/DBInterface/monitoringdb/tests/common.py
index 1a9d348958a..fd4d994efc5 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/tests/common.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/tests/common.py
@@ -1,10 +1,12 @@
import os
from .. import station_test_raw_parser as raw_parser
-def read_station_test_data():
- with open(os.path.dirname(__file__) + '/test_stationtestdata.csv') as data:
+
+def read_station_test_data(filename):
+ with open(os.path.join(os.path.dirname(__file__) , filename)) as data:
return [line for line in data.read().split('\n') if line]
+
def split_rows_as_expected(line):
if "CHECKS" in line:
values = [value for value in line.split(',')]
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/tests/t_stationtest_parser.py b/LCU/Maintenance/DBInterface/monitoringdb/tests/t_stationtest_parser.py
index 0cfbfa5c6b9..6c4a49200b6 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/tests/t_stationtest_parser.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/tests/t_stationtest_parser.py
@@ -6,13 +6,13 @@ from .common import *
class TestStationTestsParser(unittest.TestCase):
def test_parsing_one_test(self):
- station_test_data = read_station_test_data()
+ station_test_data = read_station_test_data('test_generic_stationtestdata.csv')
first_test = raw_parser.split_history_into_tests(station_test_data)[0]
self.assertEqual(first_test[-1],
"20130522,HBA,036,E_FAIL,SPX9=1,SPY9=1,SPX10=1,SPY10=1,SPX13=1,SPY13=1,SPX14=1,SPY14=1")
def test_all_test_dict_parsing(self):
- station_test_data = read_station_test_data()
+ station_test_data = read_station_test_data('test_generic_stationtestdata.csv')
for test in raw_parser.split_history_into_tests(station_test_data):
result = raw_parser.dict_from_raw_station_test(test)
@@ -47,5 +47,17 @@ class TestStationTestsParser(unittest.TestCase):
self.assertIn('resourcetype', parsed_content)
self.assertEqual(parsed_content['resourcetype'], key)
+ def test_corresponding_component_and_tile_error(self):
+ # Tests if a tile error of the same type appears with the same type of a component error then the two
+ # errors are joined
+ station_test_data = read_station_test_data('test_multiple_tileerror_and_rcuerror.csv')
+ for test in raw_parser.split_history_into_tests(station_test_data):
+ result = raw_parser.dict_from_raw_station_test(test)
+ hba_result = [error for error in result['components_error'] if error['component_type'] == 'HBA'
+ and error['component_id'] == 33
+ and error['error_type'] == 'HIGH_NOISE']
+ for result in hba_result:
+ self.assertIsNot(len(result['tile_errors']), 0)
+
if __name__=='__main__':
unittest.main()
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/tests/test_stationtestdata.csv b/LCU/Maintenance/DBInterface/monitoringdb/tests/test_generic_stationtestdata.csv
similarity index 100%
rename from LCU/Maintenance/DBInterface/monitoringdb/tests/test_stationtestdata.csv
rename to LCU/Maintenance/DBInterface/monitoringdb/tests/test_generic_stationtestdata.csv
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/tests/test_multiple_tileerror_and_rcuerror.csv b/LCU/Maintenance/DBInterface/monitoringdb/tests/test_multiple_tileerror_and_rcuerror.csv
new file mode 100644
index 00000000000..9515ee04969
--- /dev/null
+++ b/LCU/Maintenance/DBInterface/monitoringdb/tests/test_multiple_tileerror_and_rcuerror.csv
@@ -0,0 +1,20 @@
+20180508,NFO,---,VERSIONS,
+20180508,NFO,---,STATION,NAME=CS002C
+20180508,NFO,---,RUNTIME,START=08-05-2018T00:00:00,STOP=08-05-2018T02:07:21
+20180508,NFO,---,CHECKS,RV,SPU,RBC,SH1,F1,D1,O1,SP1,NS1=60,S1,SH3,F3,D3,O3,SP3,NS3=60,S3,M5,O5,SN5,SP5,NS5=60,E7,TV,TBC,TM
+20180508,NFO,---,STATISTICS,BAD_LBL=2,BAD_LBH=3,BAD_HBA=-1,BAD_HBA0=1,BAD_HBA1=6
+20180508,LBL,---,TESTSIGNAL,SUBBANDX=301,SIGNALX=81.7,SUBBANDY=301,SIGNALY=81.7
+20180508,LBL,058,RF_FAIL,Y=84.6
+20180508,LBL,064,DOWN,X=78.3,Y=78.2,Xoff=-22,Yoff=9
+20180508,LBH,---,TESTSIGNAL,SUBBANDX=301,SIGNALX=81.7,SUBBANDY=301,SIGNALY=81.4
+20180508,LBH,015,JITTER,Xproc=98.333,Ydiff=2.532,Yref=0.1
+20180508,LBH,017,DOWN,X=83.0,Y=78.6,Xoff=0,Yoff=-21
+20180508,LBH,039,RF_FAIL,X=76.7
+20180508,HBA,033,HIGH_NOISE,Xproc=15.556,Xval=85.9,Xdiff=19.027,Xref=70.1
+20180508,HBA,038,HIGH_NOISE,Yproc=5.000,Yval=69.3,Ydiff=1.775,Yref=69.2
+20180508,HBA,004,E_FAIL,X1=100.8 143 104.4 100.1 140 105.2,Y1=99.5 140 101.6 99.8 140 102.4
+20180508,HBA,028,E_FAIL,X3=75.6 143 104.3 78.1 143 104.8
+20180508,HBA,031,E_FAIL,X5=65.5 142 104.0 65.3 140 104.4,Y5=66.9 142 102.6 66.8 139 102.9
+20180508,HBA,033,E_FAIL,HNX1=84.8 4.591,HNX2=80.0 10.499,JX8=1.000,HNX9=76.4 12.931,HNX11=80.7 16.661
+20180508,HBA,039,E_FAIL,Y16=98.0 141 101.3 73.8 139 101.5
+20180508,HBA,044,E_FAIL,X15=105.8 143 105.7 65.4 143 106.0,Y15=100.5 143 102.4 65.3 142 102.8
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/urls.py b/LCU/Maintenance/DBInterface/monitoringdb/urls.py
index cde62e231b5..387ae262e39 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/urls.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/urls.py
@@ -12,6 +12,8 @@ station_test_router.register(r'^', StationTestViewSet)
rtsm_router = routers.DefaultRouter()
#RTSM
+
+rtsm_router.register(r'^/summary', RTSMObservationSummaryViewSet)
rtsm_router.register(r'^/errors_detailed', RTSMErrorsDetailedViewSet)
rtsm_router.register(r'^/errors', RTSMErrorsViewSet)
rtsm_router.register(r'^', RTSMObservationViewSet)
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py b/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py
index 126da0b71a7..007fd24bd73 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py
@@ -1,8 +1,9 @@
from .common import *
from ..models.rtsm import RTSMObservation, RTSMError
-from ..serializers.rtsm import RTSMObservationSerializer, RTSMErrorSummarySerializer, RTSMErrorSerializer
+from ..serializers.rtsm import RTSMObservationSerializer, RTSMErrorSummarySerializer, RTSMErrorSerializer,\
+ RTSMObservationSummarySerializer
from ..rtsm_test_raw_parser import parse_rtsm_test
-
+from rest_framework.serializers import Serializer
logger = logging.getLogger('views')
@@ -45,6 +46,19 @@ class RTSMObservationViewSet(viewsets.ModelViewSet):
return queryset
+class RTSMObservationSummaryViewSet(viewsets.ReadOnlyModelViewSet):
+ queryset = RTSMObservation.objects.all()
+ serializer_class = RTSMObservationSummarySerializer
+
+ def get_queryset(self):
+ queryset = RTSMObservation.objects.all()
+ for key, param in self.request.query_params.items():
+ if key in RESERVED_FILTER_NAME:
+ continue
+ queryset = queryset.filter(**{key: param})
+ return queryset
+
+
@api_view(['POST'])
def insert_raw_rtsm_test(request):
"""
--
GitLab