From 5aaa58182e64cf77ff7b856d6c1f71d0e5eee58f Mon Sep 17 00:00:00 2001
From: Mattia Mancini <mancini@astron.nl>
Date: Fri, 24 May 2019 08:33:07 +0000
Subject: [PATCH] OSB-45: refactored code to simplify the parsed structure
---
.../monitoringdb/station_test_raw_parser.py | 63 +++++++++++--------
.../monitoringdb/views/station_test_views.py | 30 +++++----
2 files changed, 53 insertions(+), 40 deletions(-)
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py b/LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py
index f47b1ae7c47..43cb0c9e0d6 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py
@@ -124,7 +124,7 @@ def collect_error_details(content):
rf_fail = parse_rffail_string('y', error_details.pop('y'))
error_details.update(rf_fail)
- element_errors = []
+ element_errors = dict()
for error_name, error_value in dict(error_details).items():
modem_comunication_error_pattern = 'e([0-9]{2})'
matched_key = re.search(modem_comunication_error_pattern, error_name)
@@ -133,12 +133,12 @@ def collect_error_details(content):
error_details.pop(error_name)
element_id = int(matched_key.group(1))
element = dict(element_id=element_id)
-
+ error_type = 'MODEM'
element_error_details = dict(error_code=error_value)
element_error = dict(element=element,
- type='MODEM',
+ type=error_type,
details=element_error_details)
- element_errors.append(element_error)
+ element_errors[(element_id, error_type)] = element_error
nosignal_error_pattern = 'E([0-9]{2})([XY])'
matched_key = re.search(nosignal_error_pattern, error_name)
@@ -148,18 +148,21 @@ def collect_error_details(content):
element_id = int(matched_key.group(1))
polarization = matched_key.group(2).lower()
element = dict(element_id=element_id)
-
+ error_type = 'NOSIGNAL'
element_error_details = {polarization: error_value}
- element_error = dict(element=element,
- type='NOSIGNAL',
- details=element_error_details)
- element_errors.append(element_error)
+ if (element_id, error_type) in element_errors:
+ element_errors[(element_id, error_type)]['details'].update(element_error_details)
+ else:
+ element_error = dict(element=element,
+ type='NOSIGNAL',
+ details=element_error_details)
+ element_errors[(element_id, error_type)] = element_error
if error_name in ['X', 'Y']:
error_details.pop(error_name)
error_name = error_name.lower()
error_details[error_name] = error_value
- error_details.update(element_errors=element_errors)
+ error_details.update(element_errors=list(element_errors.values()))
return error_details
@@ -175,19 +178,35 @@ def dict_from_component_error(content):
component = dict(component_id=parse_component_id(component_id), type=component_type.strip())
- element_errors = error_details.pop('element_errors')
component_error = dict(component=component,
details=error_details,
type=error_type.strip())
- for element_error in element_errors:
- element_error['component_error'] = dict(component_error)
-
- component_error['element_errors'] = element_errors
return component_error
+def preparse_content(raw_content):
+ content = dict()
+ for line in raw_content:
+
+ if "CHECKS" in line:
+ values = line.split(',')
+ else:
+ values = [parse_key_value_pairs(value) if ('=' in value) else value for value in
+ line.split(',')]
+ key = tuple(values[:4])
+ details = values[3:]
+
+ if key in content:
+ content[key] += details
+ else:
+ content[key] = values
+
+ return list(content.values())
+
+
+
def dict_from_raw_station_test(content):
"""
Expects a string content with the station test output
@@ -196,16 +215,9 @@ def dict_from_raw_station_test(content):
:return: a station test dict
"""
- preparsed_content = []
- for line in content:
- if "CHECKS" in line:
- values = [value for value in line.split(',')]
- else:
- values = [parse_key_value_pairs(value) if ('=' in value) else value for value in
- line.split(',')]
-
- preparsed_content.append(values)
+ preparsed_content = preparse_content(content)
result = {'component_errors': [], 'element_errors': []}
+
for row in preparsed_content:
row_type = row[3]
if row_type == "STATION":
@@ -237,9 +249,6 @@ def dict_from_raw_station_test(content):
result['element_errors'].extend(dicts_from_element_error(row))
else:
component_error_dict = dict_from_component_error(row)
- element_errors = component_error_dict.pop('element_errors')
- result['element_errors'].extend(element_errors)
-
result['component_errors'].append(component_error_dict)
return result
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/views/station_test_views.py b/LCU/Maintenance/DBInterface/monitoringdb/views/station_test_views.py
index 834bf3a1c2f..07e8816d50e 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/views/station_test_views.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/views/station_test_views.py
@@ -14,7 +14,10 @@ from ..serializers.element import ElementSerializer
from ..serializers.element_error import ElementErrorSerializer
from ..serializers.station import StationSerializer
from ..serializers.station_tests import StationTestSerializer
-from ..station_test_raw_parser import parse_raw_station_test
+from ..models.log import ActionLog
+from datetime import datetime
+
+from lofar.maintenance.monitoringdb.tasks.insert_raw_tests import insert_station_test
logger = logging.getLogger('views')
@@ -140,21 +143,25 @@ def insert_raw_station_test(request):
"""
if request.method == 'POST':
-
if 'content' in request.data:
try:
logger.info('handling raw request for %s', request)
content = request.data['content']
logger.info('handling raw request for data %s', content)
- parsed_content = parse_raw_station_test(content)
- if parsed_content is None:
- raise Exception('cannot parse test {}'.format(content))
- logger.info('data parsed successfully for test')
- station_test_ids = []
- for entry in parsed_content:
- station_test = StationTestSerializer.insert_station_test(entry)
- station_test_ids.append(station_test.pk)
+ action_log = ActionLog(who=request.META['REMOTE_ADDR'],
+ what='INSERT_REQUESTED',
+ when=datetime.now(),
+ model_name='StationTest',
+ entry_id=-1)
+
+ action_log.save()
+
+ insert_station_test.delay(action_log.pk, content)
+
+ return Response(status=status.HTTP_200_OK,
+ data='Station tests insert acknowledged.'
+ ' Check log for id %s' % action_log.pk)
except Exception as e:
logger.exception("exception occurred while parsing raw station info: %s", e)
return Response(exception=True,
@@ -167,6 +174,3 @@ def insert_raw_station_test(request):
data="the post message is not correct." +
" It has to be of the form \{'content':[RAWSTRING]\}",
status=status.HTTP_400_BAD_REQUEST)
- logger.info('Request processed correctly. Processed station test ids: %s', station_test_ids)
- return Response(status=status.HTTP_200_OK,
- data='Station tests inserted with ids {} \n'.format(station_test_ids))
--
GitLab