Skip to content
Snippets Groups Projects
Commit 828d1713 authored by Mattia Mancini's avatar Mattia Mancini
Browse files

Story SW-300: starting implementing raw station test parser to parse raw station test output

parent 59fe2d46
No related branches found
No related tags found
2 merge requests!89Monitoring maintenance Epic branch merge,!1Resolve OSB-13 "Monitoringmaintenance "
Showing
with 667 additions and 1 deletion
......@@ -1770,6 +1770,29 @@ LCU/Firmware/tools/src/rsuctl3_reset -text
LCU/Firmware/tools/src/view_images.sh -text
LCU/MaintenanceDB/CMakeLists.txt -text
LCU/MaintenanceDB/DBInterface/CMakeLists.txt -text
LCU/MaintenanceDB/DBInterface/__init__.py -text
LCU/MaintenanceDB/DBInterface/django_postgresql/__init__.py -text
LCU/MaintenanceDB/DBInterface/django_postgresql/create_db.sql -text
LCU/MaintenanceDB/DBInterface/django_postgresql/monitoringdb/__init__.py -text
LCU/MaintenanceDB/DBInterface/django_postgresql/monitoringdb/admin.py -text
LCU/MaintenanceDB/DBInterface/django_postgresql/monitoringdb/apps.py -text
LCU/MaintenanceDB/DBInterface/django_postgresql/monitoringdb/migrations/__init__.py -text
LCU/MaintenanceDB/DBInterface/django_postgresql/monitoringdb/models/__init__.py -text
LCU/MaintenanceDB/DBInterface/django_postgresql/monitoringdb/models/componenterror.py -text
LCU/MaintenanceDB/DBInterface/django_postgresql/monitoringdb/models/fixed_types.py -text
LCU/MaintenanceDB/DBInterface/django_postgresql/monitoringdb/models/stationtest.py -text
LCU/MaintenanceDB/DBInterface/django_postgresql/monitoringdb/models/tileerror.py -text
LCU/MaintenanceDB/DBInterface/django_postgresql/monitoringdb/serializers/__init__.py -text
LCU/MaintenanceDB/DBInterface/django_postgresql/monitoringdb/serializers/componenterror.py -text
LCU/MaintenanceDB/DBInterface/django_postgresql/monitoringdb/serializers/stationtest.py -text
LCU/MaintenanceDB/DBInterface/django_postgresql/monitoringdb/serializers/tileerror.py -text
LCU/MaintenanceDB/DBInterface/django_postgresql/monitoringdb/station_test_raw_parser.py -text
LCU/MaintenanceDB/DBInterface/django_postgresql/monitoringdb/tests.py -text
LCU/MaintenanceDB/DBInterface/django_postgresql/monitoringdb/views.py -text
LCU/MaintenanceDB/DBInterface/django_postgresql/requirements.txt -text
LCU/MaintenanceDB/DBInterface/django_postgresql/settings.py -text
LCU/MaintenanceDB/DBInterface/django_postgresql/urls.py -text
LCU/MaintenanceDB/DBInterface/django_postgresql/wsgi.py -text
LCU/MaintenanceDB/DBInterface/django_rest/__init__.py -text
LCU/MaintenanceDB/DBInterface/django_rest/create_db.sql -text
LCU/MaintenanceDB/DBInterface/django_rest/monitoringdb/__init__.py -text
......@@ -1790,6 +1813,7 @@ LCU/MaintenanceDB/DBInterface/django_rest/settings.py -text
LCU/MaintenanceDB/DBInterface/django_rest/urls.py -text
LCU/MaintenanceDB/DBInterface/django_rest/wsgi.py -text
LCU/MaintenanceDB/DBInterface/manage.py -text
LCU/MaintenanceDB/__init__.py -text
LCU/PPSTune/CMakeLists.txt -text
LCU/PPSTune/MANIFEST.in -text
LCU/PPSTune/README -text
......@@ -1992,6 +2016,7 @@ LCU/StationTest/xc_160_setup.sh eol=lf
LCU/StationTest/xc_160_verify.sh eol=lf
LCU/StationTest/xc_200_setup.sh eol=lf
LCU/StationTest/xc_200_verify.sh eol=lf
LCU/__init__.py -text
LCU/checkhardware/check_hardware.py -text
LCU/checkhardware/checkhardware_lib/CMakeLists.txt -text
LCU/checkhardware/checkhardware_lib/__init__.py -text
......
......@@ -3,7 +3,7 @@
lofar_package(DBInterface 1.0)
# Install files matching regex pattern in current directory and below
install(DIRECTORY django_rest
install(DIRECTORY django_postgresql
DESTINATION ${CMAKE_INSTALL_PREFIX}/webserver
USE_SOURCE_PERMISSIONS
PATTERN ".svn" EXCLUDE)
......
CREATE DATABASE MonitoringDB;
CREATE USER admin WITH PASSWORD 'fix-me';
ALTER ROLE admin SET client_encoding TO 'utf8';
ALTER ROLE admin SET default_transaction_isolation TO 'read committed';
ALTER ROLE admin SET timezone TO 'UTC';
GRANT ALL PRIVILEGES ON DATABASE MonitoringDB TO admin;
from django.contrib import admin
# Register your model here.
from django.apps import AppConfig
class MonitoringDbConfig(AppConfig):
name = 'django_postgresql.monitoringdb'
from .stationtest import *
from .componenterror import *
\ No newline at end of file
from django.db import models
from polymorphic.models import PolymorphicModel
from .stationtest import StationTest
from .fixed_types import COMPONENT_TYPES, ERROR_TYPES
class ComponentError(PolymorphicModel):
id = models.AutoField(primary_key=True)
component_id = models.IntegerField()
component_type = models.CharField(max_length=6, choices=COMPONENT_TYPES)
error_type = models.CharField(max_length=15, choices=ERROR_TYPES)
station_test = models.ForeignKey(StationTest,
related_name='component_errors',
on_delete=models.CASCADE)
class LBAComponentError(ComponentError):
lba_type = models.CharField(max_length=1)
class HBANoSignal(ComponentError):
pass
class TooLowError(ComponentError):
medianX = models.FloatField(default=-1)
medianY = models.FloatField(default=-1)
class DownError(ComponentError):
signalX = models.FloatField(default=-1)
signalY = models.FloatField(default=-1)
signalX_offset = models.FloatField(default=-1)
signalY_offset = models.FloatField(default=-1)
class RFFailError(ComponentError):
polarization = models.CharField(max_length=1)
measured_signal_nodelay = models.FloatField()
measured_signal_fulldelay = models.FloatField()
subband_used_nodelay = models.PositiveSmallIntegerField()
subband_used_fulldelay = models.PositiveSmallIntegerField()
reference_signal_nodelay = models.FloatField()
reference_signal_fulldelay = models.FloatField()
class NoiseError(ComponentError):
type = models.CharField(default='-', max_length=1)
percentageX = models.FloatField(default=-1)
percentageY = models.FloatField(default=-1)
peak_valueX = models.FloatField(default=-1)
peak_valueY = models.FloatField(default=-1)
fluctuationX = models.FloatField(default=-1)
fluctuationY = models.FloatField(default=-1)
limitX = models.FloatField(default=-1)
limitY = models.FloatField(default=-1)
class JitterError(ComponentError):
percentageX = models.FloatField(default=-1)
percentageY = models.FloatField(default=-1)
average_valueX = models.FloatField(default=-1)
average_valueY = models.FloatField(default=-1)
fluctuationX = models.FloatField(default=-1)
fluctuationY = models.FloatField(default=-1)
class OscillationError(ComponentError):
polarizationX = models.BooleanField(default=False)
polarizationY = models.BooleanField(default=False)
class SpuriousError(ComponentError):
polarizationX = models.BooleanField(default=False)
polarizationY = models.BooleanField(default=False)
class SummatorNoise(ComponentError):
polarizationX = models.BooleanField(default=False)
polarizationY = models.BooleanField(default=False)
\ No newline at end of file
STATION_TYPES = (
('c', 'core'),
('i', 'international'),
('r', 'remote')
)
COMPONENT_TYPES = (
('RSP', 'RSP'),
('TBB', 'Temporary Buffered Board'),
('RCU', 'RCU'),
('LBL', 'LBL'),
('LBH', 'LBH'),
('LBA', 'LBA'),
('HBA', 'HBA'),
)
ERROR_TYPES = (
('VERSION', 'Not expected hardware version'),
('MEMORY', 'Memory error'),
('BROKEN', 'Broken antennas connected to this RCU'),
('NOSIGNAL', 'Signal not detected'),
('TOOLOW', 'Average signal too low to perform test'),
('DOWN', 'Antenna fallen down'),
('RF_FAIL', 'Signal too high/low in tested subband'),
('LOW_NOISE', 'Average signal in subband is low compared to the other subbands'),
('HIGH_NOISE', 'Average signal in subband is high compared to the other subbands'),
('JITTER', 'Jitter error'),
('OSCILLATION', 'Oscillating antenna'),
('MODEM', 'Failure with tile connection'),
('SPURIOUS', 'Spurious signal found'),
('P_SUMMATOR', 'Too low signals on all tiles'),
('C_SUMMATOR', 'No communication with elements possible'),
('SUMMATOR_NOISE', 'oscillating summator'),
('TESTSIGNAL', 'Test signal used in RF test'),
)
POLARIZATIONS = ('X', 'Y')
LBA_TYPES = ('H', 'L')
\ No newline at end of file
"""
In this file all the model regarding the stationtest output are collected
"""
from django.db import models
from .fixed_types import STATION_TYPES
class StationTest(models.Model):
id = models.AutoField(primary_key=True)
station_name = models.CharField(max_length=6)
station_type = models.CharField(max_length=1, choices=STATION_TYPES)
start_time = models.DateTimeField()
end_time = models.DateTimeField()
performed_checks = models.CharField(max_length=50)
from django.db import models
from polymorphic.models import PolymorphicModel
from .componenterror import ComponentError
class TileError(PolymorphicModel):
id = models.AutoField(primary_key=True)
tile_id = models.SmallIntegerField(default=-1)
parent_component_error = models.ForeignKey(ComponentError,
related_name='tile_errors',
on_delete=models.CASCADE)
class TileNoSignalError(TileError):
polarization = models.CharField(max_length=1)
class TileModemError(TileError):
communication_error = models.BooleanField()
class TileOscillatingError(TileError):
polarization = models.CharField(max_length=1)
class TileSpuriousError(TileError):
polarization = models.CharField(max_length=1)
class TileNoiseError(TileError):
polarization = models.CharField(max_length=1)
type = models.CharField(max_length=1)
# measured (lowest) highest mean value over all subbands
value = models.FloatField()
# difference between lowest and highest found mean values
diff = models.FloatField()
class TileJitterError(TileError):
polarization = models.CharField(max_length=1)
fluctuation = models.FloatField()
class TileRFFailure(TileError):
polarization = models.CharField(max_length=1)
measured_signal_nodelay = models.FloatField()
measured_signal_fulldelay = models.FloatField()
subband_used_nodelay = models.PositiveSmallIntegerField()
subband_used_fulldelay = models.PositiveSmallIntegerField()
reference_signal_nodelay = models.FloatField()
reference_signal_fulldelay = models.FloatField()
\ No newline at end of file
from rest_framework import serializers
from rest_polymorphic.serializers import PolymorphicSerializer
from ..models.componenterror import ComponentError, LBAComponentError, HBANoSignal, TooLowError,\
DownError, RFFailError, NoiseError, JitterError, \
OscillationError, SpuriousError, SummatorNoise
from ..models.fixed_types import COMPONENT_TYPES, ERROR_TYPES
from ..models.fixed_types import LBA_TYPES
from ..models.tileerror import TileNoSignalError
from ..models import StationTest
from .tileerror import TileNoSignalErrorSerializer
class ComponentErrorSerializer(serializers.Serializer):
id = serializers.IntegerField(read_only=True)
station_test_id = serializers.PrimaryKeyRelatedField(queryset=StationTest.objects.all(), source='station_test.id')
component_id = serializers.IntegerField()
component_type = serializers.ChoiceField(COMPONENT_TYPES)
error_type = serializers.ChoiceField(ERROR_TYPES)
def create(self, validated_data):
"""
Create the component error given the fields
"""
return ComponentError.objects.create(**validated_data)
def modify_instance_fields(self, instance, validated_data):
instance.id = validated_data.get('id', instance.id)
instance.component_id = validated_data.get('component_id', instance.id)
instance.component_type = validated_data.get('component_type', instance.id)
instance.error_type = validated_data.get('error_type', instance.id)
return instance
def update(self, instance, validated_data):
"""
Update an existing instance
:param instance: instance to be update
:param validated_data: data contained
:type validated_data: dict
:return:
"""
instance = self.modify_instance_fields(instance, validated_data)
return instance
class LBAComponentErrorSerializer(ComponentErrorSerializer):
lba_type = serializers.ChoiceField(LBA_TYPES)
def create(self, validated_data):
"""
Create the LBA component error given the fields
"""
return LBAComponentError(**validated_data)
def update(self, instance, validated_data):
instance = self.modify_instance_fields(instance, validated_data)
instance.lba_type = validated_data.get('lba_type', instance.lba_type)
instance.save()
return instance
class HBANoSignalSerializer(ComponentErrorSerializer):
tiles = TileNoSignalErrorSerializer(many=True)
def create(self, validated_data):
instance = super(HBANoSignalSerializer, self).create(validated_data)
tiles_validated_data = validated_data.pop('tiles')
for tile_validated_data in tiles_validated_data:
tile_validated_data['parent_component_error'] = instance
tile_error_instance = TileNoSignalErrorSerializer().create(tile_validated_data)
tile_error_instance.save()
return instance
def update(self, instance, validated_data):
tiles_validated_data = validated_data.pop('tiles')
instance = self.modify_instance_fields(instance, validated_data)
instance.save()
for tile_validated_data in tiles_validated_data:
tile_id = tile_validated_data.get('id', None)
if tile_id:
tile_error_instance = TileNoSignalError.objects.get(id=tile_id, parent_component_error=instance)
tile_error_instance = TileNoSignalErrorSerializer.modify_instance_fields(tile_error_instance,
tile_validated_data)
tile_error_instance.save()
else:
TileNoSignalError.objects.create(parent_component_error=instance, **tile_validated_data)
class TooLowErrorSerializer(ComponentErrorSerializer):
medianX = serializers.FloatField()
medianY = serializers.FloatField()
def create(self, validated_data):
return TooLowError.objects.create(**validated_data)
def update(self, instance, validated_data):
instance = self.modify_instance_fields(instance, validated_data)
instance.medianX = validated_data.get('medianX', instance.medianX)
instance.medianY = validated_data.get('medianY', instance.medianY)
class DownErrorSerializer(serializers.ModelSerializer):
'signalX
'signalY
'signalX_offset
'signalY_offset
class Meta:
models = DownError
fields = ('component_id',
'component_type',
'error_type',
'signalX',
'signalY',
'signalX_offset',
'signalY_offset',
)
class RFFailErrorSerializer(serializers.ModelSerializer):
class Meta:
models = RFFailError
fields = ('component_id',
'component_type',
'error_type',
'polarization',
'measured_signal_nodelay',
'measured_signal_delay',
'subband_used_nodelay',
'subband_used_delay',
'reference_signal_nodelay',
'reference_signal_delay',
)
class NoiseErrorSerializer(serializers.ModelSerializer):
class Meta:
models = NoiseError
fields = ('component_id',
'component_type',
'error_type',
'type',
'percentageX',
'percentageY',
'peak_valueX',
'peak_valueY',
'fluctuationX',
'fluctuationY',
'limitX',
'limitY'
)
class JitterErrorSerializer(serializers.ModelSerializer):
class Meta:
models = JitterError
fields = ('component_id',
'component_type',
'error_type',
'percentageX',
'percentageY',
'average_valueX',
'average_valueY',
'fluctuationX',
'fluctuationY',
)
class OscillationErrorSerializer(serializers.ModelSerializer):
class Meta:
models = OscillationError
fields = ('component_id',
'component_type',
'error_type',
'polarizationX',
'polarizationY',)
class SpuriousErrorSerializer(serializers.ModelSerializer):
class Meta:
models = SpuriousError
fields = ('component_id',
'component_type',
'error_type',
'polarizationX',
'polarizationY',)
class SummatorNoiseSerializer(serializers.ModelSerializer):
class Meta:
models = SummatorNoise
fields = ('component_id',
'component_type',
'error_type',
'polarizationX',
'polarizationY',)
class ComponentErrorPolimorphicSerializer(PolymorphicSerializer):
model_serializer_mapping = {
ComponentError: ComponentErrorSerializer,
HBANoSignal: HBANoSignalSerializer,
TooLowError: TooLowErrorSerializer,
DownError: DownErrorSerializer,
RFFailError: RFFailErrorSerializer,
NoiseError: NoiseErrorSerializer,
JitterError: JitterErrorSerializer,
OscillationError: OscillationErrorSerializer,
SpuriousError: SpuriousErrorSerializer,
SummatorNoise: SummatorNoiseSerializer,
LBAComponentError: LBAComponentErrorSerializer,
}
\ No newline at end of file
from ..models import *
from rest_framework import serializers
from .componenterror import ComponentErrorPolimorphicSerializer
class StationTestSerializer(serializers.ModelSerializer):
component_errors = ComponentErrorPolimorphicSerializer(many=True)
class Meta:
model = StationTest
fields = ('id', 'station_name', 'station_type', 'start_time', 'end_time', 'performed_checks', 'component_errors')
def create(self, validated_data):
components_errors_data = validated_data.pop('component_errors')
station_test_instance = StationTest.objects.create(**validated_data)
station_test_instance.save()
for component_errors_data in components_errors_data:
component_errors_data['station_test'] = station_test_instance
component_error_instance = ComponentErrorPolimorphicSerializer().create(component_errors_data)
component_error_instance.save()
return station_test_instance
from rest_framework import serializers
from rest_polymorphic.serializers import PolymorphicSerializer
from ..models.fixed_types import POLARIZATIONS
from ..models.tileerror import TileError, TileJitterError, \
TileNoiseError, TileModemError, TileOscillatingError, TileSpuriousError, TileRFFailure, TileNoSignalError
class TileErrorSerializer(serializers.Serializer):
tile_id = serializers.IntegerField()
def create(self, validated_data):
TileError.objects.create(validated_data)
def modify_instance_fields(self, instance, validated_data):
instance.tile_id = validated_data.get('tile_id', instance.tile_id)
return instance
def update(self, instance, validated_data):
instance = self.modify_instance_fields(instance, validated_data)
instance.save()
return instance
class TileJitterErrorSerializer(serializers.ModelSerializer):
class Meta:
model = TileJitterError
fields = ('tile_id', 'polarization', 'fluctuation')
class TileNoiseErrorSerializer(serializers.ModelSerializer):
class Meta:
model = TileNoiseError
fields = ('tile_id', 'polarization', 'type', 'value', 'diff')
class TileModemErrorSerializer(serializers.ModelSerializer):
class Meta:
model = TileModemError
fields = ('tile_id', 'communication_error',)
class TileOscillatingErrorSerializer(serializers.ModelSerializer):
class Meta:
model = TileOscillatingError
fields = ('tile_id', 'polarization',)
class TileSpuriousErrorSerializer(serializers.ModelSerializer):
class Meta:
model = TileSpuriousError
fields = ('tile_id', 'polarization',)
class TileRFFailureSerializer(serializers.ModelSerializer):
class Meta:
model = TileRFFailure
fields = ('tile_id',
'polarization',
'measured_signal_nodelay',
'measured_signal_delay',
'subband_used_nodelay',
'subband_used_delay',
'reference_signal_nodelay',
'reference_signal_delay',
)
class TileNoSignalErrorSerializer(TileErrorSerializer):
polarization = serializers.ChoiceField(POLARIZATIONS)
def create(self, validated_data):
return TileError.objects.create(validated_data)
def update(self, instance, validated_data):
instance = super(TileNoSignalErrorSerializer, self).modify_instance_fields(instance, validated_data)
instance.polarization = validated_data.get('polarization', instance.polarization)
instance.save()
return instance
class TileErrorPolimorphicSerializer(PolymorphicSerializer):
model_serializer_mapping = {
TileError: TileErrorSerializer,
TileModemError: TileModemErrorSerializer,
TileOscillatingError: TileOscillatingErrorSerializer,
TileSpuriousError: TileSpuriousErrorSerializer,
TileNoiseError: TileNoiseErrorSerializer,
TileJitterError: TileJitterErrorSerializer,
TileRFFailure: TileRFFailureSerializer,
TileNoSignalError: TileNoSignalErrorSerializer
}
\ No newline at end of file
"""
This modules contains all the function needed to parse a raw station test output into the models used to describe it
"""
from datetime import datetime
def parse_key_value_pairs(content):
"""
Parse a key value pair returning a dict.
es 'key=value' -> {'key': 'value'}
:param content: the string to parse
:return: dict
"""
assert '=' in content
pairs = content.split('=')
return {pairs[0]:pairs[1]}
def is_component_type(component_type, values):
"""
Receive a list of values that compose a line in the station test output and returns true if
the line refers to the component_type type false otherwise
:param values: a list of values that compose a line in the station test output
:param component_type: component type
:return: True or False
"""
return values[1] == component_type
def filter_by_component_type(component_type, contents):
"""
Receive a list of rows that compose a line in the station test output and returns a list of rows
that are referred to the component type type
:param values: a list of values that compose a line in the station test output
:param component_type: component type
:return: a list of values that refers to the component_type
"""
return filter(lambda content: is_component_type(component_type, content), contents)
def parse_datetime(date, date_time):
"""
If the datetime is only the time of the current date it used the data
to generate the date time
:param date:
:param date_time:
:return:
"""
if 'T' in date_time:
return datetime.strptime(date_time, '%Y-%m-%dT%H:%M:%S')
else:
return datetime.strptime("T".join([date, date_time]), '%Y%m%dT%H:%M:%S')
def parse_from_raw_station_test(content):
"""
Expects a string content with the station test output
and output a list of Django Models
:param content: string content with the station test output
:return: a list of Django models
"""
model_list = []
station_name = ''
start_time = ''
end_time = ''
preparsed_content = []
for line in content.split('\n'):
values = [parse_key_value_pairs(value) if '=' in value else value for value in line.split(',')]
preparsed_content.append(values)
for row in filter_by_component_type('NFO', preparsed_content):
print('parsing ', row)
if row[3] == "STATION":
station_name = row[4]['NAME']
elif row[3] == "RUNTIME":
start_time = parse_datetime(row[0], row[4]['START'])
end_time = parse_datetime(row[0], row[5]['STOP'])
print(station_name, start_time, end_time)
return model_list
\ No newline at end of file
from django.test import TestCase
# Create your tests here.
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment