From 1a426b42908c51f1fb9483f76ff281cc8643423d Mon Sep 17 00:00:00 2001
From: Mattia Mancini <mancini@astron.nl>
Date: Mon, 14 May 2018 14:04:13 +0000
Subject: [PATCH] Story SW-300: fixes throughout the all branch
---
.gitattributes | 1 +
.../DBInterface/django_postgresql/settings.py | 5 +
.../monitoringdb/models/rspcomponenterror.py | 10 +-
.../monitoringdb/models/spucomponenterror.py | 2 +-
.../monitoringdb/models/tbbcomponenterror.py | 12 +--
.../serializers/componenterror.py | 1 +
.../serializers/componenterrors_generic.py | 1 +
.../serializers/hbaerrorserializers.py | 22 ++---
.../serializers/lbaerrorserializer.py | 18 ++--
.../serializers/rcuerrorserializer.py | 1 +
.../serializers/rsperrorserializer.py | 4 +
.../serializers/spuerrorserializer.py | 1 +
.../monitoringdb/serializers/stationtest.py | 9 +-
.../serializers/tbberrorserializer.py | 7 +-
.../monitoringdb/station_test_raw_parser.py | 29 ++++--
.../DBInterface/monitoringdb/views.py | 6 +-
LCU/Maintenance/MDB_tools/cli/mdb_loader.py | 91 +++++++++++++++++++
LCU/Maintenance/MDB_tools/cli/probe_mdb.py | 15 ++-
18 files changed, 189 insertions(+), 46 deletions(-)
create mode 100644 LCU/Maintenance/MDB_tools/cli/mdb_loader.py
diff --git a/.gitattributes b/.gitattributes
index decbbeae5b9..826f58aab42 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1820,6 +1820,7 @@ LCU/Maintenance/DBInterface/monitoringdb/tests/t_stationtest_parser.py -text
LCU/Maintenance/DBInterface/monitoringdb/tests/test_stationtestdata.csv -text
LCU/Maintenance/DBInterface/monitoringdb/views.py -text
LCU/Maintenance/MDB_tools/CMakeLists.txt -text
+LCU/Maintenance/MDB_tools/cli/mdb_loader.py -text
LCU/Maintenance/MDB_tools/cli/probe_mdb.py -text
LCU/Maintenance/__init__.py -text
LCU/PPSTune/CMakeLists.txt -text
diff --git a/LCU/Maintenance/DBInterface/django_postgresql/settings.py b/LCU/Maintenance/DBInterface/django_postgresql/settings.py
index 0cb313281ef..c562285c181 100644
--- a/LCU/Maintenance/DBInterface/django_postgresql/settings.py
+++ b/LCU/Maintenance/DBInterface/django_postgresql/settings.py
@@ -123,3 +123,8 @@ USE_TZ = True
# https://docs.djangoproject.com/en/2.0/howto/static-files/
STATIC_URL = '/static/'
+
+REST_FRAMEWORK = {
+ 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination',
+ 'PAGE_SIZE': 5
+}
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/rspcomponenterror.py b/LCU/Maintenance/DBInterface/monitoringdb/models/rspcomponenterror.py
index 2e2e533cd2a..bfd5f99a719 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/models/rspcomponenterror.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/models/rspcomponenterror.py
@@ -10,18 +10,18 @@ class RSPComponentError(ComponentError):
on_delete=models.CASCADE)
-class RSPVersionError(ComponentError):
- bp = models.CharField(max_length=10)
- ap = models.CharField(max_length=10)
+class RSPVersionError(RSPComponentError):
+ BP = models.CharField(max_length=10)
+ AP = models.CharField(max_length=10)
-class RSPVoltageError(ComponentError):
+class RSPVoltageError(RSPComponentError):
voltage_1_2 = models.FloatField(default=0)
voltage_2_5 = models.FloatField(default=0)
voltage_3_3 = models.FloatField(default=0)
-class RSPTemperatureError(ComponentError):
+class RSPTemperatureError(RSPComponentError):
pcb = models.FloatField(default=0)
bp = models.FloatField(default=0)
ap0 = models.FloatField(default=0)
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/spucomponenterror.py b/LCU/Maintenance/DBInterface/monitoringdb/models/spucomponenterror.py
index f62e0445bd9..f840436292b 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/models/spucomponenterror.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/models/spucomponenterror.py
@@ -10,7 +10,7 @@ class SPUComponentError(ComponentError):
on_delete=models.CASCADE)
-class SPUVoltageError(ComponentError):
+class SPUVoltageError(SPUComponentError):
voltage_rcu = models.FloatField(default=5)
voltage_lba = models.FloatField(default=8)
voltage_hba = models.FloatField(default=48)
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/tbbcomponenterror.py b/LCU/Maintenance/DBInterface/monitoringdb/models/tbbcomponenterror.py
index e58b1dfd917..42e0530ea80 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/models/tbbcomponenterror.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/models/tbbcomponenterror.py
@@ -10,22 +10,22 @@ class TBBComponentError(ComponentError):
on_delete=models.CASCADE)
-class TBBVersionError(ComponentError):
- tp = models.CharField(max_length=10)
- mp = models.CharField(max_length=10)
+class TBBVersionError(TBBComponentError):
+ TP = models.CharField(max_length=10)
+ MP = models.CharField(max_length=10)
-class TBBMemoryError(ComponentError):
+class TBBMemoryError(TBBComponentError):
pass
-class TBBVoltageError(ComponentError):
+class TBBVoltageError(TBBComponentError):
voltage_1_2 = models.FloatField(default=0)
voltage_2_5 = models.FloatField(default=0)
voltage_3_3 = models.FloatField(default=0)
-class TBBTemperatureError(ComponentError):
+class TBBTemperatureError(TBBComponentError):
pcb = models.FloatField(default=0)
tp = models.FloatField(default=0)
mp0 = models.FloatField(default=0)
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/componenterror.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/componenterror.py
index 31a0be8f0be..a035137f257 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/componenterror.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/componenterror.py
@@ -3,6 +3,7 @@ from rest_polymorphic.serializers import PolymorphicSerializer
from ..models.fixed_types import COMPONENT_TYPES, ERROR_TYPES
from ..models.stationtest import StationTest
+from ..models.componenterror import ComponentError
class ComponentErrorSerializer(serializers.Serializer):
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/componenterrors_generic.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/componenterrors_generic.py
index 06ed5538937..f6dd439b88e 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/componenterrors_generic.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/componenterrors_generic.py
@@ -39,6 +39,7 @@ class ComponentErrorPolimorphicSerializer(PolymorphicSerializer):
TBBComponentError: TBBComponentErrorSerializer,
TBBMemoryError: TBBMemoryErrorSerializer,
TBBVoltageError: TBBVoltageErrorSerializer,
+ TBBVersionError: TBBVersionErrorSerializer,
TBBTemperatureError: TBBTemperatureErrorSerializer,
SPUComponentError: SPUComponentErrorSerializer
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/hbaerrorserializers.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/hbaerrorserializers.py
index d35d38b2e8c..ff6319626dd 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/hbaerrorserializers.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/hbaerrorserializers.py
@@ -9,7 +9,7 @@ logger = logging.getLogger(__name__)
class HBAComponentErrorSerializer(ComponentErrorSerializer):
class Meta:
model = HBAComponentError
- fields = ('__all__',)
+ fields = '__all__'
def create(self, validated_data):
tile_errors_data = validated_data.pop('tile_errors')
@@ -30,9 +30,11 @@ class HBAComponentErrorSerializer(ComponentErrorSerializer):
class HBASummatorErrorSerializer(HBAComponentErrorSerializer):
class Meta:
model = HBASummatorError
- fields = ('__all__',)
+ fields = '__all__'
def create(self, validated_data):
+ if 'tile_errors' in validated_data:
+ validated_data.pop('tile_errors')
return HBASummatorError.objects.create(**validated_data)
@@ -41,7 +43,7 @@ class HBASummatorErrorSerializer(HBAComponentErrorSerializer):
class HBASummatorNoiseSerializer(HBAComponentErrorSerializer):
class Meta:
model = HBASummatorNoise
- fields = ('__all__',)
+ fields = '__all__'
def create(self, validated_data):
if 'tile_errors' in validated_data:
@@ -54,7 +56,7 @@ class HBAModemErrorSerializer(HBAComponentErrorSerializer):
class Meta:
model = HBAModemError
- fields = ('__all__',)
+ fields = '__all__'
class HBANoiseSerializer(HBAComponentErrorSerializer):
@@ -62,7 +64,7 @@ class HBANoiseSerializer(HBAComponentErrorSerializer):
class Meta:
model = HBANoiseError
- fields = ('__all__',)
+ fields = '__all__'
class HBAJitterSerializer(HBAComponentErrorSerializer):
@@ -70,7 +72,7 @@ class HBAJitterSerializer(HBAComponentErrorSerializer):
class Meta:
model = HBAJitterError
- fields = ('__all__',)
+ fields = '__all__'
class HBAOscillationErrorSerializer(HBAComponentErrorSerializer ):
@@ -78,7 +80,7 @@ class HBAOscillationErrorSerializer(HBAComponentErrorSerializer ):
class Meta:
model = HBAOscillationError
- fields = ('__all__',)
+ fields = '__all__'
class HBASpuriousSerializer(HBAComponentErrorSerializer):
@@ -86,9 +88,7 @@ class HBASpuriousSerializer(HBAComponentErrorSerializer):
class Meta:
model = HBASpuriousError
-
- def create(self, validated_data):
- return HBASpuriousError.objects.create(**validated_data)
+ fields = '__all__'
class HBARFFailSerializer(HBAComponentErrorSerializer):
@@ -96,4 +96,4 @@ class HBARFFailSerializer(HBAComponentErrorSerializer):
class Meta:
model = HBARFFailError
- fields = ('__all__',)
\ No newline at end of file
+ fields = '__all__'
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/lbaerrorserializer.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/lbaerrorserializer.py
index beb4b76e712..30b99dd9e90 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/lbaerrorserializer.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/lbaerrorserializer.py
@@ -5,20 +5,16 @@ from .componenterror import ComponentErrorSerializer
class LBAComponentErrorSerializer(ComponentErrorSerializer):
class Meta:
model = LBAComponentError
-
-class LBADownErrorSerializer(LBAComponentErrorSerializer):
- class Meta:
- model = LBADownError
- fields = ('__all__', )
+ fields = '__all__'
def create(self, validated_data):
- return LBADownError.objects.create(**validated_data)
+ return LBAComponentError.objects.create(**validated_data)
class LBADownErrorSerializer(LBAComponentErrorSerializer):
class Meta:
model = LBADownError
- fields = ('__all__', )
+ fields = '__all__'
def create(self, validated_data):
return LBADownError.objects.create(**validated_data)
@@ -27,7 +23,7 @@ class LBADownErrorSerializer(LBAComponentErrorSerializer):
class LBARFFailSerializer(LBAComponentErrorSerializer):
class Meta:
model = LBARFFailError
- fields = ('__all__', )
+ fields = '__all__'
def create(self, validated_data):
return LBARFFailError.objects.create(**validated_data)
@@ -36,6 +32,7 @@ class LBARFFailSerializer(LBAComponentErrorSerializer):
class LBAFlatErrorSerializer(LBAComponentErrorSerializer):
class Meta:
model = LBAFlatError
+ fields = '__all__'
def create(self, validated_data):
return LBAFlatError.objects.create(**validated_data)
@@ -44,6 +41,7 @@ class LBAFlatErrorSerializer(LBAComponentErrorSerializer):
class LBAShortErrorSerializer(LBAComponentErrorSerializer):
class Meta:
model = LBAShortError
+ fields = '__all__'
def create(self, validated_data):
return LBAShortError.objects.create(**validated_data)
@@ -52,6 +50,7 @@ class LBAShortErrorSerializer(LBAComponentErrorSerializer):
class LBAOscillationSerializer(LBAComponentErrorSerializer):
class Meta:
model = LBAOscillationError
+ fields = '__all__'
def create(self, validated_data):
return LBAOscillationError.objects.create(**validated_data)
@@ -60,6 +59,7 @@ class LBAOscillationSerializer(LBAComponentErrorSerializer):
class LBASpuriousSerializer(LBAComponentErrorSerializer):
class Meta:
model = LBASpuriousError
+ fields = '__all__'
def create(self, validated_data):
return LBASpuriousError.objects.create(**validated_data)
@@ -68,6 +68,7 @@ class LBASpuriousSerializer(LBAComponentErrorSerializer):
class LBANoiseSerializer(LBAComponentErrorSerializer):
class Meta:
model = LBANoiseError
+ fields = '__all__'
def create(self, validated_data):
return LBANoiseError.objects.create(**validated_data)
@@ -76,6 +77,7 @@ class LBANoiseSerializer(LBAComponentErrorSerializer):
class LBAJitterSerializer(LBAComponentErrorSerializer):
class Meta:
model = LBAJitterError
+ fields = '__all__'
def create(self, validated_data):
return LBAJitterError.objects.create(**validated_data)
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/rcuerrorserializer.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/rcuerrorserializer.py
index 08dc4f149f4..f4c6566ac5a 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/rcuerrorserializer.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/rcuerrorserializer.py
@@ -5,6 +5,7 @@ from .componenterror import ComponentErrorSerializer
class RCUComponentErrorSerializer(ComponentErrorSerializer):
class Meta:
model = RCUComponentError
+ fields = '__all__'
def create(self, validated_data):
instance = self.Meta.models.objects.create(**validated_data)
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/rsperrorserializer.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/rsperrorserializer.py
index edf2ea1a557..71c02f544ea 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/rsperrorserializer.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/rsperrorserializer.py
@@ -5,6 +5,7 @@ from .componenterror import ComponentErrorSerializer
class RSPComponentErrorSerializer(ComponentErrorSerializer):
class Meta:
model = RSPComponentError
+ fields = '__all__'
def create(self, validated_data):
instance = self.Meta.model.objects.create(**validated_data)
@@ -14,13 +15,16 @@ class RSPComponentErrorSerializer(ComponentErrorSerializer):
class RSPVersionErrorSerializer(RSPComponentErrorSerializer):
class Meta:
model = RSPVersionError
+ fields = '__all__'
class RSPVoltageErrorSerializer(RSPComponentErrorSerializer):
class Meta:
model = RSPVoltageError
+ fields = '__all__'
class RSPTemperatureErrorSerializer(RSPComponentErrorSerializer):
class Meta:
model = RSPTemperatureError
+ fields = '__all__'
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/spuerrorserializer.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/spuerrorserializer.py
index 7b56587b771..daab0eea2a5 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/spuerrorserializer.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/spuerrorserializer.py
@@ -5,6 +5,7 @@ from .componenterror import ComponentErrorSerializer
class SPUComponentErrorSerializer(ComponentErrorSerializer):
class Meta:
model = SPUComponentError
+ fields = '__all__'
def create(self, validated_data):
instance = self.Meta.models.objects.create(**validated_data)
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/stationtest.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/stationtest.py
index b8ee1828231..ec636e85b91 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/stationtest.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/stationtest.py
@@ -36,11 +36,18 @@ class StationTestSerializer(serializers.ModelSerializer):
return element
except ObjectDoesNotExist:
logger.info('station test not found inserting %s', validated_data)
+
station_test_instance = StationTest.objects.create(**validated_data)
station_test_instance.save()
for component_error in component_error_data:
component_error['station_test'] = station_test_instance
- component_error_instance = ComponentErrorPolimorphicSerializer().create(component_error)
+ try:
+ component_error_instance = ComponentErrorPolimorphicSerializer().create(dict(component_error))
+ except Exception as e:
+ print(e)
+ print(component_error)
+ import sys
+ sys.exit(1)
component_error_instance.save()
return station_test_instance
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/tbberrorserializer.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/tbberrorserializer.py
index f1e9465b4e3..94cbad2c5d5 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/tbberrorserializer.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/tbberrorserializer.py
@@ -5,6 +5,7 @@ from .componenterror import ComponentErrorSerializer
class TBBComponentErrorSerializer(ComponentErrorSerializer):
class Meta:
model = TBBComponentError
+ fields = '__all__'
def create(self, validated_data):
instance = self.Meta.model.objects.create(**validated_data)
@@ -14,18 +15,22 @@ class TBBComponentErrorSerializer(ComponentErrorSerializer):
class TBBVersionErrorSerializer(TBBComponentErrorSerializer):
class Meta:
model = TBBVersionError
+ fields = '__all__'
class TBBMemoryErrorSerializer(TBBComponentErrorSerializer):
class Meta:
model = TBBMemoryError
+ fields = '__all__'
class TBBVoltageErrorSerializer(TBBComponentErrorSerializer):
class Meta:
model = TBBVoltageError
+ fields = '__all__'
class TBBTemperatureErrorSerializer(TBBComponentErrorSerializer):
class Meta:
- model = TBBTemperatureError
\ No newline at end of file
+ model = TBBTemperatureError
+ fields = '__all__'
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py b/LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py
index ac385829b85..e7c6b9dee9d 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py
@@ -68,7 +68,16 @@ def parse_raw_station_test(content):
print('content splitted ', content.split('\n'))
station_tests = split_history_into_tests(content.split('\n'))
print('station_tests_are ', station_tests)
- return [dict_from_raw_station_test(stest) for stest in station_tests]
+ results = []
+ for stest in station_tests:
+ dict_stest = dict_from_raw_station_test(stest)
+ if 'start_time' in dict_stest:
+ results.append(dict_stest)
+ else:
+ import sys
+ print(stest)
+ sys.exit(1)
+ return results
def split_history_into_tests(content):
@@ -78,7 +87,7 @@ def split_history_into_tests(content):
next_line_columns = content[i+1].split(',')
type = next_line_columns[3]
current_test.append(line)
- if 'VERSIONS' in type or ('STATION' in type and current_test):
+ if 'VERSIONS' in type or ('STATION' in type and not current_test):
all_tests.append(current_test)
current_test = []
all_tests.append(current_test)
@@ -165,11 +174,10 @@ def dict_from_hba_component_error(content):
if error_type in ['C_SUMMATOR', 'P_SUMMATOR']:
resourcetype = 'HBASummatorError'
type = error_type[0].capitalize()
- result.update(type = type)
+ result.update(type=type)
elif error_type == 'MODEM':
resourcetype = 'HBAModemError'
- tile_errors = dicts_from_tile_error(extra_args)
- result.update(tile_errors=tile_errors)
+
elif error_type == 'RF_FAIL':
resourcetype = 'HBARFFailError'
for arg in extra_args:
@@ -180,7 +188,7 @@ def dict_from_hba_component_error(content):
subband_used_nodelay,\
subband_used_fulldelay,\
reference_signal_nodelay,\
- reference_signal_fulldelay = map(float, values.split(' '))
+ reference_signal_fulldelay = map(float, values.replace('nan', '999').split(' '))
result['measured_signal_nodelay' + polarization] = measured_signal_nodelay
result['measured_signal_fulldelay' + polarization] = measured_signal_fulldelay
@@ -310,7 +318,10 @@ def dict_from_lba_component_error(content):
key, value = arg.popitem()
polarization = 'X' if 'X' in key else 'Y'
key = key.strip('X').strip('Y')
+ key = 'val' if key == 'ref' else key
+
key = translate[key] + polarization
+
if 'value' in key:
key = 'average_' + key
result[key] = float(value)
@@ -394,6 +405,8 @@ def dict_from_raw_station_test(content):
pass
elif type == 'STATISTICS':
pass
+ elif type == 'DRIVER':
+ pass
elif type == "E_FAIL":
tile_errors = dicts_from_tile_error(row[4:])
translate_tile_error_to_component_error_type = dict(TileModemError='HBAModemError',
@@ -410,8 +423,6 @@ def dict_from_raw_station_test(content):
TileRFFailure='RF_FAIL')
for tile_error in tile_errors:
- if tile_error['resourcetype'] == 'TileError':
- print(row)
linked_type = translate_tile_error_to_component_error_type[tile_error['resourcetype']]
component_id = int(row[2])
if (component_id, linked_type) not in result['components_error']:
@@ -464,7 +475,7 @@ def dicts_from_tile_error(contents):
elif error_type == '':
resourcetype = 'TileRFFailure'
measured_signal_nodelay, measured_signal_fulldelay, subband_used_nodelay, subband_used_fulldelay, \
- reference_signal_nodelay, reference_signal_fulldelay = map(float, args.strip().split(' '))
+ reference_signal_nodelay, reference_signal_fulldelay = map(float, args.replace('nan', '999').split(' '))
item.update(measured_signal_fulldelay=measured_signal_fulldelay,
measured_signal_nodelay=measured_signal_nodelay,
subband_used_fulldelay=subband_used_fulldelay,
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/views.py b/LCU/Maintenance/DBInterface/monitoringdb/views.py
index d0a86a9f0ec..d4a50496302 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/views.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/views.py
@@ -16,7 +16,7 @@ import logging
logger = logging.getLogger(__name__)
-
+RESERVED_FILTER_NAME = ['limit', 'offset', 'format']
class RTSMObservationViewSet(viewsets.ModelViewSet):
queryset = RTSMObservation.objects.all()
serializer_class = RTSMObservationSerializer
@@ -24,6 +24,8 @@ class RTSMObservationViewSet(viewsets.ModelViewSet):
def get_queryset(self):
queryset = RTSMObservation.objects.all()
for key, param in self.request.query_params.items():
+ if key in RESERVED_FILTER_NAME:
+ continue
queryset = queryset.filter(**{key: param})
return queryset
@@ -43,6 +45,8 @@ class StationTestViewSet(viewsets.ModelViewSet):
"""
queryset = StationTest.objects.all()
for key, param in self.request.query_params.items():
+ if key in RESERVED_FILTER_NAME:
+ continue
queryset = queryset.filter(**{key: param})
return queryset
diff --git a/LCU/Maintenance/MDB_tools/cli/mdb_loader.py b/LCU/Maintenance/MDB_tools/cli/mdb_loader.py
new file mode 100644
index 00000000000..aaee02a4d8e
--- /dev/null
+++ b/LCU/Maintenance/MDB_tools/cli/mdb_loader.py
@@ -0,0 +1,91 @@
+import logging
+import argparse
+import requests
+import sys
+import json
+from glob import glob
+import os
+import re
+
+from datetime import datetime
+
+logger = logging.getLogger('probe_mdb')
+
+"""
+This program is meant to load the station tests and RTSM present in a certain directory to the database
+"""
+
+def setup_argument_parser():
+ parser = argparse.ArgumentParser(prog='probe_mdb')
+ parser.add_argument('path', help='format and path format of the file to load. es. /where/is/stored/*.dat')
+ parser.add_argument('-s', '--station_test', help='probe station tests', action='store_true', default=False)
+ parser.add_argument('-r', '--rtsm', help='probe rtsm tests', action='store_true', default=False)
+ parser.add_argument('--address', help='address of the server. default [localhost]:8000',
+ default='http://localhost:8000')
+
+ return parser
+
+
+def parse_arguments(parser):
+ args = parser.parse_args()
+ if args.station_test or args.rtsm:
+ return parser.parse_args()
+ else:
+ parser.print_help()
+ sys.exit(1)
+
+
+def obtain_file_list(path):
+ logger.info('listing file in %s', path)
+ file_list = filter(os.path.isfile, glob(path))
+ logger.debug('found files: %s', file_list)
+ return file_list
+
+
+def read_file_to_json(filename, is_rtsm):
+ rtsm_filename_pattern = '(?P<station_name>\w*)_\w*_\w*.dat'
+ try:
+ with open(filename, 'r') as input_stream:
+ result = dict(content=input_stream.read().strip('\n'))
+ if is_rtsm:
+ result.update(station_name=re.search(rtsm_filename_pattern, filename).group('station_name'))
+ return result
+ except Exception as e:
+ logging.error('cannot read file %s', filename)
+ logging.exception(e)
+
+
+def perform_request(args, content):
+ address = args.address
+ full_address = '/'.join([address, create_query_string(args)])
+ logging.info('performing request to address %s', full_address)
+ json_content = json.dumps(content)
+ logging.debug('request content %s', json_content)
+ response = requests.post(full_address, data=content)
+ logging.info('response acknowledged: status code is %s', response.status_code)
+ if response.status_code == 200:
+ return True
+ else:
+ logging.error('error sending request %s', response.reason)
+ return False
+
+
+def create_query_string(args):
+ query = ''
+ if args.station_test:
+ query += 'stationtests/insert_raw'
+ elif args.rtsm:
+ query += 'rtsm/insert_raw'
+ return query
+
+
+if __name__=='__main__':
+ logging.basicConfig(format="%(asctime)s %(levelname)s: %(message)s",level=logging.DEBUG)
+ parser = setup_argument_parser()
+ args = parse_arguments(parser)
+
+ for filename in obtain_file_list(args.path):
+ logging.info('processing file %s', filename)
+ content = read_file_to_json(filename, args.rtsm)
+ perform_request(args, content)
+ logging.info('file %s processed', filename)
\ No newline at end of file
diff --git a/LCU/Maintenance/MDB_tools/cli/probe_mdb.py b/LCU/Maintenance/MDB_tools/cli/probe_mdb.py
index 69808b34a1f..feb75b06ac9 100644
--- a/LCU/Maintenance/MDB_tools/cli/probe_mdb.py
+++ b/LCU/Maintenance/MDB_tools/cli/probe_mdb.py
@@ -9,6 +9,12 @@ from datetime import datetime
logger = logging.getLogger('probe_mdb')
+"""
+This program is meant to probe from the command line the station tests and RTSM in a certain period of
+time and on a certain station name
+"""
+
+
def setup_argument_parser():
parser = argparse.ArgumentParser(prog='probe_mdb')
parser.add_argument('--address', help='address of the server. default [localhost]:8000', default='http://localhost:8000')
@@ -61,12 +67,12 @@ def flatten_station_test_results(results):
def result_to_pandas(result):
print(result)
dataframe = json_normalize(result, meta=['start_time', 'end_time', 'station_name', 'resource_type', 'component_id'])
- print (dataframe)
+ return dataframe
def create_query_string(args):
query = ''
- if args.station:
+ if args.station_test:
query += 'rtsm?'
elif args.rtsm:
query += 'stationtests?'
@@ -103,4 +109,7 @@ if __name__=='__main__':
if args is None:
sys.exit(1)
result = perform_query(args)
- result_to_pandas('', result)
+ print(result)
+ pandas_table = result_to_pandas(result)
+ if args.to_csv:
+ pandas_table.to_csv(args.to_csv)
\ No newline at end of file
--
GitLab