diff --git a/.gitattributes b/.gitattributes
index 5c6549f611dfb69c9db0b87e0b47a0ae7fcfec73..f37db444339fdd94d39ff075699f3ff1913c93ff 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1787,8 +1787,6 @@ LCU/Maintenance/DBInterface/monitoringdb/models/componenterror.py -text
LCU/Maintenance/DBInterface/monitoringdb/models/fixed_types.py -text
LCU/Maintenance/DBInterface/monitoringdb/models/hbacomponenterror.py -text
LCU/Maintenance/DBInterface/monitoringdb/models/lbacomponenterror.py -text
-LCU/Maintenance/DBInterface/monitoringdb/models/mongoengine/__init__.py -text
-LCU/Maintenance/DBInterface/monitoringdb/models/mongoengine/station_test.py -text
LCU/Maintenance/DBInterface/monitoringdb/models/rcucomponenterror.py -text
LCU/Maintenance/DBInterface/monitoringdb/models/rspcomponenterror.py -text
LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py -text
@@ -1802,7 +1800,6 @@ LCU/Maintenance/DBInterface/monitoringdb/serializers/componenterror.py -text
LCU/Maintenance/DBInterface/monitoringdb/serializers/componenterrors_generic.py -text
LCU/Maintenance/DBInterface/monitoringdb/serializers/hbaerrorserializers.py -text
LCU/Maintenance/DBInterface/monitoringdb/serializers/lbaerrorserializer.py -text
-LCU/Maintenance/DBInterface/monitoringdb/serializers/mongoengine.py -text
LCU/Maintenance/DBInterface/monitoringdb/serializers/rcuerrorserializer.py -text
LCU/Maintenance/DBInterface/monitoringdb/serializers/rsperrorserializer.py -text
LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py -text
@@ -1814,7 +1811,6 @@ LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py -text
LCU/Maintenance/DBInterface/monitoringdb/tests/__init__.py -text
LCU/Maintenance/DBInterface/monitoringdb/tests/common.py -text
LCU/Maintenance/DBInterface/monitoringdb/tests/old_tests.py -text
-LCU/Maintenance/DBInterface/monitoringdb/tests/t_nosqlconfiguration.py -text
LCU/Maintenance/DBInterface/monitoringdb/tests/t_stationtest_model_creation.py -text
LCU/Maintenance/DBInterface/monitoringdb/tests/t_stationtest_parser.py -text
LCU/Maintenance/DBInterface/monitoringdb/tests/test_stationtestdata.csv -text
diff --git a/LCU/Maintenance/DBInterface/django_postgresql/settings.py b/LCU/Maintenance/DBInterface/django_postgresql/settings.py
index 95205da44f5b5631f9989acebf3a29c242e517f8..f866e7e0dbd7c3e44bb7f3656ee37fab94cfb57a 100644
--- a/LCU/Maintenance/DBInterface/django_postgresql/settings.py
+++ b/LCU/Maintenance/DBInterface/django_postgresql/settings.py
@@ -85,7 +85,7 @@ LOGGING = {
'console': {
'class': 'logging.StreamHandler',
'formatter': 'verbose',
- 'level': 'INFO'
+ 'level': 'DEBUG'
},
'log_file': {
'level': 'DEBUG',
@@ -107,6 +107,10 @@ LOGGING = {
'handlers': ['console'],
'level': 'DEBUG',
},
+ 'django': {
+ 'handlers': ['console'],
+ 'level': 'DEBUG',
+ },
},
}
@@ -125,20 +129,6 @@ DATABASES = {
}
}
-
-MONGO_DB_SESSION = 'default'
-
-
-MONGO_DBS = {
- 'default': {
- 'db': 'monitoringdb',
- 'host': 'mongodb://lofarmonitortest.control.lofar:27017/'
- }
-}
-
-
-def default_connection():
- mongoengine.connect(**MONGO_DBS[MONGO_DB_SESSION])
# Password validation
# https://docs.djangoproject.com/en/2.0/ref/settings/#auth-password-validators
@@ -180,5 +170,6 @@ STATIC_URL = '/static/'
REST_FRAMEWORK = {
'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination',
- 'PAGE_SIZE': 5
+ 'PAGE_SIZE': 100,
+ 'STRICT_JSON': False
}
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/hbacomponenterror.py b/LCU/Maintenance/DBInterface/monitoringdb/models/hbacomponenterror.py
index 57949a08cad99a8253b7039a948c583eee393a87..ffc17f2bdcadae0513381c18bbd2be06b6e00587 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/models/hbacomponenterror.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/models/hbacomponenterror.py
@@ -20,50 +20,50 @@ class HBAModemError(HBAComponentError):
class HBARFFailError(HBAComponentError):
- measured_signal_nodelayX = models.FloatField(default=0)
- measured_signal_fulldelayX = models.FloatField(default=0)
+ measured_signal_nodelayX = models.FloatField(null=True, default=None)
+ measured_signal_fulldelayX = models.FloatField(null=True, default=None)
- subband_used_nodelayX = models.PositiveSmallIntegerField(default=0)
- subband_used_fulldelayX = models.PositiveSmallIntegerField(default=0)
+ subband_used_nodelayX = models.PositiveSmallIntegerField(null=True,default=None)
+ subband_used_fulldelayX = models.PositiveSmallIntegerField(null=True,default=None)
- reference_signal_nodelayX = models.FloatField(default=0)
- reference_signal_fulldelayX = models.FloatField(default=0)
+ reference_signal_nodelayX = models.FloatField(null=True,default=None)
+ reference_signal_fulldelayX = models.FloatField(null=True,default=None)
- measured_signal_nodelayY = models.FloatField(default=0)
- measured_signal_fulldelayY = models.FloatField(default=0)
+ measured_signal_nodelayY = models.FloatField(default=None, null=True)
+ measured_signal_fulldelayY = models.FloatField(default=None, null=True)
- subband_used_nodelayY = models.PositiveSmallIntegerField(default=0)
- subband_used_fulldelayY = models.PositiveSmallIntegerField(default=0)
+ subband_used_nodelayY = models.PositiveSmallIntegerField(default=None, null=True)
+ subband_used_fulldelayY = models.PositiveSmallIntegerField(default=None, null=True)
- reference_signal_nodelayY = models.FloatField(default=0)
- reference_signal_fulldelayY = models.FloatField(default=0)
+ reference_signal_nodelayY = models.FloatField(default=None, null=True)
+ reference_signal_fulldelayY = models.FloatField(default=None, null=True)
class HBANoiseError(HBAComponentError):
type = models.CharField(default='-', max_length=1, choices=NOISE_ERROR_TYPE)
- percentageX = models.FloatField(default=-1)
- percentageY = models.FloatField(default=-1)
+ percentageX = models.FloatField(default=None, null=True)
+ percentageY = models.FloatField(default=None, null=True)
- peak_valueX = models.FloatField(default=-1)
- peak_valueY = models.FloatField(default=-1)
+ peak_valueX = models.FloatField(default=None, null=True)
+ peak_valueY = models.FloatField(default=None, null=True)
- fluctuationX = models.FloatField(default=-1)
- fluctuationY = models.FloatField(default=-1)
+ fluctuationX = models.FloatField(default=None, null=True)
+ fluctuationY = models.FloatField(default=None, null=True)
- limitX = models.FloatField(default=-1)
- limitY = models.FloatField(default=-1)
+ limitX = models.FloatField(default=None, null=True)
+ limitY = models.FloatField(default=None, null=True)
class HBAJitterError(HBAComponentError):
- percentageX = models.FloatField(default=-1)
- percentageY = models.FloatField(default=-1)
+ percentageX = models.FloatField(default=None, null=True)
+ percentageY = models.FloatField(default=None, null=True)
- average_valueX = models.FloatField(default=-1)
- average_valueY = models.FloatField(default=-1)
+ average_valueX = models.FloatField(default=None, null=True)
+ average_valueY = models.FloatField(default=None, null=True)
- fluctuationX = models.FloatField(default=-1)
- fluctuationY = models.FloatField(default=-1)
+ fluctuationX = models.FloatField(default=None, null=True)
+ fluctuationY = models.FloatField(default=None, null=True)
class HBAOscillationError(HBAComponentError):
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/lbacomponenterror.py b/LCU/Maintenance/DBInterface/monitoringdb/models/lbacomponenterror.py
index d8b73cbf73a7a72dd5a2a6db31aff17caf70d089..3026a489afbd79c6ba9dc843f84d8809a34828d0 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/models/lbacomponenterror.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/models/lbacomponenterror.py
@@ -5,30 +5,29 @@ from .stationtest import StationTest
class LBAComponentError(ComponentError):
- pass
lba_type = models.CharField(max_length=1)
class LBADownError(LBAComponentError):
- signalX = models.FloatField(default=-1)
- signalY = models.FloatField(default=-1)
- signalX_offset = models.FloatField(default=-1)
- signalY_offset = models.FloatField(default=-1)
+ signalX = models.FloatField(default=None, null=True)
+ signalY = models.FloatField(default=None, null=True)
+ signalX_offset = models.FloatField(default=None, null=True)
+ signalY_offset = models.FloatField(default=None, null=True)
class LBARFFailError(LBAComponentError):
- signalX = models.FloatField(default=0)
- signalY = models.FloatField(default=0)
+ signalX = models.FloatField(default=None, null=True)
+ signalY = models.FloatField(default=None, null=True)
class LBAFlatError(LBAComponentError):
- mean_signalX = models.FloatField(default=0)
- mean_signalY = models.FloatField(default=0)
+ mean_signalX = models.FloatField(default=None, null=True)
+ mean_signalY = models.FloatField(default=None, null=True)
class LBAShortError(LBAComponentError):
- mean_signalX = models.FloatField(default=0)
- mean_signalY = models.FloatField(default=0)
+ mean_signalX = models.FloatField(default=None, null=True)
+ mean_signalY = models.FloatField(default=None, null=True)
class LBAOscillationError(LBAComponentError):
@@ -44,25 +43,25 @@ class LBASpuriousError(LBAComponentError):
class LBANoiseError(LBAComponentError):
type = models.CharField(default='-', max_length=1, choices=NOISE_ERROR_TYPE)
- percentageX = models.FloatField(default=-1)
- percentageY = models.FloatField(default=-1)
+ percentageX = models.FloatField(default=None, null=True)
+ percentageY = models.FloatField(default=None, null=True)
- peak_valueX = models.FloatField(default=-1)
- peak_valueY = models.FloatField(default=-1)
+ peak_valueX = models.FloatField(default=None, null=True)
+ peak_valueY = models.FloatField(default=None, null=True)
- fluctuationX = models.FloatField(default=-1)
- fluctuationY = models.FloatField(default=-1)
+ fluctuationX = models.FloatField(default=None, null=True)
+ fluctuationY = models.FloatField(default=None, null=True)
- limitX = models.FloatField(default=-1)
- limitY = models.FloatField(default=-1)
+ limitX = models.FloatField(default=None, null=True)
+ limitY = models.FloatField(default=None, null=True)
class LBAJitterError(LBAComponentError):
- percentageX = models.FloatField(default=-1)
- percentageY = models.FloatField(default=-1)
+ percentageX = models.FloatField(default=None, null=True)
+ percentageY = models.FloatField(default=None, null=True)
- average_valueX = models.FloatField(default=-1)
- average_valueY = models.FloatField(default=-1)
+ average_valueX = models.FloatField(default=None, null=True)
+ average_valueY = models.FloatField(default=None, null=True)
- fluctuationX = models.FloatField(default=-1)
- fluctuationY = models.FloatField(default=-1)
\ No newline at end of file
+ fluctuationX = models.FloatField(default=None, null=True)
+ fluctuationY = models.FloatField(default=None, null=True)
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/mongoengine/__init__.py b/LCU/Maintenance/DBInterface/monitoringdb/models/mongoengine/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/mongoengine/station_test.py b/LCU/Maintenance/DBInterface/monitoringdb/models/mongoengine/station_test.py
deleted file mode 100644
index 51f76d9231c11044a0ad06e6eaed9bcf2c369ef0..0000000000000000000000000000000000000000
--- a/LCU/Maintenance/DBInterface/monitoringdb/models/mongoengine/station_test.py
+++ /dev/null
@@ -1,45 +0,0 @@
-import ming
-from lofar.maintenance.django_postgresql.settings import default_connection
-
-from mongoengine.document import DynamicDocument, EmbeddedDocument, DynamicEmbeddedDocument
-import mongoengine.fields as schema
-
-default_connection()
-
-
-class StationTest(EmbeddedDocument):
- station_name = schema.StringField(required=True)
- station_type = schema.StringField(required=True)
- start_time = schema.DateTimeField(required=True)
- end_time = schema.DateTimeField(required=True)
- performed_checks = schema.StringField(required=True)
-
-
-class TileError(DynamicEmbeddedDocument):
- tile_id = schema.IntField(required=True)
-
-
-class ComponentError(DynamicDocument):
- error_type = schema.StringField(required=True)
- component_type = schema.StringField(required=True)
- station_test = schema.EmbeddedDocumentField(StationTest)
-
- component_id = schema.IntField(required=True)
-
- meta = {'allow_inheritance': True,
- 'indexes': [
- {'name': 'component_error_unique',
- 'fields': ['$station_test.station_name', '-station_test.start_time', 'error_type', 'component_type',
- 'component_id', ],
- 'unique': True,
- 'cls': False
- }
- ]}
-
-
-class LBAComponentError(ComponentError):
- lba_type = schema.StringField(required=True)
-
-
-class HBAComponentError(ComponentError):
- tile_errors = schema.EmbeddedDocumentField(TileError)
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/rspcomponenterror.py b/LCU/Maintenance/DBInterface/monitoringdb/models/rspcomponenterror.py
index 93c8431a4caf2d474756a9025dfb8c1ac6deccb7..f9563c8264d9d28fc3645e31210c8b16f6217549 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/models/rspcomponenterror.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/models/rspcomponenterror.py
@@ -14,15 +14,15 @@ class RSPVersionError(RSPComponentError):
class RSPVoltageError(RSPComponentError):
- voltage_1_2 = models.FloatField(default=0)
- voltage_2_5 = models.FloatField(default=0)
- voltage_3_3 = models.FloatField(default=0)
+ voltage_1_2 = models.FloatField(default=None, null=True)
+ voltage_2_5 = models.FloatField(default=None, null=True)
+ voltage_3_3 = models.FloatField(default=None, null=True)
class RSPTemperatureError(RSPComponentError):
- pcb = models.FloatField(default=0)
- bp = models.FloatField(default=0)
- ap0 = models.FloatField(default=0)
- ap1 = models.FloatField(default=0)
- ap2 = models.FloatField(default=0)
- ap3 = models.FloatField(default=0)
+ pcb = models.FloatField(default=None, null=True)
+ bp = models.FloatField(default=None, null=True)
+ ap0 = models.FloatField(default=None, null=True)
+ ap1 = models.FloatField(default=None, null=True)
+ ap2 = models.FloatField(default=None, null=True)
+ ap3 = models.FloatField(default=None, null=True)
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py b/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py
index dd763e7f98866744e461892266310bffb3d50d5a..c634c9f5ade5caa3b101ed98d9bb9731e11e914c 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py
@@ -16,16 +16,16 @@ class RTSMObservation(models.Model):
start_time = models.DateTimeField()
end_time = models.DateTimeField()
- samples = models.IntegerField(default=0)
+ samples = models.IntegerField(default=None, null=True)
class RTSMError(models.Model):
- rcu = models.SmallIntegerField(default=-1)
- mode = models.SmallIntegerField(default=0)
+ rcu = models.SmallIntegerField(default=None, null=True)
+ mode = models.SmallIntegerField(default=None, null=True)
observation = models.ForeignKey(RTSMObservation, related_name='errors', on_delete=models.CASCADE)
error_type = models.CharField(max_length=10)
- start_frequency = models.FloatField(default=0)
- stop_frequency = models.FloatField(default=0)
+ start_frequency = models.FloatField(default=None, null=True)
+ stop_frequency = models.FloatField(default=None, null=True)
time = models.DateTimeField()
bad_spectrum = ArrayField(models.FloatField())
average_spectrum = ArrayField(models.FloatField())
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/tbbcomponenterror.py b/LCU/Maintenance/DBInterface/monitoringdb/models/tbbcomponenterror.py
index 8a6386532ab95990f0e357fb397b05c3f701cc6a..f3d810107aa6a401bceb4179537ff9d7f1ecad20 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/models/tbbcomponenterror.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/models/tbbcomponenterror.py
@@ -18,15 +18,15 @@ class TBBMemoryError(TBBComponentError):
class TBBVoltageError(TBBComponentError):
- voltage_1_2 = models.FloatField(default=0)
- voltage_2_5 = models.FloatField(default=0)
- voltage_3_3 = models.FloatField(default=0)
+ voltage_1_2 = models.FloatField(default=None, null=True)
+ voltage_2_5 = models.FloatField(default=None, null=True)
+ voltage_3_3 = models.FloatField(default=None, null=True)
class TBBTemperatureError(TBBComponentError):
- pcb = models.FloatField(default=0)
- tp = models.FloatField(default=0)
- mp0 = models.FloatField(default=0)
- mp1 = models.FloatField(default=0)
- mp2 = models.FloatField(default=0)
- mp3 = models.FloatField(default=0)
\ No newline at end of file
+ pcb = models.FloatField(default=None, null=True)
+ tp = models.FloatField(default=None, null=True)
+ mp0 = models.FloatField(default=None, null=True)
+ mp1 = models.FloatField(default=None, null=True)
+ mp2 = models.FloatField(default=None, null=True)
+ mp3 = models.FloatField(default=None, null=True)
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/componenterror.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/componenterror.py
index df2c879ce1c69f885031897fe9eaf34104fae83f..5cfd259cf1fed10d310f8b8e430c567aed1dfdf3 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/componenterror.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/componenterror.py
@@ -5,7 +5,7 @@ from ..models.stationtest import StationTest
from ..models.componenterror import ComponentError
-class ComponentErrorSerializer(serializers.Serializer):
+class ComponentErrorSerializer(serializers.ModelSerializer):
id = serializers.IntegerField(read_only=True)
station_test_id = serializers.PrimaryKeyRelatedField(queryset=StationTest.objects.all(), source='parent.id')
@@ -13,6 +13,10 @@ class ComponentErrorSerializer(serializers.Serializer):
component_type = serializers.ChoiceField(COMPONENT_TYPES)
error_type = serializers.ChoiceField(ERROR_TYPES)
+ class Meta:
+ model = ComponentError
+ fields = '__all__'
+
def create(self, validated_data):
"""
Create the component error given the fields
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/lbaerrorserializer.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/lbaerrorserializer.py
index 30b99dd9e90742ee13adfa7b20ed93d8d821d004..b7c95383cc97a650e6629a8570cec95e5c1b067f 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/lbaerrorserializer.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/lbaerrorserializer.py
@@ -1,7 +1,6 @@
from ..models.lbacomponenterror import *
from .componenterror import ComponentErrorSerializer
-
class LBAComponentErrorSerializer(ComponentErrorSerializer):
class Meta:
model = LBAComponentError
@@ -12,7 +11,7 @@ class LBAComponentErrorSerializer(ComponentErrorSerializer):
class LBADownErrorSerializer(LBAComponentErrorSerializer):
- class Meta:
+ class Meta(LBAComponentErrorSerializer.Meta):
model = LBADownError
fields = '__all__'
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/mongoengine.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/mongoengine.py
deleted file mode 100644
index a8e64f984a80ea7dab60a75aabb512e6af08e194..0000000000000000000000000000000000000000
--- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/mongoengine.py
+++ /dev/null
@@ -1,25 +0,0 @@
-from rest_framework.serializers import ModelSerializer
-import rest_framework.serializers as schema
-
-
-class StationTestSerializer(ModelSerializer):
- station_name = schema.CharField()
- station_type = schema.CharField()
- start_time = schema.DateTimeField()
- end_time = schema.DateTimeField()
- performed_checks = schema.CharField()
-
-
-class ComponentErrorSerializer(ModelSerializer):
- error_type = schema.CharField()
- component_type = schema.CharField()
- station_test = StationTestSerializer()
- component_id = schema.IntegerField()
-
-
-class TileErrorSerializer(ModelSerializer):
- tile_id = schema.IntegerField()
-
-
-class HBAComponentErrorSerializer(ComponentErrorSerializer):
- tile_errors = TileErrorSerializer(many=True)
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/tests/t_nosqlconfiguration.py b/LCU/Maintenance/DBInterface/monitoringdb/tests/t_nosqlconfiguration.py
deleted file mode 100644
index e0e516769be310773861afbae1d85edc00723aa5..0000000000000000000000000000000000000000
--- a/LCU/Maintenance/DBInterface/monitoringdb/tests/t_nosqlconfiguration.py
+++ /dev/null
@@ -1,98 +0,0 @@
-import lofar.maintenance.django_postgresql.settings as settings
-from lofar.maintenance.monitoringdb.models.ming.station_test import ComponentError, StationTest,LBAComponentError, HBAComponentError
-import unittest
-from datetime import datetime
-import pytz
-from mongoengine.errors import NotUniqueError
-
-def get_mock_dict_data():
- start_time = pytz.utc.localize(datetime(2014, 5, 12, 0, 0, 0))
- start_time2 = pytz.utc.localize(datetime(2014, 5, 25, 0, 0, 0))
-
- end_time = pytz.utc.localize(datetime(2014, 10, 12, 0, 0, 0))
-
- station_test = dict(station_name='FR001',
- station_type='I',
- start_time=start_time,
- end_time=end_time,
- performed_checks="franco lolo lello")
- station_test2 = dict(station_name='FR001',
- station_type='I',
- start_time=start_time2,
- end_time=end_time,
- performed_checks="franco lolo lello")
-
- component_errors = [
- dict(
- component_id=12,
- error_type='flagging',
- component_type='antenna',
- station_test=station_test,
- lba_type='ciccio'),
- dict(
- component_id=2,
- error_type='flagging',
- component_type='antenna',
- station_test=station_test),
- dict(
- component_id=4,
- error_type='flagging',
- component_type='antenna',
- station_test=station_test),
- dict(
- component_id=1,
- error_type='flagging',
- component_type='antenna',
- station_test=station_test),
- dict(
- component_id=12,
- error_type='flagging',
- component_type='antenna',
- station_test=station_test),
-
- ]
- new_component_errors = []
- for component_error in component_errors:
- new_component_errors.append(component_error)
- new_component_error = dict(component_error)
- new_component_error.update(station_test=station_test2)
- new_component_errors.append(new_component_error)
-
- return new_component_errors
-
-
-class TestMongoEngineDjango(unittest.TestCase):
-
- def clean_component_error_collection(self):
- ComponentError.drop_collection()
-
- def test_station_test_creation(self):
- errors = get_mock_dict_data()
- # There is a not unique error message so it should throw an exception
- with self.assertRaises(NotUniqueError) as e:
- for error in errors:
- st = StationTest(**error.pop('station_test'))
- if 'lba_type' in error:
- ce = LBAComponentError(**error, station_test=st)
- else:
- ce = HBAComponentError(**error, station_test=st)
- ce.save()
-
- def test_station_test_query(self):
- errors = get_mock_dict_data()
- try:
- for error in errors:
- st = StationTest(**error.pop('station_test'))
- if 'lba_type' in error:
- ce = LBAComponentError(**error, station_test=st)
- else:
- ce = HBAComponentError(**error, station_test=st)
- ce.save()
- except NotUniqueError as e:
- pass# it is intended behaviour
-
- lba_component_error = LBAComponentError.objects({})[0]
- self.assertEqual(lba_component_error.lba_type, 'ciccio')
-
- def tearDown(self):
- self.clean_component_error_collection()
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/urls.py b/LCU/Maintenance/DBInterface/monitoringdb/urls.py
index b71c83fffdf1248d583032ce124df0af44a713c3..dc7315516f2d52340f3c97743ac81158fbeb8da0 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/urls.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/urls.py
@@ -12,8 +12,8 @@ station_test_router.register(r'^', StationTestViewSet)
rtsm_router = routers.DefaultRouter()
#RTSM
-rtsm_router.register(r'/errors_detailed', RTSMErrorsDetailedViewSet)
-rtsm_router.register(r'/errors', RTSMErrorsViewSet)
+rtsm_router.register(r'^errors_detailed', RTSMErrorsDetailedViewSet)
+rtsm_router.register(r'^errors', RTSMErrorsViewSet)
rtsm_router.register(r'^', RTSMObservationViewSet)
diff --git a/LCU/Maintenance/MDB_tools/cli/probe_mdb.py b/LCU/Maintenance/MDB_tools/cli/probe_mdb.py
index f26f218be6f3bb6223e6958ab7a665291df9229b..d354d5e7dae1a7e5418ec46374734f16f7c72c0b 100644
--- a/LCU/Maintenance/MDB_tools/cli/probe_mdb.py
+++ b/LCU/Maintenance/MDB_tools/cli/probe_mdb.py
@@ -2,10 +2,13 @@ import argparse
import json
import logging
import sys
-from datetime import datetime
-import pprint
+from datetime import datetime, timedelta
+import beautifultable
+import blessings
+
import requests
from pandas.io.json import json_normalize
+from operator import itemgetter
logger = logging.getLogger('probe_mdb')
@@ -23,6 +26,7 @@ def setup_argument_parser():
parser.add_argument('--to_date', help='tests to date [YYYY-MM-DD]', default=None)
parser.add_argument('--station', help='select a specific station. es. CS001C', default=None)
parser.add_argument('--station_type', help='select a specific station type. es. I', choices=['C', 'R', 'I'])
+ parser.add_argument('--last-month', help='select the last month results', action='store_true')
parser.add_argument('--to_csv', help='prints results into a csv file', default=None)
return parser
@@ -30,44 +34,38 @@ def setup_argument_parser():
def parse_arguments(parser):
args = parser.parse_args()
- if args.station_test or args.rtsm:
- return parser.parse_args()
- else:
- parser.print_help()
+ return args
-def perform_query(query_string, address):
+
+def perform_query(query_string, address, next_url=None):
"""
Execute the query string to the specified address
:param query_string: the query string to be done
:param address: the address at which it will be done
+ :param next_url: if set it uses directly next_url as address. This is meant for recursive lookup.
:return: a list of dicts or an empty list
"""
- full_address = '/'.join([address, query_string])
+ if next_url is None:
+ full_address = '/'.join([address, query_string])
+ else:
+ full_address = next_url
logging.info('performing query %s', full_address)
response = requests.get(full_address)
logging.info('response acknowledged: status code is %s', response.status_code)
if response.status_code == 200:
- return json.loads(response.content)
+ parsed_content = json.loads(response.content)
+
+ extracted_content = parsed_content['results']
+ next_url = parsed_content['next']
+ if next_url is not None:
+ return extracted_content + perform_query(query_string, address, next_url)
+ else:
+ return extracted_content
else:
return []
-def flatten_station_test_results(results):
- returned_list = []
-
- for result in results:
- start_time = result['start_time']
- end_time = result['end_time']
- station_name = result['station_name']
-
- rcu_error_list = result.pop('rcu_errors')
- rcu_error_list = result.pop('spu_errors')
- rcu_error_list = result.pop('hba_errors')
- rcu_error_list = result.pop('lba_errors')
- rcu_error_list = result.pop('spu_errors')
-
-
def result_to_pandas(result):
print(result)
dataframe = json_normalize(result, meta=['start_time', 'end_time', 'station_name', 'resource_type', 'component_id'])
@@ -83,25 +81,35 @@ def get_query_string_for_time_limit(from_date, to_date):
:return: the query string
"""
query = ''
- try:
- date = datetime.strptime(from_date, '%Y-%m-%d')
- query += '&start_time__year__gte=%d' % (date.year)
- query += '&start_time__month__gte=%d' % (date.month)
- query += '&start_time__day__gte=%d' % (date.day)
-
- except ValueError:
- logger.error('format of from date not valid; YYYY-MM-DD es. 2016-15-12')
- sys.exit(1)
-
- try:
- date = datetime.strptime(to_date, '%Y-%m-%d')
- query += '&start_time__year__lte=%d' % (date.year)
- query += '&start_time__month__lte=%d' % (date.month)
- query += '&start_time__day__lte=%d' % (date.day)
-
- except ValueError:
- logger.error('format of to date not valid; YYYY-MM-DD es. 2016-15-12')
- sys.exit(1)
+ if from_date:
+ try:
+ if not isinstance(from_date, datetime):
+ date = datetime.strptime(from_date, '%Y-%m-%d')
+ else:
+ date = from_date
+ query += '&start_time__year__gte=%d' % (date.year,)
+ query += '&start_time__month__gte=%d' % (date.month,)
+ query += '&start_time__day__gte=%d' % (date.day,)
+
+ except ValueError:
+ logger.error('format of from date not valid; YYYY-MM-DD es. 2016-15-12')
+ sys.exit(1)
+
+ if to_date:
+ try:
+
+ if not isinstance(from_date, datetime):
+ date = datetime.strptime(from_date, '%Y-%m-%d')
+ else:
+ date = to_date
+
+ query += '&start_time__year__lte=%d' % (date.year)
+ query += '&start_time__month__lte=%d' % (date.month)
+ query += '&start_time__day__lte=%d' % (date.day)
+
+ except ValueError:
+ logger.error('format of to date not valid; YYYY-MM-DD es. 2016-15-12')
+ sys.exit(1)
return query
@@ -109,15 +117,13 @@ def get_query_string_for_type(station_type):
"""
Create the query string to probe a certain station time in the time range [start_time, end_time]
:param station_type: it can be RTSM or STATION_TEST
- :param start_time: date from which the results are queried
- :param end_time: date to which the results are queried
:return:
"""
query = ''
if station_type == 'RTSM':
- query += 'rtsm?'
+ query += 'rtsm/?'
elif station_type == 'STATION_TEST':
- query += 'stationtests?'
+ query += 'stationtests/?'
else:
logger.error('please specify an station_type=[RTSM|STATION_TEST]')
raise ValueError('please specify an station_type=[RTSM|STATION_TEST]')
@@ -134,7 +140,7 @@ def get_query_string_for_station_name(station_name):
return query
-class TestType():
+class TestType:
"""
Enum class
"""
@@ -149,8 +155,9 @@ def query_station_test(address, from_time, to_time, station_name=""):
"""
query_string = get_query_string_for_type(TestType.STATION_TEST)
- query_string += get_query_string_for_time_limit(from_time, to_time)
- if station_name:
+ if from_time or to_time:
+ query_string += get_query_string_for_time_limit(from_time, to_time)
+ if station_name and station_name is not None:
query_string += get_query_string_for_station_name(station_name)
return perform_query(query_string, address)
@@ -161,22 +168,185 @@ def query_rtsm(address, from_time, to_time, station_name=""):
Queries the station test for a given time span and a station name
"""
query_string = get_query_string_for_type(TestType.RTSM)
-
- query_string += get_query_string_for_time_limit(from_time, to_time)
+ if from_time or to_time:
+ query_string += get_query_string_for_time_limit(from_time, to_time)
if station_name:
query_string += get_query_string_for_station_name(station_name)
return perform_query(query_string, address)
-def print_out_station_test_summary(station_test):
- summary_fields = ['station_name', 'station_type', 'start_time', 'end_time', 'performed checks']
+def reformat_values(item):
+ """
+ Reformats some objects to better print them out
+ :param item:
+ :return:
+ """
+ if isinstance(item, datetime):
+ return str(item).replace('T', ' ').replace('-', ' ')
+ else:
+ return str(item)
+
+
+def print_out_station_test_summary(station_test, id):
+ """
+ Prints the summary of the station test
+ :param station_test: dict containing the results of the station test
+ :return:
+ """
+ summary_fields = ['station_name', 'station_type', 'start_time', 'end_time', 'performed_checks']
+ summary_header = ['Name', 'Type', 'Start', 'End', 'Checks']
+ values = [reformat_values(station_test[item]) for item in summary_fields]
+ print()
+ print('Station test #{}'.format(id))
+ table_width = 200
+ terminal = blessings.Terminal()
+ if terminal.does_styling:
+ table_width = terminal.width
+ table = beautifultable.BeautifulTable(max_width=table_width)
+ table.column_headers = summary_header
+ table.append_row(values)
+ table[0]['Start'] = table[0]['Start'].replace('-', '/').replace('T', ' ')
+ table[0]['End'] = table[0]['End'].replace('-', '/').replace('T', ' ')
+
+ print(table)
+
+
+def sort_component_errors(component_errors):
+ """
+ Sort the component errors by
+ component_type ascending
+ component_id ascending
+ :param component_errors:
+ :return:
+ """
+ sorted_components = sorted(component_errors, key=itemgetter('component_type', 'component_id'))
+ return sorted_components
+
+
+def print_out_component_errors_summary(station_test):
+ summary_fields = ['component_type', 'component_id', 'error_type']
+
+ table = beautifultable.BeautifulTable()
+ table.column_headers = summary_fields
+ for error in sort_component_errors(station_test['all_errors']):
+ values = [error.get(item, '') for item in summary_fields]
+ table.append_row(values)
+ print()
+ print('Component Error Summary')
+ print(table)
- values = [station_test[item] for item in summary_fields]
- for field_name, value in zip(summary_fields, values):
- pprint.pprint(" = ".join([field_name, value]), indent=5)
- print(field_name, value)
+def print_out_component_errors(station_test):
+ excluded_fields = {'parent', 'resourcetype', 'polymorphic_ctype', 'id', 'station_test_id', 'tile_errors',
+ 'parent_component_error'}
+ summary_fields = ['component_type', 'component_id', 'error_type']
+ tile_summary_fields = ['tile_id']
+
+ component_error_indentation = ' ' * 3
+ tile_error_indentation = ' ' * 5
+
+ print()
+ print('=' * 5 + 'Details' + '=' * 5)
+
+ for i, error in enumerate(sort_component_errors(station_test['all_errors'])):
+ component_error_string = 'Component error #{}'.format(i)
+ print(component_error_indentation + component_error_string)
+ print(component_error_indentation + '-' * len(component_error_string))
+
+ for key in summary_fields:
+ value = str(error[key])
+ print(component_error_indentation + key + ' ' + value)
+
+ details_field = error.keys() - summary_fields - excluded_fields
+ details = []
+
+ for key in details_field:
+ value = error[key]
+ if value is None:
+ continue
+ details.append((key, value))
+
+ if details:
+ for key, value in details:
+ print(component_error_indentation + key + ' ' + str(value))
+
+ tile_errors = error.get('tile_errors', None)
+ if tile_errors:
+ for i, tile_error in enumerate(tile_errors):
+ tile_error_string = 'Tile error #{} on tile id {}'.format(i, tile_error['tile_id'])
+ print(tile_error_indentation + tile_error_string)
+ print(tile_error_indentation + '-' * len(tile_error_string))
+
+ tile_keys = tile_error.keys() - tile_summary_fields - excluded_fields
+ tile_values = [tile_error[key] for key in tile_keys]
+ for tile_key, tile_value in zip(tile_keys, tile_values):
+ print(tile_error_indentation + tile_key + ' ' + str(tile_value))
+ print()
+ print('\n')
+
+
+def rtsm_sort_errors(errors):
+ """
+ Sort the rtsm errors by
+ rcu
+ time
+
+ :param component_errors:
+ :return:
+ """
+ sorted_components = sorted(errors, key=itemgetter('rcu', 'error_type', 'time'))
+ return sorted_components
+
+def print_out_rtsm_summary(rtsm_results):
+ table_width = 200
+ terminal = blessings.Terminal()
+ if terminal.does_styling:
+ table_width = terminal.width
+ table = beautifultable.BeautifulTable(max_width=table_width)
+ for rtsm_result in rtsm_results:
+ obs_id = rtsm_results['observation_id']
+ station_name = rtsm_result['station_name']
+ sorted_rtsm = rtsm_sort_errors(rtsm_result['errors'])
+
+
+ for error in rtsm_sort_errors(rtsm_result['errors']):
+ pass
+
+
+def print_out_rtsm(rtsm_results):
+ """
+ Prints the results of the RTSM
+ :param rtsm_results:
+ :return:
+ """
+ table_width = 200
+ terminal = blessings.Terminal()
+ if terminal.does_styling:
+ table_width = terminal.width
+ table = beautifultable.BeautifulTable(max_width=table_width)
+ table.column_headers = ['obs_id',
+ 'station',
+ 'time',
+ 'rcu',
+ 'mode',
+ 'error_type',
+ 'start frequency',
+ 'stop frequency'
+ ]
+
+ for rtsm_result in rtsm_results:
+ obs_id = rtsm_result['observation_id']
+ station_name = rtsm_result['station_name']
+ for error in rtsm_sort_errors(rtsm_result['errors']):
+ rcu_id = error['rcu']
+ start_frequency = error['start_frequency']
+ stop_frequency = error['stop_frequency']
+ time = error['time']
+ mode = error['mode']
+ error_type = error['error_type']
+ table.append_row([obs_id, station_name, time, rcu_id, mode, error_type, start_frequency, stop_frequency])
+ print(table)
def print_out(station_test_results, rtsm_results):
@@ -186,8 +356,16 @@ def print_out(station_test_results, rtsm_results):
:param rtsm_results:
:return:
"""
- for station_test_result in station_test_results:
- print_out_station_test_summary(station_test_result)
+
+ for i, station_test_result in enumerate(station_test_results):
+ print_out_station_test_summary(station_test_result, i)
+ print_out_component_errors_summary(station_test_result)
+ print_out_component_errors(station_test_result)
+ print('=' * 10)
+ print()
+ print('RTSM')
+ print_out_rtsm(rtsm_results)
+
def probe_mdb():
@@ -196,12 +374,19 @@ def probe_mdb():
if args is None:
sys.exit(1)
- station_test_results = query_station_test(args.address, args.from_date, args.to_date, args.station)
+ from_date = args.from_date
+ to_date = args.to_date
+
+ if args.last_month:
+ from_date = datetime.now()-timedelta(days=30)
+
+ station_test_results = query_station_test(args.address, from_date, to_date, args.station)
rtsm_results = query_rtsm(args.address, args.from_date, args.to_date, args.station)
print_out(station_test_results, rtsm_results)
+
if __name__ == '__main__':
logging.basicConfig(format="%(asctime)s %(levelname)s: %(message)s", level=logging.DEBUG)
probe_mdb()
diff --git a/LCU/Maintenance/MDB_tools/requirements.txt b/LCU/Maintenance/MDB_tools/requirements.txt
index 51f068d73014144265067528a9ba1f25bd8d948f..cf65c7d69b58a013329c4459faf0fcbeddac6efe 100644
--- a/LCU/Maintenance/MDB_tools/requirements.txt
+++ b/LCU/Maintenance/MDB_tools/requirements.txt
@@ -6,4 +6,6 @@ django-rest-polymorphic
requests
pandas
fabric
-invocations
\ No newline at end of file
+invocations
+beautifultable
+blessings
\ No newline at end of file