diff --git a/.gitattributes b/.gitattributes
index f37db444339fdd94d39ff075699f3ff1913c93ff..5c6549f611dfb69c9db0b87e0b47a0ae7fcfec73 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1787,6 +1787,8 @@ LCU/Maintenance/DBInterface/monitoringdb/models/componenterror.py -text
LCU/Maintenance/DBInterface/monitoringdb/models/fixed_types.py -text
LCU/Maintenance/DBInterface/monitoringdb/models/hbacomponenterror.py -text
LCU/Maintenance/DBInterface/monitoringdb/models/lbacomponenterror.py -text
+LCU/Maintenance/DBInterface/monitoringdb/models/mongoengine/__init__.py -text
+LCU/Maintenance/DBInterface/monitoringdb/models/mongoengine/station_test.py -text
LCU/Maintenance/DBInterface/monitoringdb/models/rcucomponenterror.py -text
LCU/Maintenance/DBInterface/monitoringdb/models/rspcomponenterror.py -text
LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py -text
@@ -1800,6 +1802,7 @@ LCU/Maintenance/DBInterface/monitoringdb/serializers/componenterror.py -text
LCU/Maintenance/DBInterface/monitoringdb/serializers/componenterrors_generic.py -text
LCU/Maintenance/DBInterface/monitoringdb/serializers/hbaerrorserializers.py -text
LCU/Maintenance/DBInterface/monitoringdb/serializers/lbaerrorserializer.py -text
+LCU/Maintenance/DBInterface/monitoringdb/serializers/mongoengine.py -text
LCU/Maintenance/DBInterface/monitoringdb/serializers/rcuerrorserializer.py -text
LCU/Maintenance/DBInterface/monitoringdb/serializers/rsperrorserializer.py -text
LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py -text
@@ -1811,6 +1814,7 @@ LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py -text
LCU/Maintenance/DBInterface/monitoringdb/tests/__init__.py -text
LCU/Maintenance/DBInterface/monitoringdb/tests/common.py -text
LCU/Maintenance/DBInterface/monitoringdb/tests/old_tests.py -text
+LCU/Maintenance/DBInterface/monitoringdb/tests/t_nosqlconfiguration.py -text
LCU/Maintenance/DBInterface/monitoringdb/tests/t_stationtest_model_creation.py -text
LCU/Maintenance/DBInterface/monitoringdb/tests/t_stationtest_parser.py -text
LCU/Maintenance/DBInterface/monitoringdb/tests/test_stationtestdata.csv -text
diff --git a/LCU/Maintenance/DBInterface/django_postgresql/settings.py b/LCU/Maintenance/DBInterface/django_postgresql/settings.py
index e1a9eb88616ca1bd4ea9068868329cfc6e948b3b..95205da44f5b5631f9989acebf3a29c242e517f8 100644
--- a/LCU/Maintenance/DBInterface/django_postgresql/settings.py
+++ b/LCU/Maintenance/DBInterface/django_postgresql/settings.py
@@ -11,7 +11,7 @@ https://docs.djangoproject.com/en/2.0/ref/settings/
"""
import os
-
+import mongoengine
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -126,9 +126,23 @@ DATABASES = {
}
+MONGO_DB_SESSION = 'default'
+
+
+MONGO_DBS = {
+ 'default': {
+ 'db': 'monitoringdb',
+ 'host': 'mongodb://lofarmonitortest.control.lofar:27017/'
+ }
+}
+
+
+def default_connection():
+ mongoengine.connect(**MONGO_DBS[MONGO_DB_SESSION])
# Password validation
# https://docs.djangoproject.com/en/2.0/ref/settings/#auth-password-validators
+
AUTH_PASSWORD_VALIDATORS = [
{
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/__init__.py b/LCU/Maintenance/DBInterface/monitoringdb/models/__init__.py
index ee04345452a2bffc9923f4f9dfe45f6c53308d4a..dc4538f4bd61150fc5caea52c224ef71ff9b0e48 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/models/__init__.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/models/__init__.py
@@ -1,8 +1,8 @@
-from .componenterror import *
-from .hbacomponenterror import *
-from .lbacomponenterror import *
-from .rcucomponenterror import *
-from .rspcomponenterror import *
-from .tbbcomponenterror import *
-from .spucomponenterror import *
-from .stationtest import *
+#from .componenterror import *
+#from .hbacomponenterror import *
+#from .lbacomponenterror import *
+#from .rcucomponenterror import *
+#from .rspcomponenterror import *
+#from .tbbcomponenterror import *
+#from .spucomponenterror import *
+#from .stationtest import *
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/mongoengine/__init__.py b/LCU/Maintenance/DBInterface/monitoringdb/models/mongoengine/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/mongoengine/station_test.py b/LCU/Maintenance/DBInterface/monitoringdb/models/mongoengine/station_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..51f76d9231c11044a0ad06e6eaed9bcf2c369ef0
--- /dev/null
+++ b/LCU/Maintenance/DBInterface/monitoringdb/models/mongoengine/station_test.py
@@ -0,0 +1,45 @@
+import ming
+from lofar.maintenance.django_postgresql.settings import default_connection
+
+from mongoengine.document import DynamicDocument, EmbeddedDocument, DynamicEmbeddedDocument
+import mongoengine.fields as schema
+
+default_connection()
+
+
+class StationTest(EmbeddedDocument):
+ station_name = schema.StringField(required=True)
+ station_type = schema.StringField(required=True)
+ start_time = schema.DateTimeField(required=True)
+ end_time = schema.DateTimeField(required=True)
+ performed_checks = schema.StringField(required=True)
+
+
+class TileError(DynamicEmbeddedDocument):
+ tile_id = schema.IntField(required=True)
+
+
+class ComponentError(DynamicDocument):
+ error_type = schema.StringField(required=True)
+ component_type = schema.StringField(required=True)
+ station_test = schema.EmbeddedDocumentField(StationTest)
+
+ component_id = schema.IntField(required=True)
+
+ meta = {'allow_inheritance': True,
+ 'indexes': [
+ {'name': 'component_error_unique',
+ 'fields': ['$station_test.station_name', '-station_test.start_time', 'error_type', 'component_type',
+ 'component_id', ],
+ 'unique': True,
+ 'cls': False
+ }
+ ]}
+
+
+class LBAComponentError(ComponentError):
+ lba_type = schema.StringField(required=True)
+
+
+class HBAComponentError(ComponentError):
+ tile_errors = schema.EmbeddedDocumentField(TileError)
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/mongoengine.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/mongoengine.py
new file mode 100644
index 0000000000000000000000000000000000000000..a8e64f984a80ea7dab60a75aabb512e6af08e194
--- /dev/null
+++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/mongoengine.py
@@ -0,0 +1,25 @@
+from rest_framework.serializers import ModelSerializer
+import rest_framework.serializers as schema
+
+
+class StationTestSerializer(ModelSerializer):
+ station_name = schema.CharField()
+ station_type = schema.CharField()
+ start_time = schema.DateTimeField()
+ end_time = schema.DateTimeField()
+ performed_checks = schema.CharField()
+
+
+class ComponentErrorSerializer(ModelSerializer):
+ error_type = schema.CharField()
+ component_type = schema.CharField()
+ station_test = StationTestSerializer()
+ component_id = schema.IntegerField()
+
+
+class TileErrorSerializer(ModelSerializer):
+ tile_id = schema.IntegerField()
+
+
+class HBAComponentErrorSerializer(ComponentErrorSerializer):
+ tile_errors = TileErrorSerializer(many=True)
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py b/LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py
index d815692faba1fddc883d44011b1f814a5c10285f..fc8105ba51bac943c77fbc86c99a020b162c7cba 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py
@@ -5,6 +5,7 @@ from datetime import datetime
import re
import logging
import pytz
+
logger = logging.getLogger('station_test.parser')
@@ -68,7 +69,7 @@ def split_history_into_tests(content):
all_tests = []
current_test = []
for i, line in enumerate(content[:-1]):
- next_line_columns = content[i+1].split(',')
+ next_line_columns = content[i + 1].split(',')
line_type = next_line_columns[3]
current_test.append(line)
if 'VERSIONS' in line_type or ('STATION' in line_type and ('VERSIONS' not in current_test[0])):
@@ -165,11 +166,11 @@ def dict_from_hba_component_error(content):
for arg in extra_args:
key, values = arg.popitem()
polarization = key.strip().capitalize()
- measured_signal_nodelay,\
- measured_signal_fulldelay,\
- subband_used_nodelay,\
- subband_used_fulldelay,\
- reference_signal_nodelay,\
+ measured_signal_nodelay, \
+ measured_signal_fulldelay, \
+ subband_used_nodelay, \
+ subband_used_fulldelay, \
+ reference_signal_nodelay, \
reference_signal_fulldelay = map(float, values.replace('nan', '999').split(' '))
result['measured_signal_nodelay' + polarization] = measured_signal_nodelay
@@ -185,7 +186,7 @@ def dict_from_hba_component_error(content):
key = 'polarization' + key.strip().capitalize()
result[key] = True
elif 'NOISE' in error_type:
- resourcetype='HBANoiseError'
+ resourcetype = 'HBANoiseError'
type = 'H' if 'HIGH' in error_type else 'L'
result.update(type=type)
for arg in extra_args:
@@ -200,7 +201,7 @@ def dict_from_hba_component_error(content):
key = 'peak_' + key
result[key] = float(value)
elif 'JITTER' in error_type:
- resourcetype='HBAJitterError'
+ resourcetype = 'HBAJitterError'
for arg in extra_args:
key, value = arg.popitem()
polarization = 'X' if 'X' in key else 'Y'
@@ -285,7 +286,7 @@ def dict_from_lba_component_error(content):
elif 'NOISE' in error_type:
type = 'H' if 'HIGH' in error_type else 'L'
- resourcetype="LBANoiseError"
+ resourcetype = "LBANoiseError"
result.update(type=type)
for arg in extra_args:
key, value = arg.popitem()
@@ -311,6 +312,7 @@ def dict_from_lba_component_error(content):
result.update(resourcetype=resourcetype)
return result
+
def dict_from_component_error(content):
"""
Parse the line that describes the component error and returns a dict
@@ -337,7 +339,7 @@ def dict_from_component_error(content):
elif 'HBA' in component_type:
result.update(dict_from_hba_component_error(content))
else:
- raise NotImplementedError('Unknown error type '+str(content))
+ raise NotImplementedError('Unknown error type ' + str(content))
return result
@@ -348,11 +350,6 @@ def dict_from_raw_station_test(content):
:param content: string content with the station test output
:return: a station test dict
"""
- model_list = []
-
- station_name = ''
- start_time = ''
- end_time = ''
preparsed_content = []
for line in content:
@@ -470,8 +467,8 @@ def dicts_from_tile_error(contents):
measured_signal_nodelay, measured_signal_fulldelay, subband_used_nodelay, subband_used_fulldelay, \
reference_signal_nodelay, reference_signal_fulldelay = map(float,
args.replace('nan', '999').
- replace('-1', '999').
- split(' '))
+ replace('-1', '999').
+ split(' '))
item.update(measured_signal_fulldelay=measured_signal_fulldelay,
measured_signal_nodelay=measured_signal_nodelay,
subband_used_fulldelay=subband_used_fulldelay,
@@ -480,4 +477,4 @@ def dicts_from_tile_error(contents):
reference_signal_nodelay=reference_signal_nodelay)
item.update(resourcetype=resourcetype)
results.append(item)
- return results
\ No newline at end of file
+ return results
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/tests/t_nosqlconfiguration.py b/LCU/Maintenance/DBInterface/monitoringdb/tests/t_nosqlconfiguration.py
new file mode 100644
index 0000000000000000000000000000000000000000..e0e516769be310773861afbae1d85edc00723aa5
--- /dev/null
+++ b/LCU/Maintenance/DBInterface/monitoringdb/tests/t_nosqlconfiguration.py
@@ -0,0 +1,98 @@
+import lofar.maintenance.django_postgresql.settings as settings
+from lofar.maintenance.monitoringdb.models.ming.station_test import ComponentError, StationTest,LBAComponentError, HBAComponentError
+import unittest
+from datetime import datetime
+import pytz
+from mongoengine.errors import NotUniqueError
+
+def get_mock_dict_data():
+ start_time = pytz.utc.localize(datetime(2014, 5, 12, 0, 0, 0))
+ start_time2 = pytz.utc.localize(datetime(2014, 5, 25, 0, 0, 0))
+
+ end_time = pytz.utc.localize(datetime(2014, 10, 12, 0, 0, 0))
+
+ station_test = dict(station_name='FR001',
+ station_type='I',
+ start_time=start_time,
+ end_time=end_time,
+ performed_checks="franco lolo lello")
+ station_test2 = dict(station_name='FR001',
+ station_type='I',
+ start_time=start_time2,
+ end_time=end_time,
+ performed_checks="franco lolo lello")
+
+ component_errors = [
+ dict(
+ component_id=12,
+ error_type='flagging',
+ component_type='antenna',
+ station_test=station_test,
+ lba_type='ciccio'),
+ dict(
+ component_id=2,
+ error_type='flagging',
+ component_type='antenna',
+ station_test=station_test),
+ dict(
+ component_id=4,
+ error_type='flagging',
+ component_type='antenna',
+ station_test=station_test),
+ dict(
+ component_id=1,
+ error_type='flagging',
+ component_type='antenna',
+ station_test=station_test),
+ dict(
+ component_id=12,
+ error_type='flagging',
+ component_type='antenna',
+ station_test=station_test),
+
+ ]
+ new_component_errors = []
+ for component_error in component_errors:
+ new_component_errors.append(component_error)
+ new_component_error = dict(component_error)
+ new_component_error.update(station_test=station_test2)
+ new_component_errors.append(new_component_error)
+
+ return new_component_errors
+
+
+class TestMongoEngineDjango(unittest.TestCase):
+
+ def clean_component_error_collection(self):
+ ComponentError.drop_collection()
+
+ def test_station_test_creation(self):
+ errors = get_mock_dict_data()
+ # There is a not unique error message so it should throw an exception
+ with self.assertRaises(NotUniqueError) as e:
+ for error in errors:
+ st = StationTest(**error.pop('station_test'))
+ if 'lba_type' in error:
+ ce = LBAComponentError(**error, station_test=st)
+ else:
+ ce = HBAComponentError(**error, station_test=st)
+ ce.save()
+
+ def test_station_test_query(self):
+ errors = get_mock_dict_data()
+ try:
+ for error in errors:
+ st = StationTest(**error.pop('station_test'))
+ if 'lba_type' in error:
+ ce = LBAComponentError(**error, station_test=st)
+ else:
+ ce = HBAComponentError(**error, station_test=st)
+ ce.save()
+ except NotUniqueError as e:
+ pass# it is intended behaviour
+
+ lba_component_error = LBAComponentError.objects({})[0]
+ self.assertEqual(lba_component_error.lba_type, 'ciccio')
+
+ def tearDown(self):
+ self.clean_component_error_collection()
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/views/station_test_views.py b/LCU/Maintenance/DBInterface/monitoringdb/views/station_test_views.py
index 42aac70f56317080e6dd56bbb66955eea1dbaeaf..94afcf2b4840cd03f5d87dd70a3e0a7b8a84dc8f 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/views/station_test_views.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/views/station_test_views.py
@@ -24,8 +24,8 @@ class ComponentErrorCountSummarySerializer(Serializer):
class ComponentErrorCountSummaryViewSet(viewsets.ModelViewSet):
- queryset = ComponentError.objects.all().\
- values('error_type', 'component_type', 'parent__station_name').\
+ queryset = ComponentError.objects.all(). \
+ values('error_type', 'component_type', 'parent__station_name'). \
annotate(count=aggregates.Count('error_type')).order_by('count')
serializer_class = ComponentErrorCountSummarySerializer
@@ -42,10 +42,11 @@ class ComponentErrorCountSummaryViewSet(viewsets.ModelViewSet):
if key in RESERVED_FILTER_NAME:
continue
queryset = queryset.filter(**{key: param})
- logger.info(queryset.values('parent__station_name','error_type', 'component_type'))
- return queryset.\
- values('error_type', 'component_type', 'parent__station_name').\
- annotate(count=aggregates.Count('error_type')).order_by('count')
+ logger.info(queryset.values('parent__station_name', 'error_type', 'component_type'))
+ return queryset. \
+ values('error_type', 'component_type', 'parent__station_name'). \
+ annotate(count=aggregates.Count('error_type')).order_by('count')
+
# Create your views here.
class StationTestViewSet(viewsets.ModelViewSet):
@@ -143,19 +144,19 @@ def insert_raw_station_test(request):
logger.exception("exception occurred while parsing raw station info %s: %s", request.data['content'], e)
return Response(exception=True,
data="the post message is not correct." +
- " It has to be of the form \{'content':[RAWSTRING]\}: %s. Request provided %s" % (e, request),
+ " It has to be of the form \{'content':[RAWSTRING]\}: %s. Request provided %s" % (
+ e, request),
status=status.HTTP_400_BAD_REQUEST)
else:
return Response(exception=True,
data="the post message is not correct." +
- " It has to be of the form \{'content':[RAWSTRING]\}", status=status.HTTP_400_BAD_REQUEST)
+ " It has to be of the form \{'content':[RAWSTRING]\}",
+ status=status.HTTP_400_BAD_REQUEST)
logger.info('request processed correctly inserted %d of %d', len(station_test_ids),
- len(station_test_ids)+len(station_test_ids_already_present))
+ len(station_test_ids) + len(station_test_ids_already_present))
return Response(status=status.HTTP_200_OK,
data='Station tests inserted with ids {} \n'.format(station_test_ids) +
'Station tests already present with ids {}'.format(
- station_test_ids_already_present
+ station_test_ids_already_present
)
)
-
-