Story SW-300: fixing view error

.gitattributes

@@ -1787,6 +1787,8 @@ LCU/Maintenance/DBInterface/monitoringdb/models/componenterror.py -text
 LCU/Maintenance/DBInterface/monitoringdb/models/fixed_types.py -text
 LCU/Maintenance/DBInterface/monitoringdb/models/hbacomponenterror.py -text
 LCU/Maintenance/DBInterface/monitoringdb/models/lbacomponenterror.py -text
+LCU/Maintenance/DBInterface/monitoringdb/models/mongoengine/__init__.py -text
+LCU/Maintenance/DBInterface/monitoringdb/models/mongoengine/station_test.py -text
 LCU/Maintenance/DBInterface/monitoringdb/models/rcucomponenterror.py -text
 LCU/Maintenance/DBInterface/monitoringdb/models/rspcomponenterror.py -text
 LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py -text
@@ -1800,6 +1802,7 @@ LCU/Maintenance/DBInterface/monitoringdb/serializers/componenterror.py -text
 LCU/Maintenance/DBInterface/monitoringdb/serializers/componenterrors_generic.py -text
 LCU/Maintenance/DBInterface/monitoringdb/serializers/hbaerrorserializers.py -text
 LCU/Maintenance/DBInterface/monitoringdb/serializers/lbaerrorserializer.py -text
+LCU/Maintenance/DBInterface/monitoringdb/serializers/mongoengine.py -text
 LCU/Maintenance/DBInterface/monitoringdb/serializers/rcuerrorserializer.py -text
 LCU/Maintenance/DBInterface/monitoringdb/serializers/rsperrorserializer.py -text
 LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py -text
@@ -1811,6 +1814,7 @@ LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py -text
 LCU/Maintenance/DBInterface/monitoringdb/tests/__init__.py -text
 LCU/Maintenance/DBInterface/monitoringdb/tests/common.py -text
 LCU/Maintenance/DBInterface/monitoringdb/tests/old_tests.py -text
+LCU/Maintenance/DBInterface/monitoringdb/tests/t_nosqlconfiguration.py -text
 LCU/Maintenance/DBInterface/monitoringdb/tests/t_stationtest_model_creation.py -text
 LCU/Maintenance/DBInterface/monitoringdb/tests/t_stationtest_parser.py -text
 LCU/Maintenance/DBInterface/monitoringdb/tests/test_stationtestdata.csv -text

LCU/Maintenance/DBInterface/django_postgresql/settings.py

@@ -11,7 +11,7 @@ https://docs.djangoproject.com/en/2.0/ref/settings/
 """
 
 import os
-
+import mongoengine
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
@@ -126,9 +126,23 @@ DATABASES = {
 }
 
 
+MONGO_DB_SESSION = 'default'
+
+
+MONGO_DBS = {
+    'default': {
+        'db': 'monitoringdb',
+        'host': 'mongodb://lofarmonitortest.control.lofar:27017/'
+    }
+}
+
+
+def default_connection():
+    mongoengine.connect(**MONGO_DBS[MONGO_DB_SESSION])
 # Password validation
 # https://docs.djangoproject.com/en/2.0/ref/settings/#auth-password-validators
 
+
 AUTH_PASSWORD_VALIDATORS = [
     {
         'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',

LCU/Maintenance/DBInterface/monitoringdb/models/__init__.py

-from .componenterror import *
-from .hbacomponenterror import *
-from .lbacomponenterror import *
-from .rcucomponenterror import *
-from .rspcomponenterror import *
-from .tbbcomponenterror import *
-from .spucomponenterror import *
-from .stationtest import *
+#from .componenterror import *
+#from .hbacomponenterror import *
+#from .lbacomponenterror import *
+#from .rcucomponenterror import *
+#from .rspcomponenterror import *
+#from .tbbcomponenterror import *
+#from .spucomponenterror import *
+#from .stationtest import *

LCU/Maintenance/DBInterface/monitoringdb/models/mongoengine/station_test.py

+import ming
+from lofar.maintenance.django_postgresql.settings import default_connection
+
+from mongoengine.document import DynamicDocument, EmbeddedDocument, DynamicEmbeddedDocument
+import mongoengine.fields as schema
+
+default_connection()
+
+
+class StationTest(EmbeddedDocument):
+    station_name = schema.StringField(required=True)
+    station_type = schema.StringField(required=True)
+    start_time = schema.DateTimeField(required=True)
+    end_time = schema.DateTimeField(required=True)
+    performed_checks = schema.StringField(required=True)
+
+
+class TileError(DynamicEmbeddedDocument):
+    tile_id = schema.IntField(required=True)
+
+
+class ComponentError(DynamicDocument):
+    error_type = schema.StringField(required=True)
+    component_type = schema.StringField(required=True)
+    station_test = schema.EmbeddedDocumentField(StationTest)
+
+    component_id = schema.IntField(required=True)
+
+    meta = {'allow_inheritance': True,
+            'indexes': [
+                {'name': 'component_error_unique',
+                 'fields': ['$station_test.station_name', '-station_test.start_time', 'error_type', 'component_type',
+                            'component_id', ],
+                 'unique': True,
+                 'cls': False
+                 }
+            ]}
+
+
+class LBAComponentError(ComponentError):
+    lba_type = schema.StringField(required=True)
+
+
+class HBAComponentError(ComponentError):
+    tile_errors = schema.EmbeddedDocumentField(TileError)
\ No newline at end of file

LCU/Maintenance/DBInterface/monitoringdb/serializers/mongoengine.py

+from rest_framework.serializers import ModelSerializer
+import rest_framework.serializers as schema
+
+
+class StationTestSerializer(ModelSerializer):
+    station_name = schema.CharField()
+    station_type = schema.CharField()
+    start_time = schema.DateTimeField()
+    end_time = schema.DateTimeField()
+    performed_checks = schema.CharField()
+
+
+class ComponentErrorSerializer(ModelSerializer):
+    error_type = schema.CharField()
+    component_type = schema.CharField()
+    station_test = StationTestSerializer()
+    component_id = schema.IntegerField()
+
+
+class TileErrorSerializer(ModelSerializer):
+    tile_id = schema.IntegerField()
+
+
+class HBAComponentErrorSerializer(ComponentErrorSerializer):
+    tile_errors = TileErrorSerializer(many=True)
\ No newline at end of file

LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py

@@ -5,6 +5,7 @@ from datetime import datetime
 import re
 import logging
 import pytz
+
 logger = logging.getLogger('station_test.parser')
 
 
@@ -68,7 +69,7 @@ def split_history_into_tests(content):
     all_tests = []
     current_test = []
     for i, line in enumerate(content[:-1]):
-        next_line_columns = content[i+1].split(',')
+        next_line_columns = content[i + 1].split(',')
         line_type = next_line_columns[3]
         current_test.append(line)
         if 'VERSIONS' in line_type or ('STATION' in line_type and ('VERSIONS' not in current_test[0])):
@@ -165,11 +166,11 @@ def dict_from_hba_component_error(content):
         for arg in extra_args:
             key, values = arg.popitem()
             polarization = key.strip().capitalize()
-            measured_signal_nodelay,\
-            measured_signal_fulldelay,\
-            subband_used_nodelay,\
-            subband_used_fulldelay,\
-            reference_signal_nodelay,\
+            measured_signal_nodelay, \
+            measured_signal_fulldelay, \
+            subband_used_nodelay, \
+            subband_used_fulldelay, \
+            reference_signal_nodelay, \
             reference_signal_fulldelay = map(float, values.replace('nan', '999').split(' '))
 
             result['measured_signal_nodelay' + polarization] = measured_signal_nodelay
@@ -185,7 +186,7 @@ def dict_from_hba_component_error(content):
             key = 'polarization' + key.strip().capitalize()
             result[key] = True
     elif 'NOISE' in error_type:
-        resourcetype='HBANoiseError'
+        resourcetype = 'HBANoiseError'
         type = 'H' if 'HIGH' in error_type else 'L'
         result.update(type=type)
         for arg in extra_args:
@@ -200,7 +201,7 @@ def dict_from_hba_component_error(content):
                 key = 'peak_' + key
             result[key] = float(value)
     elif 'JITTER' in error_type:
-        resourcetype='HBAJitterError'
+        resourcetype = 'HBAJitterError'
         for arg in extra_args:
             key, value = arg.popitem()
             polarization = 'X' if 'X' in key else 'Y'
@@ -285,7 +286,7 @@ def dict_from_lba_component_error(content):
 
     elif 'NOISE' in error_type:
         type = 'H' if 'HIGH' in error_type else 'L'
-        resourcetype="LBANoiseError"
+        resourcetype = "LBANoiseError"
         result.update(type=type)
         for arg in extra_args:
             key, value = arg.popitem()
@@ -311,6 +312,7 @@ def dict_from_lba_component_error(content):
     result.update(resourcetype=resourcetype)
     return result
 
+
 def dict_from_component_error(content):
     """
     Parse the line that describes the component error and returns a dict
@@ -337,7 +339,7 @@ def dict_from_component_error(content):
     elif 'HBA' in component_type:
         result.update(dict_from_hba_component_error(content))
     else:
-        raise NotImplementedError('Unknown error type '+str(content))
+        raise NotImplementedError('Unknown error type ' + str(content))
     return result
 
 
@@ -348,11 +350,6 @@ def dict_from_raw_station_test(content):
     :param content: string content with the station test output
     :return: a station test dict
     """
-    model_list = []
-
-    station_name = ''
-    start_time = ''
-    end_time = ''
 
     preparsed_content = []
     for line in content:
@@ -470,8 +467,8 @@ def dicts_from_tile_error(contents):
             measured_signal_nodelay, measured_signal_fulldelay, subband_used_nodelay, subband_used_fulldelay, \
             reference_signal_nodelay, reference_signal_fulldelay = map(float,
                                                                        args.replace('nan', '999').
-                                                                            replace('-1', '999').
-                                                                            split(' '))
+                                                                       replace('-1', '999').
+                                                                       split(' '))
             item.update(measured_signal_fulldelay=measured_signal_fulldelay,
                         measured_signal_nodelay=measured_signal_nodelay,
                         subband_used_fulldelay=subband_used_fulldelay,
@@ -480,4 +477,4 @@ def dicts_from_tile_error(contents):
                         reference_signal_nodelay=reference_signal_nodelay)
         item.update(resourcetype=resourcetype)
         results.append(item)
-    return results
\ No newline at end of file
+    return results

LCU/Maintenance/DBInterface/monitoringdb/tests/t_nosqlconfiguration.py

+import lofar.maintenance.django_postgresql.settings as settings
+from lofar.maintenance.monitoringdb.models.ming.station_test import ComponentError, StationTest,LBAComponentError, HBAComponentError
+import unittest
+from datetime import datetime
+import pytz
+from mongoengine.errors import NotUniqueError
+
+def get_mock_dict_data():
+    start_time = pytz.utc.localize(datetime(2014, 5, 12, 0, 0, 0))
+    start_time2 = pytz.utc.localize(datetime(2014, 5, 25, 0, 0, 0))
+
+    end_time = pytz.utc.localize(datetime(2014, 10, 12, 0, 0, 0))
+
+    station_test = dict(station_name='FR001',
+                        station_type='I',
+                        start_time=start_time,
+                        end_time=end_time,
+                        performed_checks="franco lolo lello")
+    station_test2 = dict(station_name='FR001',
+                        station_type='I',
+                        start_time=start_time2,
+                        end_time=end_time,
+                        performed_checks="franco lolo lello")
+
+    component_errors = [
+        dict(
+            component_id=12,
+            error_type='flagging',
+            component_type='antenna',
+            station_test=station_test,
+            lba_type='ciccio'),
+        dict(
+            component_id=2,
+            error_type='flagging',
+            component_type='antenna',
+            station_test=station_test),
+        dict(
+            component_id=4,
+            error_type='flagging',
+            component_type='antenna',
+            station_test=station_test),
+        dict(
+            component_id=1,
+            error_type='flagging',
+            component_type='antenna',
+            station_test=station_test),
+        dict(
+            component_id=12,
+            error_type='flagging',
+            component_type='antenna',
+            station_test=station_test),
+
+    ]
+    new_component_errors = []
+    for component_error in component_errors:
+        new_component_errors.append(component_error)
+        new_component_error = dict(component_error)
+        new_component_error.update(station_test=station_test2)
+        new_component_errors.append(new_component_error)
+
+    return new_component_errors
+
+
+class TestMongoEngineDjango(unittest.TestCase):
+
+    def clean_component_error_collection(self):
+        ComponentError.drop_collection()
+
+    def test_station_test_creation(self):
+        errors = get_mock_dict_data()
+        # There is a not unique error message so it should throw an exception
+        with self.assertRaises(NotUniqueError) as e:
+            for error in errors:
+                st = StationTest(**error.pop('station_test'))
+                if 'lba_type' in error:
+                    ce = LBAComponentError(**error, station_test=st)
+                else:
+                    ce = HBAComponentError(**error, station_test=st)
+                ce.save()
+
+    def test_station_test_query(self):
+        errors = get_mock_dict_data()
+        try:
+            for error in errors:
+                st = StationTest(**error.pop('station_test'))
+                if 'lba_type' in error:
+                    ce = LBAComponentError(**error, station_test=st)
+                else:
+                    ce = HBAComponentError(**error, station_test=st)
+                ce.save()
+        except NotUniqueError as e:
+            pass# it is intended behaviour
+
+        lba_component_error = LBAComponentError.objects({})[0]
+        self.assertEqual(lba_component_error.lba_type, 'ciccio')
+
+    def tearDown(self):
+        self.clean_component_error_collection()
\ No newline at end of file

LCU/Maintenance/DBInterface/monitoringdb/views/station_test_views.py

@@ -24,8 +24,8 @@ class ComponentErrorCountSummarySerializer(Serializer):
 
 
 class ComponentErrorCountSummaryViewSet(viewsets.ModelViewSet):
-    queryset = ComponentError.objects.all().\
-        values('error_type', 'component_type', 'parent__station_name').\
+    queryset = ComponentError.objects.all(). \
+        values('error_type', 'component_type', 'parent__station_name'). \
         annotate(count=aggregates.Count('error_type')).order_by('count')
     serializer_class = ComponentErrorCountSummarySerializer
 
@@ -42,10 +42,11 @@ class ComponentErrorCountSummaryViewSet(viewsets.ModelViewSet):
             if key in RESERVED_FILTER_NAME:
                 continue
             queryset = queryset.filter(**{key: param})
-            logger.info(queryset.values('parent__station_name','error_type', 'component_type'))
-        return queryset.\
-                values('error_type', 'component_type', 'parent__station_name').\
-                annotate(count=aggregates.Count('error_type')).order_by('count')
+            logger.info(queryset.values('parent__station_name', 'error_type', 'component_type'))
+        return queryset. \
+            values('error_type', 'component_type', 'parent__station_name'). \
+            annotate(count=aggregates.Count('error_type')).order_by('count')
+
 
 # Create your views here.
 class StationTestViewSet(viewsets.ModelViewSet):
@@ -143,19 +144,19 @@ def insert_raw_station_test(request):
                 logger.exception("exception occurred while parsing raw station info %s: %s", request.data['content'], e)
                 return Response(exception=True,
                                 data="the post message is not correct." +
-                                " It has to be of the form \{'content':[RAWSTRING]\}: %s. Request provided %s" % (e, request),
+                                     " It has to be of the form \{'content':[RAWSTRING]\}: %s. Request provided %s" % (
+                                     e, request),
                                 status=status.HTTP_400_BAD_REQUEST)
         else:
             return Response(exception=True,
                             data="the post message is not correct." +
-                            " It has to be of the form \{'content':[RAWSTRING]\}", status=status.HTTP_400_BAD_REQUEST)
+                                 " It has to be of the form \{'content':[RAWSTRING]\}",
+                            status=status.HTTP_400_BAD_REQUEST)
         logger.info('request processed correctly inserted %d of %d', len(station_test_ids),
-                    len(station_test_ids)+len(station_test_ids_already_present))
+                    len(station_test_ids) + len(station_test_ids_already_present))
         return Response(status=status.HTTP_200_OK,
                         data='Station tests inserted with ids {} \n'.format(station_test_ids) +
                              'Station tests already present with ids {}'.format(
-                                station_test_ids_already_present
+                                 station_test_ids_already_present
                              )
                         )
-
-