Story OSB-11: adding filter capabilities and cleaning up the code.

.gitattributes

@@ -1816,8 +1816,10 @@ LCU/Maintenance/DBInterface/monitoringdb/serializers/component_error.py -text
 LCU/Maintenance/DBInterface/monitoringdb/serializers/element.py -text
 LCU/Maintenance/DBInterface/monitoringdb/serializers/element_error.py -text
 LCU/Maintenance/DBInterface/monitoringdb/serializers/error_details.py -text
+LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py -text
 LCU/Maintenance/DBInterface/monitoringdb/serializers/station.py -text
 LCU/Maintenance/DBInterface/monitoringdb/serializers/station_tests.py -text
+LCU/Maintenance/DBInterface/monitoringdb/serializers/utils.py -text
 LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py -text
 LCU/Maintenance/DBInterface/monitoringdb/tests/__init__.py -text
 LCU/Maintenance/DBInterface/monitoringdb/tests/common.py -text

LCU/Maintenance/DBInterface/CMakeLists.txt

@@ -12,7 +12,7 @@ find_python_module(rest_framework REQUIRED)          #sudo pip install djangores
 find_python_module(rest_polymorphic REQUIRED)        #sudo pip install django-polymorphic
 find_python_module(polymorphic REQUIRED)             #sudo pip install django-rest-polymorphic
 find_python_module(requests REQUIRED)
-find_python_module(pandas REQUIRED)
+
 
 # includes every python file excepts for the manage.py
 FILE(GLOB_RECURSE PY_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ./*.py)

LCU/Maintenance/DBInterface/django_postgresql/settings.py

@@ -36,6 +36,7 @@ INSTALLED_APPS = [
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'rest_framework',
+    'django_filters',
     'lofar.maintenance.monitoringdb.apps.MonitoringDbConfig',
 ]
 
@@ -169,5 +170,6 @@ STATIC_URL = '/static/'
 
 REST_FRAMEWORK = {
     'DEFAULT_PAGINATION_CLASS': 'lofar.maintenance.monitoringdb.pagination.DefaultPaginationSettings',
-    'STRICT_JSON': False
+    'STRICT_JSON': False,
+    'DEFAULT_FILTER_BACKENDS': ('django_filters.rest_framework.DjangoFilterBackend',)
 }
\ No newline at end of file

LCU/Maintenance/DBInterface/monitoringdb/models/error_details.py

@@ -2,14 +2,14 @@ from django.db import models
 
 
 class ErrorDetails(models.Model):
-    x = models.BooleanField(default=None, null=True)
-    y = models.BooleanField(default=None, null=True)
+    x = models.NullBooleanField()
+    y = models.NullBooleanField()
 
     xval = models.FloatField(default=None, null=True)
     yval = models.FloatField(default=None, null=True)
 
-    xoffset = models.FloatField(default=None, null=True)
-    yoffset = models.FloatField(default=None, null=True)
+    xoff = models.FloatField(default=None, null=True)
+    yoff = models.FloatField(default=None, null=True)
 
     xval_no_delay = models.FloatField(default=None, null=True)
     yval_no_delay = models.FloatField(default=None, null=True)
@@ -40,3 +40,34 @@ class ErrorDetails(models.Model):
 
     xdiff = models.FloatField(default=None, null=True)
     ydiff = models.FloatField(default=None, null=True)
+
+    xmean = models.FloatField(default=None, null=True)
+    ymean = models.FloatField(default=None, null=True)
+
+    tp = models.CharField(max_length=10, null=True, default=None)
+    mp = models.CharField(max_length=10, null=True, default=None)
+    ap = models.CharField(max_length=10, null=True, default=None)
+    bp = models.CharField(max_length=10, null=True, default=None)
+
+    mp0 = models.CharField(max_length=10, null=True, default=None)
+    mp1 = models.CharField(max_length=10, null=True, default=None)
+    mp2 = models.CharField(max_length=10, null=True, default=None)
+    mp3 = models.CharField(max_length=10, null=True, default=None)
+
+    ap0 = models.CharField(max_length=10, null=True, default=None)
+    ap1 = models.CharField(max_length=10, null=True, default=None)
+    ap2 = models.CharField(max_length=10, null=True, default=None)
+    ap3 = models.CharField(max_length=10, null=True, default=None)
+
+    pcb = models.CharField(max_length=10, null=True, default=None)
+
+    rcu_5_0v = models.FloatField(default=None, null=True)
+    lba_8_0v = models.FloatField(default=None, null=True)
+    hba_48v = models.FloatField(default=None, null=True)
+    spu_3_3v = models.FloatField(default=None, null=True)
+
+    v1_2 = models.FloatField(default=None, null=True)
+    v2_5 = models.FloatField(default=None, null=True)
+    v3_3 = models.FloatField(default=None, null=True)
+
+    error_code = models.CharField(max_length=10, null=True, default=None)
\ No newline at end of file

LCU/Maintenance/DBInterface/monitoringdb/serializers/component.py

-from rest_framework.serializers import HyperlinkedModelSerializer
+from rest_framework.serializers import ModelSerializer
 from ..models.component import Component
 
-class ComponentSerializer(HyperlinkedModelSerializer):
+class ComponentSerializer(ModelSerializer):
     class Meta:
         model = Component
-        fields = '__all__'
\ No newline at end of file
+        fields = '__all__'
+        depth = 1
\ No newline at end of file

LCU/Maintenance/DBInterface/monitoringdb/serializers/component_error.py

-from rest_framework.serializers import HyperlinkedModelSerializer
+from rest_framework.serializers import HyperlinkedModelSerializer, ModelSerializer
 from ..models.component_error import ComponentError, Component
+from .utils import NotNullModelSerializer
 
-class ComponentErrorSerializer(HyperlinkedModelSerializer):
+
+class ComponentErrorSerializer(NotNullModelSerializer):
     class Meta:
         model = ComponentError
         fields = '__all__'
+        depth = 2

LCU/Maintenance/DBInterface/monitoringdb/serializers/element.py

-from rest_framework.serializers import HyperlinkedModelSerializer
+from rest_framework.serializers import ModelSerializer
 from ..models.element import Element
 
 
-class ElementSerializer(HyperlinkedModelSerializer):
+class ElementSerializer(ModelSerializer):
     class Meta:
         model = Element
         fields = '__all__'
+        depth = 1

LCU/Maintenance/DBInterface/monitoringdb/serializers/element_error.py

-from rest_framework.serializers import HyperlinkedModelSerializer
+from rest_framework.serializers import ModelSerializer
 from ..models.element_error import ElementError
 
 
-class ElementErrorSerializer(HyperlinkedModelSerializer):
+class ElementErrorSerializer(ModelSerializer):
     class Meta:
         model = ElementError
         fields = '__all__'
+        depth = 1
\ No newline at end of file

LCU/Maintenance/DBInterface/monitoringdb/serializers/error_details.py

-from rest_framework.serializers import HyperlinkedModelSerializer
+from rest_framework.serializers import ModelSerializer
 from ..models.error_details import ErrorDetails
 
-class ErrorDetailsSerializer(HyperlinkedModelSerializer):
+class ErrorDetailsSerializer(ModelSerializer):
     class Meta:
         model = ErrorDetails
         fields = '__all__'
+
+    def to_representation(self, instance):
+        result = super().to_representation(instance)
+
+        result = {key: value
+         for key, value in
+         filter(lambda item: item[1] is not None, result.items())}
+
+        return result
\ No newline at end of file

LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py

+from ..models.rtsm import *
+from rest_framework import serializers
+import logging
+from django.db.transaction import atomic
+from ..exceptions import ItemAlreadyExists
+from django.db.models import ObjectDoesNotExist
+
+logger = logging.getLogger('serializers')
+
+
+class RTSMErrorsAggregateSerializer(serializers.Serializer):
+    count = serializers.IntegerField()
+    percentage = serializers.FloatField()
+    error_type = serializers.CharField()
+    rcu = serializers.IntegerField()
+
+
+class RTSMErrorSerializer(serializers.ModelSerializer):
+    bad_spectrum = serializers.ListField(child=serializers.FloatField())
+    average_spectrum = serializers.ListField(child=serializers.FloatField())
+
+    class Meta:
+        model = RTSMError
+        fields = '__all__'
+
+
+class RTSMErrorSummarySerializer(serializers.ModelSerializer):
+    samples = serializers.IntegerField(source='observation.samples', read_only=True)
+    start_time = serializers.DateTimeField(source='observation.start_time', read_only=True)
+    end_time = serializers.DateTimeField(source='observation.start_time', read_only=True)
+    observation_id = serializers.IntegerField(source='observation.observation_id', read_only=True)
+    id = serializers.IntegerField()
+
+    class Meta:
+        model = RTSMErrorSummary
+        fields = '__all__'
+
+
+class RTSMObservationSummarySerializer(serializers.ModelSerializer):
+    errors_found = RTSMErrorsAggregateSerializer(many=True)
+
+    class Meta:
+        model = RTSMObservationSummary
+        fields = '__all__'
+
+
+class RTSMObservationSerializer(serializers.ModelSerializer):
+    errors_summary = RTSMErrorSummarySerializer(many=True)
+
+    class Meta:
+        model = RTSMObservation
+        fields = '__all__'
+
+    @atomic
+    def create(self, validated_data):
+        rtsm_errors = validated_data.pop('errors')
+        try:
+            start_time, station_name = validated_data['start_time'], validated_data['station_name']
+            station_type = station_name[0].capitalize()
+            station_type = station_type if station_type in ['C', 'R'] else 'I'
+            validated_data.update(station_type=station_type)
+            element = RTSMObservation.objects.filter(start_time=start_time,
+                                                     station_name=station_name).first()
+
+            if element is None:
+                logger.info('rtsm not found inserting %s', validated_data)
+            else:
+                logger.info('RTSM test for station %s started at %s is present already. Skipping insertion',
+                            start_time, station_name)
+                raise ItemAlreadyExists(element)
+        except ObjectDoesNotExist as e:
+            logger.info('rtsm not found inserting %s', validated_data)
+        RTSMObservation_instance = RTSMObservation.objects.create(**validated_data)
+        RTSMObservation_instance.save()
+        for rtsm_error in rtsm_errors:
+            rtsm_error.update(observation=RTSMObservation_instance)
+            rtsm_error_instance = RTSMError.objects.create(**rtsm_error)
+            rtsm_error_instance.save()
+
+        return RTSMObservation_instance

LCU/Maintenance/DBInterface/monitoringdb/serializers/station.py

-from rest_framework.serializers import HyperlinkedModelSerializer
+from rest_framework.serializers import ModelSerializer
 from ..models.station import Station
 
-class StationSerializer(HyperlinkedModelSerializer):
+
+class StationSerializer(ModelSerializer):
     class Meta:
         model = Station
-        fields = '__all__'
\ No newline at end of file
+        fields = '__all__'
+        depth = 1
\ No newline at end of file

LCU/Maintenance/DBInterface/monitoringdb/serializers/station_tests.py

-from rest_framework.serializers import HyperlinkedModelSerializer
+from rest_framework.serializers import ModelSerializer
+from ..models.station_test import StationTest
 
-
-class StationTestSerializer(HyperlinkedModelSerializer):
+class StationTestSerializer(ModelSerializer):
     class Meta:
-        model = 'StationTest'
-        fields = '__all__'
\ No newline at end of file
+        model = StationTest
+        fields = '__all__'
+        depth = 1
\ No newline at end of file

LCU/Maintenance/DBInterface/monitoringdb/serializers/utils.py

+from rest_framework.serializers import ModelSerializer
+from collections import OrderedDict
+
+
+class NotNullModelSerializer(ModelSerializer):
+    def filter_null_values(self, result, items):
+        for key, value in items:
+            if isinstance(value, dict):
+                result[key] = self.filter_null_values(OrderedDict(), value.items())
+            else:
+                if value is not None:
+                    result[key] = value
+        return result
+
+    def to_representation(self, instance):
+        pre_result = super(NotNullModelSerializer, self).to_representation(instance)
+
+        filtered_result = self.filter_null_values(OrderedDict(), pre_result.items())
+
+        return filtered_result
\ No newline at end of file

LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py

@@ -18,8 +18,13 @@ def parse_key_value_pairs(content):
     :return: dict
     """
     assert '=' in content
-    pairs = content.split('=')
-    return {pairs[0].lower(): pairs[1]}
+    key, value = content.split('=')
+    key = re.sub('([.-])', '_', key).lower()
+
+    if re.match('\A\d*\.\d*', key):
+        key = 'v' + key.rstrip('v')
+
+    return {key: value}
 
 
 def parse_datetime(date, date_time):
@@ -87,7 +92,7 @@ def parse_component_id(component_id_str):
     :param component_id_str: string representation of the component id as in the stationtest output
     :return: the component id integer
     """
-    return int(component_id_str) if component_id_str != '---' else None
+    return int(component_id_str) if component_id_str != '---' else -1
 
 
 def collect_error_details(content):
@@ -112,6 +117,42 @@ def collect_error_details(content):
         rf_fail = parse_rffail_string('y', error_details.pop('y'))
         error_details.update(rf_fail)
 
+    element_errors = []
+    for error_name, error_value in dict(error_details).items():
+        modem_comunication_error_pattern = 'e([0-9]{2})'
+        matched_key = re.search(modem_comunication_error_pattern, error_name)
+
+        if matched_key:
+            error_details.pop(error_name)
+            element_id = int(matched_key.group(1))
+            element = dict(element_id=element_id)
+
+            element_error_details = dict(error_code=error_value)
+            element_error = dict(element=element,
+                                 type='MODEM',
+                                 details=element_error_details)
+            element_errors.append(element_error)
+
+        nosignal_error_pattern = 'E([0-9]{2})([XY])'
+        matched_key = re.search(nosignal_error_pattern, error_name)
+
+        if matched_key:
+            error_details.pop(error_name)
+            element_id = int(matched_key.group(1))
+            polarization = matched_key.group(2).lower()
+            element = dict(element_id=element_id)
+
+            element_error_details = {polarization:error_value}
+            element_error = dict(element=element,
+                                 type='NOSIGNAL',
+                                 details=element_error_details)
+            element_errors.append(element_error)
+
+        if error_name in ['X', 'Y']:
+            error_details.pop(error_name)
+            error_name = error_name.lower()
+            error_details[error_name] = error_value
+    error_details.update(element_errors=element_errors)
     return error_details
 
 
@@ -125,9 +166,17 @@ def dict_from_component_error(content):
     error_details = collect_error_details(content)
 
     component = dict(component_id=parse_component_id(component_id), type=component_type.strip())
+
+    element_errors = error_details.pop('element_errors')
+
     component_error = dict(component=component,
                            details=error_details,
                            type=error_type)
+
+    for element_error in element_errors:
+        element_error['component_error'] = dict(component_error)
+
+    component_error['element_errors'] = element_errors
     return component_error
 
 
@@ -179,6 +228,9 @@ def dict_from_raw_station_test(content):
             result['element_errors'].extend(dicts_from_element_error(row))
         else:
             component_error_dict = dict_from_component_error(row)
+            element_errors = component_error_dict.pop('element_errors')
+            result['element_errors'].extend(element_errors)
+
             result['component_errors'].append(component_error_dict)
 
     return result
@@ -254,7 +306,6 @@ def dicts_from_element_error(contents):
 
         element_error_details = dict()
         if element_error_type in ['HN', 'LN']:
-
             value, diff = map(float, args.strip().split(' '))
             element_error_details[polarization+'val'] = value
             element_error_details[polarization+'diff'] = diff

LCU/Maintenance/DBInterface/monitoringdb/views/station_test_views.py

+from django_filters import rest_framework as filters
 from rest_framework.viewsets import ModelViewSet
 
 from .common import *
@@ -23,36 +24,59 @@ logger = logging.getLogger('views')
 class ComponentViewSet(ModelViewSet):
     serializer_class = ComponentSerializer
     queryset = Component.objects.all()
+    filter_fields = '__all__'
+
+
+class ComponentErrorFilterSet(filters.FilterSet):
+    component_id = filters.NumberFilter(field_name='component', lookup_expr='component_id')
+    component_type = filters.CharFilter(field_name='component', lookup_expr='type')
+
+    station_name = filters.CharFilter(field_name='station_test__station',
+                                      lookup_expr='name__contains')
+    station_type = filters.CharFilter(field_name='station_test__station',
+                                      lookup_expr='name__contains')
+
+    from_date = filters.DateFilter(field_name='station_test', lookup_expr='start_datetime__gt')
+    to_date = filters.DateFilter(field_name='station_test', lookup_expr='end_datetime__lt')
+    type = filters.CharFilter(field_name='type',
+                              lookup_expr='contains')
 
 
 class ComponentErrorViewSet(ModelViewSet):
     serializer_class = ComponentErrorSerializer
     queryset = ComponentError.objects.all()
+    filter_fields = '__all__'
+    filterset_class = ComponentErrorFilterSet
 
 
 class StationViewSet(ModelViewSet):
     serializer_class = StationSerializer
     queryset = Station.objects.all()
+    filter_fields = '__all__'
 
 
 class StationTestViewSet(ModelViewSet):
     serializer_class = StationTestSerializer
     queryset = StationTest.objects.all()
+    filter_fields = '__all__'
 
 
 class ErrorDetailsViewSet(ModelViewSet):
     serializer_class = ErrorDetailsSerializer
     queryset = ErrorDetails.objects.all()
+    filter_fields = '__all__'
 
 
 class ElementErrorViewSet(ModelViewSet):
     serializer_class = ElementErrorSerializer
     queryset = ElementError.objects.all()
+    filter_fields = '__all__'
 
 
 class ElementViewSet(ModelViewSet):
     serializer_class = ElementSerializer
     queryset = Element.objects.all()
+    filter_fields = '__all__'
 
 
 def insert_component_error(station_test_entry,
@@ -70,6 +94,7 @@ def insert_component_error(station_test_entry,
         print('Component entry is not present, inserting ...')
         component_entry = Component(station=station_entry,
                                     **component)
+        print(component_entry, component_error)
         component_entry.save()
 
     component_error_entry = ComponentError.objects.filter(component=component_entry,
@@ -124,7 +149,6 @@ def insert_element_error(station_test_entry,
     element_error_details = element_error.pop('details')
 
     element_entry = insert_element(component, element)
-    print(element_entry)
 
     element_error_entry = ElementError.objects.filter(element=element_entry,
                                                       component_error=component_error_entry,