diff --git a/.gitattributes b/.gitattributes
index d47f4241858699fa7e518b9aaa08b5b1f70d0a83..ea2f19e498d22b1498d79537649c9e59698f6430 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1816,8 +1816,10 @@ LCU/Maintenance/DBInterface/monitoringdb/serializers/component_error.py -text
LCU/Maintenance/DBInterface/monitoringdb/serializers/element.py -text
LCU/Maintenance/DBInterface/monitoringdb/serializers/element_error.py -text
LCU/Maintenance/DBInterface/monitoringdb/serializers/error_details.py -text
+LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py -text
LCU/Maintenance/DBInterface/monitoringdb/serializers/station.py -text
LCU/Maintenance/DBInterface/monitoringdb/serializers/station_tests.py -text
+LCU/Maintenance/DBInterface/monitoringdb/serializers/utils.py -text
LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py -text
LCU/Maintenance/DBInterface/monitoringdb/tests/__init__.py -text
LCU/Maintenance/DBInterface/monitoringdb/tests/common.py -text
diff --git a/LCU/Maintenance/DBInterface/CMakeLists.txt b/LCU/Maintenance/DBInterface/CMakeLists.txt
index 05dde07d10464b296cf57376a20a75f90c6b8fa6..a2b97ed843a26b7d2f5b0186b9a9d5973355765e 100644
--- a/LCU/Maintenance/DBInterface/CMakeLists.txt
+++ b/LCU/Maintenance/DBInterface/CMakeLists.txt
@@ -12,7 +12,7 @@ find_python_module(rest_framework REQUIRED) #sudo pip install djangores
find_python_module(rest_polymorphic REQUIRED) #sudo pip install django-polymorphic
find_python_module(polymorphic REQUIRED) #sudo pip install django-rest-polymorphic
find_python_module(requests REQUIRED)
-find_python_module(pandas REQUIRED)
+
# includes every python file excepts for the manage.py
FILE(GLOB_RECURSE PY_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ./*.py)
diff --git a/LCU/Maintenance/DBInterface/django_postgresql/settings.py b/LCU/Maintenance/DBInterface/django_postgresql/settings.py
index d067939c84501ff4384a976a1a3c4eeb9134447c..bf15401446632b5428f003c80dba9b84ccc676f4 100644
--- a/LCU/Maintenance/DBInterface/django_postgresql/settings.py
+++ b/LCU/Maintenance/DBInterface/django_postgresql/settings.py
@@ -36,6 +36,7 @@ INSTALLED_APPS = [
'django.contrib.messages',
'django.contrib.staticfiles',
'rest_framework',
+ 'django_filters',
'lofar.maintenance.monitoringdb.apps.MonitoringDbConfig',
]
@@ -169,5 +170,6 @@ STATIC_URL = '/static/'
REST_FRAMEWORK = {
'DEFAULT_PAGINATION_CLASS': 'lofar.maintenance.monitoringdb.pagination.DefaultPaginationSettings',
- 'STRICT_JSON': False
+ 'STRICT_JSON': False,
+ 'DEFAULT_FILTER_BACKENDS': ('django_filters.rest_framework.DjangoFilterBackend',)
}
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/error_details.py b/LCU/Maintenance/DBInterface/monitoringdb/models/error_details.py
index b8a9c0a83f067fdaf1673ddc2cabb59b1c86fc6d..38efb0da44888fdfe521cb83c45134c2d396b383 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/models/error_details.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/models/error_details.py
@@ -2,14 +2,14 @@ from django.db import models
class ErrorDetails(models.Model):
- x = models.BooleanField(default=None, null=True)
- y = models.BooleanField(default=None, null=True)
+ x = models.NullBooleanField()
+ y = models.NullBooleanField()
xval = models.FloatField(default=None, null=True)
yval = models.FloatField(default=None, null=True)
- xoffset = models.FloatField(default=None, null=True)
- yoffset = models.FloatField(default=None, null=True)
+ xoff = models.FloatField(default=None, null=True)
+ yoff = models.FloatField(default=None, null=True)
xval_no_delay = models.FloatField(default=None, null=True)
yval_no_delay = models.FloatField(default=None, null=True)
@@ -40,3 +40,34 @@ class ErrorDetails(models.Model):
xdiff = models.FloatField(default=None, null=True)
ydiff = models.FloatField(default=None, null=True)
+
+ xmean = models.FloatField(default=None, null=True)
+ ymean = models.FloatField(default=None, null=True)
+
+ tp = models.CharField(max_length=10, null=True, default=None)
+ mp = models.CharField(max_length=10, null=True, default=None)
+ ap = models.CharField(max_length=10, null=True, default=None)
+ bp = models.CharField(max_length=10, null=True, default=None)
+
+ mp0 = models.CharField(max_length=10, null=True, default=None)
+ mp1 = models.CharField(max_length=10, null=True, default=None)
+ mp2 = models.CharField(max_length=10, null=True, default=None)
+ mp3 = models.CharField(max_length=10, null=True, default=None)
+
+ ap0 = models.CharField(max_length=10, null=True, default=None)
+ ap1 = models.CharField(max_length=10, null=True, default=None)
+ ap2 = models.CharField(max_length=10, null=True, default=None)
+ ap3 = models.CharField(max_length=10, null=True, default=None)
+
+ pcb = models.CharField(max_length=10, null=True, default=None)
+
+ rcu_5_0v = models.FloatField(default=None, null=True)
+ lba_8_0v = models.FloatField(default=None, null=True)
+ hba_48v = models.FloatField(default=None, null=True)
+ spu_3_3v = models.FloatField(default=None, null=True)
+
+ v1_2 = models.FloatField(default=None, null=True)
+ v2_5 = models.FloatField(default=None, null=True)
+ v3_3 = models.FloatField(default=None, null=True)
+
+ error_code = models.CharField(max_length=10, null=True, default=None)
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/component.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/component.py
index 836b28f3fe44f6407c2a2b5a67a1750afb9eb445..b3a188222f08a1baa55c2eadc7c88b65c2f1d81f 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/component.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/component.py
@@ -1,7 +1,8 @@
-from rest_framework.serializers import HyperlinkedModelSerializer
+from rest_framework.serializers import ModelSerializer
from ..models.component import Component
-class ComponentSerializer(HyperlinkedModelSerializer):
+class ComponentSerializer(ModelSerializer):
class Meta:
model = Component
- fields = '__all__'
\ No newline at end of file
+ fields = '__all__'
+ depth = 1
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/component_error.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/component_error.py
index db3d12aa12a9d3b18189d5d14b792f5babfaee08..fca4cfdd124299360c886090d5e069f27181f57c 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/component_error.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/component_error.py
@@ -1,7 +1,10 @@
-from rest_framework.serializers import HyperlinkedModelSerializer
+from rest_framework.serializers import HyperlinkedModelSerializer, ModelSerializer
from ..models.component_error import ComponentError, Component
+from .utils import NotNullModelSerializer
-class ComponentErrorSerializer(HyperlinkedModelSerializer):
+
+class ComponentErrorSerializer(NotNullModelSerializer):
class Meta:
model = ComponentError
fields = '__all__'
+ depth = 2
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/element.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/element.py
index 27b3bee6931385fce9bd67ca16f390a2e31a57b9..61dbbd28835ec31479cb992467aa1a6962c60704 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/element.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/element.py
@@ -1,8 +1,9 @@
-from rest_framework.serializers import HyperlinkedModelSerializer
+from rest_framework.serializers import ModelSerializer
from ..models.element import Element
-class ElementSerializer(HyperlinkedModelSerializer):
+class ElementSerializer(ModelSerializer):
class Meta:
model = Element
fields = '__all__'
+ depth = 1
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/element_error.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/element_error.py
index 36873a66b36c99362132f548257cd275bc97c128..fad380a9bebe1d4c24fe80043d559a517b7186b3 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/element_error.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/element_error.py
@@ -1,8 +1,9 @@
-from rest_framework.serializers import HyperlinkedModelSerializer
+from rest_framework.serializers import ModelSerializer
from ..models.element_error import ElementError
-class ElementErrorSerializer(HyperlinkedModelSerializer):
+class ElementErrorSerializer(ModelSerializer):
class Meta:
model = ElementError
fields = '__all__'
+ depth = 1
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/error_details.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/error_details.py
index 7c8db3cf1fcba61b89d7870526ac6b1e3b8e459a..e1a8af9ee7903d1325eab08707ca169f6ead7d01 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/error_details.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/error_details.py
@@ -1,7 +1,16 @@
-from rest_framework.serializers import HyperlinkedModelSerializer
+from rest_framework.serializers import ModelSerializer
from ..models.error_details import ErrorDetails
-class ErrorDetailsSerializer(HyperlinkedModelSerializer):
+class ErrorDetailsSerializer(ModelSerializer):
class Meta:
model = ErrorDetails
fields = '__all__'
+
+ def to_representation(self, instance):
+ result = super().to_representation(instance)
+
+ result = {key: value
+ for key, value in
+ filter(lambda item: item[1] is not None, result.items())}
+
+ return result
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py
new file mode 100644
index 0000000000000000000000000000000000000000..b26d5eef29363a01421349f1dc546f4a2ba33191
--- /dev/null
+++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py
@@ -0,0 +1,80 @@
+from ..models.rtsm import *
+from rest_framework import serializers
+import logging
+from django.db.transaction import atomic
+from ..exceptions import ItemAlreadyExists
+from django.db.models import ObjectDoesNotExist
+
+logger = logging.getLogger('serializers')
+
+
+class RTSMErrorsAggregateSerializer(serializers.Serializer):
+ count = serializers.IntegerField()
+ percentage = serializers.FloatField()
+ error_type = serializers.CharField()
+ rcu = serializers.IntegerField()
+
+
+class RTSMErrorSerializer(serializers.ModelSerializer):
+ bad_spectrum = serializers.ListField(child=serializers.FloatField())
+ average_spectrum = serializers.ListField(child=serializers.FloatField())
+
+ class Meta:
+ model = RTSMError
+ fields = '__all__'
+
+
+class RTSMErrorSummarySerializer(serializers.ModelSerializer):
+ samples = serializers.IntegerField(source='observation.samples', read_only=True)
+ start_time = serializers.DateTimeField(source='observation.start_time', read_only=True)
+ end_time = serializers.DateTimeField(source='observation.start_time', read_only=True)
+ observation_id = serializers.IntegerField(source='observation.observation_id', read_only=True)
+ id = serializers.IntegerField()
+
+ class Meta:
+ model = RTSMErrorSummary
+ fields = '__all__'
+
+
+class RTSMObservationSummarySerializer(serializers.ModelSerializer):
+ errors_found = RTSMErrorsAggregateSerializer(many=True)
+
+ class Meta:
+ model = RTSMObservationSummary
+ fields = '__all__'
+
+
+class RTSMObservationSerializer(serializers.ModelSerializer):
+ errors_summary = RTSMErrorSummarySerializer(many=True)
+
+ class Meta:
+ model = RTSMObservation
+ fields = '__all__'
+
+ @atomic
+ def create(self, validated_data):
+ rtsm_errors = validated_data.pop('errors')
+ try:
+ start_time, station_name = validated_data['start_time'], validated_data['station_name']
+ station_type = station_name[0].capitalize()
+ station_type = station_type if station_type in ['C', 'R'] else 'I'
+ validated_data.update(station_type=station_type)
+ element = RTSMObservation.objects.filter(start_time=start_time,
+ station_name=station_name).first()
+
+ if element is None:
+ logger.info('rtsm not found inserting %s', validated_data)
+ else:
+ logger.info('RTSM test for station %s started at %s is present already. Skipping insertion',
+ start_time, station_name)
+ raise ItemAlreadyExists(element)
+ except ObjectDoesNotExist as e:
+ logger.info('rtsm not found inserting %s', validated_data)
+ RTSMObservation_instance = RTSMObservation.objects.create(**validated_data)
+ RTSMObservation_instance.save()
+ for rtsm_error in rtsm_errors:
+ rtsm_error.update(observation=RTSMObservation_instance)
+ rtsm_error_instance = RTSMError.objects.create(**rtsm_error)
+ rtsm_error_instance.save()
+
+ return RTSMObservation_instance
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/station.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/station.py
index 6d6965baa5dc6f5a54902198c6fb6c1940c30654..3908306f4fa6ddf802b6ec564e83db9097746fcf 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/station.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/station.py
@@ -1,7 +1,9 @@
-from rest_framework.serializers import HyperlinkedModelSerializer
+from rest_framework.serializers import ModelSerializer
from ..models.station import Station
-class StationSerializer(HyperlinkedModelSerializer):
+
+class StationSerializer(ModelSerializer):
class Meta:
model = Station
- fields = '__all__'
\ No newline at end of file
+ fields = '__all__'
+ depth = 1
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/station_tests.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/station_tests.py
index 155348b7f16f15fc037e65a9f428b338957f7141..bf66977bb752344434b9e59217bb1c8d7050b1fe 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/station_tests.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/station_tests.py
@@ -1,7 +1,8 @@
-from rest_framework.serializers import HyperlinkedModelSerializer
+from rest_framework.serializers import ModelSerializer
+from ..models.station_test import StationTest
-
-class StationTestSerializer(HyperlinkedModelSerializer):
+class StationTestSerializer(ModelSerializer):
class Meta:
- model = 'StationTest'
- fields = '__all__'
\ No newline at end of file
+ model = StationTest
+ fields = '__all__'
+ depth = 1
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/utils.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..ec9989f955c7d24c153c3664a3f49183d354cd8f
--- /dev/null
+++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/utils.py
@@ -0,0 +1,20 @@
+from rest_framework.serializers import ModelSerializer
+from collections import OrderedDict
+
+
+class NotNullModelSerializer(ModelSerializer):
+ def filter_null_values(self, result, items):
+ for key, value in items:
+ if isinstance(value, dict):
+ result[key] = self.filter_null_values(OrderedDict(), value.items())
+ else:
+ if value is not None:
+ result[key] = value
+ return result
+
+ def to_representation(self, instance):
+ pre_result = super(NotNullModelSerializer, self).to_representation(instance)
+
+ filtered_result = self.filter_null_values(OrderedDict(), pre_result.items())
+
+ return filtered_result
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py b/LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py
index 21c83a0042939966882b34ae21330d1124d130fa..2fab6c0e6aba8976d3020e024b39012ca870edb9 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py
@@ -18,8 +18,13 @@ def parse_key_value_pairs(content):
:return: dict
"""
assert '=' in content
- pairs = content.split('=')
- return {pairs[0].lower(): pairs[1]}
+ key, value = content.split('=')
+ key = re.sub('([.-])', '_', key).lower()
+
+ if re.match('\A\d*\.\d*', key):
+ key = 'v' + key.rstrip('v')
+
+ return {key: value}
def parse_datetime(date, date_time):
@@ -87,7 +92,7 @@ def parse_component_id(component_id_str):
:param component_id_str: string representation of the component id as in the stationtest output
:return: the component id integer
"""
- return int(component_id_str) if component_id_str != '---' else None
+ return int(component_id_str) if component_id_str != '---' else -1
def collect_error_details(content):
@@ -112,6 +117,42 @@ def collect_error_details(content):
rf_fail = parse_rffail_string('y', error_details.pop('y'))
error_details.update(rf_fail)
+ element_errors = []
+ for error_name, error_value in dict(error_details).items():
+ modem_comunication_error_pattern = 'e([0-9]{2})'
+ matched_key = re.search(modem_comunication_error_pattern, error_name)
+
+ if matched_key:
+ error_details.pop(error_name)
+ element_id = int(matched_key.group(1))
+ element = dict(element_id=element_id)
+
+ element_error_details = dict(error_code=error_value)
+ element_error = dict(element=element,
+ type='MODEM',
+ details=element_error_details)
+ element_errors.append(element_error)
+
+ nosignal_error_pattern = 'E([0-9]{2})([XY])'
+ matched_key = re.search(nosignal_error_pattern, error_name)
+
+ if matched_key:
+ error_details.pop(error_name)
+ element_id = int(matched_key.group(1))
+ polarization = matched_key.group(2).lower()
+ element = dict(element_id=element_id)
+
+ element_error_details = {polarization:error_value}
+ element_error = dict(element=element,
+ type='NOSIGNAL',
+ details=element_error_details)
+ element_errors.append(element_error)
+
+ if error_name in ['X', 'Y']:
+ error_details.pop(error_name)
+ error_name = error_name.lower()
+ error_details[error_name] = error_value
+ error_details.update(element_errors=element_errors)
return error_details
@@ -125,9 +166,17 @@ def dict_from_component_error(content):
error_details = collect_error_details(content)
component = dict(component_id=parse_component_id(component_id), type=component_type.strip())
+
+ element_errors = error_details.pop('element_errors')
+
component_error = dict(component=component,
details=error_details,
type=error_type)
+
+ for element_error in element_errors:
+ element_error['component_error'] = dict(component_error)
+
+ component_error['element_errors'] = element_errors
return component_error
@@ -179,6 +228,9 @@ def dict_from_raw_station_test(content):
result['element_errors'].extend(dicts_from_element_error(row))
else:
component_error_dict = dict_from_component_error(row)
+ element_errors = component_error_dict.pop('element_errors')
+ result['element_errors'].extend(element_errors)
+
result['component_errors'].append(component_error_dict)
return result
@@ -254,7 +306,6 @@ def dicts_from_element_error(contents):
element_error_details = dict()
if element_error_type in ['HN', 'LN']:
-
value, diff = map(float, args.strip().split(' '))
element_error_details[polarization+'val'] = value
element_error_details[polarization+'diff'] = diff
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/views/station_test_views.py b/LCU/Maintenance/DBInterface/monitoringdb/views/station_test_views.py
index 45f04c4dab6a88b76884f61bc2a4fcaf90f33840..13b73513b3b180fe8bf21edd48e832ba27de9612 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/views/station_test_views.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/views/station_test_views.py
@@ -1,3 +1,4 @@
+from django_filters import rest_framework as filters
from rest_framework.viewsets import ModelViewSet
from .common import *
@@ -23,36 +24,59 @@ logger = logging.getLogger('views')
class ComponentViewSet(ModelViewSet):
serializer_class = ComponentSerializer
queryset = Component.objects.all()
+ filter_fields = '__all__'
+
+
+class ComponentErrorFilterSet(filters.FilterSet):
+ component_id = filters.NumberFilter(field_name='component', lookup_expr='component_id')
+ component_type = filters.CharFilter(field_name='component', lookup_expr='type')
+
+ station_name = filters.CharFilter(field_name='station_test__station',
+ lookup_expr='name__contains')
+ station_type = filters.CharFilter(field_name='station_test__station',
+ lookup_expr='name__contains')
+
+ from_date = filters.DateFilter(field_name='station_test', lookup_expr='start_datetime__gt')
+ to_date = filters.DateFilter(field_name='station_test', lookup_expr='end_datetime__lt')
+ type = filters.CharFilter(field_name='type',
+ lookup_expr='contains')
class ComponentErrorViewSet(ModelViewSet):
serializer_class = ComponentErrorSerializer
queryset = ComponentError.objects.all()
+ filter_fields = '__all__'
+ filterset_class = ComponentErrorFilterSet
class StationViewSet(ModelViewSet):
serializer_class = StationSerializer
queryset = Station.objects.all()
+ filter_fields = '__all__'
class StationTestViewSet(ModelViewSet):
serializer_class = StationTestSerializer
queryset = StationTest.objects.all()
+ filter_fields = '__all__'
class ErrorDetailsViewSet(ModelViewSet):
serializer_class = ErrorDetailsSerializer
queryset = ErrorDetails.objects.all()
+ filter_fields = '__all__'
class ElementErrorViewSet(ModelViewSet):
serializer_class = ElementErrorSerializer
queryset = ElementError.objects.all()
+ filter_fields = '__all__'
class ElementViewSet(ModelViewSet):
serializer_class = ElementSerializer
queryset = Element.objects.all()
+ filter_fields = '__all__'
def insert_component_error(station_test_entry,
@@ -70,6 +94,7 @@ def insert_component_error(station_test_entry,
print('Component entry is not present, inserting ...')
component_entry = Component(station=station_entry,
**component)
+ print(component_entry, component_error)
component_entry.save()
component_error_entry = ComponentError.objects.filter(component=component_entry,
@@ -124,7 +149,6 @@ def insert_element_error(station_test_entry,
element_error_details = element_error.pop('details')
element_entry = insert_element(component, element)
- print(element_entry)
element_error_entry = ElementError.objects.filter(element=element_entry,
component_error=component_error_entry,