diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/component_error.py b/LCU/Maintenance/DBInterface/monitoringdb/models/component_error.py
index 9d62ef93337a8ec71cf7207e5559879f14a6922e..23d2bd3cf9c376ca9cdfc889b607e965e87e1dfd 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/models/component_error.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/models/component_error.py
@@ -9,4 +9,4 @@ class ComponentError(models.Model):
component = models.ForeignKey(Component, on_delete=models.DO_NOTHING)
station_test = models.ForeignKey(StationTest, on_delete=models.DO_NOTHING,
related_name='component_errors')
- details = JSONField()
\ No newline at end of file
+ details = JSONField(default={})
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/element_error.py b/LCU/Maintenance/DBInterface/monitoringdb/models/element_error.py
index 1440852dc8e76c7509a2c96bce98c73f0c4f3a5b..fb2e66833630c0703c1d7e5f4a502984514246e5 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/models/element_error.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/models/element_error.py
@@ -12,4 +12,4 @@ class ElementError(models.Model):
related_name='failing_elements')
type = models.CharField(max_length=50)
- details = JSONField()
\ No newline at end of file
+ details = JSONField(default={})
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py b/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py
index 946a52fb90277d77ad2b256a4a3cebb9bc0aaca0..1a63be8f95f9dab7c5f0b67e65c7edb22729624e 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py
@@ -23,7 +23,7 @@ class RTSMObservation(models.Model):
class RTSMObservationSummary(RTSMObservation):
@property
def errors_found(self):
- return RTSMErrorSummary.objects.defer('bad_spectrum', 'average_spectrum'). \
+ return RTSMError.objects.defer('spectra'). \
filter(observation=self).values('error_type', 'rcu'). \
annotate(count=models.Count('error_type', unique=True, output_field=models.FloatField())). \
annotate(percentage=models.ExpressionWrapper(models.F('count') * 100. / models.F('observation__samples'),
@@ -33,32 +33,16 @@ class RTSMObservationSummary(RTSMObservation):
proxy = True
-class RTSMErrorSummary(models.Model):
- rcu = models.SmallIntegerField(default=None, null=True)
- mode = models.SmallIntegerField(default=None, null=True)
- observation = models.ForeignKey(RTSMObservation, related_name='errors_summary', on_delete=models.CASCADE)
-
- error_type = models.CharField(max_length=10)
- start_frequency = models.FloatField(default=None, null=True)
- stop_frequency = models.FloatField(default=None, null=True)
- time = models.DateTimeField()
-
- class Meta:
- managed = False
- db_table = 'monitoringdb_rtsmerror'
-
-
class RTSMError(models.Model):
+ observation = models.ForeignKey(RTSMObservation, related_name='errors', on_delete=models.CASCADE)
+
rcu = models.SmallIntegerField(default=None, null=True)
mode = models.SmallIntegerField(default=None, null=True)
- observation = models.ForeignKey(RTSMObservation, related_name='errors', on_delete=models.CASCADE)
error_type = models.CharField(max_length=10)
start_frequency = models.FloatField(default=None, null=True)
stop_frequency = models.FloatField(default=None, null=True)
time = models.DateTimeField()
- bad_spectrum = ArrayField(models.FloatField())
- average_spectrum = ArrayField(models.FloatField())
def __str__(self):
return self.__unicode__()
@@ -74,6 +58,8 @@ class RTSMError(models.Model):
self.rcu,
self.error_type,)
- class Meta:
- managed = True
- db_table = 'monitoringdb_rtsmerror'
+
+class RTSMSpectrum(models.Model):
+ bad_spectrum = ArrayField(models.FloatField())
+ average_spectrum = ArrayField(models.FloatField())
+ rtsm_error = models.ForeignKey(RTSMError, related_name='spectra', on_delete=models.CASCADE)
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/component_error.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/component_error.py
index aeaaf00b44de6d6a75a372c5a3b2781c6efb3fda..a1ef7be0fd47a7a8eb625bb08834b7b4f55d0ff2 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/component_error.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/component_error.py
@@ -18,7 +18,7 @@ class ComponentErrorSerializer(NotNullModelSerializer):
class Meta:
model = ComponentError
fields = '__all__'
- depth = 2
+ depth = 3
def __init__(self, *args, **kwargs):
self.Meta.depth = kwargs.pop('depth', 2)
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py
index b26d5eef29363a01421349f1dc546f4a2ba33191..e1617cf099f2cc70e41ffe05f7ce49ca3dcb4287 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py
@@ -4,6 +4,7 @@ import logging
from django.db.transaction import atomic
from ..exceptions import ItemAlreadyExists
from django.db.models import ObjectDoesNotExist
+from .log import ActionLogSerializer
logger = logging.getLogger('serializers')
@@ -15,24 +16,24 @@ class RTSMErrorsAggregateSerializer(serializers.Serializer):
rcu = serializers.IntegerField()
-class RTSMErrorSerializer(serializers.ModelSerializer):
+class RTSMSpectrumSerializer(serializers.ModelSerializer):
bad_spectrum = serializers.ListField(child=serializers.FloatField())
average_spectrum = serializers.ListField(child=serializers.FloatField())
class Meta:
- model = RTSMError
+ model = RTSMSpectrum
fields = '__all__'
-class RTSMErrorSummarySerializer(serializers.ModelSerializer):
- samples = serializers.IntegerField(source='observation.samples', read_only=True)
- start_time = serializers.DateTimeField(source='observation.start_time', read_only=True)
- end_time = serializers.DateTimeField(source='observation.start_time', read_only=True)
- observation_id = serializers.IntegerField(source='observation.observation_id', read_only=True)
- id = serializers.IntegerField()
+class RTSMErrorSerializer(serializers.ModelSerializer):
+
+ observation = serializers.PrimaryKeyRelatedField(read_only=True)
+ def __init__(self, *args, **kwargs):
+ self.Meta.depth = kwargs.pop('depth', 1)
+ super(RTSMErrorSerializer, self).__init__(*args, **kwargs)
class Meta:
- model = RTSMErrorSummary
+ model = RTSMError
fields = '__all__'
@@ -42,10 +43,11 @@ class RTSMObservationSummarySerializer(serializers.ModelSerializer):
class Meta:
model = RTSMObservationSummary
fields = '__all__'
+ depth=1
class RTSMObservationSerializer(serializers.ModelSerializer):
- errors_summary = RTSMErrorSummarySerializer(many=True)
+ errors = RTSMErrorSerializer(many=True, depth=0)
class Meta:
model = RTSMObservation
@@ -55,26 +57,49 @@ class RTSMObservationSerializer(serializers.ModelSerializer):
def create(self, validated_data):
rtsm_errors = validated_data.pop('errors')
try:
- start_time, station_name = validated_data['start_time'], validated_data['station_name']
+
+
+ # derive the station type from the station name
+ station_name = validated_data['station_name']
station_type = station_name[0].capitalize()
station_type = station_type if station_type in ['C', 'R'] else 'I'
validated_data.update(station_type=station_type)
- element = RTSMObservation.objects.filter(start_time=start_time,
+
+ # searches if the RTSM is already present
+ observation_id = validated_data['observation_id']
+ element = RTSMObservation.objects.filter(observation_id=
+ observation_id,
station_name=station_name).first()
if element is None:
logger.info('rtsm not found inserting %s', validated_data)
else:
- logger.info('RTSM test for station %s started at %s is present already. Skipping insertion',
- start_time, station_name)
+ logger.info('RTSM test for sas id %s started at station %s is present already. Skipping insertion',
+ observation_id, station_name)
raise ItemAlreadyExists(element)
except ObjectDoesNotExist as e:
logger.info('rtsm not found inserting %s', validated_data)
+
RTSMObservation_instance = RTSMObservation.objects.create(**validated_data)
RTSMObservation_instance.save()
+ ActionLogSerializer.log_model_insert(RTSMObservation_instance)
+
for rtsm_error in rtsm_errors:
rtsm_error.update(observation=RTSMObservation_instance)
+
+ average_spectrum = rtsm_error.pop('average_spectrum')
+ bad_spectrum = rtsm_error.pop('bad_spectrum')
+
+
rtsm_error_instance = RTSMError.objects.create(**rtsm_error)
rtsm_error_instance.save()
+ ActionLogSerializer.log_model_insert(rtsm_error_instance)
+
+ rtsm_spectrum_instance = RTSMSpectrum.objects.create(average_spectrum=average_spectrum,
+ bad_spectrum=bad_spectrum,
+ rtsm_error=rtsm_error_instance)
+ rtsm_spectrum_instance.save()
+
+ ActionLogSerializer.log_model_insert(rtsm_spectrum_instance)
return RTSMObservation_instance
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/station_tests.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/station_tests.py
index 12b9dcbf38d0f791613f5b6f4090745c3398493d..7f9a54f9204f76668ebfa3f332af011b88d913dd 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/station_tests.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/station_tests.py
@@ -14,7 +14,7 @@ class StationTestSerializer(ModelSerializer):
class Meta:
model = StationTest
fields = '__all__'
- depth = 2
+ depth = 5
@staticmethod
def insert_station_test(station_test):
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/urls.py b/LCU/Maintenance/DBInterface/monitoringdb/urls.py
index d70138eb2df2eb62d2be6e2bbf177ab73f94ca9f..27f11b499d8318bc6f0daa28a5568e9a21fd058d 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/urls.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/urls.py
@@ -27,6 +27,8 @@ rtsm_router = routers.DefaultRouter()
rtsm_router.register(r'summary', RTSMObservationSummaryViewSet)
rtsm_router.register(r'errors_detailed', RTSMErrorsDetailedViewSet)
rtsm_router.register(r'errors', RTSMErrorsViewSet)
+rtsm_router.register(r'spectra', RTSMSpectrumViewSet)
+
rtsm_router.register(r'', RTSMObservationViewSet)
@@ -35,8 +37,9 @@ urlpatterns = [
url(r'^api/rtsm/', include(rtsm_router.urls)),
url(r'^api/api-auth', include('rest_framework.urls', namespace='rest_framework')),
- url(r'^api/stationtests/insert_raw', insert_raw_station_test),
- url(r'^api/rtsm/insert_raw', insert_raw_rtsm_test),
+ url(r'^api/stationtests/raw/insert', insert_raw_station_test),
+ url(r'^api/rtsm/raw/insert', insert_raw_rtsm_test),
+
url(r'^api/log/', include(log_router.urls)),
url(r'^api/docs', include_docs_urls(title='Monitoring DB API'))
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py b/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py
index bc84d8c4efcf005a6894a43247449dad0c829f60..96d8f49574a9390a9369a383ec4c7a786d041b6b 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py
@@ -1,62 +1,41 @@
from .common import *
-from ..models.rtsm import RTSMObservation, RTSMError
-from ..serializers.rtsm import RTSMObservationSerializer, RTSMErrorSummarySerializer, RTSMErrorSerializer,\
- RTSMObservationSummarySerializer
+from ..models.rtsm import RTSMObservation, RTSMError, RTSMSpectrum
+from ..serializers.rtsm import RTSMObservationSerializer, RTSMErrorSerializer,\
+ RTSMObservationSummarySerializer, RTSMSpectrumSerializer
from ..rtsm_test_raw_parser import parse_rtsm_test
-from rest_framework.serializers import Serializer
+
logger = logging.getLogger('views')
+class RTSMSpectrumViewSet(viewsets.ReadOnlyModelViewSet):
+ queryset = RTSMSpectrum.objects.all()
+ serializer_class = RTSMSpectrumSerializer
+ filter_fields = ['rtsm_error']
+
class RTSMErrorsDetailedViewSet(viewsets.ReadOnlyModelViewSet):
queryset = RTSMError.objects.all()
serializer_class = RTSMErrorSerializer
+ filter_fields = '__all__'
- def get_queryset(self):
- queryset = RTSMError.objects.all()
- for key, param in self.request.query_params.items():
- if key in RESERVED_FILTER_NAME:
- continue
- queryset = queryset.filter(**{key: param})
- return queryset
class RTSMErrorsViewSet(viewsets.ReadOnlyModelViewSet):
queryset = RTSMError.objects.all()
- serializer_class = RTSMErrorSummarySerializer
+ serializer_class = RTSMErrorSerializer
+ filter_fields = '__all__'
- def get_queryset(self):
- queryset = RTSMError.objects.all()
- for key, param in self.request.query_params.items():
- if key in RESERVED_FILTER_NAME:
- continue
- queryset = queryset.filter(**{key: param})
- return queryset
class RTSMObservationViewSet(viewsets.ReadOnlyModelViewSet):
queryset = RTSMObservation.objects.all()
serializer_class = RTSMObservationSerializer
-
- def get_queryset(self):
- queryset = RTSMObservation.objects.all()
- for key, param in self.request.query_params.items():
- if key in RESERVED_FILTER_NAME:
- continue
- queryset = queryset.filter(**{key: param})
- return queryset
+ filter_fields = '__all__'
class RTSMObservationSummaryViewSet(viewsets.ReadOnlyModelViewSet):
queryset = RTSMObservation.objects.all()
serializer_class = RTSMObservationSummarySerializer
-
- def get_queryset(self):
- queryset = RTSMObservation.objects.all()
- for key, param in self.request.query_params.items():
- if key in RESERVED_FILTER_NAME:
- continue
- queryset = queryset.filter(**{key: param})
- return queryset
+ filter_fields = '__all__'
@api_view(['POST'])
diff --git a/LCU/Maintenance/MDB_tools/cli/mdb_loader.py b/LCU/Maintenance/MDB_tools/cli/mdb_loader.py
index f492f1cc220f7c6167b9478c8e7e370db14f7a5c..999bdaa0273ca4538d36478edf0c42c5c764d123 100755
--- a/LCU/Maintenance/MDB_tools/cli/mdb_loader.py
+++ b/LCU/Maintenance/MDB_tools/cli/mdb_loader.py
@@ -52,7 +52,7 @@ def read_stationtest_rtsm_output_to_dict(file_path):
the station name to which the test refers
ex. {'content': [file_content], 'station_name'}
"""
- rtsm_filename_pattern = '(?P<station_name>\w*)_\d*_\d*.dat'
+ rtsm_filename_pattern = '.*(?P<station_name>\w+)_\d+_\d+\.dat$'
try:
with open(file_path, 'r') as input_stream:
result = dict(content=input_stream.read().strip('\n'), type='stationtest')
@@ -61,6 +61,8 @@ def read_stationtest_rtsm_output_to_dict(file_path):
result.update(station_name=
re.search(rtsm_filename_pattern, file_path).group('station_name'),
type='rtsm')
+ print(re.search(rtsm_filename_pattern, file_path).group('station_name'))
+
return result
except Exception as e:
logging.error('cannot read file %s', file_path)
@@ -169,7 +171,7 @@ def obtain_stationtest_file_list_and_process(file_path_pattern, address):
if send_stationtest_rtsm_content_to_address(address, dict(content=test)):
logger.info('processing file %s: loading test %d of %d',
file_path,
- i,
+ i + 1,
len(tests))
else:
logger.error('error on file %s', file_path)