OSB-45: end of week commit

.gitattributes

@@ -1863,7 +1863,9 @@ LCU/Maintenance/DBInterface/monitoringdb/serializers/station_tests.py -text
 LCU/Maintenance/DBInterface/monitoringdb/serializers/utils.py -text
 LCU/Maintenance/DBInterface/monitoringdb/serializers/wincc.py -text
 LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py -text
-LCU/Maintenance/DBInterface/monitoringdb/tasks.py -text
+LCU/Maintenance/DBInterface/monitoringdb/tasks/__init__.py -text
+LCU/Maintenance/DBInterface/monitoringdb/tasks/generate_plots.py -text
+LCU/Maintenance/DBInterface/monitoringdb/tasks/insert_raw_tests.py -text
 LCU/Maintenance/DBInterface/monitoringdb/tests/__init__.py -text
 LCU/Maintenance/DBInterface/monitoringdb/tests/common.py -text
 LCU/Maintenance/DBInterface/monitoringdb/tests/old_tests.py -text

LCU/Maintenance/DBInterface/django_postgresql/settings.py

@@ -30,7 +30,7 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 SECRET_KEY = 'x)5=9*2a&)32h-loh@rlt_9eyw%t$-fqao*#1j2gh^7=^bnjyy'
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = True
+DEBUG = False
 
 ALLOWED_HOSTS = ['lofarmonitortest.control.lofar', 'localhost', '127.0.0.1']
 

LCU/Maintenance/DBInterface/monitoringdb/models/__init__.py

@@ -6,3 +6,12 @@
 #from .tbbcomponenterror import *
 #from .spucomponenterror import *
 #from .stationtest import *
+__all__ = ['component',
+           'component_error',
+           'element',
+           'element_error',
+           'station',
+           'station_test',
+           'log',
+           'rtsm',
+           'wincc']
\ No newline at end of file

LCU/Maintenance/DBInterface/monitoringdb/models/component.py

@@ -5,4 +5,7 @@ from .station import Station
 class Component(models.Model):
     type = models.CharField(max_length=50)
     component_id = models.IntegerField()
-    station = models.ForeignKey(Station, on_delete=models.DO_NOTHING)
\ No newline at end of file
+    station = models.ForeignKey(Station, on_delete=models.DO_NOTHING)
+
+    class Meta:
+        unique_together = ('station', 'type', 'component_id')

LCU/Maintenance/DBInterface/monitoringdb/models/component_error.py

@@ -9,4 +9,7 @@ class ComponentError(models.Model):
     component = models.ForeignKey(Component, on_delete=models.DO_NOTHING)
     station_test = models.ForeignKey(StationTest, on_delete=models.DO_NOTHING,
                                      related_name='component_errors')
-    details = JSONField(default=dict)
\ No newline at end of file
+    details = JSONField(default=dict)
+
+    class Meta:
+        unique_together = ('station_test', 'component', 'type')

LCU/Maintenance/DBInterface/monitoringdb/models/element.py

@@ -5,3 +5,6 @@ from .component import Component
 class Element(models.Model):
     component = models.ForeignKey(Component, on_delete=models.DO_NOTHING)
     element_id = models.PositiveIntegerField()
+
+    class Meta:
+        unique_together = ('component', 'element_id')

LCU/Maintenance/DBInterface/monitoringdb/models/element_error.py

@@ -12,4 +12,7 @@ class ElementError(models.Model):
                                         related_name='failing_elements')
 
     type = models.CharField(max_length=50)
-    details = JSONField(default=dict)
\ No newline at end of file
+    details = JSONField(default=dict)
+
+    class Meta:
+        unique_together = ('component_error', 'element', 'type')

LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py

@@ -4,6 +4,7 @@ In this file all the model regarding the real time station monitor output are co
 from django.db import models
 from django.contrib.postgres.fields import ArrayField
 from .station import Station
+from .component import Component
 
 MODE_TO_COMPONENT = {
     1: 'LBL',
@@ -35,43 +36,22 @@ class RTSMObservation(models.Model):
 
 class RTSMError(models.Model):
     observation = models.ForeignKey(RTSMObservation, related_name='errors', on_delete=models.CASCADE)
-
+    component = models.ForeignKey(Component, on_delete=models.SET_NULL)
     rcu = models.SmallIntegerField(default=None, null=True)
     mode = models.SmallIntegerField(default=None, null=True)
+    percentage = models.FloatField(default=None, null=True)
 
     error_type = models.CharField(max_length=50)
-    start_frequency = models.FloatField(default=None, null=True)
-    stop_frequency = models.FloatField(default=None, null=True)
-    time = models.DateTimeField()
 
-    def __str__(self):
-        return self.__unicode__()
 
-    def __repr__(self):
-        return self.__unicode__()
-
-    def __unicode__(self):
-        return '%s - MODE %s, FROM %s MHz TO %s MHz RCU=(%s): %s' % (self.time,
-                                                                     self.mode,
-                                                                     self.start_frequency,
-                                                                     self.stop_frequency,
-                                                                     self.rcu,
-                                                                     self.error_type,)
-
-
-class RTSMErrorSummary(models.Model):
-    observation = models.ForeignKey(RTSMObservation, related_name='errors_summary', on_delete=models.CASCADE)
-    percentage = models.FloatField(default=None, null=True)
-    rcu = models.SmallIntegerField(default=None, null=True)
-    mode = models.SmallIntegerField(default=None, null=True)
-    count = models.IntegerField(default=None, null=True)
-    error_type = models.CharField(max_length=50)
-    start_frequency = models.FloatField(default=None, null=True)
-    stop_frequency = models.FloatField(default=None, null=True)
+class RTSMErrorSample(models.Model):
+    time = models.DateTimeField()
+    error = models.ForeignKey(RTSMError, related_name='samples', on_delete=models.CASCADE)
+    observation_id = models.ForeignKey(RTSMObservation, on_delete=models.CASCADE)
 
 
 class RTSMSummaryPlot(models.Model):
-    error_summary = models.ForeignKey(RTSMErrorSummary,
+    error_summary = models.ForeignKey(RTSMError,
                                       related_name='summary_plot',
                                       on_delete=models.SET_NULL,
                                       null=True)
@@ -80,6 +60,9 @@ class RTSMSummaryPlot(models.Model):
 
 
 class RTSMSpectrum(models.Model):
+    start_frequency = models.FloatField(default=None, null=True)
+    stop_frequency = models.FloatField(default=None, null=True)
+
     bad_spectrum = ArrayField(models.FloatField())
     average_spectrum = ArrayField(models.FloatField())
-    rtsm_error = models.ForeignKey(RTSMError, related_name='spectra', on_delete=models.CASCADE)
+    rtsm_error = models.ForeignKey(RTSMErrorSample, related_name='spectra', on_delete=models.CASCADE)

LCU/Maintenance/DBInterface/monitoringdb/models/station.py

@@ -4,5 +4,5 @@ from .fixed_types import STATION_TYPES
 
 class Station(models.Model):
     location = models.CharField(max_length=50, null=True, blank=True, help_text='where the station is located')
-    name = models.CharField(max_length=10, help_text='name of the station')
+    name = models.CharField(max_length=10, help_text='name of the station', unique=True)
     type = models.CharField(max_length=1, choices=STATION_TYPES, help_text='station type one of [Core[C], Remote[R], International[I]]')

LCU/Maintenance/DBInterface/monitoringdb/models/station_test.py

@@ -6,4 +6,5 @@ class StationTest(models.Model):
     start_datetime = models.DateTimeField()
     end_datetime = models.DateTimeField()
     checks = models.CharField(max_length=2000)
-    station = models.ForeignKey(Station, on_delete=models.DO_NOTHING)
\ No newline at end of file
+    station = models.ForeignKey(Station, on_delete=models.DO_NOTHING)
+

LCU/Maintenance/DBInterface/monitoringdb/rtsm_test_raw_parser.py

@@ -74,6 +74,29 @@ def preparse_rtsm_test_file(content):
     return observation
 
 
+def group_samples_by_error_type(content):
+    errors = content.pop('errors')
+    grouped_errors = dict()
+    for error in errors:
+        key = (error['mode'], error['rcu'], error['error_type'])
+        time = error.pop('time')
+        average_spectrum = error.pop('average_spectrum')
+        bad_spectrum = error.pop('bad_spectrum')
+        sample = dict(time=time,
+                      average_spectrum=average_spectrum,
+                      bad_spectrum=bad_spectrum)
+        if key in grouped_errors:
+            grouped_errors[key]['n_samples'] += 1
+            grouped_errors[key]['samples'] += [sample]
+        else:
+            grouped_errors[key] = dict(
+                n_samples = 1,
+                samples = [sample],
+                **error
+            )
+    content['errors'] = list(grouped_errors.values())
+    return content
+
 def parse_rtsm_test(content):
     """
     Expects a string content with the rtsm test output
@@ -81,4 +104,4 @@ def parse_rtsm_test(content):
     :param content: string content with the station test output
     :return: a list of Django models
     """
-    return preparse_rtsm_test_file(split_raw_rtsm_test(content))
\ No newline at end of file
+    return group_samples_by_error_type(preparse_rtsm_test_file(split_raw_rtsm_test(content)))
\ No newline at end of file

LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py

@@ -20,29 +20,28 @@ class RTSMSpectrumSerializer(serializers.ModelSerializer):
         fields = '__all__'
 
 
-class RTSMErrorSerializer(serializers.ModelSerializer):
+class RTSMErrorSampleSerializer(serializers.ModelSerializer):
     observation = serializers.PrimaryKeyRelatedField(read_only=True)
 
     def __init__(self, *args, **kwargs):
         self.Meta.depth = kwargs.pop('depth', 1)
-        super(RTSMErrorSerializer, self).__init__(*args, **kwargs)
+        super(RTSMErrorSampleSerializer, self).__init__(*args, **kwargs)
 
     class Meta:
-        model = RTSMError
+        model = RTSMErrorSample
         fields = '__all__'
 
 
-class RTSMErrorSummarySerializer(serializers.ModelSerializer):
+class RTSMErrorSerializer(serializers.ModelSerializer):
     observation = serializers.PrimaryKeyRelatedField(read_only=True)
 
     class Meta:
-        model = RTSMErrorSummary
+        model = RTSMError
         fields = '__all__'
 
 
 class RTSMObservationSerializer(serializers.ModelSerializer):
-    errors = RTSMErrorSerializer(many=True, depth=0)
-    errors_summary = RTSMErrorSummarySerializer(many=True)
+    errors = RTSMErrorSerializer(many=True)
 
     class Meta:
         model = RTSMObservation
@@ -57,7 +56,7 @@ class RTSMObservationSerializer(serializers.ModelSerializer):
             annotate(percentage=models.ExpressionWrapper(models.F('count') * 100. / models.F('observation__samples'),
                                                          output_field=models.FloatField()))
         for summary_entry in summary_entries:
-            RTSMErrorSummary(observation=RTSMObservation_entry, **summary_entry).save()
+            RTSMError(observation=RTSMObservation_entry, **summary_entry).save()
 
     @atomic
     def create(self, validated_data):
@@ -87,11 +86,11 @@ class RTSMObservationSerializer(serializers.ModelSerializer):
             raise ItemAlreadyExists(RTSMObservation_instance)
         for rtsm_error in rtsm_errors:
             rtsm_error.update(observation=RTSMObservation_instance)
-
+            print(rtsm_error)
             average_spectrum = rtsm_error.pop('average_spectrum')
             bad_spectrum = rtsm_error.pop('bad_spectrum')
 
-            rtsm_error_instance = RTSMError.objects.create(**rtsm_error)
+            rtsm_error_instance = RTSMErrorSample.objects.create(**rtsm_error)
             rtsm_error_instance.save()
             ActionLogSerializer.log_model_insert(rtsm_error_instance)
 

LCU/Maintenance/DBInterface/monitoringdb/tasks.py → LCU/Maintenance/DBInterface/monitoringdb/tasks/generate_plots.py

@@ -8,7 +8,7 @@ import matplotlib.pyplot as plt
 import numpy
 from celery import shared_task
 from django.conf import settings
-from .models.rtsm import MODE_TO_COMPONENT, RTSMErrorSummary, RTSMSummaryPlot
+from lofar.maintenance.monitoringdb.models.rtsm import MODE_TO_COMPONENT, RTSMError, RTSMSummaryPlot
 
 logger = logging.getLogger(__name__)
 
@@ -179,7 +179,7 @@ def generate_summary_plot_for_error(error_summary_id):
     """
     basePath = settings.URL_TO_STORE_RTSM_PLOTS
 
-    error_summary = RTSMErrorSummary.objects.get(pk=error_summary_id)
+    error_summary = RTSMError.objects.get(pk=error_summary_id)
     summary_plot = error_summary.summary_plot.first()
     if summary_plot is not None:
         if summary_plot.uri == None:

LCU/Maintenance/DBInterface/monitoringdb/tasks/insert_raw_tests.py

+import logging
+
+from celery import shared_task
+from django.db import transaction
+from lofar.maintenance.monitoringdb.models.component import Component
+from lofar.maintenance.monitoringdb.models.component_error import ComponentError
+from lofar.maintenance.monitoringdb.models.element import Element
+from lofar.maintenance.monitoringdb.models.element_error import ElementError
+from lofar.maintenance.monitoringdb.models.station import Station
+from lofar.maintenance.monitoringdb.models.station_test import StationTest
+from lofar.maintenance.monitoringdb.station_test_raw_parser import parse_raw_station_test
+from lofar.maintenance.monitoringdb.models.log import ActionLog
+logger = logging.getLogger(__name__)
+
+
+def create_component_errors(station: Station, station_test: StationTest, component_errors_data):
+    component_list = []
+    component_error_list = []
+    for component_error in component_errors_data:
+        component = component_error.pop('component')
+        component_entity, _ = Component.objects.get_or_create(station=station, **component)
+
+        component_list.append(component_entity)
+        component_error_entity, _ = ComponentError.objects.get_or_create(component=component_entity,
+                                                                             station_test=station_test,
+                                                                             **component_error)
+
+        component_error_list.append(component_error_entity)
+
+    return component_list, component_error_list
+
+
+def create_element_errors(station, station_test, elements_error_data):
+    element_errors = []
+    for element_error_data in elements_error_data:
+        component_error = element_error_data.pop('component_error')
+        component = component_error.pop('component')
+        element = element_error_data.pop('element')
+
+        component, _ = Component.objects.get_or_create(station=station, **component)
+        component_error, _ = ComponentError.objects.get_or_create(station_test=station_test,
+                                                                  component=component,
+                                                                  **component_error)
+        element, _ = Element.objects.get_or_create(**element, component=component)
+
+        element_error, _ = ElementError.objects.get_or_create(element=element,
+                                                              component_error=component_error,
+                                                              **element_error_data)
+
+        element_errors.append(element_error)
+
+
+def create_station(station_data):
+    station, created = Station.objects.get_or_create(**station_data)
+    return station
+
+
+def create_station_test(station: Station, station_test_data):
+    station_test, created = StationTest.objects.get_or_create(station=station, **station_test_data)
+    return station_test
+
+
+@transaction.atomic
+def create_station_tests(station_tests_data):
+    parsed_tests = parse_raw_station_test(station_tests_data)
+
+    for station_test_data in parsed_tests:
+        station_data = station_test_data.pop('station')
+        component_errors = station_test_data.pop('component_errors')
+        element_errors = station_test_data.pop('element_errors')
+
+        station = create_station(station_data)
+        station, created = Station.objects.get_or_create(station)
+
+        station_test = create_station_test(station, station_test_data)
+
+        create_component_errors(station,
+                                station_test,
+                                component_errors)
+        create_element_errors(station, station_test, element_errors)
+
+
+@shared_task()
+def insert_station_test(action_log_id: ActionLog, raw_tests):
+    action_log = ActionLog.objects.get(pk=action_log_id)
+    try:
+        parsed_content = parse_raw_station_test(raw_tests)
+        if parsed_content is None:
+            raise Exception('cannot parse test {}'.format(raw_tests))
+        create_station_tests(raw_tests)
+        logger.info('data parsed successfully for test')
+        action_log.what = 'INSERTED'
+        action_log.save()
+
+    except Exception as e:
+        action_log.what = 'INSERT_FAILED'
+        action_log.save()
+        raise e
\ No newline at end of file

LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py

@@ -8,7 +8,7 @@ import coreschema
 import pytz
 from django.db.models import Count
 from lofar.maintenance.monitoringdb.models.component_error import ComponentError
-from lofar.maintenance.monitoringdb.models.rtsm import RTSMErrorSummary, MODE_TO_COMPONENT, \
+from lofar.maintenance.monitoringdb.models.rtsm import RTSMError, MODE_TO_COMPONENT, \
     COMPONENT_TO_MODE
 from lofar.maintenance.monitoringdb.models.rtsm import RTSMObservation
 from lofar.maintenance.monitoringdb.models.station import Station
@@ -600,7 +600,7 @@ class ControllerStationTestStatistics(ValidableReadOnlyView):
                                    test_type):
 
         component_errors = ComponentError.objects.all()
-        rtsm_summary_errors = RTSMErrorSummary.objects.all()
+        rtsm_summary_errors = RTSMError.objects.all()
 
         if station_group:
             component_errors = component_errors.filter(station_test__station__type=station_group)
@@ -649,7 +649,7 @@ class ControllerStationTestStatistics(ValidableReadOnlyView):
     def compute_errors_per_type(self, from_date, to_date, central_time, station_group, test_type):
 
         component_errors = ComponentError.objects.all()
-        rtsm_summary_errors = RTSMErrorSummary.objects.all()
+        rtsm_summary_errors = RTSMError.objects.all()
 
         station_test_results = []
         rtsm_results = []
@@ -1078,7 +1078,7 @@ class ControllerStationComponentElementErrors(ValidableReadOnlyView):
         rcus_per_polarization = rcus_from_antenna_and_type(self.antenna_id,
                                                            self.component_type)
 
-        rtsm_errors = RTSMErrorSummary.objects.values('observation__pk',
+        rtsm_errors = RTSMError.objects.values('observation__pk',
                                                       'observation__start_datetime',
                                                       'observation__end_datetime',
                                                       'observation__observation_id',

LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py

 from .common import *
 from ..models.rtsm import RTSMObservation, RTSMError, RTSMSpectrum
 from ..serializers.rtsm import RTSMObservationSerializer, RTSMErrorSerializer, \
-    RTSMSpectrumSerializer, RTSMSummaryPlotSerializer, RTSMErrorSummary
+    RTSMSpectrumSerializer, RTSMSummaryPlotSerializer, RTSMError
 from ..rtsm_test_raw_parser import parse_rtsm_test
 from django.shortcuts import get_object_or_404
 from django.http import Http404, HttpResponseServerError
 import os
-from ..tasks import check_error_summary_plot
+from lofar.maintenance.monitoringdb.tasks.generate_plots import check_error_summary_plot
 
 logger = logging.getLogger('views')
 
@@ -33,7 +33,7 @@ class RTSMSummaryPlot(viewsets.ViewSet):
     """
     Get the summary plot associated to the error summary given
     """
-    queryset = RTSMErrorSummary.objects.all()
+    queryset = RTSMError.objects.all()
 
     def retrieve(self, request, pk=None):
         try: