From 53a17c26814885efd769464537c602f03288c2fa Mon Sep 17 00:00:00 2001 From: Mattia Mancini <mancini@astron.nl> Date: Fri, 24 May 2019 14:22:38 +0000 Subject: [PATCH] OSB-45: end of week commit --- .gitattributes | 4 +- .../DBInterface/django_postgresql/settings.py | 2 +- .../monitoringdb/models/__init__.py | 9 ++ .../monitoringdb/models/component.py | 5 +- .../monitoringdb/models/component_error.py | 5 +- .../monitoringdb/models/element.py | 3 + .../monitoringdb/models/element_error.py | 5 +- .../DBInterface/monitoringdb/models/rtsm.py | 41 +++----- .../monitoringdb/models/station.py | 2 +- .../monitoringdb/models/station_test.py | 3 +- .../monitoringdb/rtsm_test_raw_parser.py | 25 ++++- .../monitoringdb/serializers/rtsm.py | 19 ++-- .../monitoringdb/tasks/__init__.py | 0 .../{tasks.py => tasks/generate_plots.py} | 4 +- .../monitoringdb/tasks/insert_raw_tests.py | 98 +++++++++++++++++++ .../monitoringdb/views/controllers.py | 8 +- .../monitoringdb/views/rtsm_views.py | 6 +- 17 files changed, 183 insertions(+), 56 deletions(-) create mode 100644 LCU/Maintenance/DBInterface/monitoringdb/tasks/__init__.py rename LCU/Maintenance/DBInterface/monitoringdb/{tasks.py => tasks/generate_plots.py} (98%) create mode 100644 LCU/Maintenance/DBInterface/monitoringdb/tasks/insert_raw_tests.py diff --git a/.gitattributes b/.gitattributes index 802cce871a5..2d407a27d83 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1863,7 +1863,9 @@ LCU/Maintenance/DBInterface/monitoringdb/serializers/station_tests.py -text LCU/Maintenance/DBInterface/monitoringdb/serializers/utils.py -text LCU/Maintenance/DBInterface/monitoringdb/serializers/wincc.py -text LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py -text -LCU/Maintenance/DBInterface/monitoringdb/tasks.py -text +LCU/Maintenance/DBInterface/monitoringdb/tasks/__init__.py -text +LCU/Maintenance/DBInterface/monitoringdb/tasks/generate_plots.py -text +LCU/Maintenance/DBInterface/monitoringdb/tasks/insert_raw_tests.py -text LCU/Maintenance/DBInterface/monitoringdb/tests/__init__.py -text LCU/Maintenance/DBInterface/monitoringdb/tests/common.py -text LCU/Maintenance/DBInterface/monitoringdb/tests/old_tests.py -text diff --git a/LCU/Maintenance/DBInterface/django_postgresql/settings.py b/LCU/Maintenance/DBInterface/django_postgresql/settings.py index 28d448777c4..12b5486b5c1 100644 --- a/LCU/Maintenance/DBInterface/django_postgresql/settings.py +++ b/LCU/Maintenance/DBInterface/django_postgresql/settings.py @@ -30,7 +30,7 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) SECRET_KEY = 'x)5=9*2a&)32h-loh@rlt_9eyw%t$-fqao*#1j2gh^7=^bnjyy' # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True +DEBUG = False ALLOWED_HOSTS = ['lofarmonitortest.control.lofar', 'localhost', '127.0.0.1'] diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/__init__.py b/LCU/Maintenance/DBInterface/monitoringdb/models/__init__.py index dc4538f4bd6..8a74a41750c 100644 --- a/LCU/Maintenance/DBInterface/monitoringdb/models/__init__.py +++ b/LCU/Maintenance/DBInterface/monitoringdb/models/__init__.py @@ -6,3 +6,12 @@ #from .tbbcomponenterror import * #from .spucomponenterror import * #from .stationtest import * +__all__ = ['component', + 'component_error', + 'element', + 'element_error', + 'station', + 'station_test', + 'log', + 'rtsm', + 'wincc'] \ No newline at end of file diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/component.py b/LCU/Maintenance/DBInterface/monitoringdb/models/component.py index 2882a9a2036..a8bf7a0a9a6 100644 --- a/LCU/Maintenance/DBInterface/monitoringdb/models/component.py +++ b/LCU/Maintenance/DBInterface/monitoringdb/models/component.py @@ -5,4 +5,7 @@ from .station import Station class Component(models.Model): type = models.CharField(max_length=50) component_id = models.IntegerField() - station = models.ForeignKey(Station, on_delete=models.DO_NOTHING) \ No newline at end of file + station = models.ForeignKey(Station, on_delete=models.DO_NOTHING) + + class Meta: + unique_together = ('station', 'type', 'component_id') diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/component_error.py b/LCU/Maintenance/DBInterface/monitoringdb/models/component_error.py index de50e7bce8a..a092511788b 100644 --- a/LCU/Maintenance/DBInterface/monitoringdb/models/component_error.py +++ b/LCU/Maintenance/DBInterface/monitoringdb/models/component_error.py @@ -9,4 +9,7 @@ class ComponentError(models.Model): component = models.ForeignKey(Component, on_delete=models.DO_NOTHING) station_test = models.ForeignKey(StationTest, on_delete=models.DO_NOTHING, related_name='component_errors') - details = JSONField(default=dict) \ No newline at end of file + details = JSONField(default=dict) + + class Meta: + unique_together = ('station_test', 'component', 'type') diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/element.py b/LCU/Maintenance/DBInterface/monitoringdb/models/element.py index d0ce9e9b57c..708adfc1fff 100644 --- a/LCU/Maintenance/DBInterface/monitoringdb/models/element.py +++ b/LCU/Maintenance/DBInterface/monitoringdb/models/element.py @@ -5,3 +5,6 @@ from .component import Component class Element(models.Model): component = models.ForeignKey(Component, on_delete=models.DO_NOTHING) element_id = models.PositiveIntegerField() + + class Meta: + unique_together = ('component', 'element_id') diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/element_error.py b/LCU/Maintenance/DBInterface/monitoringdb/models/element_error.py index a7a14dcf547..2b428c90fb8 100644 --- a/LCU/Maintenance/DBInterface/monitoringdb/models/element_error.py +++ b/LCU/Maintenance/DBInterface/monitoringdb/models/element_error.py @@ -12,4 +12,7 @@ class ElementError(models.Model): related_name='failing_elements') type = models.CharField(max_length=50) - details = JSONField(default=dict) \ No newline at end of file + details = JSONField(default=dict) + + class Meta: + unique_together = ('component_error', 'element', 'type') diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py b/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py index 7800afb4696..c50dda3dad6 100644 --- a/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py +++ b/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py @@ -4,6 +4,7 @@ In this file all the model regarding the real time station monitor output are co from django.db import models from django.contrib.postgres.fields import ArrayField from .station import Station +from .component import Component MODE_TO_COMPONENT = { 1: 'LBL', @@ -35,43 +36,22 @@ class RTSMObservation(models.Model): class RTSMError(models.Model): observation = models.ForeignKey(RTSMObservation, related_name='errors', on_delete=models.CASCADE) - + component = models.ForeignKey(Component, on_delete=models.SET_NULL) rcu = models.SmallIntegerField(default=None, null=True) mode = models.SmallIntegerField(default=None, null=True) + percentage = models.FloatField(default=None, null=True) error_type = models.CharField(max_length=50) - start_frequency = models.FloatField(default=None, null=True) - stop_frequency = models.FloatField(default=None, null=True) - time = models.DateTimeField() - def __str__(self): - return self.__unicode__() - def __repr__(self): - return self.__unicode__() - - def __unicode__(self): - return '%s - MODE %s, FROM %s MHz TO %s MHz RCU=(%s): %s' % (self.time, - self.mode, - self.start_frequency, - self.stop_frequency, - self.rcu, - self.error_type,) - - -class RTSMErrorSummary(models.Model): - observation = models.ForeignKey(RTSMObservation, related_name='errors_summary', on_delete=models.CASCADE) - percentage = models.FloatField(default=None, null=True) - rcu = models.SmallIntegerField(default=None, null=True) - mode = models.SmallIntegerField(default=None, null=True) - count = models.IntegerField(default=None, null=True) - error_type = models.CharField(max_length=50) - start_frequency = models.FloatField(default=None, null=True) - stop_frequency = models.FloatField(default=None, null=True) +class RTSMErrorSample(models.Model): + time = models.DateTimeField() + error = models.ForeignKey(RTSMError, related_name='samples', on_delete=models.CASCADE) + observation_id = models.ForeignKey(RTSMObservation, on_delete=models.CASCADE) class RTSMSummaryPlot(models.Model): - error_summary = models.ForeignKey(RTSMErrorSummary, + error_summary = models.ForeignKey(RTSMError, related_name='summary_plot', on_delete=models.SET_NULL, null=True) @@ -80,6 +60,9 @@ class RTSMSummaryPlot(models.Model): class RTSMSpectrum(models.Model): + start_frequency = models.FloatField(default=None, null=True) + stop_frequency = models.FloatField(default=None, null=True) + bad_spectrum = ArrayField(models.FloatField()) average_spectrum = ArrayField(models.FloatField()) - rtsm_error = models.ForeignKey(RTSMError, related_name='spectra', on_delete=models.CASCADE) + rtsm_error = models.ForeignKey(RTSMErrorSample, related_name='spectra', on_delete=models.CASCADE) diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/station.py b/LCU/Maintenance/DBInterface/monitoringdb/models/station.py index 2be2b8a8a94..00eb3dba4cb 100644 --- a/LCU/Maintenance/DBInterface/monitoringdb/models/station.py +++ b/LCU/Maintenance/DBInterface/monitoringdb/models/station.py @@ -4,5 +4,5 @@ from .fixed_types import STATION_TYPES class Station(models.Model): location = models.CharField(max_length=50, null=True, blank=True, help_text='where the station is located') - name = models.CharField(max_length=10, help_text='name of the station') + name = models.CharField(max_length=10, help_text='name of the station', unique=True) type = models.CharField(max_length=1, choices=STATION_TYPES, help_text='station type one of [Core[C], Remote[R], International[I]]') diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/station_test.py b/LCU/Maintenance/DBInterface/monitoringdb/models/station_test.py index b915ad512c0..9d7ff07d90c 100644 --- a/LCU/Maintenance/DBInterface/monitoringdb/models/station_test.py +++ b/LCU/Maintenance/DBInterface/monitoringdb/models/station_test.py @@ -6,4 +6,5 @@ class StationTest(models.Model): start_datetime = models.DateTimeField() end_datetime = models.DateTimeField() checks = models.CharField(max_length=2000) - station = models.ForeignKey(Station, on_delete=models.DO_NOTHING) \ No newline at end of file + station = models.ForeignKey(Station, on_delete=models.DO_NOTHING) + diff --git a/LCU/Maintenance/DBInterface/monitoringdb/rtsm_test_raw_parser.py b/LCU/Maintenance/DBInterface/monitoringdb/rtsm_test_raw_parser.py index a6c7bf90793..04f5f8b589c 100644 --- a/LCU/Maintenance/DBInterface/monitoringdb/rtsm_test_raw_parser.py +++ b/LCU/Maintenance/DBInterface/monitoringdb/rtsm_test_raw_parser.py @@ -74,6 +74,29 @@ def preparse_rtsm_test_file(content): return observation +def group_samples_by_error_type(content): + errors = content.pop('errors') + grouped_errors = dict() + for error in errors: + key = (error['mode'], error['rcu'], error['error_type']) + time = error.pop('time') + average_spectrum = error.pop('average_spectrum') + bad_spectrum = error.pop('bad_spectrum') + sample = dict(time=time, + average_spectrum=average_spectrum, + bad_spectrum=bad_spectrum) + if key in grouped_errors: + grouped_errors[key]['n_samples'] += 1 + grouped_errors[key]['samples'] += [sample] + else: + grouped_errors[key] = dict( + n_samples = 1, + samples = [sample], + **error + ) + content['errors'] = list(grouped_errors.values()) + return content + def parse_rtsm_test(content): """ Expects a string content with the rtsm test output @@ -81,4 +104,4 @@ def parse_rtsm_test(content): :param content: string content with the station test output :return: a list of Django models """ - return preparse_rtsm_test_file(split_raw_rtsm_test(content)) \ No newline at end of file + return group_samples_by_error_type(preparse_rtsm_test_file(split_raw_rtsm_test(content))) \ No newline at end of file diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py index cd2a577bf40..85f26935647 100644 --- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py +++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py @@ -20,29 +20,28 @@ class RTSMSpectrumSerializer(serializers.ModelSerializer): fields = '__all__' -class RTSMErrorSerializer(serializers.ModelSerializer): +class RTSMErrorSampleSerializer(serializers.ModelSerializer): observation = serializers.PrimaryKeyRelatedField(read_only=True) def __init__(self, *args, **kwargs): self.Meta.depth = kwargs.pop('depth', 1) - super(RTSMErrorSerializer, self).__init__(*args, **kwargs) + super(RTSMErrorSampleSerializer, self).__init__(*args, **kwargs) class Meta: - model = RTSMError + model = RTSMErrorSample fields = '__all__' -class RTSMErrorSummarySerializer(serializers.ModelSerializer): +class RTSMErrorSerializer(serializers.ModelSerializer): observation = serializers.PrimaryKeyRelatedField(read_only=True) class Meta: - model = RTSMErrorSummary + model = RTSMError fields = '__all__' class RTSMObservationSerializer(serializers.ModelSerializer): - errors = RTSMErrorSerializer(many=True, depth=0) - errors_summary = RTSMErrorSummarySerializer(many=True) + errors = RTSMErrorSerializer(many=True) class Meta: model = RTSMObservation @@ -57,7 +56,7 @@ class RTSMObservationSerializer(serializers.ModelSerializer): annotate(percentage=models.ExpressionWrapper(models.F('count') * 100. / models.F('observation__samples'), output_field=models.FloatField())) for summary_entry in summary_entries: - RTSMErrorSummary(observation=RTSMObservation_entry, **summary_entry).save() + RTSMError(observation=RTSMObservation_entry, **summary_entry).save() @atomic def create(self, validated_data): @@ -87,11 +86,11 @@ class RTSMObservationSerializer(serializers.ModelSerializer): raise ItemAlreadyExists(RTSMObservation_instance) for rtsm_error in rtsm_errors: rtsm_error.update(observation=RTSMObservation_instance) - + print(rtsm_error) average_spectrum = rtsm_error.pop('average_spectrum') bad_spectrum = rtsm_error.pop('bad_spectrum') - rtsm_error_instance = RTSMError.objects.create(**rtsm_error) + rtsm_error_instance = RTSMErrorSample.objects.create(**rtsm_error) rtsm_error_instance.save() ActionLogSerializer.log_model_insert(rtsm_error_instance) diff --git a/LCU/Maintenance/DBInterface/monitoringdb/tasks/__init__.py b/LCU/Maintenance/DBInterface/monitoringdb/tasks/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/LCU/Maintenance/DBInterface/monitoringdb/tasks.py b/LCU/Maintenance/DBInterface/monitoringdb/tasks/generate_plots.py similarity index 98% rename from LCU/Maintenance/DBInterface/monitoringdb/tasks.py rename to LCU/Maintenance/DBInterface/monitoringdb/tasks/generate_plots.py index b89b200c714..31abe3ae27f 100644 --- a/LCU/Maintenance/DBInterface/monitoringdb/tasks.py +++ b/LCU/Maintenance/DBInterface/monitoringdb/tasks/generate_plots.py @@ -8,7 +8,7 @@ import matplotlib.pyplot as plt import numpy from celery import shared_task from django.conf import settings -from .models.rtsm import MODE_TO_COMPONENT, RTSMErrorSummary, RTSMSummaryPlot +from lofar.maintenance.monitoringdb.models.rtsm import MODE_TO_COMPONENT, RTSMError, RTSMSummaryPlot logger = logging.getLogger(__name__) @@ -179,7 +179,7 @@ def generate_summary_plot_for_error(error_summary_id): """ basePath = settings.URL_TO_STORE_RTSM_PLOTS - error_summary = RTSMErrorSummary.objects.get(pk=error_summary_id) + error_summary = RTSMError.objects.get(pk=error_summary_id) summary_plot = error_summary.summary_plot.first() if summary_plot is not None: if summary_plot.uri == None: diff --git a/LCU/Maintenance/DBInterface/monitoringdb/tasks/insert_raw_tests.py b/LCU/Maintenance/DBInterface/monitoringdb/tasks/insert_raw_tests.py new file mode 100644 index 00000000000..92f1f48a051 --- /dev/null +++ b/LCU/Maintenance/DBInterface/monitoringdb/tasks/insert_raw_tests.py @@ -0,0 +1,98 @@ +import logging + +from celery import shared_task +from django.db import transaction +from lofar.maintenance.monitoringdb.models.component import Component +from lofar.maintenance.monitoringdb.models.component_error import ComponentError +from lofar.maintenance.monitoringdb.models.element import Element +from lofar.maintenance.monitoringdb.models.element_error import ElementError +from lofar.maintenance.monitoringdb.models.station import Station +from lofar.maintenance.monitoringdb.models.station_test import StationTest +from lofar.maintenance.monitoringdb.station_test_raw_parser import parse_raw_station_test +from lofar.maintenance.monitoringdb.models.log import ActionLog +logger = logging.getLogger(__name__) + + +def create_component_errors(station: Station, station_test: StationTest, component_errors_data): + component_list = [] + component_error_list = [] + for component_error in component_errors_data: + component = component_error.pop('component') + component_entity, _ = Component.objects.get_or_create(station=station, **component) + + component_list.append(component_entity) + component_error_entity, _ = ComponentError.objects.get_or_create(component=component_entity, + station_test=station_test, + **component_error) + + component_error_list.append(component_error_entity) + + return component_list, component_error_list + + +def create_element_errors(station, station_test, elements_error_data): + element_errors = [] + for element_error_data in elements_error_data: + component_error = element_error_data.pop('component_error') + component = component_error.pop('component') + element = element_error_data.pop('element') + + component, _ = Component.objects.get_or_create(station=station, **component) + component_error, _ = ComponentError.objects.get_or_create(station_test=station_test, + component=component, + **component_error) + element, _ = Element.objects.get_or_create(**element, component=component) + + element_error, _ = ElementError.objects.get_or_create(element=element, + component_error=component_error, + **element_error_data) + + element_errors.append(element_error) + + +def create_station(station_data): + station, created = Station.objects.get_or_create(**station_data) + return station + + +def create_station_test(station: Station, station_test_data): + station_test, created = StationTest.objects.get_or_create(station=station, **station_test_data) + return station_test + + +@transaction.atomic +def create_station_tests(station_tests_data): + parsed_tests = parse_raw_station_test(station_tests_data) + + for station_test_data in parsed_tests: + station_data = station_test_data.pop('station') + component_errors = station_test_data.pop('component_errors') + element_errors = station_test_data.pop('element_errors') + + station = create_station(station_data) + station, created = Station.objects.get_or_create(station) + + station_test = create_station_test(station, station_test_data) + + create_component_errors(station, + station_test, + component_errors) + create_element_errors(station, station_test, element_errors) + + +@shared_task() +def insert_station_test(action_log_id: ActionLog, raw_tests): + action_log = ActionLog.objects.get(pk=action_log_id) + try: + parsed_content = parse_raw_station_test(raw_tests) + if parsed_content is None: + raise Exception('cannot parse test {}'.format(raw_tests)) + create_station_tests(raw_tests) + logger.info('data parsed successfully for test') + action_log.what = 'INSERTED' + action_log.save() + + except Exception as e: + action_log.what = 'INSERT_FAILED' + action_log.save() + raise e \ No newline at end of file diff --git a/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py b/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py index f2cb198a830..8d91dc0d5b1 100644 --- a/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py +++ b/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py @@ -8,7 +8,7 @@ import coreschema import pytz from django.db.models import Count from lofar.maintenance.monitoringdb.models.component_error import ComponentError -from lofar.maintenance.monitoringdb.models.rtsm import RTSMErrorSummary, MODE_TO_COMPONENT, \ +from lofar.maintenance.monitoringdb.models.rtsm import RTSMError, MODE_TO_COMPONENT, \ COMPONENT_TO_MODE from lofar.maintenance.monitoringdb.models.rtsm import RTSMObservation from lofar.maintenance.monitoringdb.models.station import Station @@ -600,7 +600,7 @@ class ControllerStationTestStatistics(ValidableReadOnlyView): test_type): component_errors = ComponentError.objects.all() - rtsm_summary_errors = RTSMErrorSummary.objects.all() + rtsm_summary_errors = RTSMError.objects.all() if station_group: component_errors = component_errors.filter(station_test__station__type=station_group) @@ -649,7 +649,7 @@ class ControllerStationTestStatistics(ValidableReadOnlyView): def compute_errors_per_type(self, from_date, to_date, central_time, station_group, test_type): component_errors = ComponentError.objects.all() - rtsm_summary_errors = RTSMErrorSummary.objects.all() + rtsm_summary_errors = RTSMError.objects.all() station_test_results = [] rtsm_results = [] @@ -1078,7 +1078,7 @@ class ControllerStationComponentElementErrors(ValidableReadOnlyView): rcus_per_polarization = rcus_from_antenna_and_type(self.antenna_id, self.component_type) - rtsm_errors = RTSMErrorSummary.objects.values('observation__pk', + rtsm_errors = RTSMError.objects.values('observation__pk', 'observation__start_datetime', 'observation__end_datetime', 'observation__observation_id', diff --git a/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py b/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py index 8591075bc63..d41760307a5 100644 --- a/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py +++ b/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py @@ -1,12 +1,12 @@ from .common import * from ..models.rtsm import RTSMObservation, RTSMError, RTSMSpectrum from ..serializers.rtsm import RTSMObservationSerializer, RTSMErrorSerializer, \ - RTSMSpectrumSerializer, RTSMSummaryPlotSerializer, RTSMErrorSummary + RTSMSpectrumSerializer, RTSMSummaryPlotSerializer, RTSMError from ..rtsm_test_raw_parser import parse_rtsm_test from django.shortcuts import get_object_or_404 from django.http import Http404, HttpResponseServerError import os -from ..tasks import check_error_summary_plot +from lofar.maintenance.monitoringdb.tasks.generate_plots import check_error_summary_plot logger = logging.getLogger('views') @@ -33,7 +33,7 @@ class RTSMSummaryPlot(viewsets.ViewSet): """ Get the summary plot associated to the error summary given """ - queryset = RTSMErrorSummary.objects.all() + queryset = RTSMError.objects.all() def retrieve(self, request, pk=None): try: -- GitLab