diff --git a/.gitattributes b/.gitattributes
index fc08600221760329c77f34bc1029e5883458ca02..bce0e9b734e849ec3a1c32e952a2cd05f953d272 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1806,8 +1806,8 @@ LCU/Maintenance/DBInterface/monitoringdb/models/component.py -text
LCU/Maintenance/DBInterface/monitoringdb/models/component_error.py -text
LCU/Maintenance/DBInterface/monitoringdb/models/element.py -text
LCU/Maintenance/DBInterface/monitoringdb/models/element_error.py -text
-LCU/Maintenance/DBInterface/monitoringdb/models/error_details.py -text
LCU/Maintenance/DBInterface/monitoringdb/models/fixed_types.py -text
+LCU/Maintenance/DBInterface/monitoringdb/models/log.py -text
LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py -text
LCU/Maintenance/DBInterface/monitoringdb/models/station.py -text
LCU/Maintenance/DBInterface/monitoringdb/models/station_test.py -text
@@ -1818,7 +1818,7 @@ LCU/Maintenance/DBInterface/monitoringdb/serializers/component.py -text
LCU/Maintenance/DBInterface/monitoringdb/serializers/component_error.py -text
LCU/Maintenance/DBInterface/monitoringdb/serializers/element.py -text
LCU/Maintenance/DBInterface/monitoringdb/serializers/element_error.py -text
-LCU/Maintenance/DBInterface/monitoringdb/serializers/error_details.py -text
+LCU/Maintenance/DBInterface/monitoringdb/serializers/log.py -text
LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py -text
LCU/Maintenance/DBInterface/monitoringdb/serializers/station.py -text
LCU/Maintenance/DBInterface/monitoringdb/serializers/station_tests.py -text
@@ -1827,6 +1827,7 @@ LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py -text
LCU/Maintenance/DBInterface/monitoringdb/tests/__init__.py -text
LCU/Maintenance/DBInterface/monitoringdb/tests/common.py -text
LCU/Maintenance/DBInterface/monitoringdb/tests/old_tests.py -text
+LCU/Maintenance/DBInterface/monitoringdb/tests/t_django_insert.py -text
LCU/Maintenance/DBInterface/monitoringdb/tests/t_stationtest_model_creation.py -text
LCU/Maintenance/DBInterface/monitoringdb/tests/t_stationtest_parser.py -text
LCU/Maintenance/DBInterface/monitoringdb/tests/test_generic_stationtestdata.csv -text
@@ -1834,6 +1835,8 @@ LCU/Maintenance/DBInterface/monitoringdb/tests/test_multiple_tileerror_and_rcuer
LCU/Maintenance/DBInterface/monitoringdb/urls.py -text
LCU/Maintenance/DBInterface/monitoringdb/views/__init__.py -text
LCU/Maintenance/DBInterface/monitoringdb/views/common.py -text
+LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py -text
+LCU/Maintenance/DBInterface/monitoringdb/views/logs_view.py -text
LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py -text
LCU/Maintenance/DBInterface/monitoringdb/views/station_test_views.py -text
LCU/Maintenance/MDB_WebView/CMakeLists.txt -text
diff --git a/LCU/Maintenance/DBInterface/django_postgresql/settings.py b/LCU/Maintenance/DBInterface/django_postgresql/settings.py
index b481e8f78fcfd933d35f5ffbdd3e1a0ff4211223..be5844ac85b5ee6180fa14a38f69937533dd2184 100644
--- a/LCU/Maintenance/DBInterface/django_postgresql/settings.py
+++ b/LCU/Maintenance/DBInterface/django_postgresql/settings.py
@@ -106,6 +106,10 @@ LOGGING = {
'serializers': {
'handlers': ['console'],
'level': 'DEBUG',
+ },
+ 'controllers': {
+ 'handlers': ['console'],
+ 'level': 'DEBUG'
}
},
}
@@ -121,7 +125,11 @@ DATABASES = {
'PASSWORD': 'fix-me',
'HOST': 'localhost',
'PORT': '',
+ 'TEST': {
+ 'NAME': 'test_monitoringdb'
+ }
}
+
}
# Password validation
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/component_error.py b/LCU/Maintenance/DBInterface/monitoringdb/models/component_error.py
index ead4319aeed55a3cd51a267605d9576f83822d4e..de50e7bce8a361568d225af3448adc8163b0bf5d 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/models/component_error.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/models/component_error.py
@@ -1,7 +1,7 @@
from django.db import models
from .component import Component
from .station_test import StationTest
-from .error_details import ErrorDetails
+from django.contrib.postgres.fields import JSONField
class ComponentError(models.Model):
@@ -9,7 +9,4 @@ class ComponentError(models.Model):
component = models.ForeignKey(Component, on_delete=models.DO_NOTHING)
station_test = models.ForeignKey(StationTest, on_delete=models.DO_NOTHING,
related_name='component_errors')
- details = models.ForeignKey(ErrorDetails,
- on_delete=models.CASCADE,
- related_name='details',
- null=True)
\ No newline at end of file
+ details = JSONField(default=dict)
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/element_error.py b/LCU/Maintenance/DBInterface/monitoringdb/models/element_error.py
index 5200e709481f5321e0ef08c6a62157c95d75d232..a7a14dcf547a2f19972564861b8c98ffed6c57e0 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/models/element_error.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/models/element_error.py
@@ -1,7 +1,7 @@
from django.db import models
from .element import Element
-from .error_details import ErrorDetails
-from .station_test import StationTest
+from django.contrib.postgres.fields import JSONField
+
from .component_error import ComponentError
@@ -12,4 +12,4 @@ class ElementError(models.Model):
related_name='failing_elements')
type = models.CharField(max_length=50)
- details = models.ForeignKey(ErrorDetails, on_delete=models.CASCADE, related_name='element_error_details')
\ No newline at end of file
+ details = JSONField(default=dict)
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/error_details.py b/LCU/Maintenance/DBInterface/monitoringdb/models/error_details.py
deleted file mode 100644
index ed14ff246a8b36a22e2d71ed57a5ac6b1f6b86e3..0000000000000000000000000000000000000000
--- a/LCU/Maintenance/DBInterface/monitoringdb/models/error_details.py
+++ /dev/null
@@ -1,73 +0,0 @@
-from django.db import models
-
-
-class ErrorDetails(models.Model):
- x = models.NullBooleanField()
- y = models.NullBooleanField()
-
- xval = models.FloatField(default=None, null=True)
- yval = models.FloatField(default=None, null=True)
-
- xoff = models.FloatField(default=None, null=True)
- yoff = models.FloatField(default=None, null=True)
-
- xval_no_delay = models.FloatField(default=None, null=True)
- yval_no_delay = models.FloatField(default=None, null=True)
-
- xsb_no_delay = models.FloatField(default=None, null=True)
- ysb_no_delay = models.FloatField(default=None, null=True)
-
- xref_no_delay = models.FloatField(default=None, null=True)
- yref_no_delay = models.FloatField(default=None, null=True)
-
- xref = models.FloatField(default=None, null=True)
- yref = models.FloatField(default=None, null=True)
-
- xsb_full_delay = models.FloatField(default=None, null=True)
- ysb_full_delay = models.FloatField(default=None, null=True)
-
- xref_full_delay = models.FloatField(default=None, null=True)
- yref_full_delay = models.FloatField(default=None, null=True)
-
- xval_full_delay = models.FloatField(default=None, null=True)
- yval_full_delay = models.FloatField(default=None, null=True)
-
- xlimit = models.FloatField(default=None, null=True)
- ylimit = models.FloatField(default=None, null=True)
-
- xproc = models.FloatField(default=None, null=True)
- yproc = models.FloatField(default=None, null=True)
-
- xdiff = models.FloatField(default=None, null=True)
- ydiff = models.FloatField(default=None, null=True)
-
- xmean = models.FloatField(default=None, null=True)
- ymean = models.FloatField(default=None, null=True)
-
- tp = models.CharField(max_length=10, null=True, default=None)
- mp = models.CharField(max_length=10, null=True, default=None)
- ap = models.CharField(max_length=10, null=True, default=None)
- bp = models.CharField(max_length=10, null=True, default=None)
-
- mp0 = models.CharField(max_length=10, null=True, default=None)
- mp1 = models.CharField(max_length=10, null=True, default=None)
- mp2 = models.CharField(max_length=10, null=True, default=None)
- mp3 = models.CharField(max_length=10, null=True, default=None)
-
- ap0 = models.CharField(max_length=10, null=True, default=None)
- ap1 = models.CharField(max_length=10, null=True, default=None)
- ap2 = models.CharField(max_length=10, null=True, default=None)
- ap3 = models.CharField(max_length=10, null=True, default=None)
-
- pcb = models.CharField(max_length=10, null=True, default=None)
-
- rcu5_0v = models.FloatField(default=None, null=True)
- lba8_0v = models.FloatField(default=None, null=True)
- hba48v = models.FloatField(default=None, null=True)
- spu3_3v = models.FloatField(default=None, null=True)
-
- v1_2 = models.FloatField(default=None, null=True)
- v2_5 = models.FloatField(default=None, null=True)
- v3_3 = models.FloatField(default=None, null=True)
-
- error_code = models.CharField(max_length=10, null=True, default=None)
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/log.py b/LCU/Maintenance/DBInterface/monitoringdb/models/log.py
new file mode 100644
index 0000000000000000000000000000000000000000..de8ba434bb0be45db43f9509bf7816328fbe6310
--- /dev/null
+++ b/LCU/Maintenance/DBInterface/monitoringdb/models/log.py
@@ -0,0 +1,9 @@
+import django.db.models as models
+
+
+class ActionLog(models.Model):
+ entry_id = models.IntegerField(help_text='database id of the concerned object')
+ model_name = models.CharField(max_length=100)
+ who = models.CharField(max_length=100)
+ when = models.DateTimeField(auto_now_add=True)
+ what = models.CharField(max_length=1000)
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py b/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py
index 4bf1a1cc149498529728b1c79531a06658c0f10b..860269b9b9b23b8d8888695f9bd404aa154dc0f2 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py
@@ -3,7 +3,7 @@ In this file all the model regarding the real time station monitor output are co
"""
from django.db import models
from django.contrib.postgres.fields import ArrayField
-
+from .station import Station
from .fixed_types import STATION_TYPES
@@ -11,54 +11,24 @@ from .fixed_types import STATION_TYPES
class RTSMObservation(models.Model):
observation_id = models.PositiveIntegerField(default=0)
- station_name = models.CharField(max_length=6)
- station_type = models.CharField(max_length=1, choices=STATION_TYPES)
+ station = models.ForeignKey(Station, on_delete=models.SET_NULL, null=True)
- start_time = models.DateTimeField()
- end_time = models.DateTimeField()
+ start_datetime = models.DateTimeField()
+ end_datetime = models.DateTimeField()
samples = models.IntegerField(default=None, null=True)
-class RTSMObservationSummary(RTSMObservation):
- @property
- def errors_found(self):
- return RTSMErrorSummary.objects.defer('bad_spectrum', 'average_spectrum'). \
- filter(observation=self).values('error_type', 'rcu'). \
- annotate(count=models.Count('error_type', unique=True, output_field=models.FloatField())). \
- annotate(percentage=models.ExpressionWrapper(models.F('count') * 100. / models.F('observation__samples'),
- output_field=models.FloatField()))
-
- class Meta:
- proxy = True
-
-
-class RTSMErrorSummary(models.Model):
- rcu = models.SmallIntegerField(default=None, null=True)
- mode = models.SmallIntegerField(default=None, null=True)
- observation = models.ForeignKey(RTSMObservation, related_name='errors_summary', on_delete=models.CASCADE)
-
- error_type = models.CharField(max_length=10)
- start_frequency = models.FloatField(default=None, null=True)
- stop_frequency = models.FloatField(default=None, null=True)
- time = models.DateTimeField()
-
- class Meta:
- managed = False
- db_table = 'monitoringdb_rtsmerror'
-
-
class RTSMError(models.Model):
+ observation = models.ForeignKey(RTSMObservation, related_name='errors', on_delete=models.CASCADE)
+
rcu = models.SmallIntegerField(default=None, null=True)
mode = models.SmallIntegerField(default=None, null=True)
- observation = models.ForeignKey(RTSMObservation, related_name='errors', on_delete=models.CASCADE)
- error_type = models.CharField(max_length=10)
+ error_type = models.CharField(max_length=50)
start_frequency = models.FloatField(default=None, null=True)
stop_frequency = models.FloatField(default=None, null=True)
time = models.DateTimeField()
- bad_spectrum = ArrayField(models.FloatField())
- average_spectrum = ArrayField(models.FloatField())
def __str__(self):
return self.__unicode__()
@@ -67,13 +37,26 @@ class RTSMError(models.Model):
return self.__unicode__()
def __unicode__(self):
- return '%s - MODE %d, FROM %f MHz TO %f MHz RCU=(%d): %s' % (self.time,
+ return '%s - MODE %s, FROM %s MHz TO %s MHz RCU=(%s): %s' % (self.time,
self.mode,
self.start_frequency,
self.stop_frequency,
self.rcu,
self.error_type,)
- class Meta:
- managed = True
- db_table = 'monitoringdb_rtsmerror'
+
+class RTSMErrorSummary(models.Model):
+ observation = models.ForeignKey(RTSMObservation, related_name='errors_summary', on_delete=models.CASCADE)
+ percentage = models.FloatField(default=None, null=True)
+ rcu = models.SmallIntegerField(default=None, null=True)
+ mode = models.SmallIntegerField(default=None, null=True)
+ count = models.IntegerField(default=None, null=True)
+ error_type = models.CharField(max_length=50)
+ start_frequency = models.FloatField(default=None, null=True)
+ stop_frequency = models.FloatField(default=None, null=True)
+
+
+class RTSMSpectrum(models.Model):
+ bad_spectrum = ArrayField(models.FloatField())
+ average_spectrum = ArrayField(models.FloatField())
+ rtsm_error = models.ForeignKey(RTSMError, related_name='spectra', on_delete=models.CASCADE)
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/station.py b/LCU/Maintenance/DBInterface/monitoringdb/models/station.py
index ec6eaa599dfdf63ac3cd0588ddfa12a39855c403..2be2b8a8a949b2b58cff7d1e1c9edbaa07f27eb4 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/models/station.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/models/station.py
@@ -3,6 +3,6 @@ from .fixed_types import STATION_TYPES
class Station(models.Model):
- location = models.CharField(max_length=50, null=True, blank=True)
- name = models.CharField(max_length=10)
- type = models.CharField(max_length=1, choices=STATION_TYPES)
+ location = models.CharField(max_length=50, null=True, blank=True, help_text='where the station is located')
+ name = models.CharField(max_length=10, help_text='name of the station')
+ type = models.CharField(max_length=1, choices=STATION_TYPES, help_text='station type one of [Core[C], Remote[R], International[I]]')
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/rtsm_test_raw_parser.py b/LCU/Maintenance/DBInterface/monitoringdb/rtsm_test_raw_parser.py
index 132b355b3a70fe0ce45c9eec333a02306b08cba4..a6c7bf907930bdcd40d17dfdd9c052dd6f5754f3 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/rtsm_test_raw_parser.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/rtsm_test_raw_parser.py
@@ -1,6 +1,15 @@
from datetime import datetime
import pytz
+rtsm_error_types_to_error_types = dict(HN='HIGH_NOISE',
+ SN='SUMMATOR_NOISE',
+ CR='CABLE_REFLECTION',
+ LN='LOW_NOISE',
+ J='JITTER',
+ OSC='OSCILLATION',
+ FLAT='FLAT',
+ DOWN='DOWN',
+ SHORT='SHORT')
def split_raw_rtsm_test(content):
"""
@@ -44,15 +53,15 @@ def preparse_rtsm_test_file(content):
rcu, mode, observation_id, error_type, start_frequency, stop_frequency, time = value.split(',')
result.update(rcu=int(rcu),
mode=int(mode),
- error_type=error_type,
+ error_type=rtsm_error_types_to_error_types[error_type],
start_frequency=float(start_frequency),
stop_frequency=float(stop_frequency),
time=parse_data(time))
elif 'INFO' in key:
observation_id, start_time, stop_time, samples = value.split(',')
observation.update(observation_id=int(observation_id),
- start_time=parse_data(start_time),
- end_time=parse_data(stop_time),
+ start_datetime=parse_data(start_time),
+ end_datetime=parse_data(stop_time),
samples=int(samples))
elif 'BAD' in key:
values = [float(item) for item in value.lstrip('[').rstrip(']').strip().split(' ')]
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/component_error.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/component_error.py
index 9e4129617a0bcbf3c01e578c287111b74398d90a..a1ef7be0fd47a7a8eb625bb08834b7b4f55d0ff2 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/component_error.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/component_error.py
@@ -1,6 +1,13 @@
+import logging
+
from rest_framework.serializers import PrimaryKeyRelatedField, CharField, IntegerField
-from ..models.component_error import ComponentError
+
from .utils import NotNullModelSerializer
+from ..models.component import Component
+from ..models.component_error import ComponentError
+from .log import ActionLogSerializer
+
+logger = logging.getLogger(__name__)
class ComponentErrorSerializer(NotNullModelSerializer):
@@ -11,8 +18,60 @@ class ComponentErrorSerializer(NotNullModelSerializer):
class Meta:
model = ComponentError
fields = '__all__'
- depth = 2
+ depth = 3
def __init__(self, *args, **kwargs):
self.Meta.depth = kwargs.pop('depth', 2)
super(ComponentErrorSerializer, self).__init__(*args, **kwargs)
+
+ @staticmethod
+ def insert_component_error(station_test_entry,
+ station_entry,
+ component_error):
+
+ component = component_error.pop('component')
+
+ component_entry = Component.objects.filter(station=station_entry,
+ **component).first()
+ if component_entry is None:
+ logger.debug('Component entry is not present, inserting ...')
+ component_entry = Component(station=station_entry,
+ **component)
+ logger.debug(component_entry, component_error)
+
+
+ component_entry.save()
+ # Log the action
+ ActionLogSerializer.log_model_insert(component_entry)
+
+ component_error_entry = ComponentError.objects.filter(component=component_entry,
+ station_test=station_test_entry,
+ **component_error).first()
+ if component_error_entry is None:
+ logger.debug('Component error entry is not present, inserting ...')
+
+ component_error_entry = ComponentError(component=component_entry,
+ station_test=station_test_entry,
+ **component_error)
+ component_error_entry.save()
+ # Log the action
+ ActionLogSerializer.log_model_insert(component_error_entry)
+
+ return component_error_entry
+
+ @staticmethod
+ def insert_component_errors(station_test_entry,
+ station_entry,
+ component_errors):
+ results = []
+ for component_error in component_errors:
+ try:
+ results.append(ComponentErrorSerializer.insert_component_error(
+ station_test_entry, station_entry, component_error))
+ except Exception as e:
+ logger.exception(
+ 'Cannot insert component error %s'
+ ' for station_test_entry %s and station_entry %s: %s',
+ component_error, station_test_entry, station_entry, e)
+ raise e
+ return results
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/element.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/element.py
index 61dbbd28835ec31479cb992467aa1a6962c60704..54a1edfb0bbbfb3af0aba70ce41cb866f83ec302 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/element.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/element.py
@@ -1,5 +1,6 @@
from rest_framework.serializers import ModelSerializer
from ..models.element import Element
+from .log import ActionLogSerializer
class ElementSerializer(ModelSerializer):
@@ -7,3 +8,15 @@ class ElementSerializer(ModelSerializer):
model = Element
fields = '__all__'
depth = 1
+
+ @staticmethod
+ def insert_element(component, element):
+ element_entry = Element.objects.filter(component=component,
+ **element).first()
+ if element_entry is None:
+ element_entry = Element(component=component,
+ **element)
+ element_entry.save()
+ # Log the action
+ ActionLogSerializer.log_model_insert(element_entry)
+ return element_entry
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/element_error.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/element_error.py
index 787277e8245b4ceb1e6f0f4b4b15e51c0c493952..fea8dcbe6ce8679bae76aa564c8ff35b7b9480fa 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/element_error.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/element_error.py
@@ -1,10 +1,65 @@
from rest_framework.serializers import ModelSerializer
+
+from .component_error import ComponentErrorSerializer
+from .element import ElementSerializer
from ..models.element_error import ElementError
-from .error_details import ErrorDetailsSerializer
+from .log import ActionLogSerializer
+import logging
+
+
+logger = logging.getLogger(__name__)
-class ElementErrorSerializer(ModelSerializer):
+class ElementErrorSerializer(ModelSerializer):
class Meta:
model = ElementError
fields = '__all__'
- depth = 1
\ No newline at end of file
+ depth = 1
+
+ @staticmethod
+ def insert_element_error(station_test_entry,
+ station_entry,
+ element_error):
+ component_error = element_error.pop('component_error')
+
+ component_error_entry = ComponentErrorSerializer.insert_component_error(station_test_entry,
+ station_entry,
+ component_error)
+ component = component_error_entry.component
+
+ element = element_error.pop('element')
+ element_error_details = element_error.pop('details')
+
+ element_entry = ElementSerializer.insert_element(component, element)
+
+ element_error_entry = ElementError.objects.filter(element=element_entry,
+ component_error=component_error_entry,
+ **element_error).first()
+ if element_error_entry is None:
+ element_error_entry = ElementError(element=element_entry,
+ details=element_error_details,
+ component_error=component_error_entry,
+ **element_error)
+
+ element_error_entry.save()
+ # Log the action
+ ActionLogSerializer.log_model_insert(element_error_entry)
+
+ return element_error_entry
+
+ @staticmethod
+ def insert_element_errors(station_test_entry,
+ station_entry,
+ element_errors):
+ results = []
+ for element_error in element_errors:
+ try:
+ results.append(ElementErrorSerializer.insert_element_error(station_test_entry,
+ station_entry,
+ element_error))
+ except Exception as e:
+ logger.exception(
+ 'Cannot insert element error %s'
+ ' for station_test_entry %s and station_entry %s: %s',
+ element_error, station_test_entry, station_entry, e)
+ raise e
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/error_details.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/error_details.py
deleted file mode 100644
index e1a8af9ee7903d1325eab08707ca169f6ead7d01..0000000000000000000000000000000000000000
--- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/error_details.py
+++ /dev/null
@@ -1,16 +0,0 @@
-from rest_framework.serializers import ModelSerializer
-from ..models.error_details import ErrorDetails
-
-class ErrorDetailsSerializer(ModelSerializer):
- class Meta:
- model = ErrorDetails
- fields = '__all__'
-
- def to_representation(self, instance):
- result = super().to_representation(instance)
-
- result = {key: value
- for key, value in
- filter(lambda item: item[1] is not None, result.items())}
-
- return result
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/log.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/log.py
new file mode 100644
index 0000000000000000000000000000000000000000..fbae920a4f1c0d62302eb6c3c2d002ba4a50960b
--- /dev/null
+++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/log.py
@@ -0,0 +1,33 @@
+from rest_framework.serializers import ModelSerializer
+from ..models.log import ActionLog
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class ActionLogSerializer(ModelSerializer):
+ class Meta:
+ model = ActionLog
+ fields = '__all__'
+ depth = 1
+
+ @staticmethod
+ def append_log(primary_key, model_name, who, what):
+ action_log = ActionLog(entry_id=primary_key,
+ model_name=model_name,
+ who=who,
+ what=what)
+ action_log.save()
+
+ return action_log
+
+ @staticmethod
+ def log_model_insert(element_entry, who='system'):
+ try:
+ ActionLogSerializer.append_log(element_entry.pk,
+ model_name=element_entry.__class__.__name__,
+ who=who,
+ what='Inserted')
+ except Exception as e:
+ logger.exception('Cannot log entry %s', element_entry)
+ raise e
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py
index b26d5eef29363a01421349f1dc546f4a2ba33191..8cddf2b51b544d17f7ec1482e9f7c20ba0808ca4 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py
@@ -3,78 +3,100 @@ from rest_framework import serializers
import logging
from django.db.transaction import atomic
from ..exceptions import ItemAlreadyExists
-from django.db.models import ObjectDoesNotExist
+from .log import ActionLogSerializer
+from .station import Station
logger = logging.getLogger('serializers')
-class RTSMErrorsAggregateSerializer(serializers.Serializer):
- count = serializers.IntegerField()
- percentage = serializers.FloatField()
- error_type = serializers.CharField()
- rcu = serializers.IntegerField()
-
-
-class RTSMErrorSerializer(serializers.ModelSerializer):
+class RTSMSpectrumSerializer(serializers.ModelSerializer):
bad_spectrum = serializers.ListField(child=serializers.FloatField())
average_spectrum = serializers.ListField(child=serializers.FloatField())
class Meta:
- model = RTSMError
+ model = RTSMSpectrum
fields = '__all__'
-class RTSMErrorSummarySerializer(serializers.ModelSerializer):
- samples = serializers.IntegerField(source='observation.samples', read_only=True)
- start_time = serializers.DateTimeField(source='observation.start_time', read_only=True)
- end_time = serializers.DateTimeField(source='observation.start_time', read_only=True)
- observation_id = serializers.IntegerField(source='observation.observation_id', read_only=True)
- id = serializers.IntegerField()
+class RTSMErrorSerializer(serializers.ModelSerializer):
+ observation = serializers.PrimaryKeyRelatedField(read_only=True)
+
+ def __init__(self, *args, **kwargs):
+ self.Meta.depth = kwargs.pop('depth', 1)
+ super(RTSMErrorSerializer, self).__init__(*args, **kwargs)
class Meta:
- model = RTSMErrorSummary
+ model = RTSMError
fields = '__all__'
-class RTSMObservationSummarySerializer(serializers.ModelSerializer):
- errors_found = RTSMErrorsAggregateSerializer(many=True)
+class RTSMErrorSummarySerializer(serializers.ModelSerializer):
+ observation = serializers.PrimaryKeyRelatedField(read_only=True)
class Meta:
- model = RTSMObservationSummary
+ model = RTSMErrorSummary
fields = '__all__'
class RTSMObservationSerializer(serializers.ModelSerializer):
+ errors = RTSMErrorSerializer(many=True, depth=0)
errors_summary = RTSMErrorSummarySerializer(many=True)
class Meta:
model = RTSMObservation
fields = '__all__'
+ def compute_summary(self, RTSMObservation_entry):
+ summary_entries = RTSMObservation_entry.errors.defer('spectra'). \
+ filter(observation=RTSMObservation_entry).values('error_type', 'mode', 'rcu', 'start_frequency',
+ 'stop_frequency'). \
+ annotate(count=models.Count('error_type', unique=True, output_field=models.FloatField())). \
+ annotate(percentage=models.ExpressionWrapper(models.F('count') * 100. / models.F('observation__samples'),
+ output_field=models.FloatField()))
+ for summary_entry in summary_entries:
+ RTSMErrorSummary(observation=RTSMObservation_entry, **summary_entry).save()
+
@atomic
def create(self, validated_data):
rtsm_errors = validated_data.pop('errors')
- try:
- start_time, station_name = validated_data['start_time'], validated_data['station_name']
- station_type = station_name[0].capitalize()
- station_type = station_type if station_type in ['C', 'R'] else 'I'
- validated_data.update(station_type=station_type)
- element = RTSMObservation.objects.filter(start_time=start_time,
- station_name=station_name).first()
-
- if element is None:
- logger.info('rtsm not found inserting %s', validated_data)
- else:
- logger.info('RTSM test for station %s started at %s is present already. Skipping insertion',
- start_time, station_name)
- raise ItemAlreadyExists(element)
- except ObjectDoesNotExist as e:
+
+ # derive the station type from the station name
+ station_name = validated_data.pop('station_name')
+ station_type = station_name[0].capitalize()
+ station_type = station_type if station_type in ['C', 'R'] else 'I'
+
+ station_entry = Station.objects.filter(name=station_name).first()
+ if station_entry is None:
+ station_entry = Station(name=station_name, type=station_type)
+ station_entry.save()
+ # searches if the RTSM is already present
+ observation_id = validated_data['observation_id']
+ RTSMObservation_instance = RTSMObservation.objects.filter(observation_id=observation_id,
+ station=station_entry).first()
+
+ if RTSMObservation_instance is None:
logger.info('rtsm not found inserting %s', validated_data)
- RTSMObservation_instance = RTSMObservation.objects.create(**validated_data)
- RTSMObservation_instance.save()
+ RTSMObservation_instance = RTSMObservation.objects.create(station=station_entry,
+ **validated_data)
+ RTSMObservation_instance.save()
+ ActionLogSerializer.log_model_insert(RTSMObservation_instance)
+ else:
+ raise ItemAlreadyExists(RTSMObservation_instance)
for rtsm_error in rtsm_errors:
rtsm_error.update(observation=RTSMObservation_instance)
+
+ average_spectrum = rtsm_error.pop('average_spectrum')
+ bad_spectrum = rtsm_error.pop('bad_spectrum')
+
rtsm_error_instance = RTSMError.objects.create(**rtsm_error)
rtsm_error_instance.save()
+ ActionLogSerializer.log_model_insert(rtsm_error_instance)
+
+ rtsm_spectrum_instance = RTSMSpectrum.objects.create(average_spectrum=average_spectrum,
+ bad_spectrum=bad_spectrum,
+ rtsm_error=rtsm_error_instance)
+ rtsm_spectrum_instance.save()
+ ActionLogSerializer.log_model_insert(rtsm_spectrum_instance)
+ self.compute_summary(RTSMObservation_instance)
return RTSMObservation_instance
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/station_tests.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/station_tests.py
index 6b5246a94f4031add41a18bf44bbb4a482b09643..7f9a54f9204f76668ebfa3f332af011b88d913dd 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/station_tests.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/station_tests.py
@@ -1,10 +1,59 @@
from rest_framework.serializers import ModelSerializer
from ..models.station_test import StationTest
+from ..models.station import Station
from .component_error import ComponentErrorSerializer
+from .element_error import ElementErrorSerializer
+from .log import ActionLogSerializer
+
+import logging
+
+logger = logging.getLogger(__name__)
class StationTestSerializer(ModelSerializer):
component_errors = ComponentErrorSerializer(many=True, read_only=True, depth=0)
class Meta:
model = StationTest
fields = '__all__'
- depth = 2
\ No newline at end of file
+ depth = 5
+
+ @staticmethod
+ def insert_station_test(station_test):
+ """
+ Insert the station test in the database and if necessary the component errors
+ and the element errors
+ :param station_test: json representation of the station test
+ :type station_test: dict
+ :return: the station test entry inserted
+ :rtype: StationTest
+ """
+ station = station_test.pop('station')
+ component_errors = station_test.pop('component_errors')
+ element_errors = station_test.pop('element_errors')
+ station_entry = Station.objects.filter(**station).first()
+
+ if station_entry is None:
+ logger.debug('Station is not present, inserting ...')
+ station_entry = Station(**station)
+ station_entry.save()
+ # Log the action
+ ActionLogSerializer.log_model_insert(station_entry)
+
+ station_test_entry = StationTest.objects.filter(**station_test).first()
+ if station_test_entry is None:
+ logger.debug('Station test is not present, inserting ...')
+ station_test_entry = StationTest(station=station_entry,
+ **station_test)
+ station_test_entry.save()
+ # Log the action
+ ActionLogSerializer.log_model_insert(station_test_entry)
+
+ ComponentErrorSerializer.insert_component_errors(station_test_entry,
+ station_entry,
+ component_errors)
+
+ ElementErrorSerializer.insert_element_errors(station_test_entry,
+ station_entry,
+ element_errors)
+
+ return station_test_entry
+
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py b/LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py
index 3f4ff081edde65732f4984ccc8db68d06a339d6f..f47b1ae7c47fe063d27563a25738af2cc9f602e5 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py
@@ -179,7 +179,7 @@ def dict_from_component_error(content):
component_error = dict(component=component,
details=error_details,
- type=error_type)
+ type=error_type.strip())
for element_error in element_errors:
element_error['component_error'] = dict(component_error)
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/tests/__init__.py b/LCU/Maintenance/DBInterface/monitoringdb/tests/__init__.py
index a23cebb546df0dffde79ef4731888d2364a13007..c54420b6cdc7d12fff94d0c82beb9fb366380be4 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/tests/__init__.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/tests/__init__.py
@@ -1 +1 @@
-__all__ = ['t_stationtest_model_creation', 't_stationtest_parser']
\ No newline at end of file
+__all__ = ['t_stationtest_model_creation', 't_stationtest_parser', 't_django_insert']
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/tests/t_django_insert.py b/LCU/Maintenance/DBInterface/monitoringdb/tests/t_django_insert.py
new file mode 100644
index 0000000000000000000000000000000000000000..b0561c23646dffa62c10a32bf84072d4615f06cb
--- /dev/null
+++ b/LCU/Maintenance/DBInterface/monitoringdb/tests/t_django_insert.py
@@ -0,0 +1,38 @@
+from django.test import TestCase
+
+import lofar.maintenance.django_postgresql.settings as settings
+
+import lofar.maintenance.monitoringdb.serializers.component_error as component_error
+from lofar.maintenance.monitoringdb.models.station import Station
+from lofar.maintenance.monitoringdb.models.station_test import StationTest
+from datetime import datetime
+
+from lofar.maintenance.monitoringdb.models.component_error import ComponentError
+
+
+class TestStationTestInsert(TestCase):
+ def test_insert_station_test_no_component_errors(self):
+ pass
+
+
+class TestComponentErrorInsert(TestCase):
+ def test_insert_component_error_with_details(self):
+
+ station_entry = Station(location='Here', name='CS001C', type='C')
+ station_entry.save()
+ station_test_entry = StationTest(start_datetime=datetime(2017, 4, 12, 0,0,0),
+ end_datetime=datetime(2017, 4, 12, 1,0,0),
+ checks='Some checks',
+ station=station_entry
+ )
+ station_test_entry.save()
+ test_data = {'component': {'component_id': 50, 'type': 'LBL'}, 'details': {'xoff': '11', 'yoff': '25', 'xval': 77.0, 'yval': 75.9}, 'type': 'DOWN'}
+
+ component_error_entry = component_error.ComponentErrorSerializer.insert_component_error(station_test_entry,
+ station_entry,
+ test_data)
+ component_error_entry.save()
+
+ id = component_error_entry.pk
+
+ print(ComponentError.objects.get(pk=id).details)
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/tests/t_stationtest_model_creation.py b/LCU/Maintenance/DBInterface/monitoringdb/tests/t_stationtest_model_creation.py
index e198e8af58306f4f3efc9f0e22a71b7b44378c21..ebb74d66956186c32dac7206347a3ed2ad5e2e2e 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/tests/t_stationtest_model_creation.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/tests/t_stationtest_model_creation.py
@@ -10,5 +10,6 @@ class TestStationTestModelCreation(unittest.TestCase):
result = raw_parser.dict_from_raw_station_test(test)
print(result)
+
if __name__ == '__main__':
unittest.main()
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/tests/t_stationtest_parser.py b/LCU/Maintenance/DBInterface/monitoringdb/tests/t_stationtest_parser.py
index 039e4ae7009b2c84b9564d9aeecb30e954f35f73..0932b4634680837910a3b5fd5df2fd8c75754eec 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/tests/t_stationtest_parser.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/tests/t_stationtest_parser.py
@@ -2,9 +2,7 @@ import unittest
from .common import *
-
class TestStationTestsParser(unittest.TestCase):
-
def test_element_error_details_parsing(self):
preparsed_element_error = ['20180508', 'HBA', '044', 'E_FAIL',
{'x1': '99.5 140 101.6 99.8 140 102.4'},
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/urls.py b/LCU/Maintenance/DBInterface/monitoringdb/urls.py
index 44fc3e4b92cae9f095986241610c1f9ca033f8fd..bf97b6fc0efb2329c1b571d156c6874b331e46e7 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/urls.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/urls.py
@@ -1,52 +1,47 @@
from django.conf.urls import url, include
from rest_framework import routers
+from .views.controllers import *
from .views.station_test_views import *
from .views.rtsm_views import *
-from rest_framework.schemas import get_schema_view
-from rest_framework.renderers import CoreJSONRenderer
-from rest_framework import renderers
-from openapi_codec import OpenAPICodec
-
-
-class SwaggerRenderer(renderers.BaseRenderer):
- media_type = 'application/openapi+json'
- format = 'swagger'
-
- def render(self, data, media_type=None, renderer_context=None):
- codec = OpenAPICodec()
- return codec.dump(data)
-
-
-schema_view = get_schema_view(title="Maintenance API", renderer_classes=[SwaggerRenderer, CoreJSONRenderer])
+from .views.logs_view import *
+from rest_framework.documentation import include_docs_urls
+log_router = routers.DefaultRouter()
+log_router.register(r'action_log', ActionLogViewSet)
station_test_router = routers.DefaultRouter()
-#Station test
-station_test_router.register(r'error_details', ErrorDetailsViewSet)
+# Station test
station_test_router.register(r'element_error', ElementErrorViewSet)
station_test_router.register(r'element', ElementViewSet)
station_test_router.register(r'component_error', ComponentErrorViewSet)
station_test_router.register(r'component', ComponentViewSet)
station_test_router.register(r'station', StationViewSet)
-station_test_router.register(r'station_test', StationTestViewSet)
+station_test_router.register(r'', StationTestViewSet)
rtsm_router = routers.DefaultRouter()
-#RTSM
-rtsm_router.register(r'^/summary', RTSMObservationSummaryViewSet)
-rtsm_router.register(r'^/errors_detailed', RTSMErrorsDetailedViewSet)
-rtsm_router.register(r'^/errors', RTSMErrorsViewSet)
-rtsm_router.register(r'^', RTSMObservationViewSet)
+# RTSM
+rtsm_router.register(r'errors', RTSMErrorsViewSet)
+rtsm_router.register(r'spectra', RTSMSpectrumViewSet)
+rtsm_router.register(r'', RTSMObservationViewSet)
urlpatterns = [
+
url(r'^api/stationtests/', include(station_test_router.urls)),
- url(r'^api/rtsm', include(rtsm_router.urls)),
+ url(r'^api/rtsm/', include(rtsm_router.urls)),
+ url(r'^api/log/', include(log_router.urls)),
url(r'^api/api-auth', include('rest_framework.urls', namespace='rest_framework')),
- url(r'^api/stationtests/insert_raw', insert_raw_station_test),
- url(r'^api/rtsm/insert_raw', insert_raw_rtsm_test),
- url(r'^api/schema', schema_view)
+ url(r'^api/stationtests/raw/insert', insert_raw_station_test),
+ url(r'^api/rtsm/raw/insert', insert_raw_rtsm_test),
+ url(r'^api/view/ctrl_stationoverview', ControllerStationOverview.as_view()),
+ url(r'^api/view/ctrl_stationtestsummary', ControllerStationTestsSummary.as_view()),
+ url(r'^api/view/ctrl_latest_observation', ControllerLatestObservations.as_view()),
+ url(r'^api/view/ctrl_stationtest_statistics', ControllerStationTestStatistics.as_view()),
+ url(r'^api/view/ctrl_list_component_error_types', ControllerAllComponentErrorTypes.as_view()),
+
+ url(r'^api/docs', include_docs_urls(title='Monitoring DB API'))
]
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py b/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py
new file mode 100644
index 0000000000000000000000000000000000000000..7f1c3478f5757322a832036ccca3ac4f03d226d5
--- /dev/null
+++ b/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py
@@ -0,0 +1,621 @@
+import datetime
+import logging
+from collections import OrderedDict
+from math import ceil
+
+import coreapi
+import coreschema
+from django.db.models import Count
+from rest_framework import status
+from rest_framework.response import Response
+from rest_framework.schemas import ManualSchema
+from rest_framework.views import APIView
+
+from lofar.maintenance.monitoringdb.models.component_error import ComponentError
+from lofar.maintenance.monitoringdb.models.rtsm import RTSMErrorSummary
+from lofar.maintenance.monitoringdb.models.rtsm import RTSMObservation
+from lofar.maintenance.monitoringdb.models.station import Station
+from lofar.maintenance.monitoringdb.models.station_test import StationTest
+
+logger = logging.getLogger(__name__)
+
+
+def parse_date(date):
+ expected_format = '%Y-%m-%d'
+ try:
+ parsed_date = datetime.datetime.strptime(date, expected_format)
+ return parsed_date
+ except Exception as e:
+ raise ValueError('cannot parse %s with format %s - %s' % (date, expected_format, e))
+
+
+def parse_bool(boolean_value_str):
+ boolean_value_str = boolean_value_str.lower()
+ if boolean_value_str in ['t', 'true', '1']:
+ return True
+ elif boolean_value_str in ['f', 'false', '0']:
+ return False
+ else:
+ raise ValueError('%s is neither true or false' % boolean_value_str)
+
+
+class ValidableReadOnlyView(APIView):
+ """
+ Convenience APIView class to have the validation of the query parameters on a get http request
+ """
+
+ # Override this to make the schema validation work
+ fields = []
+
+ def compute_response(self):
+ raise NotImplementedError()
+
+ @property
+ def schema(self):
+ return ManualSchema(fields=self.fields)
+
+ def validate_query_parameters(self, request):
+ """
+ Validated the request parameters and stores them as fields
+ :param request: the http request to the api call
+ :type request: rest_framework.request.Request
+ :raises ValueError: if the parameter is not valid
+ :raises KeyError: if the requested parameter is missing
+ """
+ for field in self.fields:
+ if field.required and field.name not in request.query_params:
+ raise KeyError('%s parameter is missing' % field.name)
+ elif field.name not in request.query_params:
+ continue
+ else:
+ value = self.request.query_params.get(field.name)
+ if field.type:
+ self.__setattr__(field.name, field.type(value))
+ else:
+ self.__setattr__(field.name, value)
+ errors = field.schema.validate(self.__getattribute__(field.name))
+ for error in errors:
+ raise ValueError(error.text)
+
+ def get(self, request):
+ try:
+ self.validate_query_parameters(request)
+ except ValueError as e:
+ return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
+ data='Please specify the correct parameters: %s' % (e,))
+ except KeyError as e:
+ return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
+ data='Please specify all the required parameters: %s' % (e,))
+
+ try:
+ response = self.compute_response()
+ except Exception as e:
+ logger.exception(e)
+ return Response(status=status.HTTP_500_INTERNAL_SERVER_ERROR,
+ data='exception occurred: %s' % e)
+ return response
+
+
+class ControllerStationOverview(ValidableReadOnlyView):
+ """
+ Overview of the latest tests performed on the stations
+ """
+
+ station_group = 'A'
+ errors_only = 'true'
+ n_station_tests = 4
+ n_rtsm = 4
+
+ fields = [
+ coreapi.Field(
+ "station_group",
+ required=False,
+ location='query',
+ schema=coreschema.Enum(['C', 'R', 'I', 'A'], description=
+ 'Station group to select for choices are [C|R|I|ALL]',
+ )
+ ),
+ coreapi.Field(
+ "n_station_tests",
+ required=False,
+ location='query',
+ type=int,
+ schema=coreschema.Integer(description='number of station tests to select',
+ minimum=1)
+ ),
+ coreapi.Field(
+ "n_rtsm",
+ required=False,
+ location='query',
+ type=int,
+ schema=coreschema.Integer(description='number of station tests to select',
+ minimum=1)
+ ),
+ coreapi.Field(
+ "errors_only",
+ required=False,
+ location='query',
+ type=parse_bool,
+ schema=coreschema.Boolean(
+ description='displays or not only the station with more than one error')
+ )
+ ]
+
+ def compute_response(self):
+
+ station_entities = Station.objects.all()
+ for group in self.station_group:
+ if group is not 'A':
+ station_entities = station_entities.filter(type=group)
+
+ # Since django preferes a ordered dict over a dict we make it happy... for now
+ response_payload = list()
+ for station_entity in station_entities:
+ station_payload = OrderedDict()
+
+ station_payload['station_name'] = station_entity.name
+
+ station_test_list = StationTest.objects.filter(
+ station__name=station_entity.name).order_by('-end_datetime')[:self.n_station_tests]
+ rtsm_list = RTSMObservation.objects.filter(
+ station__name=station_entity.name).order_by('-end_datetime')[:self.n_rtsm]
+
+ station_payload['station_tests'] = list()
+ for station_test in station_test_list:
+ station_test_payload = OrderedDict()
+ component_errors = station_test.component_errors
+
+ station_test_payload[
+ 'total_component_errors'] = station_test.component_errors.count()
+ station_test_payload['start_datetime'] = station_test.start_datetime
+ station_test_payload['end_datetime'] = station_test.end_datetime
+ station_test_payload['checks'] = station_test.checks
+ component_errors_summary = component_errors. \
+ values('component__type', 'type').annotate(
+ total=Count('type')).order_by('-total')
+ component_errors_summary_dict = OrderedDict()
+ for item in component_errors_summary:
+ item_component_type = item['component__type']
+ item_error_type = item['type']
+ item_error_total = item['total']
+
+ if item_component_type not in component_errors_summary_dict:
+ component_errors_summary_dict[item_component_type] = OrderedDict()
+
+ component_errors_summary_dict[item_component_type][item_error_type] = \
+ item_error_total
+ station_test_payload['component_error_summary'] = component_errors_summary_dict
+
+ station_payload['station_tests'].append(station_test_payload)
+
+ station_payload['rtsm'] = list()
+ for rtsm in rtsm_list:
+ rtsm_payload = OrderedDict()
+ rtsm_payload['observation_id'] = rtsm.observation_id
+ rtsm_payload['start_datetime'] = rtsm.start_datetime
+ rtsm_payload['end_datetime'] = rtsm.end_datetime
+
+ unique_modes = [item['mode'] for item in
+ rtsm.errors_summary.values('mode').distinct()]
+ rtsm_payload['mode'] = unique_modes
+ rtsm_payload['total_component_errors'] = rtsm.errors_summary.count()
+
+ errors_summary = OrderedDict()
+
+ errors_summary_query = rtsm.errors_summary.values('error_type').annotate(
+ total=Count('error_type'))
+
+ for error_summary in errors_summary_query:
+ errors_summary[error_summary['error_type']] = error_summary['total']
+
+ rtsm_payload['error_summary'] = errors_summary
+ station_payload['rtsm'].append(rtsm_payload)
+
+ response_payload.append(station_payload)
+ if self.errors_only:
+ response_payload = filter(
+ lambda station_entry:
+ len(station_entry['station_tests']) + len(station_entry['rtsm']) > 0,
+ response_payload)
+ response_payload = sorted(response_payload, key=lambda item: item['station_name'])
+ return Response(status=status.HTTP_200_OK, data=response_payload)
+
+
+class ControllerStationTestsSummary(ValidableReadOnlyView):
+ """
+ Overview of the latest station tests performed on the stations # lookback days before now
+ """
+ station_group = 'A'
+ errors_only = 'true'
+ lookback_time = 7
+
+ fields = [
+ coreapi.Field(
+ "station_group",
+ required=False,
+ location='query',
+ schema=coreschema.Enum(['C', 'R', 'I', 'A'],
+ description='Station group to select for choices are [C|R|I|ALL]')
+ ),
+ coreapi.Field(
+ "errors_only",
+ required=False,
+ location='query',
+ type=parse_bool,
+ schema=coreschema.Boolean(
+ description='displays or not only the station with more than one error')
+ ),
+ coreapi.Field(
+ "lookback_time",
+ required=False,
+ type=int,
+ location='query',
+ schema=coreschema.Integer(description='number of days from now (default 7)',
+ minimum=1)
+ )
+ ]
+
+ def compute_response(self):
+ self.lookback_time = datetime.timedelta(days=self.lookback_time)
+
+ station_test_list = StationTest.objects \
+ .filter(start_datetime__gte=datetime.date.today() - self.lookback_time) \
+ .order_by('-start_datetime', 'station__name')
+ for group in self.station_group:
+ if group is not 'A':
+ station_test_list = station_test_list.filter(station__type=group)
+
+ # Since django preferes a ordered dict over a dict we make it happy... for now
+ response_payload = list()
+
+ for station_test in station_test_list:
+
+ station_test_payload = OrderedDict()
+ component_errors = station_test.component_errors
+ station_test_payload['station_name'] = station_test.station.name
+
+ station_test_payload[
+ 'total_component_errors'] = station_test.component_errors.count()
+ station_test_payload['date'] = station_test.start_datetime.strftime('%Y-%m-%d')
+ station_test_payload['start_datetime'] = station_test.start_datetime
+ station_test_payload['end_datetime'] = station_test.end_datetime
+ station_test_payload['checks'] = station_test.checks
+ component_errors_summary = component_errors. \
+ values('component__type', 'type').annotate(
+ total=Count('type')).order_by('-total')
+ component_errors_summary_dict = OrderedDict()
+ for item in component_errors_summary:
+ item_component_type = item['component__type']
+ item_error_type = item['type']
+ item_error_total = item['total']
+
+ if item_component_type not in component_errors_summary_dict:
+ component_errors_summary_dict[item_component_type] = OrderedDict()
+
+ component_errors_summary_dict[item_component_type][item_error_type] = \
+ item_error_total
+ station_test_payload['component_error_summary'] = component_errors_summary_dict
+
+ response_payload.append(station_test_payload)
+
+ if self.errors_only:
+ response_payload = filter(
+ lambda station_test_entry:
+ station_test_entry['total_component_errors'] > 0,
+ response_payload)
+
+ return Response(status=status.HTTP_200_OK, data=response_payload)
+
+
+class ControllerLatestObservations(ValidableReadOnlyView):
+ """
+ Overview of the latest observations performed on the stations
+ """
+
+ station_group = 'A'
+ errors_only = 'true'
+
+ fields = [
+ coreapi.Field(
+ "station_group",
+ required=False,
+ location='query',
+ schema=coreschema.Enum(['C', 'R', 'I', 'A'], description=
+ 'Station group to select for choices are [C|R|I|A]',
+ )
+ ),
+ coreapi.Field(
+ "errors_only",
+ required=False,
+ location='query',
+ type=parse_bool,
+ schema=coreschema.Boolean(
+ description='displays or not only the station with more than one error')
+ ),
+ coreapi.Field(
+ "from_date",
+ required=True,
+ location='query',
+ schema=coreschema.String(
+ description='select rtsm from date (ex. YYYY-MM-DD)')
+ )
+ ]
+
+ def compute_response(self):
+ self.from_date = parse_date(self.from_date)
+
+ filtered_entities = RTSMObservation.objects \
+ .filter(start_datetime__gte=self.from_date)
+ if self.station_group != 'A':
+ filtered_entities = filtered_entities \
+ .filter(station__type=self.station_group)
+ if self.errors_only:
+ filtered_entities = filtered_entities.exclude(errors_summary__isnull=True)
+
+ errors_summary = filtered_entities \
+ .values('observation_id',
+ 'station__name',
+ 'start_datetime',
+ 'end_datetime',
+ 'errors_summary__error_type',
+ 'errors_summary__mode') \
+ .annotate(total=Count('errors_summary__error_type')) \
+ .order_by('observation_id', 'station__name')
+ response = dict()
+
+ for error_summary in errors_summary:
+ observation_id = error_summary['observation_id']
+ station_name = error_summary['station__name']
+ start_datetime = error_summary['start_datetime']
+ end_datetime = error_summary['end_datetime']
+ mode = error_summary['errors_summary__mode']
+ error_type = error_summary['errors_summary__error_type']
+ total = error_summary['total']
+
+ if observation_id not in response:
+ response[observation_id] = OrderedDict()
+ response[observation_id]['observation_id'] = observation_id
+ response[observation_id]['start_datetime'] = start_datetime
+ response[observation_id]['end_datetime'] = end_datetime
+
+ response[observation_id]['total_component_errors'] = 0
+ response[observation_id]['mode'] = list()
+ response[observation_id]['station_involved'] = dict()
+
+ if total == 0:
+ continue
+
+ response[observation_id]['total_component_errors'] += total
+ station_involved_summary = response[observation_id]['station_involved']
+ response[observation_id]['mode'] += [mode] \
+ if mode not in response[observation_id]['mode'] else []
+ if station_name not in station_involved_summary:
+ station_involved_summary[station_name] = OrderedDict()
+ station_involved_summary[station_name]['station_name'] = station_name
+ station_involved_summary[station_name]['n_errors'] = 0
+ station_involved_summary[station_name]['component_error_summary'] = OrderedDict()
+
+ station_involved_summary[station_name]['n_errors'] += total
+ station_involved_summary[station_name]['component_error_summary'][error_type] = total
+
+ response_payload = sorted(response.values(),
+ key=lambda item: item['start_datetime'],
+ reverse=True)
+
+ return Response(status=status.HTTP_200_OK, data=response_payload)
+
+
+class ControllerStationTestStatistics(ValidableReadOnlyView):
+ """
+
+/views/ctrl_stationtest_statistics:
+
+parameters:
+station_group [C|R|I|ALL] (optional, default ALL)
+test_type [RTSM|STATIONTEST|BOTH] (optional, default BOTH)
+from_date #DATE
+to_date #DATE
+averaging_interval: #TIMESPAN
+result:
+{
+ start_date : #DATE,
+ end_date: #DATE,
+ averaging_interval: #INTERVAL,
+ error_per_station: [{
+ time: #DATE,
+ station_name: <station_name>
+ n_errors: #nr_errors int
+ }, ...],
+ error_per_error_type: [{
+ time: #DATE,
+ error_type: <error_type>
+ n_errors: #nr_errors int
+ }, ...],
+ },
+ ....
+]
+ """
+ station_group = 'A'
+ test_type = 'B'
+
+ fields = [
+ coreapi.Field(
+ "test_type",
+ required=False,
+ location='query',
+ schema=coreschema.Enum(['R', 'S', 'B'],
+ description='select the type of test possible values are (R, RTSM),'
+ ' (S, Station test), (B, both)[DEFAULT=B]',
+ )
+ ),
+ coreapi.Field(
+ "station_group",
+ required=False,
+ location='query',
+ schema=coreschema.Enum(['C', 'R', 'I', 'A'], description=
+ 'Station group to select for choices are [C|R|I|ALL]',
+ )
+ ),
+ coreapi.Field(
+ "from_date",
+ required=True,
+ location='query',
+ schema=coreschema.String(
+ description='select tests from date (ex. YYYY-MM-DD)')
+ ),
+ coreapi.Field(
+ "to_date",
+ required=True,
+ location='query',
+ schema=coreschema.String(
+ description='select tests to date (ex. YYYY-MM-DD)')
+ ),
+ coreapi.Field(
+ "averaging_interval",
+ required=True,
+ location='query',
+ type=int,
+ schema=coreschema.Integer(
+ description='averaging interval in days')
+ )
+ ]
+
+ def compute_errors_per_station(self, from_date, to_date, central_time, station_group,
+ test_type):
+
+ component_errors = ComponentError.objects.all()
+ rtsm_summary_errors = RTSMErrorSummary.objects.all()
+
+ if station_group:
+ component_errors = component_errors.filter(station_test__station__type=station_group)
+ rtsm_summary_errors = rtsm_summary_errors.filter(
+ observation__station__type=station_group)
+
+ station_test_results = []
+ rtsm_results = []
+ if test_type in ['S', 'B']:
+ station_test_results = component_errors. \
+ filter(station_test__start_datetime__gt=from_date,
+ station_test__start_datetime__lt=to_date). \
+ values('station_test__station__name'). \
+ annotate(n_errors=Count('station_test__station__name'))
+ if test_type in ['R', 'B']:
+ rtsm_results = rtsm_summary_errors. \
+ filter(observation__start_datetime__gt=from_date,
+ observation__start_datetime__lt=to_date). \
+ values('observation__station__name'). \
+ annotate(n_errors=Count('observation__station__name'))
+
+ errors_per_station_in_bin = dict()
+ central_time_str = central_time.strftime('%Y-%m-%d')
+ if test_type in ['S', 'B']:
+ for result in station_test_results:
+ station_name = result['station_test__station__name']
+ errors_per_station_in_bin[station_name] = dict(station_name=station_name,
+ n_errors=result['n_errors'],
+ time=central_time_str)
+
+ if test_type in ['R', 'B']:
+ for result in rtsm_results:
+ station_name = result['observation__station__name']
+ if station_name not in errors_per_station_in_bin:
+ errors_per_station_in_bin[station_name] = dict(station_name=station_name,
+ n_errors=result['n_errors'],
+ time=central_time_str)
+ else:
+ errors_per_station_in_bin[station_name]['n_errors'] += result['n_errors']
+
+ return errors_per_station_in_bin.values()
+
+ def compute_errors_per_type(self, from_date, to_date, central_time, station_group, test_type):
+
+ component_errors = ComponentError.objects.all()
+ rtsm_summary_errors = RTSMErrorSummary.objects.all()
+
+ station_test_results = []
+ rtsm_results = []
+
+ central_time_str = central_time.strftime('%Y-%m-%d')
+ if station_group:
+ component_errors = component_errors.filter(station_test__station__type=station_group)
+ rtsm_summary_errors = rtsm_summary_errors.filter(
+ observation__station__type=station_group)
+
+ if test_type in ['S', 'B']:
+ station_test_results = component_errors. \
+ filter(station_test__start_datetime__gt=from_date,
+ station_test__start_datetime__lt=to_date). \
+ values('type'). \
+ annotate(n_errors=Count('type'))
+ if test_type in ['R', 'B']:
+ rtsm_results = rtsm_summary_errors. \
+ filter(observation__start_datetime__gt=from_date,
+ observation__start_datetime__lt=to_date). \
+ values('error_type'). \
+ annotate(n_errors=Count('error_type'))
+
+ errors_per_error_type_in_bin = dict()
+
+ if test_type in ['S', 'B']:
+ for result in station_test_results:
+ error_type = result['type']
+ errors_per_error_type_in_bin[error_type] = dict(error_type=error_type,
+ n_errors=result['n_errors'],
+ time=central_time_str)
+ if test_type in ['R', 'B']:
+ for result in rtsm_results:
+ error_type = result['error_type']
+ if error_type not in errors_per_error_type_in_bin:
+ errors_per_error_type_in_bin[error_type] = dict(error_type=error_type,
+ n_errors=result['n_errors'],
+ time=central_time_str)
+ else:
+ errors_per_error_type_in_bin[error_type]['n_errors'] += result['n_errors']
+
+ return errors_per_error_type_in_bin.values()
+
+ def compute_response(self):
+ from_date = parse_date(self.from_date)
+ to_date = parse_date(self.to_date)
+ averaging_interval = datetime.timedelta(days=self.averaging_interval)
+ response_payload = OrderedDict()
+
+ response_payload['start_date'] = from_date
+ response_payload['end_date'] = to_date
+ response_payload['averaging_interval'] = averaging_interval
+
+ errors_per_station = []
+ errors_per_type = []
+ n_bins = int(ceil((to_date - from_date) / averaging_interval))
+
+ for i in range(n_bins):
+ if self.station_group is 'A':
+ station_group = None
+ else:
+ station_group = self.station_group
+ errors_per_station += self.compute_errors_per_station(
+ from_date=from_date + i * averaging_interval,
+ to_date=from_date + (i + 1) * averaging_interval,
+ central_time=from_date + (i + .5) * averaging_interval,
+ station_group=station_group,
+ test_type=self.test_type)
+
+ errors_per_type += self.compute_errors_per_type(
+ from_date=from_date + i * averaging_interval,
+ to_date=from_date + (i + 1) * averaging_interval,
+ central_time=from_date + (i + .5) * averaging_interval,
+ station_group=station_group,
+ test_type=self.test_type)
+
+ response_payload['errors_per_station'] = errors_per_station
+ response_payload['errors_per_type'] = errors_per_type
+
+ return Response(status=status.HTTP_200_OK, data=response_payload)
+
+
+class ControllerAllComponentErrorTypes(ValidableReadOnlyView):
+ schema = ManualSchema(description="Returns all distinct component errors", fields=[])
+
+ def compute_response(self):
+ data = [item['type'] for item in ComponentError.objects.values('type').distinct()]
+ return Response(status=status.HTTP_200_OK, data=data)
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/views/logs_view.py b/LCU/Maintenance/DBInterface/monitoringdb/views/logs_view.py
new file mode 100644
index 0000000000000000000000000000000000000000..51b961d3ac7806b7a317819ebac15f3974fcda40
--- /dev/null
+++ b/LCU/Maintenance/DBInterface/monitoringdb/views/logs_view.py
@@ -0,0 +1,13 @@
+from rest_framework.viewsets import ReadOnlyModelViewSet
+from rest_framework.schemas import SchemaGenerator
+from lofar.maintenance.monitoringdb.serializers.log import ActionLogSerializer, ActionLog
+
+
+class ActionLogViewSet(ReadOnlyModelViewSet):
+ """
+ Action event log line
+
+ """
+ queryset = ActionLog.objects.all()
+ serializer_class = ActionLogSerializer
+ filter_fields = '__all__'
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py b/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py
index bc84d8c4efcf005a6894a43247449dad0c829f60..5e0a2560ff7ca59bd3bd6d5f677de7789d38b363 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py
@@ -1,62 +1,28 @@
from .common import *
-from ..models.rtsm import RTSMObservation, RTSMError
-from ..serializers.rtsm import RTSMObservationSerializer, RTSMErrorSummarySerializer, RTSMErrorSerializer,\
- RTSMObservationSummarySerializer
+from ..models.rtsm import RTSMObservation, RTSMError, RTSMSpectrum
+from ..serializers.rtsm import RTSMObservationSerializer, RTSMErrorSerializer, \
+ RTSMSpectrumSerializer
from ..rtsm_test_raw_parser import parse_rtsm_test
-from rest_framework.serializers import Serializer
-logger = logging.getLogger('views')
+logger = logging.getLogger('views')
-class RTSMErrorsDetailedViewSet(viewsets.ReadOnlyModelViewSet):
- queryset = RTSMError.objects.all()
- serializer_class = RTSMErrorSerializer
- def get_queryset(self):
- queryset = RTSMError.objects.all()
- for key, param in self.request.query_params.items():
- if key in RESERVED_FILTER_NAME:
- continue
- queryset = queryset.filter(**{key: param})
- return queryset
+class RTSMSpectrumViewSet(viewsets.ReadOnlyModelViewSet):
+ queryset = RTSMSpectrum.objects.all()
+ serializer_class = RTSMSpectrumSerializer
+ filter_fields = ['rtsm_error']
class RTSMErrorsViewSet(viewsets.ReadOnlyModelViewSet):
queryset = RTSMError.objects.all()
- serializer_class = RTSMErrorSummarySerializer
-
- def get_queryset(self):
- queryset = RTSMError.objects.all()
- for key, param in self.request.query_params.items():
- if key in RESERVED_FILTER_NAME:
- continue
- queryset = queryset.filter(**{key: param})
- return queryset
+ serializer_class = RTSMErrorSerializer
+ filter_fields = '__all__'
class RTSMObservationViewSet(viewsets.ReadOnlyModelViewSet):
queryset = RTSMObservation.objects.all()
serializer_class = RTSMObservationSerializer
-
- def get_queryset(self):
- queryset = RTSMObservation.objects.all()
- for key, param in self.request.query_params.items():
- if key in RESERVED_FILTER_NAME:
- continue
- queryset = queryset.filter(**{key: param})
- return queryset
-
-
-class RTSMObservationSummaryViewSet(viewsets.ReadOnlyModelViewSet):
- queryset = RTSMObservation.objects.all()
- serializer_class = RTSMObservationSummarySerializer
-
- def get_queryset(self):
- queryset = RTSMObservation.objects.all()
- for key, param in self.request.query_params.items():
- if key in RESERVED_FILTER_NAME:
- continue
- queryset = queryset.filter(**{key: param})
- return queryset
+ filter_fields = '__all__'
@api_view(['POST'])
@@ -82,15 +48,15 @@ def insert_raw_rtsm_test(request):
entry.update(station_name=station_name)
logger.info('RTSM parsed successfully for obsid %d and station %s',
- entry['observation_id'],
- entry['station_name'])
+ entry['observation_id'],
+ entry['station_name'])
try:
- rtsm_item = RTSMObservationSerializer().create(entry)
+ rtsm_item = RTSMObservationSerializer().create(dict(entry))
rtsm_id = rtsm_item.id
except ItemAlreadyExists as e:
return Response(status=status.HTTP_200_OK,
- data='RTSM test was already inserted with id {}\n'.format(e.instance_id))
+ data='RTSM test was already present with id {}\n'.format(e.instance_id))
logger.info('RTSM inserted successfully for obsid %d and station %s',
entry['observation_id'],
entry['station_name'])
@@ -98,14 +64,14 @@ def insert_raw_rtsm_test(request):
logger.exception("raw station info malformed %s: %s",
request.data['content'], e)
logger.info('station test malformed skipping insertion.')
- return Response(status=status.HTTP_200_OK, data='skipping insertion RTSM malformed')
+ return Response(status=status.HTTP_406_NOT_ACCEPTABLE, data='skipping insertion RTSM malformed')
except Exception as e:
logger.exception("exception occurred while parsing raw station info %s: %s",
request.data['content'], e)
return Response(exception=True,
data="the post message is not correct." +
- " It has to be of the form \{'content':[RAWSTRING]\}: %s. Request provided %s" % (
- e, request),
+ " It has to be of the form {content:[RAWSTRING], station_name:[STATION_NAME]}: %s. Request provided %s" % (
+ e, request),
status=status.HTTP_400_BAD_REQUEST)
else:
return Response(exception=True,
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/views/station_test_views.py b/LCU/Maintenance/DBInterface/monitoringdb/views/station_test_views.py
index d2a071832255a380bedca92d88cf8f2119d09edb..834bf3a1c2f48993765ba6eb7aafd5a24d7f8bb6 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/views/station_test_views.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/views/station_test_views.py
@@ -6,14 +6,12 @@ from ..models.component import Component
from ..models.component_error import ComponentError
from ..models.element import Element
from ..models.element_error import ElementError
-from ..models.error_details import ErrorDetails
from ..models.station import Station
from ..models.station_test import StationTest
from ..serializers.component import ComponentSerializer
from ..serializers.component_error import ComponentErrorSerializer
from ..serializers.element import ElementSerializer
from ..serializers.element_error import ElementErrorSerializer
-from ..serializers.error_details import ErrorDetailsSerializer
from ..serializers.station import StationSerializer
from ..serializers.station_tests import StationTestSerializer
from ..station_test_raw_parser import parse_raw_station_test
@@ -21,68 +19,91 @@ from ..station_test_raw_parser import parse_raw_station_test
logger = logging.getLogger('views')
-class ComponentViewSet(ReadOnlyModelViewSet):
- serializer_class = ComponentSerializer
- queryset = Component.objects.all()
- filter_fields = '__all__'
-
-
class StationTestFilterSet(filters.FilterSet):
station_name = filters.CharFilter(field_name='station__name',
- lookup_expr='contains')
+ lookup_expr='contains', help_text='name of the station')
station_type = filters.CharFilter(field_name='station__type',
- lookup_expr='contains')
+ lookup_expr='contains',
+ help_text='selects the station type:' +
+ 'one of [Core[C], Remote[R], International[I]]')
- from_date = filters.DateFilter(field_name='start_datetime', lookup_expr='gt')
- to_date = filters.DateFilter(field_name='end_datetime', lookup_expr='lt')
+ from_date = filters.DateFilter(field_name='start_datetime', lookup_expr='gt',
+ help_text='select station tests from date time')
+ to_date = filters.DateFilter(field_name='end_datetime', lookup_expr='lt',
+ help_text='select station tests until date time')
class ComponentErrorFilterSet(filters.FilterSet):
- component_id = filters.NumberFilter(field_name='component', lookup_expr='component_id')
- component_type = filters.CharFilter(field_name='component', lookup_expr='type')
+ component_id = filters.NumberFilter(field_name='component', lookup_expr='component_id',
+ help_text='select by component id')
+ component_type = filters.CharFilter(field_name='component', lookup_expr='type',
+ help_text='select by component type')
station_name = filters.CharFilter(field_name='station_test__station',
- lookup_expr='name__contains')
+ lookup_expr='name__contains',
+ help_text='station name with name like')
station_type = filters.CharFilter(field_name='station_test__station',
- lookup_expr='type__contains')
+ lookup_expr='type__contains',
+ help_text='station type like')
- from_date = filters.DateFilter(field_name='station_test', lookup_expr='start_datetime__gt')
- to_date = filters.DateFilter(field_name='station_test', lookup_expr='end_datetime__lt')
+ from_date = filters.DateFilter(field_name='station_test', lookup_expr='start_datetime__gt',
+ help_text='select component errors from date time')
+ to_date = filters.DateFilter(field_name='station_test', lookup_expr='end_datetime__lt',
+ help_text='select component errors until date time')
type = filters.CharFilter(field_name='type',
- lookup_expr='contains')
+ lookup_expr='contains',
+ help_text='component error type')
class ElementErrorFilterSet(filters.FilterSet):
- component_id = filters.NumberFilter(field_name='component_error__component', lookup_expr='component_id')
- element_id = filters.NumberFilter(field_name='element', lookup_expr='element_id')
+ component_id = filters.NumberFilter(field_name='component_error__component',
+ lookup_expr='component_id',
+ help_text='id of the parent component')
+ element_id = filters.NumberFilter(field_name='element', lookup_expr='element_id',
+ help_text='element id')
station_name = filters.CharFilter(field_name='component_error__station_test__station',
- lookup_expr='name__contains')
+ lookup_expr='name__contains',
+ help_text='name of the station')
station_type = filters.CharFilter(field_name='component_error__station_test__station',
- lookup_expr='type__contains')
+ lookup_expr='type__contains',
+ help_text='station type')
- from_date = filters.DateFilter(field_name='component_error__station_test', lookup_expr='start_datetime__gt')
- to_date = filters.DateFilter(field_name='component_error__station_test', lookup_expr='end_datetime__lt')
+ from_date = filters.DateFilter(field_name='component_error__station_test',
+ lookup_expr='start_datetime__gt',
+ help_text='select element errors from date time')
+ to_date = filters.DateFilter(field_name='component_error__station_test',
+ lookup_expr='end_datetime__lt',
+ help_text='select element errors until date time')
type = filters.CharFilter(field_name='type',
- lookup_expr='contains')
-
-
-class ComponentErrorViewSet(ReadOnlyModelViewSet):
- serializer_class = ComponentErrorSerializer
- queryset = ComponentError.objects.all().order_by('-station_test__start_datetime',
- 'type',
- 'component__component_id',
- 'component__station__name')
- filter_fields = '__all__'
- filterset_class = ComponentErrorFilterSet
+ lookup_expr='contains', help_text='element error type')
class StationViewSet(ReadOnlyModelViewSet):
+ """
+ retrieve:
+ retrieve a specific station from the database
+
+ list:
+ list all the stations present in the database
+ """
serializer_class = StationSerializer
queryset = Station.objects.all()
filter_fields = '__all__'
+class ComponentViewSet(ReadOnlyModelViewSet):
+ serializer_class = ComponentSerializer
+ queryset = Component.objects.all()
+ filter_fields = '__all__'
+
+
+class ElementViewSet(ReadOnlyModelViewSet):
+ serializer_class = ElementSerializer
+ queryset = Element.objects.all()
+ filter_fields = '__all__'
+
+
class StationTestViewSet(ReadOnlyModelViewSet):
serializer_class = StationTestSerializer
queryset = StationTest.objects.all().order_by('-start_datetime', 'station__name')
@@ -90,10 +111,14 @@ class StationTestViewSet(ReadOnlyModelViewSet):
filterset_class = StationTestFilterSet
-class ErrorDetailsViewSet(ReadOnlyModelViewSet):
- serializer_class = ErrorDetailsSerializer
- queryset = ErrorDetails.objects.all()
+class ComponentErrorViewSet(ReadOnlyModelViewSet):
+ serializer_class = ComponentErrorSerializer
+ queryset = ComponentError.objects.all().order_by('-station_test__start_datetime',
+ 'type',
+ 'component__component_id',
+ 'component__station__name')
filter_fields = '__all__'
+ filterset_class = ComponentErrorFilterSet
class ElementErrorViewSet(ReadOnlyModelViewSet):
@@ -106,146 +131,13 @@ class ElementErrorViewSet(ReadOnlyModelViewSet):
filterset_class = ElementErrorFilterSet
-class ElementViewSet(ReadOnlyModelViewSet):
- serializer_class = ElementSerializer
- queryset = Element.objects.all()
- filter_fields = '__all__'
-
-
-def insert_component_error(station_test_entry,
- station_entry,
- component_error):
-
- component = component_error.pop('component')
- if 'details' in component_error:
- component_error_details = component_error.pop('details')
- else:
- component_error_details = None
- component_entry = Component.objects.filter(station=station_entry,
- **component).first()
- if component_entry is None:
- logger.debug('Component entry is not present, inserting ...')
- component_entry = Component(station=station_entry,
- **component)
- logger.debug(component_entry, component_error)
- component_entry.save()
-
- component_error_entry = ComponentError.objects.filter(component=component_entry,
- station_test=station_test_entry,
- **component_error).first()
- if component_error_entry is None:
- logger.debug('Component error entry is not present, inserting ...')
- if component_error_details:
- component_error_details_entry = ErrorDetails(**component_error_details)
- component_error_details_entry.save()
-
- component_error_entry = ComponentError(component=component_entry,
- details=component_error_details_entry,
- station_test=station_test_entry,
- **component_error)
- else:
- component_error_entry = ComponentError(component=component_entry,
- station_test=station_test_entry,
- **component_error)
- component_error_entry.save()
- return component_error_entry
-
-
-def insert_component_errors(station_test_entry,
- station_entry,
- component_errors):
- for component_error in component_errors:
- insert_component_error(station_test_entry, station_entry, component_error)
-
-
-def insert_element(component, element):
- element_entry = Element.objects.filter(component=component,
- **element).first()
- if element_entry is None:
- element_entry = Element(component=component,
- **element)
- element_entry.save()
- return element_entry
-
-
-def insert_element_error(station_test_entry,
- station_entry,
- element_error):
- component_error = element_error.pop('component_error')
-
- component_error_entry = insert_component_error(station_test_entry,
- station_entry,
- component_error)
- component = component_error_entry.component
-
- element = element_error.pop('element')
- element_error_details = element_error.pop('details')
-
- element_entry = insert_element(component, element)
-
- element_error_entry = ElementError.objects.filter(element=element_entry,
- component_error=component_error_entry,
- **element_error).first()
- if element_error_entry is None:
- element_error_details_entry = ErrorDetails(**element_error_details)
-
- element_error_details_entry.save()
-
- element_error_entry = ElementError(element=element_entry,
- details=element_error_details_entry,
- component_error=component_error_entry,
- **element_error)
-
- element_error_entry.save()
-
- return element_error_entry
-
-
-def insert_element_errors(station_test_entry,
- station_entry,
- element_errors):
- for element in element_errors:
- insert_element_error(station_test_entry,
- station_entry,
- element)
-
-
-def insert_station_test(station_test):
- station = station_test.pop('station')
- component_errors = station_test.pop('component_errors')
- element_errors = station_test.pop('element_errors')
- station_entry = Station.objects.filter(**station).first()
-
- if station_entry is None:
- logger.debug('Station is not present, inserting ...')
- station_entry = Station(**station)
- station_entry.save()
-
- station_test_entry = StationTest.objects.filter(**station_test).first()
- if station_test_entry is None:
- logger.debug('Station test is not present, inserting ...')
- station_test_entry = StationTest(station=station_entry,
- **station_test)
- station_test_entry.save()
- insert_component_errors(station_test_entry,
- station_entry,
- component_errors)
- insert_element_errors(station_test_entry,
- station_entry,
- element_errors)
- return (1,1)
-
-
@api_view(['POST'])
def insert_raw_station_test(request):
"""
This function is meant to parse a request of the form
- {
- "content": "[STATION TEST RAW TEXT]"
- }
+ { content: [STATION TEST RAW TEXT] }
parse the content field and create all the station_test entity related into the database
- :param request: HTTP request
- :return:
+
"""
if request.method == 'POST':
@@ -259,27 +151,22 @@ def insert_raw_station_test(request):
raise Exception('cannot parse test {}'.format(content))
logger.info('data parsed successfully for test')
station_test_ids = []
- station_test_ids_already_present = []
for entry in parsed_content:
- existing_id, new_id = insert_station_test(entry)
+ station_test = StationTestSerializer.insert_station_test(entry)
+ station_test_ids.append(station_test.pk)
except Exception as e:
logger.exception("exception occurred while parsing raw station info: %s", e)
return Response(exception=True,
data="the post message is not correct." +
" It has to be of the form \{'content':[RAWSTRING]\}: %s. Request provided %s" % (
- e, request),
+ e, request),
status=status.HTTP_400_BAD_REQUEST)
else:
return Response(exception=True,
data="the post message is not correct." +
" It has to be of the form \{'content':[RAWSTRING]\}",
status=status.HTTP_400_BAD_REQUEST)
- logger.info('request processed correctly inserted %d of %d', len(station_test_ids),
- len(station_test_ids) + len(station_test_ids_already_present))
+ logger.info('Request processed correctly. Processed station test ids: %s', station_test_ids)
return Response(status=status.HTTP_200_OK,
- data='Station tests inserted with ids {} \n'.format(station_test_ids) +
- 'Station tests already present with ids {}'.format(
- station_test_ids_already_present
- )
- )
+ data='Station tests inserted with ids {} \n'.format(station_test_ids))
diff --git a/LCU/Maintenance/MDB_tools/cli/station_tests_watchdog.py b/LCU/Maintenance/MDB_tools/cli/station_tests_watchdog.py
index 4b4ae3fff66afec29a9ae91ba15e4a7ed7bce59a..d4e7d6757761f810981ebd1a93a0cf63197c93e7 100644
--- a/LCU/Maintenance/MDB_tools/cli/station_tests_watchdog.py
+++ b/LCU/Maintenance/MDB_tools/cli/station_tests_watchdog.py
@@ -172,7 +172,7 @@ def handle_event_file_created(mdb_address, event):
def main():
setup_logging_framework()
- station_tests_watchdog(sys.argv)
+ station_tests_watchdog(sys.argv[1:])
if __name__ == '__main__':
diff --git a/LCU/Maintenance/MDB_tools/lib/client.py b/LCU/Maintenance/MDB_tools/lib/client.py
index d333cbfd1ca84aadbb1d0355dca71ed767ff2f14..64036c4cfb3d9fd8ef4ba23401adbe2cba97ff7b 100644
--- a/LCU/Maintenance/MDB_tools/lib/client.py
+++ b/LCU/Maintenance/MDB_tools/lib/client.py
@@ -20,9 +20,9 @@ def read_stationtest_rtsm_output_to_dict(file_path):
"""
try:
with open(file_path, 'r') as input_stream:
- rtsm_filename_pattern = '(?P<station_name>\w*)_\d*_\d*.dat'
+ rtsm_filename_pattern = '(?P<station_name>\w*)_\d+_\d+\.dat$'
result = dict(content=input_stream.read().strip('\n'))
- if re.match(rtsm_filename_pattern, file_path): # Matches the filename to understand if it is a RTSM
+ if re.search(rtsm_filename_pattern, file_path): # Matches the filename to understand if it is a RTSM
result.update(station_name=re.search(rtsm_filename_pattern, file_path).group('station_name'))
return result
except Exception as e:
@@ -109,7 +109,7 @@ def send_stationtest_rtsm_content_to_address(address, content):
:param content: content of the API call
:return: True if the request was successful False otherwise
"""
- full_address = '/'.join([address, compose_api_url_for_given_test_type(content)])
+ full_address = '/'.join([address, compose_api_url_for_given_test_type(content['content'])])
logger.info('performing request to address %s', full_address)
logger.debug('request content %s', content)
@@ -130,9 +130,9 @@ def compose_api_url_for_given_test_type(content):
:return: the query needed to insert the raw results
"""
if is_station_test(content):
- query = 'stationtests/insert_raw'
+ query = 'stationtests/raw/insert'
elif is_rtsm_test(content):
- query = 'rtsm/insert_raw'
+ query = 'rtsm/raw/insert'
else:
raise ValueError('The path format must refer to either an RTSM or a station test.')
return query
\ No newline at end of file
diff --git a/LCU/Maintenance/MDB_tools/test/t_station_tests_watchdog_integration.py b/LCU/Maintenance/MDB_tools/test/t_station_tests_watchdog_integration.py
index 21150f2f421e504ad72347477466067712269d0f..e7a00591d582f3af3846436734c3b4fb61906d4d 100644
--- a/LCU/Maintenance/MDB_tools/test/t_station_tests_watchdog_integration.py
+++ b/LCU/Maintenance/MDB_tools/test/t_station_tests_watchdog_integration.py
@@ -37,7 +37,10 @@ class TestStationTestsWatchdog(unittest.TestCase):
:param path: the path to the file/directory to delete
"""
try:
- os.remove(path)
+ if os.path.isfile(path):
+ os.remove(path)
+ else:
+ os.rmdir(path)
except Exception as e:
logger.exception('Cannot remove temporary file/directory %s : %s', path, e)