diff --git a/.gitattributes b/.gitattributes
index ca686153876f161ac8552dcb9cb2bca4163b4ebd..7973fabe8edf96baab4547c0d274c2a01735efba 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1790,6 +1790,7 @@ LCU/Maintenance/CMakeLists.txt -text
LCU/Maintenance/DBInterface/CMakeLists.txt -text
LCU/Maintenance/DBInterface/__init__.py -text
LCU/Maintenance/DBInterface/django_postgresql/__init__.py -text
+LCU/Maintenance/DBInterface/django_postgresql/celery_settings.py -text
LCU/Maintenance/DBInterface/django_postgresql/create_db.sql -text
LCU/Maintenance/DBInterface/django_postgresql/settings.py -text
LCU/Maintenance/DBInterface/django_postgresql/urls.py -text
@@ -1823,6 +1824,7 @@ LCU/Maintenance/DBInterface/monitoringdb/serializers/station.py -text
LCU/Maintenance/DBInterface/monitoringdb/serializers/station_tests.py -text
LCU/Maintenance/DBInterface/monitoringdb/serializers/utils.py -text
LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py -text
+LCU/Maintenance/DBInterface/monitoringdb/tasks.py -text
LCU/Maintenance/DBInterface/monitoringdb/tests/__init__.py -text
LCU/Maintenance/DBInterface/monitoringdb/tests/common.py -text
LCU/Maintenance/DBInterface/monitoringdb/tests/old_tests.py -text
diff --git a/LCU/Maintenance/DBInterface/CMakeLists.txt b/LCU/Maintenance/DBInterface/CMakeLists.txt
index 99c17d8d018434ce3d5035ad8d52dc78d1a6e0f5..b5fec93c6e859ebb2102a865b7ca2caf4f2a32e9 100644
--- a/LCU/Maintenance/DBInterface/CMakeLists.txt
+++ b/LCU/Maintenance/DBInterface/CMakeLists.txt
@@ -10,6 +10,8 @@ find_python_module(django REQUIRED)
find_python_module(psycopg2 REQUIRED)
find_python_module(rest_framework REQUIRED) #sudo pip install djangorestframework
find_python_module(requests REQUIRED)
+find_python_module(celery REQUIRED)
+find_python_module(django_filters REQUIRED)
# includes every python file excepts for the manage.py
diff --git a/LCU/Maintenance/DBInterface/django_postgresql/__init__.py b/LCU/Maintenance/DBInterface/django_postgresql/__init__.py
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..782bcb57226cadb466668820c9951dd0321eba7f 100644
--- a/LCU/Maintenance/DBInterface/django_postgresql/__init__.py
+++ b/LCU/Maintenance/DBInterface/django_postgresql/__init__.py
@@ -0,0 +1,3 @@
+from .celery_settings import backend_tasks
+
+__all__ = ('backend_tasks')
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/django_postgresql/celery_settings.py b/LCU/Maintenance/DBInterface/django_postgresql/celery_settings.py
new file mode 100644
index 0000000000000000000000000000000000000000..14a925101ee2431532ef369e38f18bea13b91dfe
--- /dev/null
+++ b/LCU/Maintenance/DBInterface/django_postgresql/celery_settings.py
@@ -0,0 +1,22 @@
+from __future__ import absolute_import, unicode_literals
+import os
+from celery import Celery
+
+# set the default Django settings module for the 'celery' program.
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'lofar.maintenance.django_postgresql.settings')
+
+backend_tasks = Celery('maintenancedb')
+
+# Using a string here means the worker doesn't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+# should have a `CELERY_` prefix.
+backend_tasks.config_from_object('django.conf:settings', namespace='CELERY')
+
+# Load task modules from all registered Django app configs.
+backend_tasks.autodiscover_tasks()
+
+
+@backend_tasks.task(bind=True)
+def debug_task(self):
+ print('Request: {0!r}'.format(self.request))
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/django_postgresql/settings.py b/LCU/Maintenance/DBInterface/django_postgresql/settings.py
index 87ca345a17cc324d7969741a76ffcb16707e8eca..e257b43e2daa8a9b229589193919ec676d13da3a 100644
--- a/LCU/Maintenance/DBInterface/django_postgresql/settings.py
+++ b/LCU/Maintenance/DBInterface/django_postgresql/settings.py
@@ -177,6 +177,6 @@ REST_FRAMEWORK = {
'DEFAULT_FILTER_BACKENDS': ('django_filters.rest_framework.DjangoFilterBackend',)
}
-
+CELERY_RESULT_BACKEND = 'amqp://guest@localhost//'
# LOFAR SPECIFIC PARAMETERS
-URL_TO_RTSM_PLOTS = 'https://proxy.lofar.eu/rtsm/obs_plots/'
\ No newline at end of file
+URL_TO_STORE_RTSM_PLOTS = './'
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py b/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py
index b82a31324371eb25a69b821b219c1f72c83ffdb9..16c28c62ad3ec6fae15d014798f80bbc696ab050 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py
@@ -7,7 +7,7 @@ from .station import Station
from .fixed_types import STATION_TYPES
-MODE_TO_COMPONENT = {
+MODE_TO_COMPONENT = {
1: 'LBL',
2: 'LBL',
3: 'LBH',
@@ -17,6 +17,11 @@ MODE_TO_COMPONENT = {
7: 'HBA'
}
+COMPONENT_TO_MODE = {
+ 'LBL': [1, 2],
+ 'LBH': [3, 4],
+ 'HBA': [5, 6, 7]
+}
class RTSMObservation(models.Model):
observation_id = models.PositiveIntegerField(default=0)
@@ -66,6 +71,15 @@ class RTSMErrorSummary(models.Model):
stop_frequency = models.FloatField(default=None, null=True)
+class RTSMSummaryPlot(models.Model):
+ error_summary = models.ForeignKey(RTSMErrorSummary,
+ related_name='summary_plot',
+ on_delete=models.SET_NULL,
+ null=True)
+
+ uri = models.CharField(max_length=10000, default=None, null=True)
+
+
class RTSMSpectrum(models.Model):
bad_spectrum = ArrayField(models.FloatField())
average_spectrum = ArrayField(models.FloatField())
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py
index 8cddf2b51b544d17f7ec1482e9f7c20ba0808ca4..23ab677706c69ea23db45af20f9689ee0616af70 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py
@@ -2,9 +2,11 @@ from ..models.rtsm import *
from rest_framework import serializers
import logging
from django.db.transaction import atomic
+from django.conf import settings
from ..exceptions import ItemAlreadyExists
from .log import ActionLogSerializer
from .station import Station
+import os
logger = logging.getLogger('serializers')
@@ -100,3 +102,24 @@ class RTSMObservationSerializer(serializers.ModelSerializer):
self.compute_summary(RTSMObservation_instance)
return RTSMObservation_instance
+
+
+class FilePathSerializer(serializers.Field):
+ def __init__(self, path, *args, **kwargs):
+ self.path = path
+ super(FilePathSerializer, self).__init__(*args, **kwargs)
+
+ def to_representation(self, value):
+ full_path = os.path.join(self.path, value)
+ return full_path
+
+ def to_internal_value(self, data):
+ file_name = os.path.relpath(self.path, data)
+ return file_name
+
+
+class RTSMSummaryPlotSerializer(serializers.ModelSerializer):
+ uri = FilePathSerializer(settings.URL_TO_STORE_RTSM_PLOTS)
+ class Meta:
+ model = RTSMSummaryPlot
+ fields = '__all__'
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/tasks.py b/LCU/Maintenance/DBInterface/monitoringdb/tasks.py
new file mode 100644
index 0000000000000000000000000000000000000000..b89b200c7145766bb1c59b9676c3540c35dc2346
--- /dev/null
+++ b/LCU/Maintenance/DBInterface/monitoringdb/tasks.py
@@ -0,0 +1,255 @@
+import logging
+import os
+
+import matplotlib
+
+matplotlib.use('agg')
+import matplotlib.pyplot as plt
+import numpy
+from celery import shared_task
+from django.conf import settings
+from .models.rtsm import MODE_TO_COMPONENT, RTSMErrorSummary, RTSMSummaryPlot
+
+logger = logging.getLogger(__name__)
+
+
+class Metadata:
+ def __init__(self, **kwargs):
+ for key, value in kwargs.items():
+ self.__setattr__(key, value)
+
+
+MODE_STR = {1: 'LBA Low 10..90 MHz',
+ 2: 'LBA Low 30..90',
+ 3: 'LBA High 10..90',
+ 4: 'LBA High 30..90',
+ 5: 'HBA 110..190',
+ 6: 'HBA 170..230',
+ 7: 'HBA 210..250'}
+
+
+class ObservationMetadata(Metadata):
+ station_name = 'CSXXXC'
+
+ start_datetime = '1212-12-12 12:12:12'
+ end_datetime = '1212-12-12 12:12:12'
+ observation_id = 'XXXXXX'
+ samples = 120
+
+ @staticmethod
+ def __format_datetime(datetime):
+ return datetime.strftime("%Y-%m-%d %H:%M:%S")
+
+ def set_start_time(self, start_time):
+ self.start_datetime = ObservationMetadata.__format_datetime(start_time)
+
+ def set_end_time(self, end_time):
+ self.end_datetime = ObservationMetadata.__format_datetime(end_time)
+
+
+def set_style():
+ plt.minorticks_on()
+ plt.ylim(55, 160)
+ plt.grid()
+ # plt.rcParams['svg.fonttype'] = 'none'
+ plt.rcParams['lines.linewidth'] = 1
+ plt.rcParams['figure.autolayout'] = True
+ plt.rcParams['axes.labelsize'] = 'large'
+ plt.rcParams['path.simplify'] = True
+ plt.rcParams['savefig.transparent'] = False
+ plt.rcParams['savefig.bbox'] = 'tight'
+
+ plt.rcParams['legend.loc'] = 'upper right'
+ plt.rcParams['legend.frameon'] = True
+ plt.rcParams['legend.framealpha'] = 1
+ plt.rcParams['legend.edgecolor'] = 'black'
+
+
+def rcu_mode_to_antenna(rcu, mode):
+ component_name = MODE_TO_COMPONENT[mode]
+
+ antenna = rcu // 2
+ if component_name in ['LBH', 'HBA']:
+ polarization = 'X' if rcu % 2 == 0 else 'Y'
+ elif component_name == 'LBL':
+ antenna = rcu // 2 + 48
+ polarization = 'Y' if rcu % 2 == 0 else 'X'
+ else:
+ raise ValueError('Unknown component %s' % component_name)
+
+ return dict(polarization=polarization, antenna=antenna)
+
+
+def render_summary_text(error_metadata, observation_metadata):
+ """
+ Render the summary text to include in the picture
+ :param error_metadata: the metadata information regarding the error
+ :param observation_metadata: the metadata information regarding the observation
+ :return:
+ """
+ rendered_text = 'Station Name: {:<25s}'.format(observation_metadata.station_name)
+ rendered_text += 'Faults : {}\n'.format(error_metadata.error_type)
+
+ rendered_text += 'ObsID : {:<25d}'.format(observation_metadata.observation_id)
+ rendered_text += 'Antenna : {antenna} {polarization}\n' \
+ .format(**rcu_mode_to_antenna(error_metadata.rcu, error_metadata.mode))
+
+ rendered_text += 'Start : {:<25s}'.format(observation_metadata.start_datetime)
+ rendered_text += 'RCU : {}\n'.format(error_metadata.rcu)
+
+ rendered_text += 'Stop : {:<25s}'.format(observation_metadata.end_datetime)
+ rendered_text += 'Samples : {count:d}/{samples:d}\n'.format(count=error_metadata.count,
+ samples=observation_metadata.samples)
+
+ rendered_text += 'Mode : {:<1d} {:<23s}'.format(error_metadata.mode,
+ MODE_STR[error_metadata.mode])
+ rendered_text += 'Badness : {:.2f} %\n'.format(
+ (100. * error_metadata.count) / observation_metadata.samples)
+ return rendered_text
+
+
+def produce_plot(observation_metadata,
+ error_metadata,
+ list_bad_spectra, list_good_specta, path):
+ """
+ Produces a plot given the observation metadata the error summary metadata and
+ the list of the bad spectra for the given (error, rcu) and the average spectrum
+ of the rest of the array
+ :param observation_metadata: the metadata information regarding the observation
+ :param error_metadata: the metadata information regarding the error
+ :param list_bad_spectra: a list containing the bad spectra for the given error, rcu couple
+ :param list_good_specta: the average spectrum of the rest of the array
+ :param path: the path where to store the file
+ """
+ plt.figure(figsize=(12, 9))
+ set_style()
+
+ plt.xlabel('MHz')
+ plt.ylabel('dB')
+
+ frequency_sampling = len(list_bad_spectra[0][0])
+ n_spectra = len(list_bad_spectra)
+ frequency = numpy.linspace(error_metadata.start_frequency,
+ error_metadata.stop_frequency,
+ frequency_sampling)
+ bad_spectra_cube = numpy.zeros((n_spectra, frequency_sampling))
+ average_spectra_cube = numpy.zeros((n_spectra, frequency_sampling))
+
+ for i, spectrum in enumerate(list_bad_spectra):
+ bad_spectra_cube[i, :] = spectrum[0]
+ for i, spectrum in enumerate(list_good_specta):
+ average_spectra_cube[i, :] = spectrum[0]
+
+ min_bad_spectrum = numpy.min(bad_spectra_cube, axis=0)
+ max_bad_spectrum = numpy.max(bad_spectra_cube, axis=0)
+ average_bad_spectrum = numpy.median(bad_spectra_cube, axis=0)
+
+ average_good_spectrum = numpy.median(average_spectra_cube, axis=0)
+
+ plt.fill_between(frequency, min_bad_spectrum, max_bad_spectrum, color='red',
+ alpha=.3)
+
+ plt.plot(frequency, average_bad_spectrum, color='red', label='median bad spectra')
+
+ plt.plot(frequency, average_good_spectrum, color='blue', label='median all spectra')
+
+ summary_text = render_summary_text(observation_metadata=observation_metadata,
+ error_metadata=error_metadata)
+
+ plt.text(0.02, .98, summary_text, bbox=dict(facecolor='yellow', alpha=0.2),
+ family='monospace', verticalalignment='top',
+ horizontalalignment='left',
+ transform=plt.axes().transAxes)
+ plt.legend()
+ plt.savefig(path)
+ plt.close()
+
+
+def generate_summary_plot_for_error(error_summary_id):
+ """
+ Given a error summary id generates the plot in the specific location given by the settings
+ parameter URL_TO_STORE_RTSM_PLOTS and the file name derived in the following fashion
+ %(observation_id)d_%(station_name)s_rcu%(rcu)d_%(error_type)s.png
+
+ Finally stores in the database the file name to directly access the file
+
+ :param error_summary_id: database id of the error summary
+ :return: the database id of the RTSMSummaryPlot instance
+ :raise Exception: raise if there is an exception
+ """
+ basePath = settings.URL_TO_STORE_RTSM_PLOTS
+
+ error_summary = RTSMErrorSummary.objects.get(pk=error_summary_id)
+ summary_plot = error_summary.summary_plot.first()
+ if summary_plot is not None:
+ if summary_plot.uri == None:
+ return 'another worker is taken care of the task...'
+
+ summary_plot = RTSMSummaryPlot(error_summary=error_summary, uri=None)
+ summary_plot.save()
+
+ try:
+ observation_metadata = ObservationMetadata()
+ observation = error_summary.observation
+ observation_metadata.observation_id = observation.observation_id
+ observation_metadata.station_name = observation.station.name
+ observation_metadata.set_start_time(observation.start_datetime)
+ observation_metadata.set_end_time(observation.end_datetime)
+
+ observation_metadata.samples = observation.samples
+ errors = observation.errors.filter(mode=error_summary.mode,
+ rcu=error_summary.rcu,
+ error_type=error_summary.error_type)
+
+ list_bad_spectra = errors.values_list('spectra__bad_spectrum')
+ list_good_spectra = errors.values_list('spectra__average_spectrum')
+ file_name = '%(observation_id)d_%(station_name)s_rcu%(rcu)d_%(error_type)s.png' % dict(
+ observation_id=observation_metadata.observation_id,
+ station_name=observation_metadata.station_name,
+ rcu=error_summary.rcu,
+ error_type=error_summary.error_type
+ )
+
+ full_path = os.path.join(basePath, file_name)
+ produce_plot(observation_metadata,
+ error_summary,
+ list_bad_spectra, list_good_spectra,
+ full_path)
+ summary_plot.uri = file_name
+ summary_plot.save()
+ return summary_plot.pk
+ except Exception as e:
+ logger.exception('exception %s occurred skipping...', e)
+ summary_plot.delete()
+ raise e
+
+
+@shared_task()
+def check_error_summary_plot(error_summary_id):
+ """
+ Checks if the error summary plot is presents otherwise it generates one
+ :param self: shared_task
+ :type self: celery.shared_task()
+ :param error_summary_id: database id of the RTSMErrorSummary to check
+ :return:
+ """
+ logger.debug('looking for rtsm error summary %s', error_summary_id)
+ error_summary = RTSMErrorSummary.objects.get(pk=error_summary_id)
+ summary_plot = error_summary.summary_plot.first()
+ logger.debug('summary error found %s', summary_plot)
+
+ if summary_plot is None:
+ return generate_summary_plot_for_error(error_summary_id)
+ else:
+ base_path = settings.URL_TO_STORE_RTSM_PLOTS
+ if summary_plot.uri is None:
+ return 'another worker is taking care of the task...'
+ full_path = os.path.join(base_path, summary_plot.uri)
+ if not os.path.exists(full_path):
+ return generate_summary_plot_for_error(error_summary_id)
+ elif os.path.isdir(full_path):
+ raise Exception('%s is a directory' % full_path)
+ else:
+ logger.debug('summary error %s is complete no need to generate additional plot',
+ summary_plot.pk)
+ return summary_plot.pk
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/urls.py b/LCU/Maintenance/DBInterface/monitoringdb/urls.py
index 31bd3275b091b1eefb3c17a1de49de95a23ee646..67ff6ed75212ca8b731bc90651ec984a16c3d8a7 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/urls.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/urls.py
@@ -5,6 +5,7 @@ from .views.station_test_views import *
from .views.rtsm_views import *
from .views.logs_view import *
from rest_framework.documentation import include_docs_urls
+from django.urls import path
log_router = routers.DefaultRouter()
log_router.register(r'action_log', ActionLogViewSet)
@@ -25,6 +26,7 @@ rtsm_router = routers.DefaultRouter()
# RTSM
rtsm_router.register(r'errors', RTSMErrorsViewSet)
rtsm_router.register(r'spectra', RTSMSpectrumViewSet)
+rtsm_router.register(r'error_summary_plot', RTSMSummaryPlot, base_name='rtsm-summary-plot')
rtsm_router.register(r'', RTSMObservationViewSet)
@@ -43,6 +45,6 @@ urlpatterns = [
url(r'^api/view/ctrl_stationtest_statistics', ControllerStationTestStatistics.as_view()),
url(r'^api/view/ctrl_list_component_error_types', ControllerAllComponentErrorTypes.as_view()),
url(r'^api/view/ctrl_station_component_errors', ControllerStationComponentErrors.as_view()),
-
+ url(r'^api/view/ctrl_station_component_element_errors', ControllerStationComponentElementErrors.as_view()),
url(r'^api/docs', include_docs_urls(title='Monitoring DB API'))
]
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/views/common.py b/LCU/Maintenance/DBInterface/monitoringdb/views/common.py
index 899d1642fcfe70895ed02d1ab89021424a9334d8..db345cc545f47e8eb44947f03a288b88861b6862 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/views/common.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/views/common.py
@@ -1,8 +1,10 @@
from rest_framework import viewsets, status
from rest_framework.response import Response
+from django.http import HttpResponse
from rest_framework.decorators import api_view
import logging
from ..exceptions import ItemAlreadyExists
+from django.core.exceptions import ObjectDoesNotExist
RESERVED_FILTER_NAME = ['limit', 'offset', 'format', 'page_size']
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py b/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py
index dc4b0bd734fac16b8fcf1a00e566c7b1e89fe42a..65f5eb839e7419e1c5a753221adf3ef6ac9cf937 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py
@@ -7,18 +7,19 @@ import coreapi
import coreschema
from django.db.models import Count
from lofar.maintenance.monitoringdb.models.component_error import ComponentError
-from lofar.maintenance.monitoringdb.models.rtsm import RTSMErrorSummary, MODE_TO_COMPONENT
+from lofar.maintenance.monitoringdb.models.rtsm import RTSMErrorSummary, MODE_TO_COMPONENT, COMPONENT_TO_MODE
from lofar.maintenance.monitoringdb.models.rtsm import RTSMObservation
from lofar.maintenance.monitoringdb.models.station import Station
from lofar.maintenance.monitoringdb.models.station_test import StationTest
from rest_framework import status
+from rest_framework.reverse import reverse
from rest_framework.response import Response
from rest_framework.schemas import ManualSchema
from rest_framework.views import APIView
import pytz
-logger = logging.getLogger(__name__)
+logger = logging.getLogger(__name__)
def parse_date(date):
expected_format = '%Y-%m-%d'
@@ -92,9 +93,12 @@ class ValidableReadOnlyView(APIView):
self.__setattr__(field.name, value)
errors = field.schema.validate(self.__getattribute__(field.name))
for error in errors:
- raise ValueError(error.text)
+ raise ValueError(" ".join([field.name, error.text]))
def get(self, request):
+ # Store the request as attribute
+ self.request = request
+
try:
self.validate_query_parameters(request)
except ValueError as e:
@@ -469,6 +473,9 @@ class ControllerStationTestStatistics(ValidableReadOnlyView):
station_group = 'A'
test_type = 'B'
error_types = []
+ from_date = None
+ to_date = None
+ averaging_interval = None
fields = [
coreapi.Field(
@@ -728,7 +735,9 @@ class ControllerStationComponentErrors(ValidableReadOnlyView):
.filter(start_datetime__gte=self.from_date,
end_datetime__lte=self.to_date)
- failing_component_types = station_tests.distinct('component_errors__component__type').exclude(component_errors__component__type__isnull=True).values_list('component_errors__component__type')
+ failing_component_types = station_tests.distinct('component_errors__component__type').\
+ exclude(component_errors__component__type__isnull=True).\
+ values_list('component_errors__component__type')
for failing_component_type in failing_component_types:
failing_component_type = failing_component_type[0]
@@ -761,69 +770,6 @@ class ControllerStationComponentErrors(ValidableReadOnlyView):
return response_payload
- def compose_image_storage_url(self,station_name, observation_id, start_date, rcu, component_type, error_type, both_polarization):
- """
- WARNING -----------------------------------------------------------
- This is an ugly function that has to be removed as soon as possible
- and it is meant to provide a url to the RTSM plots to have it displayed
- in the webview. A way to proper implement this part is still missing
- hence this function.
- DONT BLAME THE PLAYER BLAME THE GAME!
- -------------------------------------------------------------------
- :param observation_id: id of the observation (SASID)
- :param start_date: start date of the observation
- :param rcu: rcu number
- :param error_type: type of error
- :return:
- """
- # the sample link address looks like
- # https://proxy.lofar.eu/rtsm/obs_plots/20181108_1542_683264/683264_SE607C_rcu175_sn.png
- baseURL = settings.URL_TO_RTSM_PLOTS
-
- TO_SHORT_ERROR_TYPE = dict(SUMMATOR_NOISE='sn',
- HIGH_NOISE='hn',
- LOW_NOISE='ln',
- FLAT='flat',
- SHORT='short',
- OSCILLATION='osc',
- DOWN='down'
- )
-
- if component_type in ['LBL', 'LBH'] and both_polarization:
- TO_SHORT_ERROR_TYPE.pop('FLAT')
- type = 'ant'
- component_id = rcu // 2
- if component_type == 'LBL':
- component_id += 48
- else:
- type = 'rcu'
- component_id = rcu
-
- # If the error is not in the dict above there is not such plot on disk.
- # Hence, returns 'url not present'
- try:
- short_error_type = TO_SHORT_ERROR_TYPE[error_type]
- except KeyError:
- return 'url not present'
-
- imageURL = '%(baseURL)s/%(start_date)s_%(observation_id)s/%(observation_id)s_%(station_name)s_%(type)s%(component_id)s_%(error_type)s.png' % dict(
- baseURL=baseURL,
- observation_id=observation_id,
- start_date=start_date.strftime('%Y%m%d_%H%M'),
- station_name=station_name,
- component_id=component_id,
- type=type,
- error_type=short_error_type
- )
- return imageURL
-
- def find_other_polarization_given_rcu_number(self, rcu_number):
- # if it is even
- if rcu_number % 2 == 0:
- return rcu_number + 1
- else:
- return rcu_number - 1
-
def collect_rtsm_errors(self):
station_entry = Station.objects.filter(name=self.station_name).first()
response_payload = OrderedDict()
@@ -839,15 +785,13 @@ class ControllerStationComponentErrors(ValidableReadOnlyView):
rtsm_errors_per_component_type = list()
- response_payload[MODE_TO_COMPONENT[observing_mode]] = rtsm_errors_per_component_type
-
for rtsm_observation in rtsm_observations.order_by('-start_datetime'):
rtsm_summary = OrderedDict()
- rtsm_errors_per_component_type.append(rtsm_summary)
rtsm_summary['test_type'] = 'R'
rtsm_summary['start_date'] = rtsm_observation.start_datetime
rtsm_summary['end_date'] = rtsm_observation.end_datetime
+ rtsm_summary['observation_id'] = rtsm_observation.observation_id
component_errors_dict = OrderedDict()
rtsm_summary['component_errors'] = component_errors_dict
@@ -855,11 +799,17 @@ class ControllerStationComponentErrors(ValidableReadOnlyView):
.filter(mode=observing_mode)\
.values('error_type', 'start_frequency',
'stop_frequency', 'percentage',
- 'error_type', 'count', 'rcu')
+ 'error_type', 'count', 'rcu', 'pk')
+ if component_errors.count() == 0 and rtsm_observation.errors_summary.count() !=0:
+ continue
+ else:
+ rtsm_errors_per_component_type.append(rtsm_summary)
if self.error_types:
component_errors = component_errors.filter(error_type__in=self.error_types)
+
for component_error in component_errors:
+
component_id = component_error['rcu']
details = dict(percentage = component_error['percentage'],
start_frequency = component_error['start_frequency'],
@@ -867,27 +817,25 @@ class ControllerStationComponentErrors(ValidableReadOnlyView):
count = component_error['count'])
error_type = component_error['error_type']
# CHECKS IF THE ERROR IS PRESENT IN BOTH RCUS (hence, both polarizations of the antenna)
- both_polarization = len(component_errors.filter(rcu=self.find_other_polarization_given_rcu_number(component_id),
- error_type=error_type)) == 1
-
- details['url'] = self.compose_image_storage_url(self.station_name,
- rtsm_observation.observation_id,
- rtsm_observation.start_datetime,
- component_id,
- MODE_TO_COMPONENT[observing_mode],
- error_type,
- both_polarization)
+ url_to_plot = reverse('rtsm-summary-plot-detail', (component_error['pk'],), request=self.request)
+ details['url'] = url_to_plot
if component_id not in component_errors:
component_errors_dict[str(component_id)] = list()
component_errors_dict[str(component_id)] += [dict(error_type=error_type, details=details)]
- #self.decorate_component_errors_with_url(component_errors_dict, rtsm_observation.observation_id, rtsm_observation.start_datetime, MODE_TO_COMPONENT[observing_mode])
+ component_name = MODE_TO_COMPONENT[observing_mode]
+ if component_name not in response_payload:
+ response_payload[component_name] = rtsm_errors_per_component_type
+ else:
+ response_payload[component_name] += rtsm_errors_per_component_type
+
return response_payload
def compute_response(self):
self.from_date = parse_date(self.from_date)
self.to_date = parse_date(self.to_date)
+
station_test_errors = {}
rtsm_errors = {}
@@ -904,4 +852,241 @@ class ControllerStationComponentErrors(ValidableReadOnlyView):
payload[component_type] = sorted(station_test_errors_per_type + rtsm_errors_per_type, key=lambda item: item['start_date'], reverse=True)
- return Response(status=status.HTTP_200_OK, data=payload)
\ No newline at end of file
+ return Response(status=status.HTTP_200_OK, data=payload)
+
+
+class ControllerStationComponentElementErrors(ValidableReadOnlyView):
+ station_name= None # required
+ from_date= None # required
+ to_date= None # required
+ component_type= None # required
+ antenna_id= None # required
+ test_type= "A"
+
+ fields = [
+ coreapi.Field(
+ 'station_name',
+ required=True,
+ location='query',
+ schema=coreschema.String(description='name of the station to select')
+ ),
+ coreapi.Field(
+ 'from_date',
+ required=True,
+ location='query',
+ schema=coreschema.String(description='select tests from date (ex. YYYY-MM-DD)')
+ ),
+ coreapi.Field(
+ 'to_date',
+ required=True,
+ location='query',
+ schema=coreschema.String(description='select tests from date (ex. YYYY-MM-DD)')
+ ),
+ coreapi.Field(
+ 'component_type',
+ required=True,
+ location='query',
+ schema=coreschema.Enum(
+ ['HBA', 'LBH', 'LBL'],
+ description='select the antenna type. Possible values are (HBA, LBH, LBL)'
+ )
+ ),
+ coreapi.Field(
+ 'antenna_id',
+ required=True,
+ location='query',
+ type=int,
+ schema=coreschema.Integer(description='Select the antenna id')
+ ),
+ coreapi.Field(
+ 'test_type',
+ required=False,
+ location='query',
+ schema=coreschema.Enum(
+ ['R', 'S', 'A'],
+ description='select the type of test possible values are (R, RTSM),'
+ ' (S, Station test), (A, all)[DEFAULT=A]',
+ )
+ )
+ ]
+
+ def rcu_from_antenna_type_polarization(self, antenna_id, type, polarization):
+ """
+ Compute the rcu number for a given antenna number, type and polarization
+ :param antenna_id: id of the antenna
+ :param type: type of the antenna
+ :param polarization: polarization either [X, Y]
+ :return: the rcu id
+ :rtype: int
+ """
+ if polarization not in ['X', 'Y']:
+ raise ValueError('Polarization has to be either X or Y: %s not recognized' % polarization)
+
+ if type == 'LBH':
+ rcu_id = antenna_id * 2
+ rcu_id += 0 if polarization == 'X' else 1
+ elif type == 'LBL':
+ rcu_id = (antenna_id - 48) * 2
+ rcu_id += 1 if polarization == 'X' else 0
+ elif type == 'HBA':
+ rcu_id = antenna_id * 2
+ rcu_id += 0 if polarization == 'X' else 1
+ else:
+ rcu_id = -1
+ return rcu_id
+
+ def rcus_from_antenna_and_type(self, antenna_id, type):
+ rcu_x = self.rcu_from_antenna_type_polarization(antenna_id, type, 'X')
+ rcu_y = self.rcu_from_antenna_type_polarization(antenna_id, type, 'Y')
+
+ rcus={rcu_x:'X', rcu_y:'Y'}
+ return rcus
+
+ def compute_ok_rtsm_list(self):
+ mode=COMPONENT_TO_MODE[self.component_type]
+
+ rcus_per_polarization = self.rcus_from_antenna_and_type(self.antenna_id,
+ self.component_type)
+
+ good_observation_list = RTSMObservation.objects.filter(errors_summary__mode__in=mode,
+ start_datetime__gt=self.from_date,
+ end_datetime__lt=self.to_date,
+ station__name=self.station_name).\
+ exclude(errors_summary__rcu__in=list(rcus_per_polarization.keys())).\
+ order_by('-start_datetime').values('pk',
+ 'observation_id',
+ 'start_datetime',
+ 'end_datetime')
+ result = []
+ for observation in good_observation_list:
+ entry = dict(test_id=observation['observation_id'],
+ db_id=observation['pk'],
+ start_date=observation['start_datetime'],
+ end_date=observation['end_datetime'],
+ test_type='R',
+ component_errors=dict())
+ result.append(entry)
+ return result
+
+ def compute_ok_station_test(self):
+ good_station_test = StationTest.objects.filter(start_datetime__gt=self.from_date,
+ end_datetime__lt=self.to_date,
+ station__name=self.station_name).\
+ exclude(component_errors__component__component_id=self.antenna_id).\
+ values('pk', 'start_datetime', 'end_datetime').order_by('-start_datetime')
+ result = []
+ for station_test in good_station_test:
+ entry = dict(test_id=station_test['pk'],
+ db_id=station_test['pk'],
+ start_date=station_test['start_datetime'],
+ end_date=station_test['end_datetime'],
+ test_type='S',
+ component_errors=dict())
+ result.append(entry)
+ return result
+
+ def compute_rtsm_errors_list(self):
+ errors = dict()
+ rcus_per_polarization = self.rcus_from_antenna_and_type(self.antenna_id,
+ self.component_type)
+
+ rtsm_errors = RTSMErrorSummary.objects.values('observation__pk',
+ 'observation__start_datetime',
+ 'observation__end_datetime',
+ 'observation__observation_id',
+ 'observation__station__name',
+ 'pk',
+ 'rcu',
+ 'mode',
+ 'error_type',
+ 'percentage',
+ 'count',
+ 'observation__samples').filter(
+ observation__start_datetime__gt=self.from_date,
+ observation__end_datetime__lt=self.to_date,
+ observation__station__name=self.station_name,
+ mode__in=COMPONENT_TO_MODE[self.component_type],
+ rcu__in=list(rcus_per_polarization.keys())).order_by('-observation__start_datetime')
+ for item in rtsm_errors:
+ observation_pk = item['observation__pk']
+ if observation_pk not in errors.keys():
+ errors[observation_pk] = dict(test_id=item['observation__observation_id'],
+ db_id=item['observation__pk'],
+ start_date=item[
+ 'observation__start_datetime'],
+ end_date=item[
+ 'observation__end_datetime'],
+ test_type='R',
+ component_errors=dict())
+ rcu = item['rcu']
+ polarization = rcus_per_polarization[rcu]
+ if polarization not in errors[observation_pk]:
+ errors[observation_pk]['component_errors'][polarization] = dict(
+ rcu=rcu,
+ errors = dict()
+ )
+ error_type = item['error_type']
+ percentage = item['percentage']
+ count = item['count']
+ mode = item['mode']
+ samples = item['observation__samples']
+
+ url_to_plot = reverse('rtsm-summary-plot-detail', (item['pk'],), request=self.request)
+ errors[observation_pk]['component_errors'][polarization]['errors'][error_type] = dict(samples=samples,
+ percentage=percentage,
+ count=count,
+ mode=mode,
+ url=url_to_plot)
+ return list(errors.values())
+
+ def compute_station_tests_error_list(self):
+ errors = dict()
+ rcus_per_polarization = self.rcus_from_antenna_and_type(self.antenna_id,
+ self.component_type)
+ component_errors = ComponentError.objects.filter(
+ station_test__start_datetime__gt=self.from_date,
+ station_test__end_datetime__lt=self.to_date,
+ station_test__station__name=self.station_name,
+ component__type=self.component_type,
+ component__component_id=self.antenna_id).order_by('-station_test__start_datetime')
+ for component_error in component_errors:
+ station_test_pk = component_error.station_test.pk
+ if station_test_pk not in errors.keys():
+ errors[station_test_pk] = dict(test_id=station_test_pk,
+ db_id=station_test_pk,
+ start_date=component_error.station_test.start_datetime,
+ end_date=component_error.station_test.end_datetime,
+ test_type='S',
+ component_errors=dict())
+ for rcu_id, polarization in rcus_per_polarization.items():
+ if polarization not in errors[station_test_pk]:
+ errors[station_test_pk]['component_errors'][polarization] = dict(
+ rcu=rcu_id,
+ errors=dict())
+ error_type = component_error.type
+ details = component_error.details
+ errors_per_error_polarization = errors[station_test_pk]['component_errors'][polarization]['errors']
+ errors_per_error_polarization[error_type] = dict(details=details, element_errors=dict())
+
+
+ for element in component_error.failing_elements.values('element__element_id', 'details'):
+ element_id = element['element__element_id']
+ errors_per_error_polarization[error_type]['element_errors'][element_id] = element['details']
+
+ return list(errors.values())
+
+ def compute_response(self):
+ self.from_date = parse_date(self.from_date)
+ self.to_date = parse_date(self.to_date)
+ rtsm_errors_list = []
+ station_test_list = []
+
+ if self.test_type == 'R' or self.test_type == 'A':
+ rtsm_errors_list = self.compute_rtsm_errors_list() + self.compute_ok_rtsm_list()
+
+ if self.test_type == 'S' or self.test_type == 'A':
+ station_test_list = self.compute_station_tests_error_list() + self.compute_ok_station_test()
+
+ combined = rtsm_errors_list + station_test_list
+ combined_and_sorted = sorted(combined, key=lambda test: test['start_date'], reverse=True)
+ return Response(status=status.HTTP_200_OK, data=dict(errors=combined_and_sorted))
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py b/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py
index 5e0a2560ff7ca59bd3bd6d5f677de7789d38b363..8591075bc630b610160e1976ff36340ce554bc01 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py
@@ -1,8 +1,12 @@
from .common import *
from ..models.rtsm import RTSMObservation, RTSMError, RTSMSpectrum
from ..serializers.rtsm import RTSMObservationSerializer, RTSMErrorSerializer, \
- RTSMSpectrumSerializer
+ RTSMSpectrumSerializer, RTSMSummaryPlotSerializer, RTSMErrorSummary
from ..rtsm_test_raw_parser import parse_rtsm_test
+from django.shortcuts import get_object_or_404
+from django.http import Http404, HttpResponseServerError
+import os
+from ..tasks import check_error_summary_plot
logger = logging.getLogger('views')
@@ -25,6 +29,36 @@ class RTSMObservationViewSet(viewsets.ReadOnlyModelViewSet):
filter_fields = '__all__'
+class RTSMSummaryPlot(viewsets.ViewSet):
+ """
+ Get the summary plot associated to the error summary given
+ """
+ queryset = RTSMErrorSummary.objects.all()
+
+ def retrieve(self, request, pk=None):
+ try:
+ entity = get_object_or_404(self.queryset, pk=pk)
+ summary_plot = entity.summary_plot.first()
+
+ if summary_plot is None:
+ raise ObjectDoesNotExist()
+
+ uri = RTSMSummaryPlotSerializer(summary_plot).data['uri']
+
+ except ObjectDoesNotExist as e:
+ check_error_summary_plot.delay(pk)
+ raise Http404()
+
+ if uri and os.path.exists(uri) and os.path.isfile(uri):
+ with open(uri, 'rb') as f_stream:
+ image = f_stream.read()
+ return HttpResponse(image, status=status.HTTP_200_OK, content_type='image/gif')
+ else:
+ check_error_summary_plot.delay(pk)
+ print(uri)
+ raise Http404()
+
+
@api_view(['POST'])
def insert_raw_rtsm_test(request):
"""