From 21f2e147576d5e57d0542ed67811558dfa995812 Mon Sep 17 00:00:00 2001
From: Mattia Mancini <mancini@astron.nl>
Date: Thu, 22 Nov 2018 14:27:40 +0000
Subject: [PATCH] OSB-34: minor fixes and implementation of the plot generation
once a plot is requested.
I made sure that if a time span contains observation with different mode but with modes regarding the same
component the results are still returned for both. Additionally, a certain mode may not be for a long period.
So givin a mode a, it can happen that a certain component in the RTSM is not tested and result as green. This
a long standing issue. To solve this issue the RTSM should provide the mode in which the observation is perfomed. However, to mitigate the issue I am now excluding all the observations that do not have errors in a specific mode but have errors in another mode. This can still include spurious errors if such observation does not presents any RTSM error! ('luckly' it is very rare).
---
.../monitoringdb/views/controllers.py | 302 ++++++++++++------
1 file changed, 208 insertions(+), 94 deletions(-)
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py b/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py
index 47a2fa6440a..90afd381296 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py
@@ -7,18 +7,19 @@ import coreapi
import coreschema
from django.db.models import Count
from lofar.maintenance.monitoringdb.models.component_error import ComponentError
-from lofar.maintenance.monitoringdb.models.rtsm import RTSMErrorSummary, MODE_TO_COMPONENT
+from lofar.maintenance.monitoringdb.models.rtsm import RTSMErrorSummary, MODE_TO_COMPONENT, COMPONENT_TO_MODE
from lofar.maintenance.monitoringdb.models.rtsm import RTSMObservation
from lofar.maintenance.monitoringdb.models.station import Station
from lofar.maintenance.monitoringdb.models.station_test import StationTest
from rest_framework import status
+from rest_framework.reverse import reverse
from rest_framework.response import Response
from rest_framework.schemas import ManualSchema
from rest_framework.views import APIView
import pytz
-logger = logging.getLogger(__name__)
+logger = logging.getLogger(__name__)
def parse_date(date):
expected_format = '%Y-%m-%d'
@@ -92,9 +93,12 @@ class ValidableReadOnlyView(APIView):
self.__setattr__(field.name, value)
errors = field.schema.validate(self.__getattribute__(field.name))
for error in errors:
- raise ValueError(error.text)
+ raise ValueError(" ".join([field.name, error.text]))
def get(self, request):
+ # Store the request as attribute
+ self.request = request
+
try:
self.validate_query_parameters(request)
except ValueError as e:
@@ -761,69 +765,6 @@ class ControllerStationComponentErrors(ValidableReadOnlyView):
return response_payload
- def compose_image_storage_url(self,station_name, observation_id, start_date, rcu, component_type, error_type, both_polarization):
- """
- WARNING -----------------------------------------------------------
- This is an ugly function that has to be removed as soon as possible
- and it is meant to provide a url to the RTSM plots to have it displayed
- in the webview. A way to proper implement this part is still missing
- hence this function.
- DONT BLAME THE PLAYER BLAME THE GAME!
- -------------------------------------------------------------------
- :param observation_id: id of the observation (SASID)
- :param start_date: start date of the observation
- :param rcu: rcu number
- :param error_type: type of error
- :return:
- """
- # the sample link address looks like
- # https://proxy.lofar.eu/rtsm/obs_plots/20181108_1542_683264/683264_SE607C_rcu175_sn.png
- baseURL = settings.URL_TO_RTSM_PLOTS
-
- TO_SHORT_ERROR_TYPE = dict(SUMMATOR_NOISE='sn',
- HIGH_NOISE='hn',
- LOW_NOISE='ln',
- FLAT='flat',
- SHORT='short',
- OSCILLATION='osc',
- DOWN='down'
- )
-
- if component_type in ['LBL', 'LBH'] and both_polarization:
- TO_SHORT_ERROR_TYPE.pop('FLAT')
- type = 'ant'
- component_id = rcu // 2
- if component_type == 'LBL':
- component_id += 48
- else:
- type = 'rcu'
- component_id = rcu
-
- # If the error is not in the dict above there is not such plot on disk.
- # Hence, returns 'url not present'
- try:
- short_error_type = TO_SHORT_ERROR_TYPE[error_type]
- except KeyError:
- return 'url not present'
-
- imageURL = '%(baseURL)s/%(start_date)s_%(observation_id)s/%(observation_id)s_%(station_name)s_%(type)s%(component_id)s_%(error_type)s.png' % dict(
- baseURL=baseURL,
- observation_id=observation_id,
- start_date=start_date.strftime('%Y%m%d_%H%M'),
- station_name=station_name,
- component_id=component_id,
- type=type,
- error_type=short_error_type
- )
- return imageURL
-
- def find_other_polarization_given_rcu_number(self, rcu_number):
- # if it is even
- if rcu_number % 2 == 0:
- return rcu_number + 1
- else:
- return rcu_number - 1
-
def collect_rtsm_errors(self):
station_entry = Station.objects.filter(name=self.station_name).first()
response_payload = OrderedDict()
@@ -839,15 +780,13 @@ class ControllerStationComponentErrors(ValidableReadOnlyView):
rtsm_errors_per_component_type = list()
- response_payload[MODE_TO_COMPONENT[observing_mode]] = rtsm_errors_per_component_type
-
for rtsm_observation in rtsm_observations.order_by('-start_datetime'):
rtsm_summary = OrderedDict()
- rtsm_errors_per_component_type.append(rtsm_summary)
rtsm_summary['test_type'] = 'R'
rtsm_summary['start_date'] = rtsm_observation.start_datetime
rtsm_summary['end_date'] = rtsm_observation.end_datetime
+ rtsm_summary['observation_id'] = rtsm_observation.observation_id
component_errors_dict = OrderedDict()
rtsm_summary['component_errors'] = component_errors_dict
@@ -855,11 +794,17 @@ class ControllerStationComponentErrors(ValidableReadOnlyView):
.filter(mode=observing_mode)\
.values('error_type', 'start_frequency',
'stop_frequency', 'percentage',
- 'error_type', 'count', 'rcu')
+ 'error_type', 'count', 'rcu', 'pk')
+ if component_errors.count() == 0 and rtsm_observation.errors_summary.count() !=0:
+ continue
+ else:
+ rtsm_errors_per_component_type.append(rtsm_summary)
if self.error_types:
component_errors = component_errors.filter(error_type__in=self.error_types)
+
for component_error in component_errors:
+
component_id = component_error['rcu']
details = dict(percentage = component_error['percentage'],
start_frequency = component_error['start_frequency'],
@@ -867,27 +812,25 @@ class ControllerStationComponentErrors(ValidableReadOnlyView):
count = component_error['count'])
error_type = component_error['error_type']
# CHECKS IF THE ERROR IS PRESENT IN BOTH RCUS (hence, both polarizations of the antenna)
- both_polarization = len(component_errors.filter(rcu=self.find_other_polarization_given_rcu_number(component_id),
- error_type=error_type)) == 1
-
- details['url'] = self.compose_image_storage_url(self.station_name,
- rtsm_observation.observation_id,
- rtsm_observation.start_datetime,
- component_id,
- MODE_TO_COMPONENT[observing_mode],
- error_type,
- both_polarization)
+ url_to_plot = reverse('rtsm-summary-plot', (component_error['pk'],), request=self.request)
+ details['url'] = url_to_plot
if component_id not in component_errors:
component_errors_dict[str(component_id)] = list()
component_errors_dict[str(component_id)] += [dict(error_type=error_type, details=details)]
- #self.decorate_component_errors_with_url(component_errors_dict, rtsm_observation.observation_id, rtsm_observation.start_datetime, MODE_TO_COMPONENT[observing_mode])
+ component_name = MODE_TO_COMPONENT[observing_mode]
+ if component_name not in response_payload:
+ response_payload[component_name] = rtsm_errors_per_component_type
+ else:
+ response_payload[component_name] += rtsm_errors_per_component_type
+
return response_payload
def compute_response(self):
self.from_date = parse_date(self.from_date)
self.to_date = parse_date(self.to_date)
+
station_test_errors = {}
rtsm_errors = {}
@@ -906,26 +849,197 @@ class ControllerStationComponentErrors(ValidableReadOnlyView):
return Response(status=status.HTTP_200_OK, data=payload)
-from ..tasks import greetings, check_observation_plots
-class ControllerTestCeleryQueue(ValidableReadOnlyView):
- observation_database_id = None
+class ControllerStationComponentElementErrors(ValidableReadOnlyView):
+ station_name= None # required
+ from_date= None # required
+ to_date= None # required
+ component_type= None # required
+ antenna_id= None # required
+ test_type= "A"
+
fields = [
coreapi.Field(
- 'observation_database_id',
+ 'station_name',
+ required=True,
+ location='query',
+ schema=coreschema.String(description='name of the station to select')
+ ),
+ coreapi.Field(
+ 'from_date',
required=True,
+ location='query',
+ schema=coreschema.String(description='select tests from date (ex. YYYY-MM-DD)')
+ ),
+ coreapi.Field(
+ 'to_date',
+ required=True,
+ location='query',
+ schema=coreschema.String(description='select tests from date (ex. YYYY-MM-DD)')
+ ),
+ coreapi.Field(
+ 'component_type',
+ required=True,
+ location='query',
+ schema=coreschema.Enum(
+ ['HBA', 'LBH', 'LBL'],
+ description='select the antenna type. Possible values are (HBA, LBH, LBL)'
+ )
+ ),
+ coreapi.Field(
+ 'antenna_id',
+ required=True,
+ location='query',
type=int,
+ schema=coreschema.Integer(description='Select the antenna id')
+ ),
+ coreapi.Field(
+ 'test_type',
+ required=False,
location='query',
+ schema=coreschema.Enum(
+ ['R', 'S', 'A'],
+ description='select the type of test possible values are (R, RTSM),'
+ ' (S, Station test), (A, all)[DEFAULT=A]',
+ )
+ )
+ ]
- schema=coreschema.Integer(description='observation_id')
- )]
- def compute_response(self):
- from celery.result import AsyncResult
- task_id = 'print_last_observation_%s' % self.observation_database_id
+ def rcu_from_antenna_type_polarization(self, antenna_id, type, polarization):
+ """
+ Compute the rcu number for a given antenna number, type and polarization
+ :param antenna_id: id of the antenna
+ :param type: type of the antenna
+ :param polarization: polarization either [X, Y]
+ :return: the rcu id
+ :rtype: int
+ """
+ if polarization not in ['X', 'Y']:
+ raise ValueError('Polarization has to be either X or Y: %s not recognized' % polarization)
+
+ if type == 'LBH':
+ rcu_id = antenna_id * 2
+ rcu_id += 1 if polarization == 'X' else 0
+ elif type == 'LBL':
+ rcu_id = (antenna_id - 48) * 2
+ rcu_id += 0 if polarization == 'X' else 1
+ elif type == 'HBA':
+ rcu_id = antenna_id
+ rcu_id += 1 if polarization == 'X' else 0
+ else:
+ rcu_id = -1
+ return rcu_id
+
+ def rcus_from_antenna_and_type(self, antenna_id, type):
+ rcu_x = self.rcu_from_antenna_type_polarization(antenna_id, type, 'X')
+ rcu_y = self.rcu_from_antenna_type_polarization(antenna_id, type, 'Y')
+
+ rcus={rcu_x:'X', rcu_y:'Y'}
+ return rcus
+
+ def compute_rtsm_errors_list(self):
+ errors = dict()
+ rcus_per_polarization = self.rcus_from_antenna_and_type(self.antenna_id,
+ self.component_type)
+
+ rtsm_errors = RTSMErrorSummary.objects.values('observation__pk',
+ 'observation__start_datetime',
+ 'observation__end_datetime',
+ 'observation__observation_id',
+ 'observation__station__name',
+ 'pk',
+ 'rcu',
+ 'mode',
+ 'error_type',
+ 'percentage',
+ 'count',
+ 'observation__samples').filter(
+ observation__start_datetime__gt=self.from_date,
+ observation__end_datetime__lt=self.to_date,
+ observation__station__name=self.station_name,
+ mode__in=COMPONENT_TO_MODE[self.component_type],
+ rcu__in=list(rcus_per_polarization.keys())).order_by('-observation__start_datetime')
+ for item in rtsm_errors:
+ observation_pk = item['observation__pk']
+ if observation_pk not in errors.keys():
+ errors[observation_pk] = dict(test_id=item['observation__observation_id'],
+ db_id=item['observation__pk'],
+ start_date=item[
+ 'observation__start_datetime'],
+ end_date=item[
+ 'observation__end_datetime'],
+ test_type='R',
+ component_errors=dict())
+ rcu = item['rcu']
+ polarization = rcus_per_polarization[rcu]
+ if polarization not in errors[observation_pk]:
+ errors[observation_pk]['component_errors'][polarization] = dict(
+ rcu=rcu,
+ errors = dict(),
+ element_errors = dict()
+ )
+ error_type = item['error_type']
+ percentage = item['percentage']
+ count = item['count']
+ mode = item['mode']
+ samples = item['observation__samples']
+
+ url_to_plot = reverse('rtsm-summary-plot', (item['pk'],), request=self.request)
+ errors[observation_pk]['component_errors'][polarization]['errors'][error_type] = dict(samples=samples,
+ percentage=percentage,
+ count=count,
+ mode=mode,
+ url=url_to_plot)
+ return list(errors.values())
+
+ def compute_station_tests_error_list(self):
+ errors = dict()
+ rcus_per_polarization = self.rcus_from_antenna_and_type(self.antenna_id,
+ self.component_type)
+ component_errors = ComponentError.objects.filter(
+ station_test__start_datetime__gt=self.from_date,
+ station_test__end_datetime__lt=self.to_date,
+ station_test__station__name=self.station_name,
+ component__type=self.component_type,
+ component__component_id__in=list(rcus_per_polarization.keys())).order_by('-station_test__start_datetime')
+ for component_error in component_errors:
+ station_test_pk = component_error.station_test.pk
+ if station_test_pk not in errors.keys():
+ errors[station_test_pk] = dict(test_id=station_test_pk,
+ db_id=station_test_pk,
+ start_date=component_error.station_test.start_datetime,
+ end_date=component_error.station_test.end_datetime,
+ test_type='S',
+ component_errors=dict())
+ component_id = component_error.component.component_id
+ polarization = rcus_per_polarization[component_id]
+ if polarization not in errors[station_test_pk]:
+ errors[station_test_pk]['component_errors'][polarization] = dict(
+ rcu=component_id,
+ errors=dict())
+ error_type = component_error.type
+ details = component_error.details
+ errors_per_error_polarization = errors[station_test_pk]['component_errors'][polarization]['errors']
+ errors_per_error_polarization[error_type] = dict(details=details, element_errors=dict())
+
+
+ for element in component_error.failing_elements.values('element__element_id', 'details'):
+ element_id = element['element__element_id']
+ errors_per_error_polarization[error_type]['element_errors'][element_id] = element['details']
+
+ return list(errors.values())
- stuff = check_observation_plots.apply_async((self.observation_database_id,))
- stuff = check_observation_plots.apply_async((self.observation_database_id,))
+ def compute_response(self):
+ self.from_date = parse_date(self.from_date)
+ self.to_date = parse_date(self.to_date)
+ rtsm_errors_list = []
+ station_test_list = []
- #print(aresult.get(timeout=10))
+ if self.test_type == 'R' or self.test_type == 'A':
+ rtsm_errors_list = self.compute_rtsm_errors_list()
+ if self.test_type == 'S' or self.test_type == 'A':
+ station_test_list = self.compute_station_tests_error_list()
- return Response(status=status.HTTP_200_OK, data='command sent: %s' % stuff)
\ No newline at end of file
+ combined = rtsm_errors_list + station_test_list
+ combined_and_sorted = sorted(combined, key=lambda test: test['start_date'], reverse=True)
+ return Response(status=status.HTTP_200_OK, data=dict(errors=combined_and_sorted))
--
GitLab