diff --git a/.gitattributes b/.gitattributes index d3d32395e5a7e33b884cbbab4d06435fa37d42ef..7973fabe8edf96baab4547c0d274c2a01735efba 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1790,6 +1790,7 @@ LCU/Maintenance/CMakeLists.txt -text LCU/Maintenance/DBInterface/CMakeLists.txt -text LCU/Maintenance/DBInterface/__init__.py -text LCU/Maintenance/DBInterface/django_postgresql/__init__.py -text +LCU/Maintenance/DBInterface/django_postgresql/celery_settings.py -text LCU/Maintenance/DBInterface/django_postgresql/create_db.sql -text LCU/Maintenance/DBInterface/django_postgresql/settings.py -text LCU/Maintenance/DBInterface/django_postgresql/urls.py -text @@ -1823,6 +1824,7 @@ LCU/Maintenance/DBInterface/monitoringdb/serializers/station.py -text LCU/Maintenance/DBInterface/monitoringdb/serializers/station_tests.py -text LCU/Maintenance/DBInterface/monitoringdb/serializers/utils.py -text LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py -text +LCU/Maintenance/DBInterface/monitoringdb/tasks.py -text LCU/Maintenance/DBInterface/monitoringdb/tests/__init__.py -text LCU/Maintenance/DBInterface/monitoringdb/tests/common.py -text LCU/Maintenance/DBInterface/monitoringdb/tests/old_tests.py -text @@ -1912,6 +1914,7 @@ LCU/Maintenance/MDB_WebView/maintenancedb_view/src/utils/LOFARDefinitions.js -te LCU/Maintenance/MDB_WebView/maintenancedb_view/src/utils/autoLoader.js -text LCU/Maintenance/MDB_WebView/maintenancedb_view/src/utils/constants.js -text LCU/Maintenance/MDB_WebView/maintenancedb_view/src/utils/grid.js -text +LCU/Maintenance/MDB_WebView/maintenancedb_view/src/utils/highlightClass.js -text LCU/Maintenance/MDB_WebView/maintenancedb_view/src/utils/utils.js -text LCU/Maintenance/MDB_tools/CMakeLists.txt -text LCU/Maintenance/MDB_tools/bin/mdb_loader.py -text diff --git a/LCU/Maintenance/DBInterface/CMakeLists.txt b/LCU/Maintenance/DBInterface/CMakeLists.txt index 99c17d8d018434ce3d5035ad8d52dc78d1a6e0f5..b5fec93c6e859ebb2102a865b7ca2caf4f2a32e9 100644 --- a/LCU/Maintenance/DBInterface/CMakeLists.txt +++ b/LCU/Maintenance/DBInterface/CMakeLists.txt @@ -10,6 +10,8 @@ find_python_module(django REQUIRED) find_python_module(psycopg2 REQUIRED) find_python_module(rest_framework REQUIRED) #sudo pip install djangorestframework find_python_module(requests REQUIRED) +find_python_module(celery REQUIRED) +find_python_module(django_filters REQUIRED) # includes every python file excepts for the manage.py diff --git a/LCU/Maintenance/DBInterface/django_postgresql/__init__.py b/LCU/Maintenance/DBInterface/django_postgresql/__init__.py index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..782bcb57226cadb466668820c9951dd0321eba7f 100644 --- a/LCU/Maintenance/DBInterface/django_postgresql/__init__.py +++ b/LCU/Maintenance/DBInterface/django_postgresql/__init__.py @@ -0,0 +1,3 @@ +from .celery_settings import backend_tasks + +__all__ = ('backend_tasks') \ No newline at end of file diff --git a/LCU/Maintenance/DBInterface/django_postgresql/celery_settings.py b/LCU/Maintenance/DBInterface/django_postgresql/celery_settings.py new file mode 100644 index 0000000000000000000000000000000000000000..14a925101ee2431532ef369e38f18bea13b91dfe --- /dev/null +++ b/LCU/Maintenance/DBInterface/django_postgresql/celery_settings.py @@ -0,0 +1,22 @@ +from __future__ import absolute_import, unicode_literals +import os +from celery import Celery + +# set the default Django settings module for the 'celery' program. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'lofar.maintenance.django_postgresql.settings') + +backend_tasks = Celery('maintenancedb') + +# Using a string here means the worker doesn't have to serialize +# the configuration object to child processes. +# - namespace='CELERY' means all celery-related configuration keys +# should have a `CELERY_` prefix. +backend_tasks.config_from_object('django.conf:settings', namespace='CELERY') + +# Load task modules from all registered Django app configs. +backend_tasks.autodiscover_tasks() + + +@backend_tasks.task(bind=True) +def debug_task(self): + print('Request: {0!r}'.format(self.request)) \ No newline at end of file diff --git a/LCU/Maintenance/DBInterface/django_postgresql/settings.py b/LCU/Maintenance/DBInterface/django_postgresql/settings.py index 87ca345a17cc324d7969741a76ffcb16707e8eca..e257b43e2daa8a9b229589193919ec676d13da3a 100644 --- a/LCU/Maintenance/DBInterface/django_postgresql/settings.py +++ b/LCU/Maintenance/DBInterface/django_postgresql/settings.py @@ -177,6 +177,6 @@ REST_FRAMEWORK = { 'DEFAULT_FILTER_BACKENDS': ('django_filters.rest_framework.DjangoFilterBackend',) } - +CELERY_RESULT_BACKEND = 'amqp://guest@localhost//' # LOFAR SPECIFIC PARAMETERS -URL_TO_RTSM_PLOTS = 'https://proxy.lofar.eu/rtsm/obs_plots/' \ No newline at end of file +URL_TO_STORE_RTSM_PLOTS = './' \ No newline at end of file diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py b/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py index cfcaef773e79bc0de642704af10f495e2121788c..16c28c62ad3ec6fae15d014798f80bbc696ab050 100644 --- a/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py +++ b/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py @@ -71,6 +71,15 @@ class RTSMErrorSummary(models.Model): stop_frequency = models.FloatField(default=None, null=True) +class RTSMSummaryPlot(models.Model): + error_summary = models.ForeignKey(RTSMErrorSummary, + related_name='summary_plot', + on_delete=models.SET_NULL, + null=True) + + uri = models.CharField(max_length=10000, default=None, null=True) + + class RTSMSpectrum(models.Model): bad_spectrum = ArrayField(models.FloatField()) average_spectrum = ArrayField(models.FloatField()) diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py index 8cddf2b51b544d17f7ec1482e9f7c20ba0808ca4..23ab677706c69ea23db45af20f9689ee0616af70 100644 --- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py +++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py @@ -2,9 +2,11 @@ from ..models.rtsm import * from rest_framework import serializers import logging from django.db.transaction import atomic +from django.conf import settings from ..exceptions import ItemAlreadyExists from .log import ActionLogSerializer from .station import Station +import os logger = logging.getLogger('serializers') @@ -100,3 +102,24 @@ class RTSMObservationSerializer(serializers.ModelSerializer): self.compute_summary(RTSMObservation_instance) return RTSMObservation_instance + + +class FilePathSerializer(serializers.Field): + def __init__(self, path, *args, **kwargs): + self.path = path + super(FilePathSerializer, self).__init__(*args, **kwargs) + + def to_representation(self, value): + full_path = os.path.join(self.path, value) + return full_path + + def to_internal_value(self, data): + file_name = os.path.relpath(self.path, data) + return file_name + + +class RTSMSummaryPlotSerializer(serializers.ModelSerializer): + uri = FilePathSerializer(settings.URL_TO_STORE_RTSM_PLOTS) + class Meta: + model = RTSMSummaryPlot + fields = '__all__' diff --git a/LCU/Maintenance/DBInterface/monitoringdb/tasks.py b/LCU/Maintenance/DBInterface/monitoringdb/tasks.py new file mode 100644 index 0000000000000000000000000000000000000000..b89b200c7145766bb1c59b9676c3540c35dc2346 --- /dev/null +++ b/LCU/Maintenance/DBInterface/monitoringdb/tasks.py @@ -0,0 +1,255 @@ +import logging +import os + +import matplotlib + +matplotlib.use('agg') +import matplotlib.pyplot as plt +import numpy +from celery import shared_task +from django.conf import settings +from .models.rtsm import MODE_TO_COMPONENT, RTSMErrorSummary, RTSMSummaryPlot + +logger = logging.getLogger(__name__) + + +class Metadata: + def __init__(self, **kwargs): + for key, value in kwargs.items(): + self.__setattr__(key, value) + + +MODE_STR = {1: 'LBA Low 10..90 MHz', + 2: 'LBA Low 30..90', + 3: 'LBA High 10..90', + 4: 'LBA High 30..90', + 5: 'HBA 110..190', + 6: 'HBA 170..230', + 7: 'HBA 210..250'} + + +class ObservationMetadata(Metadata): + station_name = 'CSXXXC' + + start_datetime = '1212-12-12 12:12:12' + end_datetime = '1212-12-12 12:12:12' + observation_id = 'XXXXXX' + samples = 120 + + @staticmethod + def __format_datetime(datetime): + return datetime.strftime("%Y-%m-%d %H:%M:%S") + + def set_start_time(self, start_time): + self.start_datetime = ObservationMetadata.__format_datetime(start_time) + + def set_end_time(self, end_time): + self.end_datetime = ObservationMetadata.__format_datetime(end_time) + + +def set_style(): + plt.minorticks_on() + plt.ylim(55, 160) + plt.grid() + # plt.rcParams['svg.fonttype'] = 'none' + plt.rcParams['lines.linewidth'] = 1 + plt.rcParams['figure.autolayout'] = True + plt.rcParams['axes.labelsize'] = 'large' + plt.rcParams['path.simplify'] = True + plt.rcParams['savefig.transparent'] = False + plt.rcParams['savefig.bbox'] = 'tight' + + plt.rcParams['legend.loc'] = 'upper right' + plt.rcParams['legend.frameon'] = True + plt.rcParams['legend.framealpha'] = 1 + plt.rcParams['legend.edgecolor'] = 'black' + + +def rcu_mode_to_antenna(rcu, mode): + component_name = MODE_TO_COMPONENT[mode] + + antenna = rcu // 2 + if component_name in ['LBH', 'HBA']: + polarization = 'X' if rcu % 2 == 0 else 'Y' + elif component_name == 'LBL': + antenna = rcu // 2 + 48 + polarization = 'Y' if rcu % 2 == 0 else 'X' + else: + raise ValueError('Unknown component %s' % component_name) + + return dict(polarization=polarization, antenna=antenna) + + +def render_summary_text(error_metadata, observation_metadata): + """ + Render the summary text to include in the picture + :param error_metadata: the metadata information regarding the error + :param observation_metadata: the metadata information regarding the observation + :return: + """ + rendered_text = 'Station Name: {:<25s}'.format(observation_metadata.station_name) + rendered_text += 'Faults : {}\n'.format(error_metadata.error_type) + + rendered_text += 'ObsID : {:<25d}'.format(observation_metadata.observation_id) + rendered_text += 'Antenna : {antenna} {polarization}\n' \ + .format(**rcu_mode_to_antenna(error_metadata.rcu, error_metadata.mode)) + + rendered_text += 'Start : {:<25s}'.format(observation_metadata.start_datetime) + rendered_text += 'RCU : {}\n'.format(error_metadata.rcu) + + rendered_text += 'Stop : {:<25s}'.format(observation_metadata.end_datetime) + rendered_text += 'Samples : {count:d}/{samples:d}\n'.format(count=error_metadata.count, + samples=observation_metadata.samples) + + rendered_text += 'Mode : {:<1d} {:<23s}'.format(error_metadata.mode, + MODE_STR[error_metadata.mode]) + rendered_text += 'Badness : {:.2f} %\n'.format( + (100. * error_metadata.count) / observation_metadata.samples) + return rendered_text + + +def produce_plot(observation_metadata, + error_metadata, + list_bad_spectra, list_good_specta, path): + """ + Produces a plot given the observation metadata the error summary metadata and + the list of the bad spectra for the given (error, rcu) and the average spectrum + of the rest of the array + :param observation_metadata: the metadata information regarding the observation + :param error_metadata: the metadata information regarding the error + :param list_bad_spectra: a list containing the bad spectra for the given error, rcu couple + :param list_good_specta: the average spectrum of the rest of the array + :param path: the path where to store the file + """ + plt.figure(figsize=(12, 9)) + set_style() + + plt.xlabel('MHz') + plt.ylabel('dB') + + frequency_sampling = len(list_bad_spectra[0][0]) + n_spectra = len(list_bad_spectra) + frequency = numpy.linspace(error_metadata.start_frequency, + error_metadata.stop_frequency, + frequency_sampling) + bad_spectra_cube = numpy.zeros((n_spectra, frequency_sampling)) + average_spectra_cube = numpy.zeros((n_spectra, frequency_sampling)) + + for i, spectrum in enumerate(list_bad_spectra): + bad_spectra_cube[i, :] = spectrum[0] + for i, spectrum in enumerate(list_good_specta): + average_spectra_cube[i, :] = spectrum[0] + + min_bad_spectrum = numpy.min(bad_spectra_cube, axis=0) + max_bad_spectrum = numpy.max(bad_spectra_cube, axis=0) + average_bad_spectrum = numpy.median(bad_spectra_cube, axis=0) + + average_good_spectrum = numpy.median(average_spectra_cube, axis=0) + + plt.fill_between(frequency, min_bad_spectrum, max_bad_spectrum, color='red', + alpha=.3) + + plt.plot(frequency, average_bad_spectrum, color='red', label='median bad spectra') + + plt.plot(frequency, average_good_spectrum, color='blue', label='median all spectra') + + summary_text = render_summary_text(observation_metadata=observation_metadata, + error_metadata=error_metadata) + + plt.text(0.02, .98, summary_text, bbox=dict(facecolor='yellow', alpha=0.2), + family='monospace', verticalalignment='top', + horizontalalignment='left', + transform=plt.axes().transAxes) + plt.legend() + plt.savefig(path) + plt.close() + + +def generate_summary_plot_for_error(error_summary_id): + """ + Given a error summary id generates the plot in the specific location given by the settings + parameter URL_TO_STORE_RTSM_PLOTS and the file name derived in the following fashion + %(observation_id)d_%(station_name)s_rcu%(rcu)d_%(error_type)s.png + + Finally stores in the database the file name to directly access the file + + :param error_summary_id: database id of the error summary + :return: the database id of the RTSMSummaryPlot instance + :raise Exception: raise if there is an exception + """ + basePath = settings.URL_TO_STORE_RTSM_PLOTS + + error_summary = RTSMErrorSummary.objects.get(pk=error_summary_id) + summary_plot = error_summary.summary_plot.first() + if summary_plot is not None: + if summary_plot.uri == None: + return 'another worker is taken care of the task...' + + summary_plot = RTSMSummaryPlot(error_summary=error_summary, uri=None) + summary_plot.save() + + try: + observation_metadata = ObservationMetadata() + observation = error_summary.observation + observation_metadata.observation_id = observation.observation_id + observation_metadata.station_name = observation.station.name + observation_metadata.set_start_time(observation.start_datetime) + observation_metadata.set_end_time(observation.end_datetime) + + observation_metadata.samples = observation.samples + errors = observation.errors.filter(mode=error_summary.mode, + rcu=error_summary.rcu, + error_type=error_summary.error_type) + + list_bad_spectra = errors.values_list('spectra__bad_spectrum') + list_good_spectra = errors.values_list('spectra__average_spectrum') + file_name = '%(observation_id)d_%(station_name)s_rcu%(rcu)d_%(error_type)s.png' % dict( + observation_id=observation_metadata.observation_id, + station_name=observation_metadata.station_name, + rcu=error_summary.rcu, + error_type=error_summary.error_type + ) + + full_path = os.path.join(basePath, file_name) + produce_plot(observation_metadata, + error_summary, + list_bad_spectra, list_good_spectra, + full_path) + summary_plot.uri = file_name + summary_plot.save() + return summary_plot.pk + except Exception as e: + logger.exception('exception %s occurred skipping...', e) + summary_plot.delete() + raise e + + +@shared_task() +def check_error_summary_plot(error_summary_id): + """ + Checks if the error summary plot is presents otherwise it generates one + :param self: shared_task + :type self: celery.shared_task() + :param error_summary_id: database id of the RTSMErrorSummary to check + :return: + """ + logger.debug('looking for rtsm error summary %s', error_summary_id) + error_summary = RTSMErrorSummary.objects.get(pk=error_summary_id) + summary_plot = error_summary.summary_plot.first() + logger.debug('summary error found %s', summary_plot) + + if summary_plot is None: + return generate_summary_plot_for_error(error_summary_id) + else: + base_path = settings.URL_TO_STORE_RTSM_PLOTS + if summary_plot.uri is None: + return 'another worker is taking care of the task...' + full_path = os.path.join(base_path, summary_plot.uri) + if not os.path.exists(full_path): + return generate_summary_plot_for_error(error_summary_id) + elif os.path.isdir(full_path): + raise Exception('%s is a directory' % full_path) + else: + logger.debug('summary error %s is complete no need to generate additional plot', + summary_plot.pk) + return summary_plot.pk diff --git a/LCU/Maintenance/DBInterface/monitoringdb/urls.py b/LCU/Maintenance/DBInterface/monitoringdb/urls.py index 51e3c6f0495fbd0889f6e03a2522d927036016dc..67ff6ed75212ca8b731bc90651ec984a16c3d8a7 100644 --- a/LCU/Maintenance/DBInterface/monitoringdb/urls.py +++ b/LCU/Maintenance/DBInterface/monitoringdb/urls.py @@ -5,6 +5,7 @@ from .views.station_test_views import * from .views.rtsm_views import * from .views.logs_view import * from rest_framework.documentation import include_docs_urls +from django.urls import path log_router = routers.DefaultRouter() log_router.register(r'action_log', ActionLogViewSet) @@ -25,6 +26,7 @@ rtsm_router = routers.DefaultRouter() # RTSM rtsm_router.register(r'errors', RTSMErrorsViewSet) rtsm_router.register(r'spectra', RTSMSpectrumViewSet) +rtsm_router.register(r'error_summary_plot', RTSMSummaryPlot, base_name='rtsm-summary-plot') rtsm_router.register(r'', RTSMObservationViewSet) diff --git a/LCU/Maintenance/DBInterface/monitoringdb/views/common.py b/LCU/Maintenance/DBInterface/monitoringdb/views/common.py index 899d1642fcfe70895ed02d1ab89021424a9334d8..db345cc545f47e8eb44947f03a288b88861b6862 100644 --- a/LCU/Maintenance/DBInterface/monitoringdb/views/common.py +++ b/LCU/Maintenance/DBInterface/monitoringdb/views/common.py @@ -1,8 +1,10 @@ from rest_framework import viewsets, status from rest_framework.response import Response +from django.http import HttpResponse from rest_framework.decorators import api_view import logging from ..exceptions import ItemAlreadyExists +from django.core.exceptions import ObjectDoesNotExist RESERVED_FILTER_NAME = ['limit', 'offset', 'format', 'page_size'] diff --git a/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py b/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py index 578282f44c35e2799b95ef184b973337da879f66..65f5eb839e7419e1c5a753221adf3ef6ac9cf937 100644 --- a/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py +++ b/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py @@ -12,13 +12,14 @@ from lofar.maintenance.monitoringdb.models.rtsm import RTSMObservation from lofar.maintenance.monitoringdb.models.station import Station from lofar.maintenance.monitoringdb.models.station_test import StationTest from rest_framework import status +from rest_framework.reverse import reverse from rest_framework.response import Response from rest_framework.schemas import ManualSchema from rest_framework.views import APIView import pytz -logger = logging.getLogger(__name__) +logger = logging.getLogger(__name__) def parse_date(date): expected_format = '%Y-%m-%d' @@ -95,6 +96,9 @@ class ValidableReadOnlyView(APIView): raise ValueError(" ".join([field.name, error.text])) def get(self, request): + # Store the request as attribute + self.request = request + try: self.validate_query_parameters(request) except ValueError as e: @@ -469,6 +473,9 @@ class ControllerStationTestStatistics(ValidableReadOnlyView): station_group = 'A' test_type = 'B' error_types = [] + from_date = None + to_date = None + averaging_interval = None fields = [ coreapi.Field( @@ -728,7 +735,9 @@ class ControllerStationComponentErrors(ValidableReadOnlyView): .filter(start_datetime__gte=self.from_date, end_datetime__lte=self.to_date) - failing_component_types = station_tests.distinct('component_errors__component__type').exclude(component_errors__component__type__isnull=True).values_list('component_errors__component__type') + failing_component_types = station_tests.distinct('component_errors__component__type').\ + exclude(component_errors__component__type__isnull=True).\ + values_list('component_errors__component__type') for failing_component_type in failing_component_types: failing_component_type = failing_component_type[0] @@ -761,69 +770,6 @@ class ControllerStationComponentErrors(ValidableReadOnlyView): return response_payload - def compose_image_storage_url(self,station_name, observation_id, start_date, rcu, component_type, error_type, both_polarization): - """ - WARNING ----------------------------------------------------------- - This is an ugly function that has to be removed as soon as possible - and it is meant to provide a url to the RTSM plots to have it displayed - in the webview. A way to proper implement this part is still missing - hence this function. - DONT BLAME THE PLAYER BLAME THE GAME! - ------------------------------------------------------------------- - :param observation_id: id of the observation (SASID) - :param start_date: start date of the observation - :param rcu: rcu number - :param error_type: type of error - :return: - """ - # the sample link address looks like - # https://proxy.lofar.eu/rtsm/obs_plots/20181108_1542_683264/683264_SE607C_rcu175_sn.png - baseURL = settings.URL_TO_RTSM_PLOTS - - TO_SHORT_ERROR_TYPE = dict(SUMMATOR_NOISE='sn', - HIGH_NOISE='hn', - LOW_NOISE='ln', - FLAT='flat', - SHORT='short', - OSCILLATION='osc', - DOWN='down' - ) - - if component_type in ['LBL', 'LBH'] and both_polarization: - TO_SHORT_ERROR_TYPE.pop('FLAT') - type = 'ant' - component_id = rcu // 2 - if component_type == 'LBL': - component_id += 48 - else: - type = 'rcu' - component_id = rcu - - # If the error is not in the dict above there is not such plot on disk. - # Hence, returns 'url not present' - try: - short_error_type = TO_SHORT_ERROR_TYPE[error_type] - except KeyError: - return 'url not present' - - imageURL = '%(baseURL)s/%(start_date)s_%(observation_id)s/%(observation_id)s_%(station_name)s_%(type)s%(component_id)s_%(error_type)s.png' % dict( - baseURL=baseURL, - observation_id=observation_id, - start_date=start_date.strftime('%Y%m%d_%H%M'), - station_name=station_name, - component_id=component_id, - type=type, - error_type=short_error_type - ) - return imageURL - - def find_other_polarization_given_rcu_number(self, rcu_number): - # if it is even - if rcu_number % 2 == 0: - return rcu_number + 1 - else: - return rcu_number - 1 - def collect_rtsm_errors(self): station_entry = Station.objects.filter(name=self.station_name).first() response_payload = OrderedDict() @@ -839,15 +785,13 @@ class ControllerStationComponentErrors(ValidableReadOnlyView): rtsm_errors_per_component_type = list() - response_payload[MODE_TO_COMPONENT[observing_mode]] = rtsm_errors_per_component_type - for rtsm_observation in rtsm_observations.order_by('-start_datetime'): rtsm_summary = OrderedDict() - rtsm_errors_per_component_type.append(rtsm_summary) rtsm_summary['test_type'] = 'R' rtsm_summary['start_date'] = rtsm_observation.start_datetime rtsm_summary['end_date'] = rtsm_observation.end_datetime + rtsm_summary['observation_id'] = rtsm_observation.observation_id component_errors_dict = OrderedDict() rtsm_summary['component_errors'] = component_errors_dict @@ -855,11 +799,17 @@ class ControllerStationComponentErrors(ValidableReadOnlyView): .filter(mode=observing_mode)\ .values('error_type', 'start_frequency', 'stop_frequency', 'percentage', - 'error_type', 'count', 'rcu') + 'error_type', 'count', 'rcu', 'pk') + if component_errors.count() == 0 and rtsm_observation.errors_summary.count() !=0: + continue + else: + rtsm_errors_per_component_type.append(rtsm_summary) if self.error_types: component_errors = component_errors.filter(error_type__in=self.error_types) + for component_error in component_errors: + component_id = component_error['rcu'] details = dict(percentage = component_error['percentage'], start_frequency = component_error['start_frequency'], @@ -867,26 +817,25 @@ class ControllerStationComponentErrors(ValidableReadOnlyView): count = component_error['count']) error_type = component_error['error_type'] # CHECKS IF THE ERROR IS PRESENT IN BOTH RCUS (hence, both polarizations of the antenna) - both_polarization = len(component_errors.filter(rcu=self.find_other_polarization_given_rcu_number(component_id), - error_type=error_type)) == 1 - - details['url'] = self.compose_image_storage_url(self.station_name, - rtsm_observation.observation_id, - rtsm_observation.start_datetime, - component_id, - MODE_TO_COMPONENT[observing_mode], - error_type, - both_polarization) + url_to_plot = reverse('rtsm-summary-plot-detail', (component_error['pk'],), request=self.request) + details['url'] = url_to_plot if component_id not in component_errors: component_errors_dict[str(component_id)] = list() component_errors_dict[str(component_id)] += [dict(error_type=error_type, details=details)] + component_name = MODE_TO_COMPONENT[observing_mode] + if component_name not in response_payload: + response_payload[component_name] = rtsm_errors_per_component_type + else: + response_payload[component_name] += rtsm_errors_per_component_type + return response_payload def compute_response(self): self.from_date = parse_date(self.from_date) self.to_date = parse_date(self.to_date) + station_test_errors = {} rtsm_errors = {} @@ -975,13 +924,13 @@ class ControllerStationComponentElementErrors(ValidableReadOnlyView): if type == 'LBH': rcu_id = antenna_id * 2 - rcu_id += 1 if polarization == 'X' else 0 + rcu_id += 0 if polarization == 'X' else 1 elif type == 'LBL': rcu_id = (antenna_id - 48) * 2 - rcu_id += 0 if polarization == 'X' else 1 - elif type == 'HBA': - rcu_id = antenna_id rcu_id += 1 if polarization == 'X' else 0 + elif type == 'HBA': + rcu_id = antenna_id * 2 + rcu_id += 0 if polarization == 'X' else 1 else: rcu_id = -1 return rcu_id @@ -993,6 +942,49 @@ class ControllerStationComponentElementErrors(ValidableReadOnlyView): rcus={rcu_x:'X', rcu_y:'Y'} return rcus + def compute_ok_rtsm_list(self): + mode=COMPONENT_TO_MODE[self.component_type] + + rcus_per_polarization = self.rcus_from_antenna_and_type(self.antenna_id, + self.component_type) + + good_observation_list = RTSMObservation.objects.filter(errors_summary__mode__in=mode, + start_datetime__gt=self.from_date, + end_datetime__lt=self.to_date, + station__name=self.station_name).\ + exclude(errors_summary__rcu__in=list(rcus_per_polarization.keys())).\ + order_by('-start_datetime').values('pk', + 'observation_id', + 'start_datetime', + 'end_datetime') + result = [] + for observation in good_observation_list: + entry = dict(test_id=observation['observation_id'], + db_id=observation['pk'], + start_date=observation['start_datetime'], + end_date=observation['end_datetime'], + test_type='R', + component_errors=dict()) + result.append(entry) + return result + + def compute_ok_station_test(self): + good_station_test = StationTest.objects.filter(start_datetime__gt=self.from_date, + end_datetime__lt=self.to_date, + station__name=self.station_name).\ + exclude(component_errors__component__component_id=self.antenna_id).\ + values('pk', 'start_datetime', 'end_datetime').order_by('-start_datetime') + result = [] + for station_test in good_station_test: + entry = dict(test_id=station_test['pk'], + db_id=station_test['pk'], + start_date=station_test['start_datetime'], + end_date=station_test['end_datetime'], + test_type='S', + component_errors=dict()) + result.append(entry) + return result + def compute_rtsm_errors_list(self): errors = dict() rcus_per_polarization = self.rcus_from_antenna_and_type(self.antenna_id, @@ -1003,6 +995,7 @@ class ControllerStationComponentElementErrors(ValidableReadOnlyView): 'observation__end_datetime', 'observation__observation_id', 'observation__station__name', + 'pk', 'rcu', 'mode', 'error_type', @@ -1030,19 +1023,20 @@ class ControllerStationComponentElementErrors(ValidableReadOnlyView): if polarization not in errors[observation_pk]: errors[observation_pk]['component_errors'][polarization] = dict( rcu=rcu, - errors = dict(), - element_errors = dict() + errors = dict() ) error_type = item['error_type'] percentage = item['percentage'] count = item['count'] mode = item['mode'] samples = item['observation__samples'] + + url_to_plot = reverse('rtsm-summary-plot-detail', (item['pk'],), request=self.request) errors[observation_pk]['component_errors'][polarization]['errors'][error_type] = dict(samples=samples, percentage=percentage, count=count, mode=mode, - url='empty') + url=url_to_plot) return list(errors.values()) def compute_station_tests_error_list(self): @@ -1054,7 +1048,7 @@ class ControllerStationComponentElementErrors(ValidableReadOnlyView): station_test__end_datetime__lt=self.to_date, station_test__station__name=self.station_name, component__type=self.component_type, - component__component_id__in=list(rcus_per_polarization.keys())).order_by('-station_test__start_datetime') + component__component_id=self.antenna_id).order_by('-station_test__start_datetime') for component_error in component_errors: station_test_pk = component_error.station_test.pk if station_test_pk not in errors.keys(): @@ -1064,21 +1058,20 @@ class ControllerStationComponentElementErrors(ValidableReadOnlyView): end_date=component_error.station_test.end_datetime, test_type='S', component_errors=dict()) - component_id = component_error.component.component_id - polarization = rcus_per_polarization[component_id] - if polarization not in errors[station_test_pk]: - errors[station_test_pk]['component_errors'][polarization] = dict( - rcu=component_id, - errors=dict()) - error_type = component_error.type - details = component_error.details - errors_per_error_polarization = errors[station_test_pk]['component_errors'][polarization]['errors'] - errors_per_error_polarization[error_type] = dict(details=details, element_errors=dict()) - - - for element in component_error.failing_elements.values('element__element_id', 'details'): - element_id = element['element__element_id'] - errors_per_error_polarization[error_type]['element_errors'][element_id] = element['details'] + for rcu_id, polarization in rcus_per_polarization.items(): + if polarization not in errors[station_test_pk]: + errors[station_test_pk]['component_errors'][polarization] = dict( + rcu=rcu_id, + errors=dict()) + error_type = component_error.type + details = component_error.details + errors_per_error_polarization = errors[station_test_pk]['component_errors'][polarization]['errors'] + errors_per_error_polarization[error_type] = dict(details=details, element_errors=dict()) + + + for element in component_error.failing_elements.values('element__element_id', 'details'): + element_id = element['element__element_id'] + errors_per_error_polarization[error_type]['element_errors'][element_id] = element['details'] return list(errors.values()) @@ -1089,10 +1082,11 @@ class ControllerStationComponentElementErrors(ValidableReadOnlyView): station_test_list = [] if self.test_type == 'R' or self.test_type == 'A': - rtsm_errors_list = self.compute_rtsm_errors_list() + rtsm_errors_list = self.compute_rtsm_errors_list() + self.compute_ok_rtsm_list() + if self.test_type == 'S' or self.test_type == 'A': - station_test_list = self.compute_station_tests_error_list() + station_test_list = self.compute_station_tests_error_list() + self.compute_ok_station_test() combined = rtsm_errors_list + station_test_list combined_and_sorted = sorted(combined, key=lambda test: test['start_date'], reverse=True) - return Response(status=status.HTTP_200_OK, data=dict(errors=combined_and_sorted)) \ No newline at end of file + return Response(status=status.HTTP_200_OK, data=dict(errors=combined_and_sorted)) diff --git a/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py b/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py index 5e0a2560ff7ca59bd3bd6d5f677de7789d38b363..8591075bc630b610160e1976ff36340ce554bc01 100644 --- a/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py +++ b/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py @@ -1,8 +1,12 @@ from .common import * from ..models.rtsm import RTSMObservation, RTSMError, RTSMSpectrum from ..serializers.rtsm import RTSMObservationSerializer, RTSMErrorSerializer, \ - RTSMSpectrumSerializer + RTSMSpectrumSerializer, RTSMSummaryPlotSerializer, RTSMErrorSummary from ..rtsm_test_raw_parser import parse_rtsm_test +from django.shortcuts import get_object_or_404 +from django.http import Http404, HttpResponseServerError +import os +from ..tasks import check_error_summary_plot logger = logging.getLogger('views') @@ -25,6 +29,36 @@ class RTSMObservationViewSet(viewsets.ReadOnlyModelViewSet): filter_fields = '__all__' +class RTSMSummaryPlot(viewsets.ViewSet): + """ + Get the summary plot associated to the error summary given + """ + queryset = RTSMErrorSummary.objects.all() + + def retrieve(self, request, pk=None): + try: + entity = get_object_or_404(self.queryset, pk=pk) + summary_plot = entity.summary_plot.first() + + if summary_plot is None: + raise ObjectDoesNotExist() + + uri = RTSMSummaryPlotSerializer(summary_plot).data['uri'] + + except ObjectDoesNotExist as e: + check_error_summary_plot.delay(pk) + raise Http404() + + if uri and os.path.exists(uri) and os.path.isfile(uri): + with open(uri, 'rb') as f_stream: + image = f_stream.read() + return HttpResponse(image, status=status.HTTP_200_OK, content_type='image/gif') + else: + check_error_summary_plot.delay(pk) + print(uri) + raise Http404() + + @api_view(['POST']) def insert_raw_rtsm_test(request): """ diff --git a/LCU/Maintenance/MDB_WebView/maintenancedb_view/src/components/LatestObservations.js b/LCU/Maintenance/MDB_WebView/maintenancedb_view/src/components/LatestObservations.js index 530012f72fc8b41242c0c3a00757b028c133f783..8bd24331a9b1fed54c786e8e68bbc650cc50ae14 100644 --- a/LCU/Maintenance/MDB_WebView/maintenancedb_view/src/components/LatestObservations.js +++ b/LCU/Maintenance/MDB_WebView/maintenancedb_view/src/components/LatestObservations.js @@ -4,6 +4,7 @@ import { unique_id } from '../utils/utils.js' import AutoLoadWrapper from '../utils/autoLoader.js' import * as moment from 'moment'; import { datetime_format } from '../utils/constants' +import { highlightClass, unHighlightClass } from '../utils/highlightClass' import ObservationInspectTag from './ObservationInspectTag.js' // CSS import './LatestObservations.css' @@ -46,6 +47,17 @@ class SORow extends Component { togglePopover(){ this.setState({popoverOpen: !this.state.popoverOpen}); } + + componentDidUpdate(prevProps, prevState) { + if (this.state.popoverOpen !== prevState.popoverOpen) { + if (this.state.popoverOpen) { + highlightClass('obs-'+this.props.data.observation_id); + } else { + unHighlightClass('obs-'+this.props.data.observation_id); + } + } + } + render() { const data = this.props.data; const station_involved_list = this.getStationInvolvedList(); @@ -59,7 +71,7 @@ class SORow extends Component { return ( <tr id={this.id} className="hoverable"> - <ObservationInspectTag observationId={this.props.data.observation_id} /> + <td><ObservationInspectTag observationId={this.props.data.observation_id} /></td> <td>{ moment.utc(start_datetime).format(datetime_format) }</td> <td>{ station_involved_list.length }</td> <td>{ this.renderStationsWithProblems(station_involved_list) }</td> diff --git a/LCU/Maintenance/MDB_WebView/maintenancedb_view/src/components/StationOverview.js b/LCU/Maintenance/MDB_WebView/maintenancedb_view/src/components/StationOverview.js index 551053e73fe8592c87f66c0bd4e9d98640326361..2e423eefd723ecc722ef4b353da603532eacd754 100644 --- a/LCU/Maintenance/MDB_WebView/maintenancedb_view/src/components/StationOverview.js +++ b/LCU/Maintenance/MDB_WebView/maintenancedb_view/src/components/StationOverview.js @@ -1,4 +1,4 @@ -import React, {Component} from 'react'; +import React, {Component, PureComponent} from 'react'; import {withRouter} from "react-router"; import {Table, Popover, PopoverHeader, PopoverBody} from 'reactstrap'; import {unique_id} from '../utils/utils.js' @@ -12,7 +12,7 @@ import './StationOverview.css' /** * Badge; class to render a badge with label and pill. */ -class Badge extends Component { +class Badge extends PureComponent { getClass() { let cnt = this.props.count; @@ -28,28 +28,49 @@ class Badge extends Component { render() { let props = this.props; - return (<div id={props.myid} className={"so-badge"} onMouseOver={props.togglePopOver} onMouseOut={props.togglePopOver}> + return (<div id={props.myid} className={"so-badge "+this.props.className} onMouseOver={props.togglePopOver} onMouseOut={props.togglePopOver}> {props.label} <span className={this.getClass()}>{props.count}</span> </div>); } } + +const SOPopover = ({target, isOpen, togglePopover, title, lines}) => { + + if (!isOpen) { + return null; + } + + return (<Popover placement="auto" isOpen={isOpen} target={target} toggle={togglePopover}> + <PopoverHeader> + {title} + </PopoverHeader> + <PopoverBody> + <Table borderless size="sm"> + <tbody> + {lines.map( (obj, id) => <tr> + <th>{obj.th}</th> + <td>{obj.td}</td> + </tr>) + } + </tbody> + </Table> + </PopoverBody> + </Popover>); +} + + /** * StationTestBadge; class to render one stationtest badge in the SORow. */ class StationTestBadgeC extends Component { - constructor(props) { - super(props); + id = unique_id(); - this.id = unique_id(); - this.togglePopover = this.togglePopover.bind(this); - this.onClick = this.onClick.bind(this); - this.state = { - popoverOpen: false - }; - } + state = { + popoverOpen: false + }; getClass() { let total = this.props.data.total_component_errors; @@ -63,70 +84,47 @@ class StationTestBadgeC extends Component { return `so-stationtestbadge ${color}`; } - onClick() { + onClick = () => { let station = this.props.station; this.props.history.push(`/station_overview?station=${station}`); } - togglePopover() { + togglePopover = () => { this.setState({ popoverOpen: !this.state.popoverOpen }); } - renderPopOver() { + popoverLines() { let data = this.props.data; let summary = data.component_error_summary; - let rows = []; + + let lines = [ + { th: 'Start:' , td: moment.utc(data.start_datetime).format(datetime_format)}, + { th: 'End:', td: moment.utc(data.end_datetime).format(datetime_format)}, + { th: 'Checks:', td: data.checks} + ]; let components = Object.keys(summary).sort(); components.forEach((component) => { let comp_sum = summary[component]; let errors = Object.keys(comp_sum).sort(); - rows.push(<tr key={component}> - <th>{component}</th> - <td>{errors.map((e, id) => <Badge key={id} count={comp_sum[e]} label={e}/>)}</td> - </tr>); + lines.push({ + th: component, td: errors.map((e, id) => <Badge key={id} count={comp_sum[e]} label={e}/>) + }); }); - return (<Popover placement="auto" isOpen={this.state.popoverOpen} target={this.id} toggle={this.togglePopover}> - <PopoverHeader> - {this.props.station} - </PopoverHeader> - <PopoverBody> - <Table borderless size="sm"> - <tbody> - <tr> - <th>Start:</th> - <td>{moment.utc(data.start_datetime).format(datetime_format)}</td> - </tr> - <tr> - <th>End:</th> - <td>{moment.utc(data.end_datetime).format(datetime_format)}</td> - </tr> - <tr> - <th>Checks:</th> - <td>{data.checks}</td> - </tr> - {rows} - </tbody> - </Table> - </PopoverBody> - </Popover>); + return lines; } render() { - let popOver = ""; - - if (this.state.popoverOpen) { - popOver = this.renderPopOver(); - } + let lines = this.state.popoverOpen ? this.popoverLines() : []; return (<div> <div id={this.id} onClick={this.onClick} onMouseOver={this.togglePopover} onMouseOut={this.togglePopover} className={this.getClass()}> {this.props.data.total_component_errors} </div> - {popOver} + <SOPopover target={this.id} isOpen={this.state.popoverOpen} togglePopover={this.togglePopover} title={this.props.data.observation_id} lines={lines} /> </div>); } } @@ -138,15 +136,11 @@ const StationTestBadge = withRouter(StationTestBadgeC); */ class RTSMBadge extends Component { - constructor(props) { - super(props); + id = unique_id(); - this.id = unique_id(); - this.togglePopover = this.togglePopover.bind(this); - this.state = { - popoverOpen: false - }; - } + state = { + popoverOpen: false + }; getClass() { let total = this.props.data.total_component_errors; @@ -160,59 +154,39 @@ class RTSMBadge extends Component { return "so-pill " + color; } - togglePopover() { + togglePopover = () => { this.setState({ popoverOpen: !this.state.popoverOpen }); } - renderPopOver() { + popoverLines() { let data = this.props.data; - let summary = data.error_summary; - let badges = ""; - - let errors = Object.keys(summary).sort(); - badges = errors.map((e, i) => <Badge key={i} count={summary[e]} label={e}/>); - - return (<Popover placement="auto" isOpen={this.state.popoverOpen} target={this.id} toggle={this.togglePopover}> - <PopoverHeader> - {data.observation_id} - </PopoverHeader> - <PopoverBody> - <Table borderless size="sm"> - <tbody> - <tr> - <th>Start:</th> - <td>{moment.utc(data.start_datetime).format(datetime_format)}</td> - </tr> - <tr> - <th>End:</th> - <td>{moment.utc(data.end_datetime).format(datetime_format)}</td> - </tr> - <tr> - <th>Mode:</th> - <td>{data.mode}</td> - </tr> - <tr> - <th>Errors:</th> - <td>{badges}</td> - </tr> - </tbody> - </Table> - </PopoverBody> - </Popover>); + + let errors = Object.keys(data.error_summary).sort(); + let badges = errors.map((e, i) => <Badge key={i} count={data.error_summary[e]} label={e}/>); + + let lines = [ + { th: 'Start:' , td: moment.utc(data.start_datetime).format(datetime_format)}, + { th: 'End:', td: moment.utc(data.end_datetime).format(datetime_format)}, + { th: 'Mode:', td: data.mode }, + { th: 'Errors:', td: badges} + ]; + + return lines; } render() { - let popOver = ""; let data = this.props.data; + let lines = []; if (this.state.popoverOpen) { - popOver = this.renderPopOver(); + lines = this.popoverLines(); } return (<React.Fragment> - <Badge myid={this.id} className="so-rtsmbadge" togglePopOver={this.togglePopover} count={data.total_component_errors} label={data.observation_id}/> {popOver} + <Badge myid={this.id} className={'obs-'+data.observation_id} togglePopOver={this.togglePopover} count={data.total_component_errors} label={data.observation_id}/> + <SOPopover target={this.id} isOpen={this.state.popoverOpen} togglePopover={this.togglePopover} title={data.observation_id} lines={lines} /> </React.Fragment>); } } diff --git a/LCU/Maintenance/MDB_WebView/maintenancedb_view/src/components/StationTestView.js b/LCU/Maintenance/MDB_WebView/maintenancedb_view/src/components/StationTestView.js index 741261fec8c75f96f69bf7cd3b05e249bed08105..1553575993b913bc42f1e354b9a7c400114de703 100644 --- a/LCU/Maintenance/MDB_WebView/maintenancedb_view/src/components/StationTestView.js +++ b/LCU/Maintenance/MDB_WebView/maintenancedb_view/src/components/StationTestView.js @@ -198,6 +198,7 @@ class TestLineC extends Component { // Determine if this row needs to be highlighted this.doHighlight = this.shouldHighlight(); + const cls = this.doHighlight ? "stv-testline highlight" : "stv-testline"; return <tr className={cls}> @@ -405,6 +406,7 @@ class ComponentClass extends Component { * StationTestView class. */ class StationTestViewC extends Component { + constructor(props) { super(props); this.toggle = this.toggle.bind(this); diff --git a/LCU/Maintenance/MDB_WebView/maintenancedb_view/src/utils/highlightClass.js b/LCU/Maintenance/MDB_WebView/maintenancedb_view/src/utils/highlightClass.js new file mode 100644 index 0000000000000000000000000000000000000000..cd97e626f41caab4a1c0719c5db8bfd2c40b870d --- /dev/null +++ b/LCU/Maintenance/MDB_WebView/maintenancedb_view/src/utils/highlightClass.js @@ -0,0 +1,35 @@ + +let styleElement = null; +let curClassName = null; + + +/* Boiler plate for a Grid panel. + Looks like a React component but is ordinary function! */ +function highlightClass(className) { + + if (!styleElement) { + styleElement = document.createElement('style') + document.head.appendChild(styleElement) + } + + for (let i=0; i<styleElement.sheet.cssRules.length; i++){ + styleElement.sheet.deleteRule(0); + } + + // Insert a CSS Rule to the sheet at position 0. + styleElement.sheet.insertRule(`.${className} { background-color: #8d8d8d; color: white; }`, 0); + + + curClassName = className; +} + +function unHighlightClass(className) { + + if (styleElement.sheet.cssRules.length > 0 && className === curClassName) { + styleElement.sheet.deleteRule(0); + + curClassName = null; + } +} + +export { highlightClass, unHighlightClass };