tasks.py

import logging
import os

import matplotlib

matplotlib.use('agg')
import matplotlib.pyplot as plt
import numpy
from celery import shared_task
from django.conf import settings
from .models.rtsm import MODE_TO_COMPONENT, RTSMErrorSummary, RTSMSummaryPlot

logger = logging.getLogger(__name__)


class Metadata:
    def __init__(self, **kwargs):
        for key, value in kwargs.items():
            self.__setattr__(key, value)


MODE_STR = {1: 'LBA Low 10..90 MHz',
            2: 'LBA Low 30..90',
            3: 'LBA High 10..90',
            4: 'LBA High 30..90',
            5: 'HBA 110..190',
            6: 'HBA 170..230',
            7: 'HBA 210..250'}


class ObservationMetadata(Metadata):
    station_name = 'CSXXXC'

    start_datetime = '1212-12-12 12:12:12'
    end_datetime = '1212-12-12 12:12:12'
    observation_id = 'XXXXXX'
    samples = 120

    @staticmethod
    def __format_datetime(datetime):
        return datetime.strftime("%Y-%m-%d %H:%M:%S")

    def set_start_time(self, start_time):
        self.start_datetime = ObservationMetadata.__format_datetime(start_time)

    def set_end_time(self, end_time):
        self.end_datetime = ObservationMetadata.__format_datetime(end_time)


def set_style():
    plt.minorticks_on()
    plt.ylim(55, 160)
    plt.grid()
    #    plt.rcParams['svg.fonttype'] = 'none'
    plt.rcParams['lines.linewidth'] = 1
    plt.rcParams['figure.autolayout'] = True
    plt.rcParams['axes.labelsize'] = 'large'
    plt.rcParams['path.simplify'] = True
    plt.rcParams['savefig.transparent'] = False
    plt.rcParams['savefig.bbox'] = 'tight'

    plt.rcParams['legend.loc'] = 'upper right'
    plt.rcParams['legend.frameon'] = True
    plt.rcParams['legend.framealpha'] = 1
    plt.rcParams['legend.edgecolor'] = 'black'


def rcu_mode_to_antenna(rcu, mode):
    component_name = MODE_TO_COMPONENT[mode]

    antenna = rcu // 2
    if component_name in ['LBH', 'HBA']:
        polarization = 'X' if rcu % 2 == 0 else 'Y'
    elif component_name == 'LBL':
        antenna = rcu // 2 + 48
        polarization = 'Y' if rcu % 2 == 0 else 'X'
    else:
        raise ValueError('Unknown component %s' % component_name)

    return dict(polarization=polarization, antenna=antenna)


def render_summary_text(error_metadata, observation_metadata):
    """
    Render the summary text to include in the picture
    :param error_metadata: the metadata information regarding the error
    :param observation_metadata: the metadata information regarding the observation
    :return:
    """
    rendered_text = 'Station Name: {:<25s}'.format(observation_metadata.station_name)
    rendered_text += 'Faults      : {}\n'.format(error_metadata.error_type)

    rendered_text += 'ObsID       : {:<25d}'.format(observation_metadata.observation_id)
    rendered_text += 'Antenna     : {antenna} {polarization}\n' \
        .format(**rcu_mode_to_antenna(error_metadata.rcu, error_metadata.mode))

    rendered_text += 'Start       : {:<25s}'.format(observation_metadata.start_datetime)
    rendered_text += 'RCU         : {}\n'.format(error_metadata.rcu)

    rendered_text += 'Stop        : {:<25s}'.format(observation_metadata.end_datetime)
    rendered_text += 'Samples     : {count:d}/{samples:d}\n'.format(count=error_metadata.count,
                                                                    samples=observation_metadata.samples)

    rendered_text += 'Mode        : {:<1d} {:<23s}'.format(error_metadata.mode,
                                                           MODE_STR[error_metadata.mode])
    rendered_text += 'Badness     : {:.2f} %\n'.format(
        (100. * error_metadata.count) / observation_metadata.samples)
    return rendered_text


def produce_plot(observation_metadata,
                 error_metadata,
                 list_bad_spectra, list_good_specta, path):
    """
    Produces a plot given the observation metadata the error summary metadata and
    the list of the bad spectra for the given (error, rcu) and the average spectrum
    of the rest of the array
    :param observation_metadata: the metadata information regarding the observation
    :param error_metadata: the metadata information regarding the error
    :param list_bad_spectra: a list containing the bad spectra for the given error, rcu couple
    :param list_good_specta: the average spectrum of the rest of the array
    :param path: the path where to store the file
    """
    plt.figure(figsize=(12, 9))
    set_style()

    plt.xlabel('MHz')
    plt.ylabel('dB')

    frequency_sampling = len(list_bad_spectra[0][0])
    n_spectra = len(list_bad_spectra)
    frequency = numpy.linspace(error_metadata.start_frequency,
                               error_metadata.stop_frequency,
                               frequency_sampling)
    bad_spectra_cube = numpy.zeros((n_spectra, frequency_sampling))
    average_spectra_cube = numpy.zeros((n_spectra, frequency_sampling))

    for i, spectrum in enumerate(list_bad_spectra):
        bad_spectra_cube[i, :] = spectrum[0]
    for i, spectrum in enumerate(list_good_specta):
        average_spectra_cube[i, :] = spectrum[0]

    min_bad_spectrum = numpy.min(bad_spectra_cube, axis=0)
    max_bad_spectrum = numpy.max(bad_spectra_cube, axis=0)
    average_bad_spectrum = numpy.median(bad_spectra_cube, axis=0)

    average_good_spectrum = numpy.median(average_spectra_cube, axis=0)

    plt.fill_between(frequency, min_bad_spectrum, max_bad_spectrum, color='red',
                     alpha=.3)

    plt.plot(frequency, average_bad_spectrum, color='red', label='median bad spectra')

    plt.plot(frequency, average_good_spectrum, color='blue', label='median all spectra')

    summary_text = render_summary_text(observation_metadata=observation_metadata,
                                       error_metadata=error_metadata)

    plt.text(0.02, .98, summary_text, bbox=dict(facecolor='yellow', alpha=0.2),
             family='monospace', verticalalignment='top',
             horizontalalignment='left',
             transform=plt.axes().transAxes)
    plt.legend()
    plt.savefig(path)
    plt.close()


def generate_summary_plot_for_error(error_summary_id):
    """
    Given a error summary id generates the plot in the specific location given by the settings
    parameter URL_TO_STORE_RTSM_PLOTS and the file name derived in the following fashion
    %(observation_id)d_%(station_name)s_rcu%(rcu)d_%(error_type)s.png

    Finally stores in the database the file name to directly access the file

    :param error_summary_id: database id of the error summary
    :return: the database id of the RTSMSummaryPlot instance
    :raise Exception: raise if there is an exception
    """
    basePath = settings.URL_TO_STORE_RTSM_PLOTS

    error_summary = RTSMErrorSummary.objects.get(pk=error_summary_id)
    summary_plot = error_summary.summary_plot.first()
    if summary_plot is not None:
        if summary_plot.uri == None:
            return 'another worker is taken care of the task...'

    summary_plot = RTSMSummaryPlot(error_summary=error_summary, uri=None)
    summary_plot.save()

    try:
        observation_metadata = ObservationMetadata()
        observation = error_summary.observation
        observation_metadata.observation_id = observation.observation_id
        observation_metadata.station_name = observation.station.name
        observation_metadata.set_start_time(observation.start_datetime)
        observation_metadata.set_end_time(observation.end_datetime)

        observation_metadata.samples = observation.samples
        errors = observation.errors.filter(mode=error_summary.mode,
                                           rcu=error_summary.rcu,
                                           error_type=error_summary.error_type)

        list_bad_spectra = errors.values_list('spectra__bad_spectrum')
        list_good_spectra = errors.values_list('spectra__average_spectrum')
        file_name = '%(observation_id)d_%(station_name)s_rcu%(rcu)d_%(error_type)s.png' % dict(
            observation_id=observation_metadata.observation_id,
            station_name=observation_metadata.station_name,
            rcu=error_summary.rcu,
            error_type=error_summary.error_type
        )

        full_path = os.path.join(basePath, file_name)
        produce_plot(observation_metadata,
                     error_summary,
                     list_bad_spectra, list_good_spectra,
                     full_path)
        summary_plot.uri = file_name
        summary_plot.save()
        return summary_plot.pk
    except Exception as e:
        logger.exception('exception %s occurred skipping...', e)
        summary_plot.delete()
        raise e


@shared_task()
def check_error_summary_plot(error_summary_id):
    """
    Checks if the error summary plot is presents otherwise it generates one
    :param self: shared_task
    :type self: celery.shared_task()
    :param error_summary_id:  database id of the RTSMErrorSummary to check
    :return:
    """
    logger.debug('looking for rtsm error summary %s', error_summary_id)
    error_summary = RTSMErrorSummary.objects.get(pk=error_summary_id)
    summary_plot = error_summary.summary_plot.first()
    logger.debug('summary error found %s', summary_plot)

    if summary_plot is None:
        return generate_summary_plot_for_error(error_summary_id)
    else:
        base_path = settings.URL_TO_STORE_RTSM_PLOTS
        if summary_plot.uri is None:
            return 'another worker is taking care of the task...'
        full_path = os.path.join(base_path, summary_plot.uri)
        if not os.path.exists(full_path):
            return generate_summary_plot_for_error(error_summary_id)
        elif os.path.isdir(full_path):
            raise Exception('%s is a directory' % full_path)
        else:
            logger.debug('summary error %s is complete no need to generate additional plot',
                         summary_plot.pk)
            return summary_plot.pk