import logging
import os
import matplotlib
matplotlib.use('agg')
import matplotlib.pyplot as plt
import numpy
from celery import shared_task
from django.conf import settings
from .models.rtsm import MODE_TO_COMPONENT, RTSMErrorSummary, RTSMSummaryPlot
logger = logging.getLogger(__name__)
class Metadata:
def __init__(self, **kwargs):
for key, value in kwargs.items():
self.__setattr__(key, value)
MODE_STR = {1: 'LBA Low 10..90 MHz',
2: 'LBA Low 30..90',
3: 'LBA High 10..90',
4: 'LBA High 30..90',
5: 'HBA 110..190',
6: 'HBA 170..230',
7: 'HBA 210..250'}
class ObservationMetadata(Metadata):
station_name = 'CSXXXC'
start_datetime = '1212-12-12 12:12:12'
end_datetime = '1212-12-12 12:12:12'
observation_id = 'XXXXXX'
samples = 120
@staticmethod
def __format_datetime(datetime):
return datetime.strftime("%Y-%m-%d %H:%M:%S")
def set_start_time(self, start_time):
self.start_datetime = ObservationMetadata.__format_datetime(start_time)
def set_end_time(self, end_time):
self.end_datetime = ObservationMetadata.__format_datetime(end_time)
def set_style():
plt.minorticks_on()
plt.ylim(55, 160)
plt.grid()
# plt.rcParams['svg.fonttype'] = 'none'
plt.rcParams['lines.linewidth'] = 1
plt.rcParams['figure.autolayout'] = True
plt.rcParams['axes.labelsize'] = 'large'
plt.rcParams['path.simplify'] = True
plt.rcParams['savefig.transparent'] = False
plt.rcParams['savefig.bbox'] = 'tight'
plt.rcParams['legend.loc'] = 'upper right'
plt.rcParams['legend.frameon'] = True
plt.rcParams['legend.framealpha'] = 1
plt.rcParams['legend.edgecolor'] = 'black'
def rcu_mode_to_antenna(rcu, mode):
component_name = MODE_TO_COMPONENT[mode]
antenna = rcu // 2
if component_name in ['LBH', 'HBA']:
polarization = 'X' if rcu % 2 == 0 else 'Y'
elif component_name == 'LBL':
antenna = rcu // 2 + 48
polarization = 'Y' if rcu % 2 == 0 else 'X'
else:
raise ValueError('Unknown component %s' % component_name)
return dict(polarization=polarization, antenna=antenna)
def render_summary_text(error_metadata, observation_metadata):
"""
Render the summary text to include in the picture
:param error_metadata: the metadata information regarding the error
:param observation_metadata: the metadata information regarding the observation
:return:
"""
rendered_text = 'Station Name: {:<25s}'.format(observation_metadata.station_name)
rendered_text += 'Faults : {}\n'.format(error_metadata.error_type)
rendered_text += 'ObsID : {:<25d}'.format(observation_metadata.observation_id)
rendered_text += 'Antenna : {antenna} {polarization}\n' \
.format(**rcu_mode_to_antenna(error_metadata.rcu, error_metadata.mode))
rendered_text += 'Start : {:<25s}'.format(observation_metadata.start_datetime)
rendered_text += 'RCU : {}\n'.format(error_metadata.rcu)
rendered_text += 'Stop : {:<25s}'.format(observation_metadata.end_datetime)
rendered_text += 'Samples : {count:d}/{samples:d}\n'.format(count=error_metadata.count,
samples=observation_metadata.samples)
rendered_text += 'Mode : {:<1d} {:<23s}'.format(error_metadata.mode,
MODE_STR[error_metadata.mode])
rendered_text += 'Badness : {:.2f} %\n'.format(
(100. * error_metadata.count) / observation_metadata.samples)
return rendered_text
def produce_plot(observation_metadata,
error_metadata,
list_bad_spectra, list_good_specta, path):
"""
Produces a plot given the observation metadata the error summary metadata and
the list of the bad spectra for the given (error, rcu) and the average spectrum
of the rest of the array
:param observation_metadata: the metadata information regarding the observation
:param error_metadata: the metadata information regarding the error
:param list_bad_spectra: a list containing the bad spectra for the given error, rcu couple
:param list_good_specta: the average spectrum of the rest of the array
:param path: the path where to store the file
"""
plt.figure(figsize=(12, 9))
set_style()
plt.xlabel('MHz')
plt.ylabel('dB')
frequency_sampling = len(list_bad_spectra[0][0])
n_spectra = len(list_bad_spectra)
frequency = numpy.linspace(error_metadata.start_frequency,
error_metadata.stop_frequency,
frequency_sampling)
bad_spectra_cube = numpy.zeros((n_spectra, frequency_sampling))
average_spectra_cube = numpy.zeros((n_spectra, frequency_sampling))
for i, spectrum in enumerate(list_bad_spectra):
bad_spectra_cube[i, :] = spectrum[0]
for i, spectrum in enumerate(list_good_specta):
average_spectra_cube[i, :] = spectrum[0]
min_bad_spectrum = numpy.min(bad_spectra_cube, axis=0)
max_bad_spectrum = numpy.max(bad_spectra_cube, axis=0)
average_bad_spectrum = numpy.median(bad_spectra_cube, axis=0)
average_good_spectrum = numpy.median(average_spectra_cube, axis=0)
plt.fill_between(frequency, min_bad_spectrum, max_bad_spectrum, color='red',
alpha=.3)
plt.plot(frequency, average_bad_spectrum, color='red', label='median bad spectra')
plt.plot(frequency, average_good_spectrum, color='blue', label='median all spectra')
summary_text = render_summary_text(observation_metadata=observation_metadata,
error_metadata=error_metadata)
plt.text(0.02, .98, summary_text, bbox=dict(facecolor='yellow', alpha=0.2),
family='monospace', verticalalignment='top',
horizontalalignment='left',
transform=plt.axes().transAxes)
plt.legend()
plt.savefig(path)
plt.close()
def generate_summary_plot_for_error(error_summary_id):
"""
Given a error summary id generates the plot in the specific location given by the settings
parameter URL_TO_STORE_RTSM_PLOTS and the file name derived in the following fashion
%(observation_id)d_%(station_name)s_rcu%(rcu)d_%(error_type)s.png
Finally stores in the database the file name to directly access the file
:param error_summary_id: database id of the error summary
:return: the database id of the RTSMSummaryPlot instance
:raise Exception: raise if there is an exception
"""
basePath = settings.URL_TO_STORE_RTSM_PLOTS
error_summary = RTSMErrorSummary.objects.get(pk=error_summary_id)
summary_plot = error_summary.summary_plot.first()
if summary_plot is not None:
if summary_plot.uri == None:
return 'another worker is taken care of the task...'
summary_plot = RTSMSummaryPlot(error_summary=error_summary, uri=None)
summary_plot.save()
try:
observation_metadata = ObservationMetadata()
observation = error_summary.observation
observation_metadata.observation_id = observation.observation_id
observation_metadata.station_name = observation.station.name
observation_metadata.set_start_time(observation.start_datetime)
observation_metadata.set_end_time(observation.end_datetime)
observation_metadata.samples = observation.samples
errors = observation.errors.filter(mode=error_summary.mode,
rcu=error_summary.rcu,
error_type=error_summary.error_type)
list_bad_spectra = errors.values_list('spectra__bad_spectrum')
list_good_spectra = errors.values_list('spectra__average_spectrum')
file_name = '%(observation_id)d_%(station_name)s_rcu%(rcu)d_%(error_type)s.png' % dict(
observation_id=observation_metadata.observation_id,
station_name=observation_metadata.station_name,
rcu=error_summary.rcu,
error_type=error_summary.error_type
)
full_path = os.path.join(basePath, file_name)
produce_plot(observation_metadata,
error_summary,
list_bad_spectra, list_good_spectra,
full_path)
summary_plot.uri = file_name
summary_plot.save()
return summary_plot.pk
except Exception as e:
logger.exception('exception %s occurred skipping...', e)
summary_plot.delete()
raise e
@shared_task()
def check_error_summary_plot(error_summary_id):
"""
Checks if the error summary plot is presents otherwise it generates one
:param self: shared_task
:type self: celery.shared_task()
:param error_summary_id: database id of the RTSMErrorSummary to check
:return:
"""
logger.debug('looking for rtsm error summary %s', error_summary_id)
error_summary = RTSMErrorSummary.objects.get(pk=error_summary_id)
summary_plot = error_summary.summary_plot.first()
logger.debug('summary error found %s', summary_plot)
if summary_plot is None:
return generate_summary_plot_for_error(error_summary_id)
else:
base_path = settings.URL_TO_STORE_RTSM_PLOTS
if summary_plot.uri is None:
return 'another worker is taking care of the task...'
full_path = os.path.join(base_path, summary_plot.uri)
if not os.path.exists(full_path):
return generate_summary_plot_for_error(error_summary_id)
elif os.path.isdir(full_path):
raise Exception('%s is a directory' % full_path)
else:
logger.debug('summary error %s is complete no need to generate additional plot',
summary_plot.pk)
return summary_plot.pk