controllers.py

import datetime
import logging
from collections import OrderedDict
from collections import defaultdict
from math import ceil
from typing import List

import coreapi
import coreschema
import pytz
from django.db.models import Count, Prefetch
from django.utils import timezone
from rest_framework import status
from rest_framework.response import Response
from rest_framework.reverse import reverse
from rest_framework.schemas import ManualSchema
from rest_framework.views import APIView

from lofar.maintenance.monitoringdb.models.component_error import ComponentError
from lofar.maintenance.monitoringdb.models.rtsm import RTSMError
from lofar.maintenance.monitoringdb.models.rtsm import RTSMObservation
from lofar.maintenance.monitoringdb.models.station import Station
from lofar.maintenance.monitoringdb.models.station_test import StationTest
from lofar.maintenance.monitoringdb.models.test import GenericTest
from lofar.maintenance.monitoringdb.models.wincc import WinCCAntennaStatus, \
    ComponentStatus, StationStatus, \
    latest_status_per_station_and_component_type_antenna_id, \
    latest_status_per_station_and_component_type
from lofar.maintenance.monitoringdb.serializers.wincc import serialize_station_status

logger = logging.getLogger(__name__)
from lofar.maintenance.monitoringdb.models.rtsm import MODE_TO_FREQ_RANGE


def parse_date(date):
    expected_format = '%Y-%m-%d'
    try:
        parsed_date = datetime.datetime.strptime(date, expected_format)
        return pytz.utc.localize(parsed_date)
    except Exception as e:
        raise ValueError('cannot parse %s with format %s - %s' % (date, expected_format, e))


def parse_bool(boolean_value_str):
    boolean_value_str = boolean_value_str.lower()
    if boolean_value_str in ['t', 'true', '1']:
        return True
    elif boolean_value_str in ['f', 'false', '0']:
        return False
    else:
        raise ValueError('%s is neither true or false' % boolean_value_str)


def parse_array(array_str):
    parsed_array = list(array_str.strip().lstrip('[').rstrip(']').split(','))
    if len(parsed_array) == 1 and parsed_array[0] == '':
        return []
    else:
        return parsed_array


def _get_unique_error_types():
    """
    List the unique error types found in the database
    :return: the list containing the unique error types
    :rtype: list
    """
    try:
        return [item['type'] for item in ComponentError.objects.values('type').distinct()]
    except:
        return []


def rcu_from_antenna_type_polarization(antenna_id, type, polarization):
    """
    Compute the rcu number for a given antenna number, type and polarization
    :param antenna_id: id of the antenna
    :param type: type of the antenna
    :param polarization: polarization either [X, Y]
    :return: the rcu id
    :rtype: int
    """
    if polarization not in ['X', 'Y']:
        raise ValueError('Polarization has to be either X or Y: %s not recognized' % polarization)

    if type == 'LBH':
        rcu_id = antenna_id * 2
        rcu_id += 0 if polarization == 'X' else 1
    elif type == 'LBL':
        rcu_id = (antenna_id - 48) * 2
        rcu_id += 1 if polarization == 'X' else 0
    elif type == 'HBA':
        rcu_id = antenna_id * 2
        rcu_id += 0 if polarization == 'X' else 1
    else:
        rcu_id = -1
    return rcu_id


def antenna_id_polarization_from_rcu_type_polarization(rcu, type):
    """
    Compute the antenna id for a given rcu, type and polarization
    :param rcu: id of the rcu
    :param type: type of the antenna
    :return: the antenna id and polarization
    :rtype: (int, str)
    """
    polarization_index = rcu % 2

    if type in ['LBH', 'HBA']:
        antenna_id = rcu - polarization_index
        antenna_id /= 2.
        polarization = 'Y' if polarization_index > 0 else 'X'
    elif type == 'LBL':
        antenna_id = (rcu - polarization_index) / 2. + 48
        polarization = 'X' if polarization_index > 0 else 'Y'
    else:
        antenna_id = -1
        polarization = ''

    return int(antenna_id), polarization


def rcus_from_antenna_and_type(antenna_id, type):
    rcu_x = rcu_from_antenna_type_polarization(antenna_id, type, 'X')
    rcu_y = rcu_from_antenna_type_polarization(antenna_id, type, 'Y')

    rcus = {rcu_x: 'X', rcu_y: 'Y'}
    return rcus


class ValidableReadOnlyView(APIView):
    """
    Convenience APIView class to have the validation of the query parameters on a get http request
    """

    # Override this to make the schema validation work
    fields = []
    description = ''

    def compute_response(self):
        raise NotImplementedError()

    @property
    def schema(self):
        return ManualSchema(fields=self.fields, description=self.description)

    @staticmethod
    def __is_required_and_not_query_param(field, request):
        return field.required and field.name not in request.query_params

    @staticmethod
    def __is_not_required_and_not_query_param(field, request):
        return (not field.required) and (field.name not in request.query_params)

    def __set_parsed_parameter(self, field):
        value = self.request.query_params.get(field.name)
        if field.type:
            self.__setattr__(field.name, field.type(value))
        else:
            self.__setattr__(field.name, value)

    def __validate_against_schema(self, field):

        errors = field.schema.validate(self.__getattribute__(field.name))
        for error in errors:
            raise ValueError(" ".join([field.name, error.text]))

    def validate_query_parameters(self, request):
        """
        Validated the request parameters and stores them as fields
        :param request: the http request to the api call
        :type request: rest_framework.request.Request
        :raises ValueError: if the parameter is not valid
        :raises KeyError: if the requested parameter is missing
        """
        for field in self.fields:
            if ValidableReadOnlyView.__is_required_and_not_query_param(field, request):
                raise KeyError('%s parameter is missing' % field.name)
            elif ValidableReadOnlyView.__is_not_required_and_not_query_param(field, request):
                continue
            else:
                self.__set_parsed_parameter(field)
                self.__validate_against_schema(field)

    def get(self, request):
        # Store the request as attribute
        self.request = request
        try:
            self.validate_query_parameters(request)
        except ValueError as e:
            return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
                            data='Please specify the correct parameters: %s' % (e,))
        except KeyError as e:
            return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
                            data='Please specify all the required parameters: %s' % (e,))

        try:
            response = self.compute_response()

        except ValueError as e:
            return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
                            data='Please specify the correct parameters: %s' % (e,))
        except Exception as e:
            logger.exception(e)
            return Response(status=status.HTTP_500_INTERNAL_SERVER_ERROR,
                            data='exception occurred: %s' % e)
        return response


def compute_error_summary(station_test: StationTest, selected_error_types=None):
    component_error_summary = dict()
    total = 0

    for component_error in station_test.component_errors.all():
        component_type = component_error.component.type
        component_error_type = component_error.type

        if selected_error_types and component_error_type not in selected_error_types:
            continue
        total += 1
        if component_type not in component_error_summary:
            component_error_summary[component_type] = {component_error_type: 1}
        elif component_error_type not in component_error_summary[component_type]:
            component_error_summary[component_type][component_error_type] = 1
        else:
            component_error_summary[component_type][component_error_type] += 1

    return component_error_summary, total


from django.db.models import Window, F
from django.db.models.functions import Rank


class ControllerStationOverview(ValidableReadOnlyView):
    description = "Overview of the latest tests performed on the stations"

    station_group = 'A'
    errors_only = 'true'
    n_station_tests = 4
    n_rtsm = 4
    error_types = []

    fields = [
        coreapi.Field(
            "station_group",
            required=False,
            location='query',
            schema=coreschema.Enum(['C', 'R', 'I', 'A'], description=
            'Station group to select for choices are [C|R|I|ALL]',
                                   )
        ),
        coreapi.Field(
            "n_station_tests",
            required=False,
            location='query',
            type=int,
            schema=coreschema.Integer(description='number of station tests to select',
                                      minimum=1)
        ),
        coreapi.Field(
            "n_rtsm",
            required=False,
            location='query',
            type=int,
            schema=coreschema.Integer(description='number of station tests to select',
                                      minimum=1)
        ),
        coreapi.Field(
            "errors_only",
            required=False,
            location='query',
            type=parse_bool,
            schema=coreschema.Boolean(
                description='displays or not only the station with more than one error')
        ),
        coreapi.Field(
            "error_types",
            required=False,
            location='query',
            type=parse_array,
            schema=coreschema.Array(description='select the error types to filter for',
                                    items=coreschema.Enum(_get_unique_error_types()),
                                    unique_items=True)
        )
    ]

    def get_last_station_test_per_station(self, selected_stations):
        expected_tests = len(selected_stations) * self.n_station_tests
        station_test_instances = StationTest.objects.order_by().filter(
            station__name__in=selected_stations). \
                                     annotate(order=Window(
            expression=Rank(),
            partition_by=[F('station')],
            order_by=F('start_datetime').desc())
        ).order_by('order', 'station__name')[:expected_tests]. \
            select_related('station').prefetch_related('component_errors',
                                                       'component_errors__component')

        st_per_station = defaultdict(list)

        for ind, station_test in enumerate(station_test_instances):
            station_name = station_test.station.name

            test_summary = dict()
            test_summary.update(start_datetime=station_test.start_datetime)
            test_summary.update(end_datetime=station_test.end_datetime)
            test_summary.update(checks=station_test.checks)

            component_error_summary, total = compute_error_summary(station_test, self.error_types)
            test_summary.update(total_component_errors=total)
            test_summary.update(component_error_summary=component_error_summary)
            if station_name in st_per_station and len(
                    st_per_station[station_name]) >= self.n_station_tests:
                continue
            st_per_station[station_name] += [test_summary]
        return st_per_station

    def get_last_rtsm(self, selected_stations):

        expected_tests = len(selected_stations) * self.n_rtsm

        rtsm_instances = RTSMObservation.objects.filter(station__name__in=selected_stations). \
                             annotate(order=Window(
            expression=Rank(),
            partition_by=[F('station')],
            order_by=F('start_datetime').desc())
        ).order_by('order')[:expected_tests].prefetch_related('errors', 'station')

        rtsm_per_station = defaultdict(list)
        for rtsm in rtsm_instances:

            station_name = rtsm.station.name
            observation_id = rtsm.sas_id
            start_datetime = rtsm.start_datetime
            end_datetime = rtsm.end_datetime

            rtsm_summary = dict(observation_id=observation_id,
                                start_datetime=start_datetime,
                                end_datetime=end_datetime)
            error_summary = dict()

            count = 0
            for error in rtsm.errors.all():
                if self.error_types and error.error_type not in self.error_types:
                    continue

                if error.error_type not in error_summary:
                    error_summary[error.error_type] = 1
                    count = 1

                error_summary[error.error_type] += 1
                count += 1

            rtsm_summary.update(total_component_errors=count,
                                error_summary=error_summary)

            rtsm_per_station[station_name].append(rtsm_summary)

        return rtsm_per_station

    def compute_response(self):
        station_entities = Station.objects.all()

        if 'A' not in self.station_group:
            station_entities = station_entities.filter(type__in=self.station_group)

        station_entities = list(zip(*station_entities.values_list('name').order_by('name')))[0]

        # Since django preferes a ordered dict over a dict we make it happy... for now
        rtsm_per_station = self.get_last_rtsm(station_entities)
        st_per_station = self.get_last_station_test_per_station(station_entities)
        response_payload = [
            dict(
                station_name=station,
                rtsm=rtsm_per_station[station],
                station_tests=st_per_station[station]
            )

            for station in station_entities
        ]

        return Response(status=status.HTTP_200_OK, data=response_payload)


class ControllerStationTestsSummary(ValidableReadOnlyView):
    description = "Overview of the latest station tests performed on" \
                  " the stations a [loopback_time] days ago from now"
    station_group = 'A'
    errors_only = 'true'
    lookback_time = 7
    error_types = []

    fields = [
        coreapi.Field(
            "station_group",
            required=False,
            location='query',
            schema=coreschema.Enum(['C', 'R', 'I', 'A'],
                                   description='Station group to select for choices are [C|R|I|ALL]')
        ),
        coreapi.Field(
            "errors_only",
            required=False,
            location='query',
            type=parse_bool,
            schema=coreschema.Boolean(
                description='displays or not only the station with more than one error')
        ),
        coreapi.Field(
            "lookback_time",
            required=False,
            type=int,
            location='query',
            schema=coreschema.Integer(description='number of days from now (default 7)',
                                      minimum=1)
        ),
        coreapi.Field(
            "error_types",
            required=False,
            location='query',
            type=parse_array,
            schema=coreschema.Array(description='select the error types to filter for',
                                    items=coreschema.Enum(_get_unique_error_types()),
                                    unique_items=True)
        )
    ]

    def compute_response(self):

        from_date = timezone.now() - timezone.timedelta(days=self.lookback_time)

        station_test_entities = StationTest.objects.filter(start_datetime__gt=from_date). \
            select_related('station')

        if 'A' not in self.station_group:
            station_test_entities = station_test_entities.filter(station__type__in=
                                                                 self.station_group)

        station_test_entities = \
            station_test_entities.prefetch_related('component_errors',
                                                   'component_errors__component').order_by(
                '-start_datetime')

        # Since django preferes a ordered dict over a dict we make it happy... for now
        response_payload = list()

        for station_test in station_test_entities:
            station_test_summary = dict()
            station_test_summary.update(station_name=station_test.station.name)
            station_test_summary.update(
                total_component_errors=station_test.component_errors.count())
            station_test_summary.update(start_datetime=station_test.start_datetime)
            station_test_summary.update(end_datetime=station_test.end_datetime)
            station_test_summary.update(date=station_test.start_datetime.strftime("%Y-%m-%d"))

            component_error_summary, total = compute_error_summary(station_test, self.error_types)
            station_test_summary.update(total_component_errors=total)
            station_test_summary.update(component_error_summary=component_error_summary)
            response_payload.append(station_test_summary)

        if self.errors_only:
            response_payload = filter(
                lambda station_test_entry:
                station_test_entry['total_component_errors'] > 0,
                response_payload)

        return Response(status=status.HTTP_200_OK, data=response_payload)


class ControllerLatestObservations(ValidableReadOnlyView):
    description = "Overview of the latest observations performed on the stations"

    station_group = 'A'
    errors_only = 'true'
    error_types = []

    fields = [
        coreapi.Field(
            "station_group",
            required=False,
            location='query',
            schema=coreschema.Enum(['C', 'R', 'I', 'A'], description=
            'Station group to select for choices are [C|R|I|A]',
                                   )
        ),
        coreapi.Field(
            "errors_only",
            required=False,
            location='query',
            type=parse_bool,
            schema=coreschema.Boolean(
                description='displays or not only the station with more than one error')
        ),
        coreapi.Field(
            "from_date",
            required=True,
            location='query',
            schema=coreschema.String(
                description='select rtsm from date (ex. YYYY-MM-DD)')
        ),
        coreapi.Field(
            "error_types",
            required=False,
            location='query',
            type=parse_array,
            schema=coreschema.Array(description='select the error types to filter for',
                                    items=coreschema.Enum(_get_unique_error_types()),
                                    unique_items=True)
        )
    ]

    def compute_response(self):
        self.from_date = parse_date(self.from_date)

        filtered_entities = RTSMObservation.objects \
            .filter(start_datetime__gte=self.from_date)
        if self.station_group != 'A':
            filtered_entities = filtered_entities \
                .filter(station__type=self.station_group)
        if self.errors_only:
            filtered_entities = filtered_entities.exclude(errors__isnull=True)

        errors_summary = filtered_entities \
            .values('sas_id',
                    'station__name',
                    'start_datetime',
                    'end_datetime',
                    'errors__error_type',
                    'errors__mode') \
            .annotate(total=Count('errors__error_type')) \
            .order_by('sas_id', 'station__name')

        if self.error_types:
            errors_summary = errors_summary.filter(errors__error_type__in=self.error_types)
        response = dict()

        for error_summary in errors_summary:
            observation_id = error_summary['sas_id']
            station_name = error_summary['station__name']
            start_datetime = error_summary['start_datetime']
            end_datetime = error_summary['end_datetime']
            mode = error_summary['errors__mode']
            error_type = error_summary['errors__error_type']
            total = error_summary['total']

            if observation_id not in response:
                response[observation_id] = OrderedDict()
                response[observation_id]['observation_id'] = observation_id
                response[observation_id]['start_datetime'] = start_datetime
                response[observation_id]['end_datetime'] = end_datetime

                response[observation_id]['total_component_errors'] = 0
                response[observation_id]['mode'] = list()
                response[observation_id]['station_involved'] = dict()

            if total == 0:
                continue

            response[observation_id]['total_component_errors'] += total
            station_involved_summary = response[observation_id]['station_involved']
            response[observation_id]['mode'] += [mode] \
                if mode not in response[observation_id]['mode'] else []
            if station_name not in station_involved_summary:
                station_involved_summary[station_name] = OrderedDict()
                station_involved_summary[station_name]['station_name'] = station_name
                station_involved_summary[station_name]['n_errors'] = 0
                station_involved_summary[station_name]['component_error_summary'] = OrderedDict()

            station_involved_summary[station_name]['n_errors'] += total
            station_involved_summary[station_name]['component_error_summary'][error_type] = total

        response_payload = sorted(response.values(),
                                  key=lambda item: item['start_datetime'],
                                  reverse=True)

        return Response(status=status.HTTP_200_OK, data=response_payload)


class ControllerStationTestStatistics(ValidableReadOnlyView):
    description = "Statistical summary of both or either the station test and RTSM"
    station_group = 'A'
    test_type = 'B'
    error_types = []
    from_date = None
    to_date = None
    averaging_interval = None

    fields = [
        coreapi.Field(
            "test_type",
            required=False,
            location='query',
            schema=coreschema.Enum(['R', 'S', 'B'],
                                   description='select the type of test possible values are (R, RTSM),'
                                               ' (S, Station test), (B, both)[DEFAULT=B]',
                                   )
        ),
        coreapi.Field(
            "station_group",
            required=False,
            location='query',
            schema=coreschema.Enum(['C', 'R', 'I', 'A'], description=
            'Station group to select for choices are [C|R|I|ALL]',
                                   )
        ),
        coreapi.Field(
            "from_date",
            required=True,
            location='query',
            schema=coreschema.String(
                description='select tests from date (ex. YYYY-MM-DD)')
        ),
        coreapi.Field(
            "to_date",
            required=True,
            location='query',
            schema=coreschema.String(
                description='select tests to date (ex. YYYY-MM-DD)')
        ),
        coreapi.Field(
            "averaging_interval",
            required=True,
            location='query',
            type=int,
            schema=coreschema.Integer(
                description='averaging interval in days')
        ),
        coreapi.Field(
            "error_types",
            required=False,
            location='query',
            type=parse_array,
            schema=coreschema.Array(description='select the error types to filter for',
                                    items=coreschema.Enum(_get_unique_error_types()),
                                    unique_items=True)
        )
    ]

    def compute_errors_per_station(self, from_date, to_date, central_time, station_group,
                                   test_type):

        component_errors = ComponentError.objects.all()
        rtsm_summary_errors = RTSMError.objects.all()

        if station_group:
            component_errors = component_errors.filter(station_test__station__type=station_group)
            rtsm_summary_errors = rtsm_summary_errors.filter(
                observation__station__type=station_group)
        if self.error_types:
            component_errors = component_errors.filter(type__in=self.error_types)
            rtsm_summary_errors = rtsm_summary_errors.filter(error_type__in=self.error_types)

        station_test_results = []
        rtsm_results = []
        if test_type in ['S', 'B']:
            station_test_results = component_errors. \
                filter(station_test__start_datetime__gt=from_date,
                       station_test__start_datetime__lt=to_date). \
                values('station_test__station__name'). \
                annotate(n_errors=Count('station_test__station__name'))
        if test_type in ['R', 'B']:
            rtsm_results = rtsm_summary_errors. \
                filter(observation__start_datetime__gt=from_date,
                       observation__start_datetime__lt=to_date). \
                values('observation__station__name'). \
                annotate(n_errors=Count('observation__station__name'))

        errors_per_station_in_bin = dict()
        central_time_str = central_time.strftime('%Y-%m-%d')
        if test_type in ['S', 'B']:
            for result in station_test_results:
                station_name = result['station_test__station__name']
                errors_per_station_in_bin[station_name] = dict(station_name=station_name,
                                                               n_errors=result['n_errors'],
                                                               time=central_time_str)

        if test_type in ['R', 'B']:
            for result in rtsm_results:
                station_name = result['observation__station__name']
                if station_name not in errors_per_station_in_bin:
                    errors_per_station_in_bin[station_name] = dict(station_name=station_name,
                                                                   n_errors=result['n_errors'],
                                                                   time=central_time_str)
                else:
                    errors_per_station_in_bin[station_name]['n_errors'] += result['n_errors']

        return errors_per_station_in_bin.values()

    def compute_errors_per_type(self, from_date, to_date, central_time, station_group, test_type):

        component_errors = ComponentError.objects.all()
        rtsm_summary_errors = RTSMError.objects.all()

        station_test_results = []
        rtsm_results = []

        central_time_str = central_time.strftime('%Y-%m-%d')
        if station_group:
            component_errors = component_errors.filter(station_test__station__type=station_group)
            rtsm_summary_errors = rtsm_summary_errors.filter(
                observation__station__type=station_group)

        if self.error_types:
            component_errors = component_errors.filter(type__in=self.error_types)
            rtsm_summary_errors = rtsm_summary_errors.filter(error_type__in=self.error_types)

        if test_type in ['S', 'B']:
            station_test_results = component_errors. \
                filter(station_test__start_datetime__gt=from_date,
                       station_test__start_datetime__lt=to_date). \
                values('type'). \
                annotate(n_errors=Count('type'))
        if test_type in ['R', 'B']:
            rtsm_results = rtsm_summary_errors. \
                filter(observation__start_datetime__gt=from_date,
                       observation__start_datetime__lt=to_date). \
                values('error_type'). \
                annotate(n_errors=Count('error_type'))

        errors_per_error_type_in_bin = dict()

        if test_type in ['S', 'B']:
            for result in station_test_results:
                error_type = result['type']
                errors_per_error_type_in_bin[error_type] = dict(error_type=error_type,
                                                                n_errors=result['n_errors'],
                                                                time=central_time_str)
        if test_type in ['R', 'B']:
            for result in rtsm_results:
                error_type = result['error_type']
                if error_type not in errors_per_error_type_in_bin:
                    errors_per_error_type_in_bin[error_type] = dict(error_type=error_type,
                                                                    n_errors=result['n_errors'],
                                                                    time=central_time_str)
                else:
                    errors_per_error_type_in_bin[error_type]['n_errors'] += result['n_errors']

        return errors_per_error_type_in_bin.values()

    def compute_response(self):
        from_date = parse_date(self.from_date)
        to_date = parse_date(self.to_date) + datetime.timedelta(days=1)
        averaging_interval = datetime.timedelta(days=self.averaging_interval)
        response_payload = OrderedDict()

        response_payload['start_date'] = from_date
        response_payload['end_date'] = to_date
        response_payload['averaging_interval'] = averaging_interval

        errors_per_station = []
        errors_per_type = []
        n_bins = int(ceil((to_date - from_date) / averaging_interval))

        for i in range(n_bins):
            if self.station_group is 'A':
                station_group = None
            else:
                station_group = self.station_group
            errors_per_station += self.compute_errors_per_station(
                from_date=from_date + i * averaging_interval,
                to_date=from_date + (i + 1) * averaging_interval,
                central_time=from_date + (i + .5) * averaging_interval,
                station_group=station_group,
                test_type=self.test_type)

            errors_per_type += self.compute_errors_per_type(
                from_date=from_date + i * averaging_interval,
                to_date=from_date + (i + 1) * averaging_interval,
                central_time=from_date + (i + .5) * averaging_interval,
                station_group=station_group,
                test_type=self.test_type)

        response_payload['errors_per_station'] = errors_per_station
        response_payload['errors_per_type'] = errors_per_type

        return Response(status=status.HTTP_200_OK, data=response_payload)


class ControllerAllComponentErrorTypes(ValidableReadOnlyView):
    description = "Lists all the presents component error types"

    def compute_response(self):
        data = [item['type'] for item in ComponentError.objects.values('type').distinct()]
        return Response(status=status.HTTP_200_OK, data=data)


def gather_statuses_for_test(generic_tests: List[GenericTest]):
    statuses = {station_status.id: station_status
                for station_status in StationStatus.objects.filter(generictest__in=generic_tests). \
                    prefetch_related(
        Prefetch('component_status', queryset=ComponentStatus.objects.
                 filter(status_code__gt=10).
                 select_related('component'),
                 to_attr='component_statuses'))}
    return statuses


class ControllerStationComponentErrors(ValidableReadOnlyView):
    description = "Provides a summary per station of the component errors"
    # required parameters
    station_name = None
    from_date = None
    to_date = None
    # optional parameters
    test_type = 'B'
    error_types = []

    fields = [
        coreapi.Field(
            'station_name',
            required=True,
            location='query',
            schema=coreschema.String(description='name of the station to select')
        ),
        coreapi.Field(
            'from_date',
            required=True,
            location='query',
            schema=coreschema.String(description='select tests from date (ex. YYYY-MM-DD)')
        ),
        coreapi.Field(
            'to_date',
            required=True,
            location='query',
            schema=coreschema.String(description='select tests from date (ex. YYYY-MM-DD)')
        ),
        coreapi.Field(
            'test_type',
            required=False,
            location='query',
            schema=coreschema.Enum(
                ['R', 'S', 'B'],
                description='select the type of test possible values are (R, RTSM),'
                            ' (S, Station test), (B, both)[DEFAULT=B]',
            )
        ),
        coreapi.Field(
            "error_types",
            required=False,
            location='query',
            type=parse_array,
            schema=coreschema.Array(description='select the error types to filter for',
                                    items=coreschema.Enum(_get_unique_error_types()),
                                    unique_items=True)
        )
    ]


    def collect_station_test_errors(self):

        tests = StationTest.objects.filter(station__name=self.station_name).\
            filter(start_datetime__range=(self.from_date, self.to_date))

        component_errors = ComponentError.objects.filter(station_test__in=tests). \
            select_related('station_test', 'component'). \
            select_related('station_test__station_status'). \
            order_by('-station_test__start_datetime', 'component__station__name')

        if self.error_types:
            component_errors = component_errors.filter(type__in=self.error_types)
        response_payload = OrderedDict()

        statuses = gather_statuses_for_test(tests)

        for component_error in component_errors:

            component_id = component_error.component.component_id
            component_type = component_error.component.type

            test_id = component_error.station_test.pk

            component_error_summary = dict(error_type=component_error.type,
                                           details=component_error.details)

            if component_type not in response_payload:
                response_payload[component_type] = dict()

            per_component_type = response_payload[component_type]

            if test_id not in per_component_type:
                status_id = component_error.station_test.station_status.pk
                status = statuses[status_id]
                per_component_type[test_id] = dict(
                    start_date=component_error.station_test.start_datetime,
                    end_date=component_error.station_test.end_datetime,
                    test_type='S',
                    status=serialize_station_status(status.component_statuses, component_type),
                    component_errors=dict()
                )

            per_test = per_component_type[test_id]['component_errors']

            if component_id not in per_test:
                per_test[component_id] = [component_error_summary]
            else:
                per_test[component_id] += [component_error_summary]

        for type in response_payload:
            tests_with_no_errors_for_type = tests.exclude(component_errors__component__type=type)
            tests_with_no_errors = tests.filter(component_errors__isnull=True)

            for test in tests_with_no_errors | tests_with_no_errors_for_type:
                test_id = test.pk
                status_id = test.station_status.pk
                status = statuses[status_id]

                response_payload[type][test_id] = dict(
                    start_date=test.start_datetime,
                    end_date=test.end_datetime,
                    test_type='S',
                    status=serialize_station_status(status.component_statuses, type),
                    component_errors=dict())

            response_payload[type] = list(response_payload[type].values())

        return response_payload

    def collect_rtsm_errors(self):
        tests = RTSMObservation.objects.filter(station__name=self.station_name). \
            filter(start_datetime__range=(self.from_date, self.to_date))

        statuses = gather_statuses_for_test(tests)

        rtsm_errors = RTSMError.objects.filter(observation__station__name=self.station_name). \
            filter(observation__start_datetime__range=(self.from_date, self.to_date)). \
            select_related('observation', 'component'). \
            prefetch_related('observation__station_status',
                             'observation__station_status__component_status')
        if self.error_types:
            rtsm_errors = rtsm_errors.filter(error_type__in=self.error_types)
        response_payload = OrderedDict()

        for component_error in rtsm_errors:
            component_id = component_error.component.component_id
            component_type = component_error.component.type
            test_id = component_error.observation.pk

            polarization = component_error.polarization
            error_type = component_error.error_type
            url_to_plot = reverse('rtsm-summary-plot-detail', (component_error.pk,),
                                  request=self.request)
            start_frequency, end_frequency = MODE_TO_FREQ_RANGE[component_error.mode]
            component_error_summary = dict(error_type=error_type,
                                           rcu_id=component_error.rcu,
                                           polarization=polarization,
                                           details=dict(
                                               url=url_to_plot,
                                               component_id=component_id,
                                               percentage=component_error.percentage * 100.,
                                               n_samples=component_error.observation.samples,
                                               start_frequency=start_frequency,
                                               stop_frequency=end_frequency,
                                               mode=component_error.mode
                                           ))

            if component_type not in response_payload:
                response_payload[component_type] = dict()

            per_component_type = response_payload[component_type]

            if test_id not in per_component_type:
                if component_error.observation.station_status is None:
                    per_component_type[test_id] = dict(
                        start_date=component_error.observation.start_datetime,
                        end_date=component_error.observation.end_datetime,
                        test_type='R',
                        status={},
                        component_errors=dict()
                    )

                else:
                    status_id = component_error.observation.station_status.pk
                    status = statuses[status_id]

                    per_component_type[test_id] = dict(
                        start_date=component_error.observation.start_datetime,
                        end_date=component_error.observation.end_datetime,
                        test_type='R',
                        status=serialize_station_status(status.component_statuses, component_type),
                        component_errors=dict()
                    )

            per_test = per_component_type[test_id]['component_errors']

            if component_id not in per_test:
                per_test[component_id] = {error_type: {polarization: component_error_summary}}
            elif error_type not in per_test[component_id]:
                per_test[component_id][error_type] = {polarization: component_error_summary}
            else:
                per_test[component_id][error_type][polarization] = component_error_summary

        for type in response_payload:
            tests_with_no_errors_for_type = tests.exclude(errors__component__type=type)
            tests_with_no_errors = tests.filter(errors__isnull=True)

            for test in tests_with_no_errors | tests_with_no_errors_for_type:
                test_id = test.pk