controllers.py

import datetime
import logging
from collections import OrderedDict
from math import ceil

import coreapi
import coreschema
from django.db.models import Count
from rest_framework import status
from rest_framework.response import Response
from rest_framework.schemas import ManualSchema
from rest_framework.views import APIView

from lofar.maintenance.monitoringdb.models.component_error import ComponentError
from lofar.maintenance.monitoringdb.models.rtsm import RTSMErrorSummary
from lofar.maintenance.monitoringdb.models.rtsm import RTSMObservation
from lofar.maintenance.monitoringdb.models.station import Station
from lofar.maintenance.monitoringdb.models.station_test import StationTest

logger = logging.getLogger(__name__)


def parse_date(date):
    expected_format = '%Y-%m-%d'
    try:
        parsed_date = datetime.datetime.strptime(date, expected_format)
        return parsed_date
    except Exception as e:
        raise ValueError('cannot parse %s with format %s - %s' % (date, expected_format, e))


def parse_bool(boolean_value_str):
    boolean_value_str = boolean_value_str.lower()
    if boolean_value_str in ['t', 'true', '1']:
        return True
    elif boolean_value_str in ['f', 'false', '0']:
        return False
    else:
        raise ValueError('%s is neither true or false' % boolean_value_str)


class ValidableReadOnlyView(APIView):
    """
    Convenience APIView class to have the validation of the query parameters on a get http request
    """

    # Override this to make the schema validation work
    fields = []

    def compute_response(self):
        raise NotImplementedError()

    @property
    def schema(self):
        return ManualSchema(fields=self.fields)

    def validate_query_parameters(self, request):
        """
        Validated the request parameters and stores them as fields
        :param request: the http request to the api call
        :type request: rest_framework.request.Request
        :raises ValueError: if the parameter is not valid
        :raises KeyError: if the requested parameter is missing
        """
        for field in self.fields:
            if field.required and field.name not in request.query_params:
                raise KeyError('%s parameter is missing' % field.name)
            elif field.name not in request.query_params:
                continue
            else:
                value = self.request.query_params.get(field.name)
                if field.type:
                    self.__setattr__(field.name, field.type(value))
                else:
                    self.__setattr__(field.name, value)
                errors = field.schema.validate(self.__getattribute__(field.name))
                for error in errors:
                    raise ValueError(error.text)

    def get(self, request):
        try:
            self.validate_query_parameters(request)
        except ValueError as e:
            return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
                            data='Please specify the correct parameters: %s' % (e,))
        except KeyError as e:
            return Response(status=status.HTTP_406_NOT_ACCEPTABLE,
                            data='Please specify all the required parameters: %s' % (e,))

        try:
            response = self.compute_response()
        except Exception as e:
            logger.exception(e)
            return Response(status=status.HTTP_500_INTERNAL_SERVER_ERROR,
                            data='exception occurred: %s' % e)
        return response


class ControllerStationOverview(ValidableReadOnlyView):
    """
    Overview of the latest tests performed on the stations
    """

    station_group = 'A'
    errors_only = 'true'
    n_station_tests = 4
    n_rtsm = 4

    fields = [
        coreapi.Field(
            "station_group",
            required=False,
            location='query',
            schema=coreschema.Enum(['C', 'R', 'I', 'A'], description=
            'Station group to select for choices are [C|R|I|ALL]',
                                   )
        ),
        coreapi.Field(
            "n_station_tests",
            required=False,
            location='query',
            type=int,
            schema=coreschema.Integer(description='number of station tests to select',
                                      minimum=1)
        ),
        coreapi.Field(
            "n_rtsm",
            required=False,
            location='query',
            type=int,
            schema=coreschema.Integer(description='number of station tests to select',
                                      minimum=1)
        ),
        coreapi.Field(
            "errors_only",
            required=False,
            location='query',
            type=parse_bool,
            schema=coreschema.Boolean(
                description='displays or not only the station with more than one error')
        )
    ]

    def compute_response(self):

        station_entities = Station.objects.all()
        for group in self.station_group:
            if group is not 'A':
                station_entities = station_entities.filter(type=group)

        # Since django preferes a ordered dict over a dict we make it happy... for now
        response_payload = list()
        for station_entity in station_entities:
            station_payload = OrderedDict()

            station_payload['station_name'] = station_entity.name

            station_test_list = StationTest.objects.filter(
                station__name=station_entity.name).order_by('-end_datetime')[:self.n_station_tests]
            rtsm_list = RTSMObservation.objects.filter(
                station__name=station_entity.name).order_by('-end_datetime')[:self.n_rtsm]

            station_payload['station_tests'] = list()
            for station_test in station_test_list:
                station_test_payload = OrderedDict()
                component_errors = station_test.component_errors

                station_test_payload[
                    'total_component_errors'] = station_test.component_errors.count()
                station_test_payload['start_datetime'] = station_test.start_datetime
                station_test_payload['end_datetime'] = station_test.end_datetime
                station_test_payload['checks'] = station_test.checks
                component_errors_summary = component_errors. \
                    values('component__type', 'type').annotate(
                    total=Count('type')).order_by('-total')
                component_errors_summary_dict = OrderedDict()
                for item in component_errors_summary:
                    item_component_type = item['component__type']
                    item_error_type = item['type']
                    item_error_total = item['total']

                    if item_component_type not in component_errors_summary_dict:
                        component_errors_summary_dict[item_component_type] = OrderedDict()

                    component_errors_summary_dict[item_component_type][item_error_type] = \
                        item_error_total
                station_test_payload['component_error_summary'] = component_errors_summary_dict

                station_payload['station_tests'].append(station_test_payload)

            station_payload['rtsm'] = list()
            for rtsm in rtsm_list:
                rtsm_payload = OrderedDict()
                rtsm_payload['observation_id'] = rtsm.observation_id
                rtsm_payload['start_datetime'] = rtsm.start_datetime
                rtsm_payload['end_datetime'] = rtsm.end_datetime

                unique_modes = [item['mode'] for item in
                                rtsm.errors_summary.values('mode').distinct()]
                rtsm_payload['mode'] = unique_modes
                rtsm_payload['total_component_errors'] = rtsm.errors_summary.count()

                errors_summary = OrderedDict()

                errors_summary_query = rtsm.errors_summary.values('error_type').annotate(
                    total=Count('error_type'))

                for error_summary in errors_summary_query:
                    errors_summary[error_summary['error_type']] = error_summary['total']

                rtsm_payload['error_summary'] = errors_summary
                station_payload['rtsm'].append(rtsm_payload)

            response_payload.append(station_payload)
        if self.errors_only:
            response_payload = filter(
                lambda station_entry:
                len(station_entry['station_tests']) + len(station_entry['rtsm']) > 0,
                response_payload)
        response_payload = sorted(response_payload, key=lambda item: item['station_name'])
        return Response(status=status.HTTP_200_OK, data=response_payload)


class ControllerStationTestsSummary(ValidableReadOnlyView):
    """
    Overview of the latest station tests performed on the stations # lookback days before now
    """
    station_group = 'A'
    errors_only = 'true'
    lookback_time = 7

    fields = [
        coreapi.Field(
            "station_group",
            required=False,
            location='query',
            schema=coreschema.Enum(['C', 'R', 'I', 'A'],
                                   description='Station group to select for choices are [C|R|I|ALL]')
        ),
        coreapi.Field(
            "errors_only",
            required=False,
            location='query',
            type=parse_bool,
            schema=coreschema.Boolean(
                description='displays or not only the station with more than one error')
        ),
        coreapi.Field(
            "lookback_time",
            required=False,
            type=int,
            location='query',
            schema=coreschema.Integer(description='number of days from now (default 7)',
                                      minimum=1)
        )
    ]

    def compute_response(self):
        self.lookback_time = datetime.timedelta(days=self.lookback_time)

        station_test_list = StationTest.objects \
            .filter(start_datetime__gte=datetime.date.today() - self.lookback_time) \
            .order_by('-start_datetime', 'station__name')
        for group in self.station_group:
            if group is not 'A':
                station_test_list = station_test_list.filter(station__type=group)

        # Since django preferes a ordered dict over a dict we make it happy... for now
        response_payload = list()

        for station_test in station_test_list:

            station_test_payload = OrderedDict()
            component_errors = station_test.component_errors
            station_test_payload['station_name'] = station_test.station.name

            station_test_payload[
                'total_component_errors'] = station_test.component_errors.count()
            station_test_payload['date'] = station_test.start_datetime.strftime('%Y-%m-%d')
            station_test_payload['start_datetime'] = station_test.start_datetime
            station_test_payload['end_datetime'] = station_test.end_datetime
            station_test_payload['checks'] = station_test.checks
            component_errors_summary = component_errors. \
                values('component__type', 'type').annotate(
                total=Count('type')).order_by('-total')
            component_errors_summary_dict = OrderedDict()
            for item in component_errors_summary:
                item_component_type = item['component__type']
                item_error_type = item['type']
                item_error_total = item['total']

                if item_component_type not in component_errors_summary_dict:
                    component_errors_summary_dict[item_component_type] = OrderedDict()

                component_errors_summary_dict[item_component_type][item_error_type] = \
                    item_error_total
            station_test_payload['component_error_summary'] = component_errors_summary_dict

            response_payload.append(station_test_payload)

        if self.errors_only:
            response_payload = filter(
                lambda station_test_entry:
                station_test_entry['total_component_errors'] > 0,
                response_payload)

        return Response(status=status.HTTP_200_OK, data=response_payload)


class ControllerLatestObservations(ValidableReadOnlyView):
    """
    Overview of the latest observations performed on the stations
    """

    station_group = 'A'
    errors_only = 'true'

    fields = [
        coreapi.Field(
            "station_group",
            required=False,
            location='query',
            schema=coreschema.Enum(['C', 'R', 'I', 'A'], description=
            'Station group to select for choices are [C|R|I|A]',
                                   )
        ),
        coreapi.Field(
            "errors_only",
            required=False,
            location='query',
            type=parse_bool,
            schema=coreschema.Boolean(
                description='displays or not only the station with more than one error')
        ),
        coreapi.Field(
            "from_date",
            required=True,
            location='query',
            schema=coreschema.String(
                description='select rtsm from date (ex. YYYY-MM-DD)')
        )
    ]

    def compute_response(self):
        self.from_date = parse_date(self.from_date)

        filtered_entities = RTSMObservation.objects \
            .filter(start_datetime__gte=self.from_date)
        if self.station_group != 'A':
            filtered_entities = filtered_entities \
                .filter(station__type=self.station_group)
        if self.errors_only:
            filtered_entities = filtered_entities.exclude(errors_summary__isnull=True)

        errors_summary = filtered_entities \
            .values('observation_id',
                    'station__name',
                    'start_datetime',
                    'end_datetime',
                    'errors_summary__error_type',
                    'errors_summary__mode') \
            .annotate(total=Count('errors_summary__error_type')) \
            .order_by('observation_id', 'station__name')
        response = dict()

        for error_summary in errors_summary:
            observation_id = error_summary['observation_id']
            station_name = error_summary['station__name']
            start_datetime = error_summary['start_datetime']
            end_datetime = error_summary['end_datetime']
            mode = error_summary['errors_summary__mode']
            error_type = error_summary['errors_summary__error_type']
            total = error_summary['total']

            if observation_id not in response:
                response[observation_id] = OrderedDict()
                response[observation_id]['observation_id'] = observation_id
                response[observation_id]['start_datetime'] = start_datetime
                response[observation_id]['end_datetime'] = end_datetime

                response[observation_id]['total_component_errors'] = 0
                response[observation_id]['mode'] = list()
                response[observation_id]['station_involved'] = dict()

            if total == 0:
                continue

            response[observation_id]['total_component_errors'] += total
            station_involved_summary = response[observation_id]['station_involved']
            response[observation_id]['mode'] += [mode] \
                if mode not in response[observation_id]['mode'] else []
            if station_name not in station_involved_summary:
                station_involved_summary[station_name] = OrderedDict()
                station_involved_summary[station_name]['station_name'] = station_name
                station_involved_summary[station_name]['n_errors'] = 0
                station_involved_summary[station_name]['component_error_summary'] = OrderedDict()

            station_involved_summary[station_name]['n_errors'] += total
            station_involved_summary[station_name]['component_error_summary'][error_type] = total

        response_payload = sorted(response.values(),
                                  key=lambda item: item['start_datetime'],
                                  reverse=True)

        return Response(status=status.HTTP_200_OK, data=response_payload)


class ControllerStationTestStatistics(ValidableReadOnlyView):
    """

/views/ctrl_stationtest_statistics:

parameters:
station_group [C|R|I|ALL] (optional, default ALL)
test_type [RTSM|STATIONTEST|BOTH] (optional, default BOTH)
from_date #DATE
to_date #DATE
averaging_interval: #TIMESPAN
result:
{
  start_date : #DATE,
  end_date: #DATE,
  averaging_interval: #INTERVAL,
  error_per_station: [{
    time: #DATE,
    station_name: <station_name>
    n_errors:     #nr_errors int
    }, ...],
   error_per_error_type:  [{
    time: #DATE,
    error_type: <error_type>
    n_errors:     #nr_errors int
    }, ...],
 },
 ....
]
    """
    station_group = 'A'
    test_type = 'B'

    fields = [
        coreapi.Field(
            "test_type",
            required=False,
            location='query',
            schema=coreschema.Enum(['R', 'S', 'B'],
                                   description='select the type of test possible values are (R, RTSM),'
                                               ' (S, Station test), (B, both)[DEFAULT=B]',
                                   )
        ),
        coreapi.Field(
            "station_group",
            required=False,
            location='query',
            schema=coreschema.Enum(['C', 'R', 'I', 'A'], description=
            'Station group to select for choices are [C|R|I|ALL]',
                                   )
        ),
        coreapi.Field(
            "from_date",
            required=True,
            location='query',
            schema=coreschema.String(
                description='select tests from date (ex. YYYY-MM-DD)')
        ),
        coreapi.Field(
            "to_date",
            required=True,
            location='query',
            schema=coreschema.String(
                description='select tests to date (ex. YYYY-MM-DD)')
        ),
        coreapi.Field(
            "averaging_interval",
            required=True,
            location='query',
            type=int,
            schema=coreschema.Integer(
                description='averaging interval in days')
        )
    ]

    def compute_errors_per_station(self, from_date, to_date, central_time, station_group,
                                   test_type):

        component_errors = ComponentError.objects.all()
        rtsm_summary_errors = RTSMErrorSummary.objects.all()

        if station_group:
            component_errors = component_errors.filter(station_test__station__type=station_group)
            rtsm_summary_errors = rtsm_summary_errors.filter(
                observation__station__type=station_group)

        station_test_results = []
        rtsm_results = []
        if test_type in ['S', 'B']:
            station_test_results = component_errors. \
                filter(station_test__start_datetime__gt=from_date,
                       station_test__start_datetime__lt=to_date). \
                values('station_test__station__name'). \
                annotate(n_errors=Count('station_test__station__name'))
        if test_type in ['R', 'B']:
            rtsm_results = rtsm_summary_errors. \
                filter(observation__start_datetime__gt=from_date,
                       observation__start_datetime__lt=to_date). \
                values('observation__station__name'). \
                annotate(n_errors=Count('observation__station__name'))

        errors_per_station_in_bin = dict()
        central_time_str = central_time.strftime('%Y-%m-%d')
        if test_type in ['S', 'B']:
            for result in station_test_results:
                station_name = result['station_test__station__name']
                errors_per_station_in_bin[station_name] = dict(station_name=station_name,
                                                               n_errors=result['n_errors'],
                                                               time=central_time_str)

        if test_type in ['R', 'B']:
            for result in rtsm_results:
                station_name = result['observation__station__name']
                if station_name not in errors_per_station_in_bin:
                    errors_per_station_in_bin[station_name] = dict(station_name=station_name,
                                                                   n_errors=result['n_errors'],
                                                                   time=central_time_str)
                else:
                    errors_per_station_in_bin[station_name]['n_errors'] += result['n_errors']

        return errors_per_station_in_bin.values()

    def compute_errors_per_type(self, from_date, to_date, central_time, station_group, test_type):

        component_errors = ComponentError.objects.all()
        rtsm_summary_errors = RTSMErrorSummary.objects.all()

        station_test_results = []
        rtsm_results = []

        central_time_str = central_time.strftime('%Y-%m-%d')
        if station_group:
            component_errors = component_errors.filter(station_test__station__type=station_group)
            rtsm_summary_errors = rtsm_summary_errors.filter(
                observation__station__type=station_group)

        if test_type in ['S', 'B']:
            station_test_results = component_errors. \
                filter(station_test__start_datetime__gt=from_date,
                       station_test__start_datetime__lt=to_date). \
                values('type'). \
                annotate(n_errors=Count('type'))
        if test_type in ['R', 'B']:
            rtsm_results = rtsm_summary_errors. \
                filter(observation__start_datetime__gt=from_date,
                       observation__start_datetime__lt=to_date). \
                values('error_type'). \
                annotate(n_errors=Count('error_type'))

        errors_per_error_type_in_bin = dict()

        if test_type in ['S', 'B']:
            for result in station_test_results:
                error_type = result['type']
                errors_per_error_type_in_bin[error_type] = dict(error_type=error_type,
                                                                n_errors=result['n_errors'],
                                                                time=central_time_str)
        if test_type in ['R', 'B']:
            for result in rtsm_results:
                error_type = result['error_type']
                if error_type not in errors_per_error_type_in_bin:
                    errors_per_error_type_in_bin[error_type] = dict(error_type=error_type,
                                                                    n_errors=result['n_errors'],
                                                                    time=central_time_str)
                else:
                    errors_per_error_type_in_bin[error_type]['n_errors'] += result['n_errors']

        return errors_per_error_type_in_bin.values()

    def compute_response(self):
        from_date = parse_date(self.from_date)
        to_date = parse_date(self.to_date)
        averaging_interval = datetime.timedelta(days=self.averaging_interval)
        response_payload = OrderedDict()

        response_payload['start_date'] = from_date
        response_payload['end_date'] = to_date
        response_payload['averaging_interval'] = averaging_interval

        errors_per_station = []
        errors_per_type = []
        n_bins = int(ceil((to_date - from_date) / averaging_interval))

        for i in range(n_bins):
            if self.station_group is 'A':
                station_group = None
            else:
                station_group = self.station_group
            errors_per_station += self.compute_errors_per_station(
                from_date=from_date + i * averaging_interval,
                to_date=from_date + (i + 1) * averaging_interval,
                central_time=from_date + (i + .5) * averaging_interval,
                station_group=station_group,
                test_type=self.test_type)

            errors_per_type += self.compute_errors_per_type(
                from_date=from_date + i * averaging_interval,
                to_date=from_date + (i + 1) * averaging_interval,
                central_time=from_date + (i + .5) * averaging_interval,
                station_group=station_group,
                test_type=self.test_type)

        response_payload['errors_per_station'] = errors_per_station
        response_payload['errors_per_type'] = errors_per_type

        return Response(status=status.HTTP_200_OK, data=response_payload)


class ControllerAllComponentErrorTypes(ValidableReadOnlyView):
    schema = ManualSchema(description="Returns all distinct component errors", fields=[])

    def compute_response(self):
        data = [item['type'] for item in ComponentError.objects.values('type').distinct()]
        return Response(status=status.HTTP_200_OK, data=data)