common.py

"""
This file contains common constructs used by database models in other modules
"""

import logging
import typing

logger = logging.getLogger(__name__)

from django.db.models import Model, CharField, DateTimeField, IntegerField, UniqueConstraint
from django.contrib.postgres.fields import ArrayField, JSONField
from django.contrib.postgres.indexes import GinIndex
from lofar.common.json_utils import validate_json_against_its_schema, validate_json_against_schema, add_defaults_to_json_object_for_schema, get_default_json_object_for_schema
from lofar.sas.tmss.tmss.exceptions import SchemaValidationException
from django.urls import reverse as reverse_url
import json
import jsonschema
from datetime import timedelta
from django.utils.functional import cached_property
from lofar.sas.tmss.tmss.exceptions import TMSSException

#
# Mixins
#

class RefreshFromDbInvalidatesCachedPropertiesMixin():
    """Helper Mixin class which invalidates all 'cached_property' attributes on a model upon refreshing from the db"""
    def refresh_from_db(self, *args, **kwargs):
        self.invalidate_cached_properties()
        return super().refresh_from_db(*args, **kwargs)

    def invalidate_cached_properties(self):
        for key, value in self.__class__.__dict__.items():
            if isinstance(value, cached_property):
                self.__dict__.pop(key, None)


class ProjectPropertyMixin(RefreshFromDbInvalidatesCachedPropertiesMixin):
    @cached_property
    def project(self): # -> Project:
        '''return the related project of this task
        '''
        if not hasattr(self, 'path_to_project'):
            return TMSSException("Please define a 'path_to_project' attribute on the %s object for the ProjectPropertyMixin to function." % type(self))
        obj = self
        for attr in self.path_to_project.split('__'):
            obj = getattr(obj, attr)
            if attr == 'project':
                return obj
            if obj and not isinstance(obj, Model):  # ManyToMany fields
                obj = obj.first()
            if obj is None:
                logger.warning("The element '%s' in the path_to_project of the %s object returned None for pk=%s" % (attr, type(self), self.pk))
                return None


# abstract models

class BasicCommon(Model):
    # todo: we cannot use foreign keys in the array here, so we have to keep the Tags table up to date by trigger or so.
    # todo: we could switch to a manytomany field instead?
    tags = ArrayField(CharField(max_length=128), size=8, blank=True, help_text='User-defined search keywords for object.', default=list)
    created_at = DateTimeField(auto_now_add=True, help_text='Moment of object creation.')
    updated_at = DateTimeField(auto_now=True, help_text='Moment of last object update.')

    class Meta:
        abstract = True
        indexes = [GinIndex(fields=['tags'])]


class NamedCommon(BasicCommon):
    name = CharField(max_length=128, help_text='Human-readable name of this object.', null=False) # todo: check if we want to have this primary_key=True
    description = CharField(max_length=255, help_text='A longer description of this object.', blank=True, default="")

    def __str__(self):
        return self.name

    class Meta:
        abstract = True


# todo: check if we want to generally use this flavour, i.e. make everything named addressable by name rather than int id. (This then does not allow for multiple items of same name, of course.)
class NamedCommonPK(NamedCommon):
    name = CharField(max_length=128, help_text='Human-readable name of this object.', null=False, primary_key=True)

    class Meta:
        abstract = True


class AbstractChoice(Model):
    """
    Abstract class for all derived 'choices' models.
    We define a 'choice' as an item that you can pick from a predefined list.
    In the derived classes, we use an enum.Enum to define such a predefined list here in code.
    All values of the enums are then put automagically into the database in the populate module, which
    is/can_be/should_be called in the last migration step to populate the database with inital values
    for our 'static choices'.

    Design decision: Django also provides the 'choices' property on fields which sort of limits the number of choices
    one can make, and which sort of does some validation. In our opinion the validation is done in the wrong place, and
    no data consistency is enforced.
    So, we decided to follow Django's own hint, see https://docs.djangoproject.com/en/2.0/ref/models/fields/#choices
    "you’re probably better off using a proper database table with a ForeignKey"

    You can find the derived AbstractChoice classes being used as ForeignKey in other models, thus enforcing data
    consistency at database level.
    """
    value = CharField(max_length=128, editable=True, null=False, blank=False, unique=True, primary_key=True)

    class Meta:
        abstract = True

    def __str__(self):
        return self.value


class Template(NamedCommon):
    version = IntegerField(editable=False, null=False, help_text='Version of this template (with respect to other templates of the same name)')
    schema = JSONField(help_text='Schema for the configurable parameters needed to use this template.')

    class Meta:
        abstract = True
        constraints = [UniqueConstraint(fields=['name', 'version'], name='%(class)s_unique_name_version')]

    def get_default_json_document_for_schema(self) -> dict:
        '''get a json document object (dict) which complies with this template's schema and with all the defaults filled in.'''
        return get_default_json_object_for_schema(self.schema, cache=TemplateSchemaMixin._schema_cache, max_cache_age=TemplateSchemaMixin._MAX_SCHEMA_CACHE_AGE)

    def validate_and_annotate_schema(self):
        '''validate this template's schema, check for the required properties '$id', '$schema', 'title', 'description',
        and annotate this schema with the template's name, description and version.'''
        try:
            if isinstance(self.schema, str):
                self.schema = json.loads(self.schema)
        except json.JSONDecodeError as e:
            raise SchemaValidationException(str(e))

        # sync up the template properties with the schema
        self.schema['title'] = self.name
        self.schema['description'] = self.description
        self.schema['version'] = self.version

        # check for missing properties
        missing_properties = [property for property in ['$id', '$schema', 'title', 'description'] if property not in self.schema]
        if missing_properties:
            raise SchemaValidationException("Missing required properties '%s' for %s name='%s' version=%s in schema:\n%s" % (', '.join(missing_properties),
                                                                                                                             self.__class__.__name__, self.name, self.version,
                                                                                                                             json.dumps(self.schema, indent=2)))

        # check for valid url's
        invalid_url_properties = [property for property in ['$id', '$schema'] if not self.schema[property].startswith('http')]
        if invalid_url_properties:
            raise SchemaValidationException("Properties '%s' should contain a valid URL's for %s name='%s' version=%s in schema:\n%s" % (', '.join(invalid_url_properties),
                                                                                                                                         self.__class__.__name__, self.name, self.version,
                                                                                                                                         json.dumps(self.schema, indent=2)))

        try:
            # construct full url for $id of this schema
            path = reverse_url('get_template_json_schema', kwargs={'template': self._meta.model_name,
                                                                  'name': self.name,
                                                                  'version': self.version}).rstrip('/')
            parts = self.schema['$id'].split('/')
            scheme_host = '%s//%s' % (parts[0], parts[2])
            id_url = '%s%s#' % (scheme_host, path)
            self.schema['$id'] = id_url
        except Exception as e:
            logger.error("Could not override schema $id with auto-generated url: %s", e)

        # this template's schema has a schema of its own (usually the draft-06 meta schema). Validate it.
        validate_json_against_its_schema(self.schema)

    def validate_document(self, json_doc: typing.Union[str, dict]) -> bool:
        '''validate the given json_doc against the template's schema
           If no exception if thrown, then the given json_string validates against the given schema.
           :raises SchemaValidationException if the json_string does not validate against the schema '''
        validate_json_against_schema(json_doc, self.schema, cache=TemplateSchemaMixin._schema_cache, max_cache_age=TemplateSchemaMixin._MAX_SCHEMA_CACHE_AGE)


    @property
    def is_used(self) -> bool:
        '''Is this template used by any of its related objects?'''
        for rel_obj in self._meta.related_objects:
            if rel_obj.related_model.objects.filter(**{rel_obj.field.attname: self}).count() > 0:
                return True
        return False

    def auto_set_version_number(self):
        '''A template cannot/shouldnot be updated if it is already being used.
        So, update the version number if the template is already used, else keep it.'''
        if self.pk is None:
            # this is a new instance. auto-assign new unique version number
            self.version = self.__class__.objects.filter(name=self.name).count() + 1
        else:
            # this is a known template. Check if it is being used.
            if self.is_used:
                # yes, this template is used by others, so "editing"/updating is forbidden,
                # so create new instance (by setting pk=None) and assign new unique version number
                self.pk = None
                self.version = self.__class__.objects.filter(name=self.name).count() + 1

    def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
        self.auto_set_version_number()
        self.validate_and_annotate_schema()
        super().save(force_insert or self.pk is None, force_update, using, update_fields)


# concrete models

class Tags(Model):
    # todo: figure out how to keep this in sync with tags columns (->BasicCommon)
    # todo: Or remove this altogether without keeping track of tags?
    title = CharField(max_length=128)
    description = CharField(max_length=255)


class TemplateSchemaMixin():
    '''The TemplateSchemaMixin class can be mixed in to models which do validate and add defaults to json documents given a json-schema.
    It uses an internal cache with a max age to minimize the number of requests to schema's, subschema's or referenced (sub)schema's.'''
    _schema_cache = {}
    _MAX_SCHEMA_CACHE_AGE = timedelta(minutes=1)

    def validate_json_against_this_templates_schema(self, document_attr:str, template_attr:str) -> None:
        '''
        annotate, validate and add defaults to the JSON document in the model instance using the schema of the given template.
        '''
        try:
            # fetch the actual JSON document and template-model-instance
            document = getattr(self, document_attr)
            template = getattr(self, template_attr)

            if document is not None and template is not None:
                if isinstance(document, str):
                    document = json.loads(document)

                validate_json_against_schema(document, template.schema, cache=self._schema_cache, max_cache_age=self._MAX_SCHEMA_CACHE_AGE)
        except AttributeError:
            pass
        except json.JSONDecodeError as e:
            raise SchemaValidationException("Invalid JSON.\nError: %s \ndata: %s" % (str(e), document))
        except jsonschema.ValidationError as e:
            raise SchemaValidationException(str(e))

    def annotate_validate_add_defaults_to_doc_using_template(self, document_attr:str, template_attr:str) -> None:
        '''
        annotate, validate and add defaults to the JSON document in the model instance using the schema of the given template.
        '''
        try:
            # fetch the actual JSON document and template-model-instance
            document = getattr(self, document_attr)
            template = getattr(self, template_attr)

            if document is not None and template is not None:
                try:
                    if isinstance(document, str):
                        document = json.loads(document)

                    # always annotate the json data document with a $schema URI to the schema that it is based on.
                    # this enables all users using this document (inside or outside of TMSS) to do their own validation and usage of editors which use the schema as UI template
                    document['$schema'] = template.schema['$id']
                except (KeyError, TypeError, AttributeError) as e:
                    raise SchemaValidationException("Cannot set $schema in json_doc to the schema's $id.\nError: %s \njson_doc: %s\nschema: %s" % (str(e), document, template.schema))

                # add defaults for missing properies, and validate on the fly
                # use the class's _schema_cache
                document = add_defaults_to_json_object_for_schema(document, template.schema, cache=self._schema_cache, max_cache_age=self._MAX_SCHEMA_CACHE_AGE)

            # update the model instance with the updated and validated document
            setattr(self, document_attr, document)
        except AttributeError:
            pass
        except json.JSONDecodeError as e:
            raise SchemaValidationException("Invalid JSON.\nError: %s \ndata: %s" % (str(e), document))
        except jsonschema.ValidationError as e:
            raise SchemaValidationException(str(e))