Commit 0afea0a8 authored by Jorrit Schaap's avatar Jorrit Schaap

TMSS-272: made working example with reusable schemas. Validation works both...

TMSS-272: made working example with reusable schemas. Validation works both server-side and client-side in the json-editor widget. For the json-editor widget we need to resolve absolute URL refs as a workaround
parent 11f06539
......@@ -13,6 +13,7 @@ from django.core.exceptions import ValidationError
from django.urls import reverse as reverse_path
from rest_framework import status
import datetime
import json
#
# Common
......@@ -242,7 +243,7 @@ class Template(NamedCommon):
tail = hash+tail
# construct the common json schema path for this ref
schema_path = reverse_path('get_common_json_schema', kwargs={'name': schema_name, 'version': schema_version})
schema_path = reverse_path('common_json_schema', kwargs={'name': schema_name, 'version': schema_version})
# and construct the proper ref url
updated_schema[key] = base_url + schema_path + tail
......@@ -263,6 +264,44 @@ class Template(NamedCommon):
return schema
@staticmethod
def resolved_refs(schema):
'''return the given schema with all $ref fields updated so they point to the given base_url'''
if isinstance(schema, dict):
updated_schema = {}
for key, value in schema.items():
if key == "$ref" and isinstance(value, str):
if value.startswith('#'):
# reference to local document, no need for http injection
updated_schema[key] = value
else:
try:
# deduct referred schema name and version from ref-value
head, hash, tail = value.partition('#')
import requests
reffered_schema = json.loads(requests.get(value).text)
parts = tail.strip('/').split('/')
for part in parts:
reffered_schema = reffered_schema[part]
return reffered_schema
except:
# aparently the reference is not conform the expected lofar common json schema path...
# so, just accept the original value and assume that the user uploaded a proper schema
updated_schema[key] = value
elif isinstance(value, dict):
updated_schema[key] = Template.resolved_refs(value)
elif isinstance(value, list):
updated_schema[key] = [Template.resolved_refs(item) for item in value]
else:
updated_schema[key] = value
return updated_schema
if isinstance(schema, list):
return [Template.resolved_refs(item) for item in schema]
return schema
def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
self.schema = Template.update_tmss_common_json_schema_refs(self.schema)
super().save(force_insert, force_update, using, update_fields)
......
......@@ -23,7 +23,7 @@ from datetime import datetime, timezone
from lofar.sas.tmss.tmss.tmssapp import models
from lofar.sas.tmss.tmss.tmssapp.models.specification import *
from lofar.sas.tmss.tmss.tmssapp.models.scheduling import *
from lofar.sas.tmss.tmss.tmssapp.validation import get_default_json_object_for_schema
from lofar.sas.tmss.tmss.tmssapp.validation import get_default_json_object_for_schema, validate_json_against_its_schema
from lofar.common import isTestEnvironment, isDevelopmentEnvironment
working_dir = os.path.dirname(os.path.abspath(__file__))
......@@ -45,22 +45,79 @@ def populate_settings(apps, schema_editor):
def populate_lofar_json_schemas(apps, schema_editor):
_populate_base_schema()
_populate_scheduling_unit_schema()
# populate task schema's
_populate_preprocessing_schema()
_populate_observation_with_stations_schema()
_populate_calibrator_addon_schema()
_populate_dataproduct_specifications_templates()
_populate_taskrelation_selection_templates()
_populate_dataproduct_feedback_templates()
_populate_obscontrol_schema()
_populate_pipelinecontrol_schema()
_populate_connectors()
_populate_qa_files_subtask_template()
_populate_qa_plots_subtask_template()
models.CommonSchemaTemplate.objects.create(name="base",
description='email address schema',
version='1',
schema={"$id": "http://127.0.0.1:8000/api/schemas/common/base/1/#",
"$schema": "http://json-schema.org/draft-06/schema#",
"definitions": {
"email" : {
"type": "string",
"format": "email",
"pattern": "@example\\.com$",
"default": "" }
} } )
models.SubtaskTemplate.objects.create(name="user_with_ref",
description='user schema',
version='1',
type=SubtaskType.objects.get(value=SubtaskType.Choices.OTHER.value),
schema={"$id": "http://127.0.0.1:8000/api/schemas/subtask_template/user_with_ref/1/#",
"$schema": "http://json-schema.org/draft-06/schema#",
"type": "object",
"default": {},
"properties": {
"name": {
"type": "string",
"minLength": 2,
"default": ""
},
"email": {
"$ref": "http://127.0.0.1:8000/api/schemas/common/base/1/#/definitions/email"
}
},
"required": ["name", "email"],
"additionalProperties": False
} )
models.SubtaskTemplate.objects.create(name="user_without_ref",
description='user schema',
version='1',
type=SubtaskType.objects.get(value=SubtaskType.Choices.OTHER.value),
schema={"$id": "http://127.0.0.1:8000/api/schemas/subtask_template/user_without_ref/1/#",
"$schema": "http://json-schema.org/draft-06/schema#",
"type": "object",
"properties": {
"name": {
"type": "string",
"minLength": 2
},
"email": {
"type": "string",
"format": "email",
"pattern": "@example\\.com$"
}
},
"required": ["name", "email"],
"additionalProperties": True
} )
# _populate_base_schema()
# _populate_scheduling_unit_schema()
# # populate task schema's
# _populate_preprocessing_schema()
# _populate_observation_with_stations_schema()
# _populate_calibrator_addon_schema()
#
# _populate_dataproduct_specifications_templates()
# _populate_taskrelation_selection_templates()
# _populate_dataproduct_feedback_templates()
# _populate_obscontrol_schema()
# _populate_pipelinecontrol_schema()
# _populate_connectors()
#
# _populate_qa_files_subtask_template()
# _populate_qa_plots_subtask_template()
def populate_test_data():
......@@ -69,6 +126,7 @@ def populate_test_data():
scheduling unit json
:return:
"""
return #TODO: remove
try:
# only add (with expensive setup time) example data when developing/testing and we're not unittesting
if isTestEnvironment() or isDevelopmentEnvironment():
......
......@@ -80,7 +80,7 @@ class DataproductFeedbackTemplateSerializer(AbstractTemplateSerializer):
class SubtaskSerializer(RelationalHyperlinkedModelSerializer):
# If this is OK then we can extend API with NO url ('flat' values) on more places if required
cluster_value = serializers.StringRelatedField(source='cluster', label='cluster_value', read_only=True)
specifications_doc = JSONEditorField()
specifications_doc = JSONEditorField('specifications_template')
class Meta:
model = models.Subtask
......
......@@ -4,7 +4,7 @@ This file contains the serializers (for the elsewhere defined data models)
from rest_framework import serializers
from .. import models
from .widgets import JSONEditorField, JSONEditorSchemaField
from .widgets import JSONEditorField
from django.contrib.auth.models import User
from django.core.exceptions import ImproperlyConfigured
from rest_framework import decorators
......@@ -91,7 +91,7 @@ class TagsSerializer(RelationalHyperlinkedModelSerializer):
class AbstractTemplateSerializer(RelationalHyperlinkedModelSerializer):
schema = JSONEditorSchemaField()
schema = JSONEditorField()
class Meta:
abstract = True
......
......@@ -3,6 +3,8 @@ This file contains customized UI elements for use in the viewsets (based on the
"""
from rest_framework import serializers
import requests
import re
import json
from lofar.sas.tmss.tmss.tmssapp import models
......@@ -10,28 +12,46 @@ class JSONEditorField(serializers.JSONField):
"""
An enhanced JSONField that provides a nice editor widget with validation against the $schema in the json field value.
"""
def to_representation(self, value):
'''same as super().to_representation, but with the josdejong_jsoneditor_widget.html injected in the render style based on the requests accepted_media_type'''
if self.context['request'].accepted_media_type == "text/html":
schema_url = value.get('$schema', 'http://json-schema.org/draft-06/schema#')
schema = requests.get(schema_url).text
self.style = {'template': 'josdejong_jsoneditor_widget.html',
'schema': schema}
else:
self.style = {}
return super().to_representation(value)
def __init__(self, schema_source: str=None, *args, **kwargs):
self.schema_source = schema_source
super().__init__(*args, **kwargs)
class JSONEditorSchemaField(JSONEditorField):
"""
An enhanced JSONfield that provides a nice editor widget with validation against the $schema in the json field value.
Checks all $ref fields in the json value, and makes sure they point to the correct tmms host.
"""
def to_representation(self, value):
'''create representation of the json-schema-value,
with all common json schema refs pointing to the correct host,
and inject the josdejong_jsoneditor_widget.html in the render style based on the requests accepted_media_type'''
self.style = {}
base_url = "%s://%s" % (self.context['request'].scheme, self.context['request'].get_host())
updated_value = models.Template.update_tmss_common_json_schema_refs(value, base_url)
return super().to_representation(updated_value)
if not isinstance(self.parent.instance, list):
try:
template = getattr(self.parent.instance, self.schema_source)
template_model_name = re.sub(r'(?<!^)(?=[A-Z])', '_', template.__class__.__name__).lower()
schema_uri = ("%s/api/schemas/%s/%s/%s#" % (base_url,
template_model_name,
template.name,
template.version)).replace(' ', '%20')
value['$schema'] = schema_uri
schema = models.Template.update_tmss_common_json_schema_refs(template.schema, base_url)
schema = json.dumps(schema, indent=2)
except (AttributeError, TypeError):
try:
schema_url = value['$schema']
schema = requests.get(schema_url).text
except KeyError:
schema = None
# the josdejong_jsoneditor_widget cannot resolve absolute URL's in the schema
# although this should be possible according to the JSON schema standard.
# so, let's do the resolving here and feed the resolved schema to the josdejong_jsoneditor_widget
schema = json.dumps(models.Template.resolved_refs(json.loads(schema)))
self.style = {'template': 'josdejong_jsoneditor_widget.html',
'schema': schema}
value = models.Template.update_tmss_common_json_schema_refs(value, base_url)
return super().to_representation(value)
......@@ -42,6 +42,21 @@ def task_specify_observation(request, pk=None):
operation_description="Get the JSON schema with the given <name> and <version> as application/json content response.")
def get_common_json_schema(request, name:str, version:str):
template = get_object_or_404(models.CommonSchemaTemplate, name=name, version=version)
return JsonResponse(template.schema)
schema = template.schema
schema['$id'] = request.get_raw_uri()
return JsonResponse(schema, json_dumps_params={"indent":2})
@permission_classes([AllowAny])
@authentication_classes([AllowAny])
@swagger_auto_schema(responses={200: 'JSON schema',
404: 'the schema with requested <name> and <version> is not available'},
operation_description="Get the JSON schema with the given <name> and <version> as application/json content response.")
def get_subtask_template_json_schema(request, name:str, version:str):
template = get_object_or_404(models.SubtaskTemplate, name=name, version=version)
schema = template.schema
schema['$id'] = request.get_raw_uri()
return JsonResponse(schema, json_dumps_params={"indent":2})
......@@ -111,6 +111,17 @@ class SubtaskTemplateViewSet(LOFARViewSet):
subtask_template = get_object_or_404(models.SubtaskTemplate, pk=pk)
return JsonResponse(subtask_template.schema)
@swagger_auto_schema(responses={200: 'The schema as a JSON object',
403: 'forbidden'},
operation_description="Get the schema as a JSON object.")
@action(methods=['get'], detail=True)
def resolved_schema(self, request, pk=None):
template = get_object_or_404(models.SubtaskTemplate, pk=pk)
base_url = "%s://%s" % (request.scheme, request.get_host())
schema = models.Template.update_tmss_common_json_schema_refs(template.schema, base_url)
schema = models.Template.flatten_schema(schema)
return JsonResponse(schema)
@swagger_auto_schema(responses={200: 'JSON object with all the defaults from the schema filled in',
403: 'forbidden'},
operation_description="Get a JSON object with all the defaults from the schema filled in.")
......
......@@ -45,6 +45,26 @@ class CommonSchemaTemplateViewSet(LOFARViewSet):
queryset = models.CommonSchemaTemplate.objects.all()
serializer_class = serializers.CommonSchemaTemplateSerializer
@swagger_auto_schema(responses={200: 'The schema as a JSON object',
403: 'forbidden'},
operation_description="Get the schema as a JSON object.")
@action(methods=['get'], detail=True)
def schema(self, request, pk=None):
template = get_object_or_404(models.CommonSchemaTemplate, pk=pk)
base_url = "%s://%s" % (request.scheme, request.get_host())
schema = models.Template.update_tmss_common_json_schema_refs(template.schema, base_url)
return JsonResponse(schema)
@swagger_auto_schema(responses={200: 'The schema as a JSON object',
403: 'forbidden'},
operation_description="Get the schema as a JSON object.")
@action(methods=['get'], detail=True)
def resolved_schema(self, request, pk=None):
template = get_object_or_404(models.CommonSchemaTemplate, pk=pk)
base_url = "%s://%s" % (request.scheme, request.get_host())
schema = models.Template.update_tmss_common_json_schema_refs(template.schema, base_url)
schema = models.Template.flatten_schema(schema)
return JsonResponse(schema)
class GeneratorTemplateViewSet(LOFARViewSet):
queryset = models.GeneratorTemplate.objects.all()
......
......@@ -55,7 +55,8 @@ urlpatterns = [
re_path(r'^swagger(?P<format>\.json|\.yaml)$', swagger_schema_view.without_ui(cache_timeout=0), name='schema-json'),
path('swagger/', swagger_schema_view.with_ui('swagger', cache_timeout=0), name='schema-swagger-ui'),
path('redoc/', swagger_schema_view.with_ui('redoc', cache_timeout=0), name='schema-redoc'),
path('schemas/<str:name>/<str:version>/', views.get_common_json_schema, name='get_common_json_schema')
path('schemas/common/<str:name>/<str:version>/', views.get_common_json_schema, name='common_json_schema'),
path('schemas/subtask_template/<str:name>/<str:version>/', views.get_subtask_template_json_schema, name='subtask_template_json_schema')
]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment