diff --git a/SAS/TMSS/src/tmss/tmssapp/models/specification.py b/SAS/TMSS/src/tmss/tmssapp/models/specification.py
index 36368a34a8be2b928bde7ddedc2ad543306a90e7..ec983056bd0a5e74a26e2b3b8014d855b4072f73 100644
--- a/SAS/TMSS/src/tmss/tmssapp/models/specification.py
+++ b/SAS/TMSS/src/tmss/tmssapp/models/specification.py
@@ -13,6 +13,7 @@ from django.core.exceptions import ValidationError
from django.urls import reverse as reverse_path
from rest_framework import status
import datetime
+import json
#
# Common
@@ -242,7 +243,7 @@ class Template(NamedCommon):
tail = hash+tail
# construct the common json schema path for this ref
- schema_path = reverse_path('get_common_json_schema', kwargs={'name': schema_name, 'version': schema_version})
+ schema_path = reverse_path('common_json_schema', kwargs={'name': schema_name, 'version': schema_version})
# and construct the proper ref url
updated_schema[key] = base_url + schema_path + tail
@@ -263,6 +264,44 @@ class Template(NamedCommon):
return schema
+ @staticmethod
+ def resolved_refs(schema):
+ '''return the given schema with all $ref fields updated so they point to the given base_url'''
+ if isinstance(schema, dict):
+ updated_schema = {}
+ for key, value in schema.items():
+ if key == "$ref" and isinstance(value, str):
+ if value.startswith('#'):
+ # reference to local document, no need for http injection
+ updated_schema[key] = value
+ else:
+ try:
+ # deduct referred schema name and version from ref-value
+ head, hash, tail = value.partition('#')
+ import requests
+ reffered_schema = json.loads(requests.get(value).text)
+ parts = tail.strip('/').split('/')
+ for part in parts:
+ reffered_schema = reffered_schema[part]
+
+ return reffered_schema
+ except:
+ # aparently the reference is not conform the expected lofar common json schema path...
+ # so, just accept the original value and assume that the user uploaded a proper schema
+ updated_schema[key] = value
+ elif isinstance(value, dict):
+ updated_schema[key] = Template.resolved_refs(value)
+ elif isinstance(value, list):
+ updated_schema[key] = [Template.resolved_refs(item) for item in value]
+ else:
+ updated_schema[key] = value
+ return updated_schema
+
+ if isinstance(schema, list):
+ return [Template.resolved_refs(item) for item in schema]
+
+ return schema
+
def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
self.schema = Template.update_tmss_common_json_schema_refs(self.schema)
super().save(force_insert, force_update, using, update_fields)
diff --git a/SAS/TMSS/src/tmss/tmssapp/populate.py b/SAS/TMSS/src/tmss/tmssapp/populate.py
index 51f46faaa1ee73cf57151de089e63779412ba8f8..624464911d6c87b49f26dd24ac43a3562b35f1d7 100644
--- a/SAS/TMSS/src/tmss/tmssapp/populate.py
+++ b/SAS/TMSS/src/tmss/tmssapp/populate.py
@@ -23,7 +23,7 @@ from datetime import datetime, timezone
from lofar.sas.tmss.tmss.tmssapp import models
from lofar.sas.tmss.tmss.tmssapp.models.specification import *
from lofar.sas.tmss.tmss.tmssapp.models.scheduling import *
-from lofar.sas.tmss.tmss.tmssapp.validation import get_default_json_object_for_schema
+from lofar.sas.tmss.tmss.tmssapp.validation import get_default_json_object_for_schema, validate_json_against_its_schema
from lofar.common import isTestEnvironment, isDevelopmentEnvironment
working_dir = os.path.dirname(os.path.abspath(__file__))
@@ -45,22 +45,79 @@ def populate_settings(apps, schema_editor):
def populate_lofar_json_schemas(apps, schema_editor):
- _populate_base_schema()
- _populate_scheduling_unit_schema()
- # populate task schema's
- _populate_preprocessing_schema()
- _populate_observation_with_stations_schema()
- _populate_calibrator_addon_schema()
-
- _populate_dataproduct_specifications_templates()
- _populate_taskrelation_selection_templates()
- _populate_dataproduct_feedback_templates()
- _populate_obscontrol_schema()
- _populate_pipelinecontrol_schema()
- _populate_connectors()
-
- _populate_qa_files_subtask_template()
- _populate_qa_plots_subtask_template()
+ models.CommonSchemaTemplate.objects.create(name="base",
+ description='email address schema',
+ version='1',
+ schema={"$id": "http://127.0.0.1:8000/api/schemas/common/base/1/#",
+ "$schema": "http://json-schema.org/draft-06/schema#",
+ "definitions": {
+ "email" : {
+ "type": "string",
+ "format": "email",
+ "pattern": "@example\\.com$",
+ "default": "" }
+ } } )
+
+ models.SubtaskTemplate.objects.create(name="user_with_ref",
+ description='user schema',
+ version='1',
+ type=SubtaskType.objects.get(value=SubtaskType.Choices.OTHER.value),
+ schema={"$id": "http://127.0.0.1:8000/api/schemas/subtask_template/user_with_ref/1/#",
+ "$schema": "http://json-schema.org/draft-06/schema#",
+ "type": "object",
+ "default": {},
+ "properties": {
+ "name": {
+ "type": "string",
+ "minLength": 2,
+ "default": ""
+ },
+ "email": {
+ "$ref": "http://127.0.0.1:8000/api/schemas/common/base/1/#/definitions/email"
+ }
+ },
+ "required": ["name", "email"],
+ "additionalProperties": False
+ } )
+
+ models.SubtaskTemplate.objects.create(name="user_without_ref",
+ description='user schema',
+ version='1',
+ type=SubtaskType.objects.get(value=SubtaskType.Choices.OTHER.value),
+ schema={"$id": "http://127.0.0.1:8000/api/schemas/subtask_template/user_without_ref/1/#",
+ "$schema": "http://json-schema.org/draft-06/schema#",
+ "type": "object",
+ "properties": {
+ "name": {
+ "type": "string",
+ "minLength": 2
+ },
+ "email": {
+ "type": "string",
+ "format": "email",
+ "pattern": "@example\\.com$"
+ }
+ },
+ "required": ["name", "email"],
+ "additionalProperties": True
+ } )
+
+ # _populate_base_schema()
+ # _populate_scheduling_unit_schema()
+ # # populate task schema's
+ # _populate_preprocessing_schema()
+ # _populate_observation_with_stations_schema()
+ # _populate_calibrator_addon_schema()
+ #
+ # _populate_dataproduct_specifications_templates()
+ # _populate_taskrelation_selection_templates()
+ # _populate_dataproduct_feedback_templates()
+ # _populate_obscontrol_schema()
+ # _populate_pipelinecontrol_schema()
+ # _populate_connectors()
+ #
+ # _populate_qa_files_subtask_template()
+ # _populate_qa_plots_subtask_template()
def populate_test_data():
@@ -69,6 +126,7 @@ def populate_test_data():
scheduling unit json
:return:
"""
+ return #TODO: remove
try:
# only add (with expensive setup time) example data when developing/testing and we're not unittesting
if isTestEnvironment() or isDevelopmentEnvironment():
diff --git a/SAS/TMSS/src/tmss/tmssapp/serializers/scheduling.py b/SAS/TMSS/src/tmss/tmssapp/serializers/scheduling.py
index da872b18f0985a99ddde4664be1c51248a67f438..10275de12d8c5bafa6cb4cde41c7299ffef0cee7 100644
--- a/SAS/TMSS/src/tmss/tmssapp/serializers/scheduling.py
+++ b/SAS/TMSS/src/tmss/tmssapp/serializers/scheduling.py
@@ -80,7 +80,7 @@ class DataproductFeedbackTemplateSerializer(AbstractTemplateSerializer):
class SubtaskSerializer(RelationalHyperlinkedModelSerializer):
# If this is OK then we can extend API with NO url ('flat' values) on more places if required
cluster_value = serializers.StringRelatedField(source='cluster', label='cluster_value', read_only=True)
- specifications_doc = JSONEditorField()
+ specifications_doc = JSONEditorField('specifications_template')
class Meta:
model = models.Subtask
diff --git a/SAS/TMSS/src/tmss/tmssapp/serializers/specification.py b/SAS/TMSS/src/tmss/tmssapp/serializers/specification.py
index 4dd4e15540022da6f68e7022f61b2b957a0b3464..bcb1bf786ad75d285a3b2d62d629b2b7d0491c54 100644
--- a/SAS/TMSS/src/tmss/tmssapp/serializers/specification.py
+++ b/SAS/TMSS/src/tmss/tmssapp/serializers/specification.py
@@ -4,7 +4,7 @@ This file contains the serializers (for the elsewhere defined data models)
from rest_framework import serializers
from .. import models
-from .widgets import JSONEditorField, JSONEditorSchemaField
+from .widgets import JSONEditorField
from django.contrib.auth.models import User
from django.core.exceptions import ImproperlyConfigured
from rest_framework import decorators
@@ -91,7 +91,7 @@ class TagsSerializer(RelationalHyperlinkedModelSerializer):
class AbstractTemplateSerializer(RelationalHyperlinkedModelSerializer):
- schema = JSONEditorSchemaField()
+ schema = JSONEditorField()
class Meta:
abstract = True
diff --git a/SAS/TMSS/src/tmss/tmssapp/serializers/widgets.py b/SAS/TMSS/src/tmss/tmssapp/serializers/widgets.py
index 4aeaf1c6f9162e21aff11f506ef72d8ccde17a64..4294cd106ece24b054d924a1f697dfd30e173f7d 100644
--- a/SAS/TMSS/src/tmss/tmssapp/serializers/widgets.py
+++ b/SAS/TMSS/src/tmss/tmssapp/serializers/widgets.py
@@ -3,6 +3,8 @@ This file contains customized UI elements for use in the viewsets (based on the
"""
from rest_framework import serializers
import requests
+import re
+import json
from lofar.sas.tmss.tmss.tmssapp import models
@@ -10,28 +12,46 @@ class JSONEditorField(serializers.JSONField):
"""
An enhanced JSONField that provides a nice editor widget with validation against the $schema in the json field value.
"""
- def to_representation(self, value):
- '''same as super().to_representation, but with the josdejong_jsoneditor_widget.html injected in the render style based on the requests accepted_media_type'''
- if self.context['request'].accepted_media_type == "text/html":
- schema_url = value.get('$schema', 'http://json-schema.org/draft-06/schema#')
- schema = requests.get(schema_url).text
- self.style = {'template': 'josdejong_jsoneditor_widget.html',
- 'schema': schema}
- else:
- self.style = {}
-
- return super().to_representation(value)
+ def __init__(self, schema_source: str=None, *args, **kwargs):
+ self.schema_source = schema_source
+ super().__init__(*args, **kwargs)
-class JSONEditorSchemaField(JSONEditorField):
- """
- An enhanced JSONfield that provides a nice editor widget with validation against the $schema in the json field value.
- Checks all $ref fields in the json value, and makes sure they point to the correct tmms host.
- """
def to_representation(self, value):
'''create representation of the json-schema-value,
with all common json schema refs pointing to the correct host,
and inject the josdejong_jsoneditor_widget.html in the render style based on the requests accepted_media_type'''
+ self.style = {}
+
base_url = "%s://%s" % (self.context['request'].scheme, self.context['request'].get_host())
- updated_value = models.Template.update_tmss_common_json_schema_refs(value, base_url)
- return super().to_representation(updated_value)
+
+ if not isinstance(self.parent.instance, list):
+ try:
+ template = getattr(self.parent.instance, self.schema_source)
+ template_model_name = re.sub(r'(?<!^)(?=[A-Z])', '_', template.__class__.__name__).lower()
+ schema_uri = ("%s/api/schemas/%s/%s/%s#" % (base_url,
+ template_model_name,
+ template.name,
+ template.version)).replace(' ', '%20')
+
+ value['$schema'] = schema_uri
+
+ schema = models.Template.update_tmss_common_json_schema_refs(template.schema, base_url)
+ schema = json.dumps(schema, indent=2)
+ except (AttributeError, TypeError):
+ try:
+ schema_url = value['$schema']
+ schema = requests.get(schema_url).text
+ except KeyError:
+ schema = None
+
+ # the josdejong_jsoneditor_widget cannot resolve absolute URL's in the schema
+ # although this should be possible according to the JSON schema standard.
+ # so, let's do the resolving here and feed the resolved schema to the josdejong_jsoneditor_widget
+ schema = json.dumps(models.Template.resolved_refs(json.loads(schema)))
+
+ self.style = {'template': 'josdejong_jsoneditor_widget.html',
+ 'schema': schema}
+
+ value = models.Template.update_tmss_common_json_schema_refs(value, base_url)
+ return super().to_representation(value)
diff --git a/SAS/TMSS/src/tmss/tmssapp/views.py b/SAS/TMSS/src/tmss/tmssapp/views.py
index 7ff34651a69a821cfe8c92fdbbac658b60b6f329..8bd5b7514f9930a8f7fd3ec33cdb129999a7cad4 100644
--- a/SAS/TMSS/src/tmss/tmssapp/views.py
+++ b/SAS/TMSS/src/tmss/tmssapp/views.py
@@ -42,6 +42,21 @@ def task_specify_observation(request, pk=None):
operation_description="Get the JSON schema with the given <name> and <version> as application/json content response.")
def get_common_json_schema(request, name:str, version:str):
template = get_object_or_404(models.CommonSchemaTemplate, name=name, version=version)
- return JsonResponse(template.schema)
+ schema = template.schema
+ schema['$id'] = request.get_raw_uri()
+ return JsonResponse(schema, json_dumps_params={"indent":2})
+
+
+@permission_classes([AllowAny])
+@authentication_classes([AllowAny])
+@swagger_auto_schema(responses={200: 'JSON schema',
+ 404: 'the schema with requested <name> and <version> is not available'},
+ operation_description="Get the JSON schema with the given <name> and <version> as application/json content response.")
+def get_subtask_template_json_schema(request, name:str, version:str):
+ template = get_object_or_404(models.SubtaskTemplate, name=name, version=version)
+ schema = template.schema
+ schema['$id'] = request.get_raw_uri()
+ return JsonResponse(schema, json_dumps_params={"indent":2})
+
diff --git a/SAS/TMSS/src/tmss/tmssapp/viewsets/scheduling.py b/SAS/TMSS/src/tmss/tmssapp/viewsets/scheduling.py
index 3b1b00922cb89377ba0bb9a3d32eebc482b97363..14eac2af4afad9a0a1db3bd2a37f5c8b34e625f9 100644
--- a/SAS/TMSS/src/tmss/tmssapp/viewsets/scheduling.py
+++ b/SAS/TMSS/src/tmss/tmssapp/viewsets/scheduling.py
@@ -111,6 +111,17 @@ class SubtaskTemplateViewSet(LOFARViewSet):
subtask_template = get_object_or_404(models.SubtaskTemplate, pk=pk)
return JsonResponse(subtask_template.schema)
+ @swagger_auto_schema(responses={200: 'The schema as a JSON object',
+ 403: 'forbidden'},
+ operation_description="Get the schema as a JSON object.")
+ @action(methods=['get'], detail=True)
+ def resolved_schema(self, request, pk=None):
+ template = get_object_or_404(models.SubtaskTemplate, pk=pk)
+ base_url = "%s://%s" % (request.scheme, request.get_host())
+ schema = models.Template.update_tmss_common_json_schema_refs(template.schema, base_url)
+ schema = models.Template.flatten_schema(schema)
+ return JsonResponse(schema)
+
@swagger_auto_schema(responses={200: 'JSON object with all the defaults from the schema filled in',
403: 'forbidden'},
operation_description="Get a JSON object with all the defaults from the schema filled in.")
diff --git a/SAS/TMSS/src/tmss/tmssapp/viewsets/specification.py b/SAS/TMSS/src/tmss/tmssapp/viewsets/specification.py
index 7fcd0731eebeec300bb11bf3af357c74bb29cd7b..822efa47080acf062bba93354b3a024fa0a14fd9 100644
--- a/SAS/TMSS/src/tmss/tmssapp/viewsets/specification.py
+++ b/SAS/TMSS/src/tmss/tmssapp/viewsets/specification.py
@@ -45,6 +45,26 @@ class CommonSchemaTemplateViewSet(LOFARViewSet):
queryset = models.CommonSchemaTemplate.objects.all()
serializer_class = serializers.CommonSchemaTemplateSerializer
+ @swagger_auto_schema(responses={200: 'The schema as a JSON object',
+ 403: 'forbidden'},
+ operation_description="Get the schema as a JSON object.")
+ @action(methods=['get'], detail=True)
+ def schema(self, request, pk=None):
+ template = get_object_or_404(models.CommonSchemaTemplate, pk=pk)
+ base_url = "%s://%s" % (request.scheme, request.get_host())
+ schema = models.Template.update_tmss_common_json_schema_refs(template.schema, base_url)
+ return JsonResponse(schema)
+
+ @swagger_auto_schema(responses={200: 'The schema as a JSON object',
+ 403: 'forbidden'},
+ operation_description="Get the schema as a JSON object.")
+ @action(methods=['get'], detail=True)
+ def resolved_schema(self, request, pk=None):
+ template = get_object_or_404(models.CommonSchemaTemplate, pk=pk)
+ base_url = "%s://%s" % (request.scheme, request.get_host())
+ schema = models.Template.update_tmss_common_json_schema_refs(template.schema, base_url)
+ schema = models.Template.flatten_schema(schema)
+ return JsonResponse(schema)
class GeneratorTemplateViewSet(LOFARViewSet):
queryset = models.GeneratorTemplate.objects.all()
diff --git a/SAS/TMSS/src/tmss/urls.py b/SAS/TMSS/src/tmss/urls.py
index a0c065425bd5e32255b07068c1334016a098132e..52bbe7db2b48430700674fa063edd0af0233fb1f 100644
--- a/SAS/TMSS/src/tmss/urls.py
+++ b/SAS/TMSS/src/tmss/urls.py
@@ -55,7 +55,8 @@ urlpatterns = [
re_path(r'^swagger(?P<format>\.json|\.yaml)$', swagger_schema_view.without_ui(cache_timeout=0), name='schema-json'),
path('swagger/', swagger_schema_view.with_ui('swagger', cache_timeout=0), name='schema-swagger-ui'),
path('redoc/', swagger_schema_view.with_ui('redoc', cache_timeout=0), name='schema-redoc'),
- path('schemas/<str:name>/<str:version>/', views.get_common_json_schema, name='get_common_json_schema')
+ path('schemas/common/<str:name>/<str:version>/', views.get_common_json_schema, name='common_json_schema'),
+ path('schemas/subtask_template/<str:name>/<str:version>/', views.get_subtask_template_json_schema, name='subtask_template_json_schema')
]