From c333bbebf728b93ab9b6d938903d6e5d0b7ba0aa Mon Sep 17 00:00:00 2001
From: goei <JsXLRu>
Date: Thu, 23 Jul 2020 19:56:59 +0200
Subject: [PATCH] TMSS-272: Use RefResolver to use local refernce to common
part, pointing, of the tmss schema
---
LCS/PyCommon/json_utils.py | 84 +------------
SAS/TMSS/src/tmss/tmssapp/adapters/parset.py | 2 +-
SAS/TMSS/src/tmss/tmssapp/populate.py | 2 +-
SAS/TMSS/src/tmss/tmssapp/schemas/base.json | 8 +-
.../task-observation-with-stations.json | 4 +-
SAS/TMSS/src/tmss/tmssapp/subtasks.py | 4 +-
SAS/TMSS/src/tmss/tmssapp/validation.py | 115 +++++++++++++++++-
SAS/TMSS/src/tmss/tmssapp/views.py | 2 +-
.../src/tmss/tmssapp/viewsets/scheduling.py | 2 +-
.../tmss/tmssapp/viewsets/specification.py | 2 +-
SAS/TMSS/test/t_parset_adapter.py | 2 +-
SAS/TMSS/test/tmss_test_data_django_models.py | 2 +-
SAS/TMSS/test/tmss_test_data_rest.py | 2 +-
13 files changed, 131 insertions(+), 100 deletions(-)
diff --git a/LCS/PyCommon/json_utils.py b/LCS/PyCommon/json_utils.py
index 801e694c143..553b3ff1495 100644
--- a/LCS/PyCommon/json_utils.py
+++ b/LCS/PyCommon/json_utils.py
@@ -15,10 +15,9 @@
# You should have received a copy of the GNU General Public License along
# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-from jsonschema import validators, Draft6Validator, RefResolver
+from jsonschema import validators, Draft6Validator
from copy import deepcopy
-import os
-import json
+
def _extend_with_default(validator_class):
"""
@@ -78,89 +77,18 @@ def get_default_json_object_for_schema(schema: str) -> dict:
return add_defaults_to_json_object_for_schema({}, schema)
-def add_defaults_to_json_object_for_schema(json_object: dict, schema: str) -> dict:
- '''return a copy of the json object with defaults filled in accoring to the schema for all the missing properties'''
+def add_defaults_to_json_object_for_schema(json_object: dict, schema: str, ref_resolver=None) -> dict:
+ '''return a copy of the json object with defaults filled in according to the schema for all the missing properties'''
copy_of_json_object = deepcopy(json_object)
- if check_ref_in_schema(schema):
- resolver = create_local_ref_resolver(BASE_DIR + "/sas/tmss/tmss/tmssapp/schemas/", "base.json")
- _DefaultValidatingDraft6Validator(schema=schema, resolver=resolver).validate(copy_of_json_object)
+ if ref_resolver is not None:
+ _DefaultValidatingDraft6Validator(schema=schema, resolver=ref_resolver).validate(copy_of_json_object)
else:
_DefaultValidatingDraft6Validator(schema).validate(copy_of_json_object)
return copy_of_json_object
-def validate_only(json_object, schema):
- if check_ref_in_schema(schema):
- resolver = create_local_ref_resolver(BASE_DIR + "/sas/tmss/tmss/tmssapp/schemas/", "base.json")
- # hmm the validation will run the schema of the 'base' not the complete.....zucht
- print("validate with resolver")
- Draft6Validator(schema=schema, resolver=resolver).validate(json_object)
- else:
- Draft6Validator(schema=schema).validate(json_object)
-
-
-def item_generator(json_input, lookup_key):
- if isinstance(json_input, dict):
- for k, v in json_input.items():
- if k == lookup_key:
- yield v
- else:
- yield from item_generator(v, lookup_key)
- elif isinstance(json_input, list):
- for item in json_input:
- yield from item_generator(item, lookup_key)
-def check_ref_in_schema(schema):
- """
- Stupid lazy method for now to determine if there is a reference in the schema
- :param schema:
- :return:
- """
- # print("check")
- # if len(list(item_generator(schema, "$ref"))) > 0:
- # return True
- # else:
- # return False
- ref_to_base = False
- for ref in list(item_generator(schema, "$ref")):
- if "base.json" in ref:
- ref_to_base = True
- print("Is there a reference to base ", ref_to_base)
- return ref_to_base
-
-
-def create_local_ref_resolver(abs_path_to_schema, schema_file_name):
- """
- Create jsonschema.RefResolver class for all schemas located in given path.
- Apparently if you using jsonschema you have to set a reference resolver to referred schemas
- for local files:
- https://stackoverflow.com/questions/25145160/json-schema-ref-does-not-work-for-relative-path
- https://stackoverflow.com/questions/53968770/how-to-set-up-local-file-references-in-python-jsonschema-document
- https://stackoverflow.com/questions/42159346/jsonschema-refresolver-to-resolve-multiple-refs-in-python
- :param abs_path_to_schema: Absolute path to location of all schemas
- :return resolver_object: RefResolver object
- """
- schema_store = {}
- with open(os.path.join(abs_path_to_schema, schema_file_name), 'r') as fp:
- def_schema = json.load(fp)
-
- file_name = os.listdir(abs_path_to_schema)
- for file_name in file_name:
- file_full_path = os.path.join(abs_path_to_schema, file_name)
- if file_full_path.endswith(".json"):
- with open(file_full_path, "r") as fp:
- schema = json.load(fp)
- if "id" in schema:
- schema_store["$id"] = schema
- print(schema_store)
- resolver = RefResolver.from_schema(def_schema, store=schema_store)
- print("Created resolver with schema_store for %s" % schema_file_name)
- return resolver
-
-# First hack is to check if I can load the schema task-calibrator-addon.json which refs to definitions.json
-# /home/goei/lofar/build/gnucxx11_opt/installed/lib/python3.6/site-packages/lofar/sas/tmss/tmss/tmssapp/schemas/scheduling-unit.json
-BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
diff --git a/SAS/TMSS/src/tmss/tmssapp/adapters/parset.py b/SAS/TMSS/src/tmss/tmssapp/adapters/parset.py
index 64ae86d3cdb..533c8eab83c 100644
--- a/SAS/TMSS/src/tmss/tmssapp/adapters/parset.py
+++ b/SAS/TMSS/src/tmss/tmssapp/adapters/parset.py
@@ -20,7 +20,7 @@
from lofar.sas.tmss.tmss.tmssapp import models
from lofar.parameterset import parameterset
from lofar.common.datetimeutils import formatDatetime
-from lofar.common.json_utils import add_defaults_to_json_object_for_schema
+from lofar.sas.tmss.tmss.tmssapp.validation import add_defaults_to_json_object_for_schema
from lofar.sas.tmss.tmss.exceptions import *
from datetime import datetime
diff --git a/SAS/TMSS/src/tmss/tmssapp/populate.py b/SAS/TMSS/src/tmss/tmssapp/populate.py
index 951a60718bc..8d0a34bb992 100644
--- a/SAS/TMSS/src/tmss/tmssapp/populate.py
+++ b/SAS/TMSS/src/tmss/tmssapp/populate.py
@@ -23,7 +23,7 @@ from datetime import datetime, timezone
from lofar.sas.tmss.tmss.tmssapp import models
from lofar.sas.tmss.tmss.tmssapp.models.specification import *
from lofar.sas.tmss.tmss.tmssapp.models.scheduling import *
-from lofar.common.json_utils import *
+from lofar.sas.tmss.tmss.tmssapp.validation import get_default_json_object_for_schema
from lofar.common import isTestEnvironment, isDevelopmentEnvironment
working_dir = os.path.dirname(os.path.abspath(__file__))
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/base.json b/SAS/TMSS/src/tmss/tmssapp/schemas/base.json
index 4ea9040d1e6..30d593ef7ab 100644
--- a/SAS/TMSS/src/tmss/tmssapp/schemas/base.json
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/base.json
@@ -1,5 +1,5 @@
{
- "$id": "definitions.json",
+ "$id": "base.json",
"definitions": {
"pointing": {
"type": "object",
@@ -38,11 +38,7 @@
"description": "Second angle [rad] (e.g. DEC)",
"default": 22
}
- },
- "required": [
- "angle1",
- "angle2"
- ]
+ }
}
}
}
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/task-observation-with-stations.json b/SAS/TMSS/src/tmss/tmssapp/schemas/task-observation-with-stations.json
index 2d75067369c..9ca6c62edb5 100644
--- a/SAS/TMSS/src/tmss/tmssapp/schemas/task-observation-with-stations.json
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/task-observation-with-stations.json
@@ -155,7 +155,7 @@
"tile_beam": {
"title": "Tile beam",
"description": "HBA only",
- "$ref": "base.json#/definitions/pointing"
+ "$ref": "base.json#"
},
"SAPs": {
"type": "array",
@@ -179,7 +179,7 @@
"digital_pointing": {
"title": "Digital pointing",
"default": {},
- "$ref": "#/definitions/pointing"
+ "$ref": "base.json#/definitions/pointing"
},
"subbands": {
"type": "array",
diff --git a/SAS/TMSS/src/tmss/tmssapp/subtasks.py b/SAS/TMSS/src/tmss/tmssapp/subtasks.py
index 361de1d280d..2aa92edfa1a 100644
--- a/SAS/TMSS/src/tmss/tmssapp/subtasks.py
+++ b/SAS/TMSS/src/tmss/tmssapp/subtasks.py
@@ -6,14 +6,14 @@ from collections.abc import Iterable
from lofar.common.datetimeutils import formatDatetime
from lofar.common import isProductionEnvironment
-from lofar.common.json_utils import add_defaults_to_json_object_for_schema, get_default_json_object_for_schema
+from lofar.sas.tmss.tmss.tmssapp.validation import add_defaults_to_json_object_for_schema, get_default_json_object_for_schema
from lofar.common.lcu_utils import get_current_stations
from lofar.sas.tmss.tmss.exceptions import SubtaskCreationException, SubtaskSchedulingException
from datetime import datetime, timedelta
from lofar.common.datetimeutils import parseDatetime
-from lofar.common.json_utils import add_defaults_to_json_object_for_schema
+from lofar.sas.tmss.tmss.tmssapp.validation import add_defaults_to_json_object_for_schema
from lofar.sas.tmss.tmss.tmssapp.models import *
from lofar.sas.resourceassignment.resourceassigner.rarpc import RARPC
from lofar.sas.resourceassignment.resourceassignmentservice.rpc import RADBRPC
diff --git a/SAS/TMSS/src/tmss/tmssapp/validation.py b/SAS/TMSS/src/tmss/tmssapp/validation.py
index d2ba9dc573d..21785954c63 100644
--- a/SAS/TMSS/src/tmss/tmssapp/validation.py
+++ b/SAS/TMSS/src/tmss/tmssapp/validation.py
@@ -1,7 +1,15 @@
import json
import jsonschema
+from jsonschema import Draft6Validator, RefResolver
+import os
from lofar.sas.tmss.tmss.exceptions import *
-from lofar.common.json_utils import validate_only
+from lofar.common.json_utils import add_defaults_to_json_object_for_schema as json_utils_add_defaults_to_json
+
+
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+TMSS_SCHEMA_PATH = BASE_DIR + "/tmssapp/schemas/"
+TMSS_COMMON_BASE_SCHEMA = "base.json"
+# TODO: validation is not the correct name any more rename to schema.py or schema_utils.py ?
def validate_json_against_schema(json_string: str, schema: str):
'''validate the given json_string against the given schema.
@@ -28,8 +36,107 @@ def validate_json_against_schema(json_string: str, schema: str):
# now do the actual validation
try:
- jsonschema.validate(json_object, schema_object)
- # validate_only(json_object, schema_object)
- # should this be changed??
+ validate_json_object_with_schema(json_object, schema_object)
except jsonschema.ValidationError as e:
raise SchemaValidationException(str(e))
+
+
+def get_default_json_object_for_schema(schema: str) -> dict:
+ """
+ TMSS wrapper for TMSS 'add_defaults_to_json_object_for_schema'
+ :param schema:
+ :return: json_object with default values of the schema
+ """
+ return add_defaults_to_json_object_for_schema({}, schema)
+
+
+def add_defaults_to_json_object_for_schema(json_object: dict, schema: str) -> dict:
+ """
+ TMSS wrapper for 'json_utils.add_defaults_to_json_object_for_schema' to use RefResolver when needed
+ :param schema:
+ :return: json_object with default values of the schema added to the given json_object
+ """
+ if check_ref_in_schema(schema):
+ resolver = create_local_ref_resolver()
+ copy_of_json_object = json_utils_add_defaults_to_json(json_object, schema, resolver)
+ else:
+ copy_of_json_object = json_utils_add_defaults_to_json(json_object, schema)
+ return copy_of_json_object
+
+
+def validate_json_object_with_schema(json_object, schema):
+ """
+ Validate the given json_object with schema
+ When required use RefResolver
+ :param json_object:
+ :param schema:
+ :return:
+ """
+ if check_ref_in_schema(schema):
+ resolver = create_local_ref_resolver()
+ Draft6Validator(schema=schema, resolver=resolver).validate(json_object)
+ else:
+ Draft6Validator(schema=schema).validate(json_object)
+
+
+def item_generator(json_input, lookup_key):
+ """
+ A function to find a key in nested json structure
+ :param json_input:
+ :param lookup_key:
+ :return: values of the key found in json
+ """
+ if isinstance(json_input, dict):
+ for k, v in json_input.items():
+ if k == lookup_key:
+ yield v
+ else:
+ yield from item_generator(v, lookup_key)
+ elif isinstance(json_input, list):
+ for item in json_input:
+ yield from item_generator(item, lookup_key)
+
+
+def check_ref_in_schema(schema):
+ """
+ Check if there is a reference to the base schema (base.json) for the given schema
+ :param schema:
+ :return: True or False
+ """
+ ref_to_base = False
+ for ref in list(item_generator(schema, "$ref")):
+ if TMSS_COMMON_BASE_SCHEMA in ref:
+ ref_to_base = True
+ return ref_to_base
+
+
+def create_local_ref_resolver(abs_path_to_schema=TMSS_SCHEMA_PATH, base_schema_file_name=TMSS_COMMON_BASE_SCHEMA):
+ """
+ Create jsonschema.RefResolver class for all schemas located in given path.
+ Apparently if you using jsonschema you have to set a reference resolver to referred schemas
+ for local files:
+ https://stackoverflow.com/questions/25145160/json-schema-ref-does-not-work-for-relative-path
+ https://stackoverflow.com/questions/53968770/how-to-set-up-local-file-references-in-python-jsonschema-document
+ https://stackoverflow.com/questions/42159346/jsonschema-refresolver-to-resolve-multiple-refs-in-python
+ :param abs_path_to_schema: Absolute path to location of all schemas
+ :param base_schema_file_name: The name of the common base schema where other schema's refer too
+ :return resolver_object: RefResolver object
+ """
+ schema_store = {}
+ with open(os.path.join(abs_path_to_schema, base_schema_file_name), 'r') as fp:
+ def_schema = json.load(fp)
+
+ file_name_list = os.listdir(abs_path_to_schema)
+ for file_name in file_name_list:
+ file_full_path = os.path.join(abs_path_to_schema, file_name)
+ if file_full_path.endswith(".json"):
+ with open(file_full_path, "r") as fp:
+ schema = json.load(fp)
+ if "$id" in schema:
+ schema_store[schema["$id"]] = schema
+ # wondering if schema_store is still required ... ??
+ # https://stackoverflow.com/questions/42159346/jsonschema-refresolver-to-resolve-multiple-refs-in-python
+ base_uri = "file:///" + abs_path_to_schema + base_schema_file_name
+ resolver = RefResolver(base_uri=base_uri, referrer=def_schema, store=schema_store)
+ return resolver
+
diff --git a/SAS/TMSS/src/tmss/tmssapp/views.py b/SAS/TMSS/src/tmss/tmssapp/views.py
index bfb670fd87b..695f70ba616 100644
--- a/SAS/TMSS/src/tmss/tmssapp/views.py
+++ b/SAS/TMSS/src/tmss/tmssapp/views.py
@@ -3,7 +3,7 @@ import os
from django.http import HttpResponse, JsonResponse
from django.shortcuts import get_object_or_404, render
from lofar.sas.tmss.tmss.tmssapp import models
-from lofar.common.json_utils import get_default_json_object_for_schema
+from lofar.sas.tmss.tmss.tmssapp.validation import get_default_json_object_for_schema
from lofar.sas.tmss.tmss.tmssapp.adapters.parset import convert_to_parset
diff --git a/SAS/TMSS/src/tmss/tmssapp/viewsets/scheduling.py b/SAS/TMSS/src/tmss/tmssapp/viewsets/scheduling.py
index e1ce15a72ab..3b1b00922cb 100644
--- a/SAS/TMSS/src/tmss/tmssapp/viewsets/scheduling.py
+++ b/SAS/TMSS/src/tmss/tmssapp/viewsets/scheduling.py
@@ -24,7 +24,7 @@ from lofar.sas.tmss.tmss.tmssapp import models
from lofar.sas.tmss.tmss.tmssapp import serializers
from datetime import datetime
-from lofar.common.json_utils import get_default_json_object_for_schema
+from lofar.sas.tmss.tmss.tmssapp.validation import get_default_json_object_for_schema
from lofar.common.datetimeutils import formatDatetime
from lofar.sas.tmss.tmss.tmssapp.adapters.parset import convert_to_parset
from drf_yasg.renderers import _SpecRenderer
diff --git a/SAS/TMSS/src/tmss/tmssapp/viewsets/specification.py b/SAS/TMSS/src/tmss/tmssapp/viewsets/specification.py
index 93ab6971734..312d47f6c72 100644
--- a/SAS/TMSS/src/tmss/tmssapp/viewsets/specification.py
+++ b/SAS/TMSS/src/tmss/tmssapp/viewsets/specification.py
@@ -21,7 +21,7 @@ from lofar.sas.tmss.tmss.tmssapp import models
from lofar.sas.tmss.tmss.tmssapp import serializers
from datetime import datetime
-from lofar.common.json_utils import get_default_json_object_for_schema
+from lofar.sas.tmss.tmss.tmssapp.validation import get_default_json_object_for_schema
from lofar.common.datetimeutils import formatDatetime
from lofar.sas.tmss.tmss.tmssapp.tasks import *
from lofar.sas.tmss.tmss.tmssapp.subtasks import *
diff --git a/SAS/TMSS/test/t_parset_adapter.py b/SAS/TMSS/test/t_parset_adapter.py
index 5d1c9052402..984075abc5c 100755
--- a/SAS/TMSS/test/t_parset_adapter.py
+++ b/SAS/TMSS/test/t_parset_adapter.py
@@ -40,7 +40,7 @@ rest_data_creator = TMSSRESTTestDataCreator(BASE_URL, AUTH)
from lofar.sas.tmss.tmss.tmssapp import models
from lofar.sas.tmss.tmss.tmssapp.adapters.parset import convert_to_parset
-from lofar.common.json_utils import get_default_json_object_for_schema
+from lofar.sas.tmss.tmss.tmssapp.validation import get_default_json_object_for_schema
class ParsetAdapterTest(unittest.TestCase):
def test_01(self):
diff --git a/SAS/TMSS/test/tmss_test_data_django_models.py b/SAS/TMSS/test/tmss_test_data_django_models.py
index 2d8a16334f7..06a72d47782 100644
--- a/SAS/TMSS/test/tmss_test_data_django_models.py
+++ b/SAS/TMSS/test/tmss_test_data_django_models.py
@@ -28,7 +28,7 @@ which is automatically destroyed at the end of the unittest session.
#######################################################
from lofar.sas.tmss.tmss.tmssapp import models
-from lofar.common.json_utils import get_default_json_object_for_schema
+from lofar.sas.tmss.tmss.tmssapp.validation import get_default_json_object_for_schema
from datetime import datetime
import uuid
diff --git a/SAS/TMSS/test/tmss_test_data_rest.py b/SAS/TMSS/test/tmss_test_data_rest.py
index ada401713f6..784e73f6edd 100644
--- a/SAS/TMSS/test/tmss_test_data_rest.py
+++ b/SAS/TMSS/test/tmss_test_data_rest.py
@@ -25,7 +25,7 @@ from datetime import datetime
import uuid
import requests
import json
-from lofar.common.json_utils import get_default_json_object_for_schema
+from lofar.sas.tmss.tmss.tmssapp.validation import get_default_json_object_for_schema
class TMSSRESTTestDataCreator():
def __init__(self, django_api_url: str, auth: requests.auth.HTTPBasicAuth):
--
GitLab