From 4af57fdc62f5b4f37fa119a751d87eef822ff681 Mon Sep 17 00:00:00 2001
From: Jorrit Schaap <schaap@astron.nl>
Date: Fri, 28 Aug 2020 12:39:15 +0200
Subject: [PATCH] TMSS-272: renamed schema files to more consistent file naming
convention. Added populate script and config file. Deploy all schema files in
share.
---
SAS/TMSS/client/bin/tmss_populate | 4 +-
SAS/TMSS/client/lib/populate.py | 75 +++++++++++-----
.../src/tmss/tmssapp/schemas/CMakeLists.txt | 17 +---
SAS/TMSS/src/tmss/tmssapp/schemas/Readme.txt | 6 ++
...son => common_schema_template-base-1.json} | 0
.../tmss/tmssapp/schemas/empty-schema.json | 8 --
...ling_unit_template-scheduling_unit-1.json} | 0
...on => subtask_template-observation-1.json} | 0
....json => subtask_template-pipeline-1.json} | 0
....json => subtask_template-qa_files-1.json} | 0
....json => subtask_template-qa_plots-1.json} | 0
...sk_relation_selection_template-SAP-1.json} | 0
...sk_template-calibrator_observation-1.json} | 0
...sk_template-preprocessing_pipeline-1.json} | 0
...> task_template-target_observation-1.json} | 0
.../src/tmss/tmssapp/schemas/templates.json | 86 +++++++++++++++++++
SAS/TMSS/test/test_utils.py | 3 +
17 files changed, 153 insertions(+), 46 deletions(-)
create mode 100644 SAS/TMSS/src/tmss/tmssapp/schemas/Readme.txt
rename SAS/TMSS/src/tmss/tmssapp/schemas/{base.json => common_schema_template-base-1.json} (100%)
delete mode 100644 SAS/TMSS/src/tmss/tmssapp/schemas/empty-schema.json
rename SAS/TMSS/src/tmss/tmssapp/schemas/{scheduling-unit.json => scheduling_unit_template-scheduling_unit-1.json} (100%)
rename SAS/TMSS/src/tmss/tmssapp/schemas/{subtask-observation-control.json => subtask_template-observation-1.json} (100%)
rename SAS/TMSS/src/tmss/tmssapp/schemas/{subtask-pipeline-control.json => subtask_template-pipeline-1.json} (100%)
rename SAS/TMSS/src/tmss/tmssapp/schemas/{subtask-qa-files.json => subtask_template-qa_files-1.json} (100%)
rename SAS/TMSS/src/tmss/tmssapp/schemas/{subtask-qa-plots.json => subtask_template-qa_plots-1.json} (100%)
rename SAS/TMSS/src/tmss/tmssapp/schemas/{task-relation-sap.json => task_relation_selection_template-SAP-1.json} (100%)
rename SAS/TMSS/src/tmss/tmssapp/schemas/{task-calibrator-addon.json => task_template-calibrator_observation-1.json} (100%)
rename SAS/TMSS/src/tmss/tmssapp/schemas/{task-preprocessing.json => task_template-preprocessing_pipeline-1.json} (100%)
rename SAS/TMSS/src/tmss/tmssapp/schemas/{task-observation-with-stations.json => task_template-target_observation-1.json} (100%)
create mode 100644 SAS/TMSS/src/tmss/tmssapp/schemas/templates.json
diff --git a/SAS/TMSS/client/bin/tmss_populate b/SAS/TMSS/client/bin/tmss_populate
index b0dbfbb70ce..375f9112e40 100755
--- a/SAS/TMSS/client/bin/tmss_populate
+++ b/SAS/TMSS/client/bin/tmss_populate
@@ -18,8 +18,8 @@
# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
import logging
-from lofar.sas.tmss.client.populate import populate_schemas
+from lofar.sas.tmss.client.populate import populate_schemas_main
if __name__ == "__main__":
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO)
- populate_schemas()
+ populate_schemas_main()
diff --git a/SAS/TMSS/client/lib/populate.py b/SAS/TMSS/client/lib/populate.py
index d6b0d9a6622..6e2eac8336d 100644
--- a/SAS/TMSS/client/lib/populate.py
+++ b/SAS/TMSS/client/lib/populate.py
@@ -1,31 +1,62 @@
+import logging
+logger = logging.getLogger(__name__)
+
import json
-import argparse
-from pprint import pprint
from lofar.sas.tmss.client.tmss_http_rest_client import TMSSsession
-from lofar.common.datetimeutils import parseDatetime
+import os
+def populate_schemas_main():
+ from optparse import OptionParser
-def populate_schemas():
- parser = argparse.ArgumentParser()
- args = parser.parse_args()
+ # Check the invocation arguments
+ parser = OptionParser('%prog [options]', description='upload the templates to TMSS')
+ parser.add_option('-d', '--dir', dest='schema_dir', type='string',
+ default=os.path.expandvars('$LOFARROOT/share/tmss/schemas'),
+ help='''directory path containing the schemas, default: '%default')''')
+ parser.add_option('-f', '--file', dest='templates_file', type='string',
+ default='templates.json',
+ help='''json file containing the list of templates with name, description, vesions, and template type. default: '%default')''')
+ (options, args) = parser.parse_args()
+ return populate_schemas(options.schema_dir, options.templates_file)
+def populate_schemas(schema_dir: str=None, templates_filename: str=None):
try:
- with TMSSsession.create_from_dbcreds_for_ldap() as session:
- session.post_template(template_path='common_schema_template',
- name="base",
- description='email address schema',
- version='1',
- schema={
- "$id": "http://127.0.0.1:8000/api/schemas/common/base/1/#",
- "$schema": "http://json-schema.org/draft-06/schema#",
- "definitions": {
- "email": {
- "type": "string",
- "format": "email",
- "pattern": "@example\\.com$",
- "default": ""}
- }})
+ if schema_dir is None:
+ schema_dir = os.path.expandvars('$LOFARROOT/share/tmss/schemas')
+
+ if templates_filename is None:
+ templates_filename = 'templates.json'
+
+ with TMSSsession.create_from_dbcreds_for_ldap() as client:
+ templates_filepath = os.path.join(schema_dir, templates_filename)
+ logger.info("Reading templates in: %s", templates_filepath)
+ with open(templates_filepath) as templates_file:
+ templates = json.loads(templates_file.read())
+
+ for template in templates:
+ with open(os.path.join(schema_dir, template.pop('file_name'))) as schema_file:
+ try:
+ json_schema = json.loads(schema_file.read())
+
+ template_path = template.pop('template')
+ name = template.pop('name', json_schema.get('title', '<no name>'))
+ description = template.pop('description', json_schema.get('description', '<no description>'))
+ version = template.pop('version', '1')
+
+ if template_path == 'subtask_template' and 'type' in template:
+ # override plain-text type by its url
+ template['type'] = client.get_path_as_json_object('subtask_type/'+template.pop('type'))['url']
+
+ logger.info("Uploading template template='%s' name='%s' version='%s'", template_path, name, version)
+ client.post_template(template_path=template_path,
+ name=name,
+ description=description,
+ version=version,
+ schema=json_schema,
+ **template)
+ except Exception as e:
+ logger.error(e)
except Exception as e:
- print(e)
+ logger.error(e)
exit(1)
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/CMakeLists.txt b/SAS/TMSS/src/tmss/tmssapp/schemas/CMakeLists.txt
index 88f2d9744d2..32542a45d0f 100644
--- a/SAS/TMSS/src/tmss/tmssapp/schemas/CMakeLists.txt
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/CMakeLists.txt
@@ -1,18 +1,7 @@
include(PythonInstall)
-set(_json_schema_files
- base.json
- scheduling-unit.json
- task-calibrator-addon.json
- task-observation-with-stations.json
- task-preprocessing.json
- task-relation-sap.json
- subtask-observation-control.json
- subtask-pipeline-control.json
- subtask-qa-files.json
- subtask-qa-plots.json
- empty-schema.json
- )
+file(GLOB json_schema_files *.json)
+lofar_add_data_files(${json_schema_files} DESTINATION tmss/schemas)
-lofar_add_data_files(${_json_schema_files} DESTINATION tmss/schemas)
+lofar_add_data_files(Readme.txt DESTINATION tmss/schemas)
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/Readme.txt b/SAS/TMSS/src/tmss/tmssapp/schemas/Readme.txt
new file mode 100644
index 00000000000..d1425abc5ee
--- /dev/null
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/Readme.txt
@@ -0,0 +1,6 @@
+For easy administration of the various templates, please use the following file naming convention: <snake_cased_template_name>_<name>_<version>.json
+
+These json files (should) contain a valid json schema, which can be uploaded to TMSS.
+
+Because there are various different types of Template models in TMSS, each with possible extra parameters,
+we've created this litte helper program tmss_populate which can upload all templates defined in the templates.json file.
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/base.json b/SAS/TMSS/src/tmss/tmssapp/schemas/common_schema_template-base-1.json
similarity index 100%
rename from SAS/TMSS/src/tmss/tmssapp/schemas/base.json
rename to SAS/TMSS/src/tmss/tmssapp/schemas/common_schema_template-base-1.json
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/empty-schema.json b/SAS/TMSS/src/tmss/tmssapp/schemas/empty-schema.json
deleted file mode 100644
index 8f8bbe6ce1a..00000000000
--- a/SAS/TMSS/src/tmss/tmssapp/schemas/empty-schema.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{
- "$id": "empty-schema.json",
- "type": "object",
- "$schema": "http://json-schema.org/draft-06/schema#",
- "definitions": {},
- "additionalProperties": false,
- "properties": {}
-}
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/scheduling-unit.json b/SAS/TMSS/src/tmss/tmssapp/schemas/scheduling_unit_template-scheduling_unit-1.json
similarity index 100%
rename from SAS/TMSS/src/tmss/tmssapp/schemas/scheduling-unit.json
rename to SAS/TMSS/src/tmss/tmssapp/schemas/scheduling_unit_template-scheduling_unit-1.json
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/subtask-observation-control.json b/SAS/TMSS/src/tmss/tmssapp/schemas/subtask_template-observation-1.json
similarity index 100%
rename from SAS/TMSS/src/tmss/tmssapp/schemas/subtask-observation-control.json
rename to SAS/TMSS/src/tmss/tmssapp/schemas/subtask_template-observation-1.json
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/subtask-pipeline-control.json b/SAS/TMSS/src/tmss/tmssapp/schemas/subtask_template-pipeline-1.json
similarity index 100%
rename from SAS/TMSS/src/tmss/tmssapp/schemas/subtask-pipeline-control.json
rename to SAS/TMSS/src/tmss/tmssapp/schemas/subtask_template-pipeline-1.json
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/subtask-qa-files.json b/SAS/TMSS/src/tmss/tmssapp/schemas/subtask_template-qa_files-1.json
similarity index 100%
rename from SAS/TMSS/src/tmss/tmssapp/schemas/subtask-qa-files.json
rename to SAS/TMSS/src/tmss/tmssapp/schemas/subtask_template-qa_files-1.json
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/subtask-qa-plots.json b/SAS/TMSS/src/tmss/tmssapp/schemas/subtask_template-qa_plots-1.json
similarity index 100%
rename from SAS/TMSS/src/tmss/tmssapp/schemas/subtask-qa-plots.json
rename to SAS/TMSS/src/tmss/tmssapp/schemas/subtask_template-qa_plots-1.json
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/task-relation-sap.json b/SAS/TMSS/src/tmss/tmssapp/schemas/task_relation_selection_template-SAP-1.json
similarity index 100%
rename from SAS/TMSS/src/tmss/tmssapp/schemas/task-relation-sap.json
rename to SAS/TMSS/src/tmss/tmssapp/schemas/task_relation_selection_template-SAP-1.json
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/task-calibrator-addon.json b/SAS/TMSS/src/tmss/tmssapp/schemas/task_template-calibrator_observation-1.json
similarity index 100%
rename from SAS/TMSS/src/tmss/tmssapp/schemas/task-calibrator-addon.json
rename to SAS/TMSS/src/tmss/tmssapp/schemas/task_template-calibrator_observation-1.json
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/task-preprocessing.json b/SAS/TMSS/src/tmss/tmssapp/schemas/task_template-preprocessing_pipeline-1.json
similarity index 100%
rename from SAS/TMSS/src/tmss/tmssapp/schemas/task-preprocessing.json
rename to SAS/TMSS/src/tmss/tmssapp/schemas/task_template-preprocessing_pipeline-1.json
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/task-observation-with-stations.json b/SAS/TMSS/src/tmss/tmssapp/schemas/task_template-target_observation-1.json
similarity index 100%
rename from SAS/TMSS/src/tmss/tmssapp/schemas/task-observation-with-stations.json
rename to SAS/TMSS/src/tmss/tmssapp/schemas/task_template-target_observation-1.json
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/templates.json b/SAS/TMSS/src/tmss/tmssapp/schemas/templates.json
new file mode 100644
index 00000000000..4e16a362927
--- /dev/null
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/templates.json
@@ -0,0 +1,86 @@
+[
+ {
+ "file_name": "common_schema_template-base-1.json",
+ "template": "common_schema_template",
+ "name": "base",
+ "version": 1
+ },
+ {
+ "file_name": "scheduling_unit_template-scheduling_unit-1.json",
+ "template": "scheduling_unit_template",
+ "name": "scheduling unit",
+ "version": 1,
+ "description": "This schema defines the structure of all tasks in a scheduling unit"
+ },
+ {
+ "file_name": "task_relation_selection_template-SAP-1.json",
+ "template": "task_relation_selection_template",
+ "name": "SAP",
+ "version": 1,
+ "description": "Select by SAP"
+ },
+ {
+ "file_name": "task_template-calibrator_observation-1.json",
+ "template": "task_template",
+ "name": "calibrator observation",
+ "version": 1,
+ "validation_code_js": "",
+ "description": "This schema defines the parameters for a calibrator observation."
+ },
+ {
+ "file_name": "task_template-target_observation-1.json",
+ "template": "task_template",
+ "name": "target observation",
+ "version": 1,
+ "validation_code_js": "",
+ "description": "This schema defines the parameters for a target observation."
+ },
+ {
+ "file_name": "task_template-preprocessing_pipeline-1.json",
+ "template": "task_template",
+ "name": "preprocessing pipeline",
+ "version": 1,
+ "validation_code_js": "",
+ "description": "This schema defines the parameters for a preprocessing pipeline."
+ },
+ {
+ "file_name": "subtask_template-observation-1.json",
+ "template": "subtask_template",
+ "name": "observation control",
+ "version": 1,
+ "description": "This schema defines the parameters to setup and control the observation subtask.",
+ "type": "observation",
+ "realtime": true,
+ "queue": false
+ },
+ {
+ "file_name": "subtask_template-pipeline-1.json",
+ "template": "subtask_template",
+ "name": "pipeline control",
+ "version": 1,
+ "description": "This schema defines the parameters to setup and control a pipeline.",
+ "type": "pipeline",
+ "realtime": true,
+ "queue": false
+ },
+ {
+ "file_name": "subtask_template-qa_files-1.json",
+ "template": "subtask_template",
+ "name": "QA file conversion",
+ "description": "This schema defines the parameters to do a file conversion from the observation/pipeline output data to a QA-h5 file.",
+ "version": 1,
+ "type": "qa_files",
+ "realtime": true,
+ "queue": false
+ },
+ {
+ "file_name": "subtask_template-qa_plots-1.json",
+ "template": "subtask_template",
+ "name": "QA plots",
+ "description": "This schema defines the parameters to create QA plots from a QA-h5 file.",
+ "version": 1,
+ "type": "qa_files",
+ "realtime": true,
+ "queue": false
+ }
+]
\ No newline at end of file
diff --git a/SAS/TMSS/test/test_utils.py b/SAS/TMSS/test/test_utils.py
index 5f515a7c97a..0a6297162e0 100644
--- a/SAS/TMSS/test/test_utils.py
+++ b/SAS/TMSS/test/test_utils.py
@@ -353,6 +353,9 @@ def main_test_environment():
with RATestEnvironment(exchange=options.exchange, broker=options.broker):
with TMSSTestEnvironment(host=options.host, preferred_django_port=options.port, exchange=options.exchange, broker=options.broker) as instance:
+ from lofar.sas.tmss.client.populate import populate_schemas
+ populate_schemas()
+
from lofar.sas.tmss.tmss.tmssapp.populate import populate_test_data
populate_test_data()
--
GitLab