TMSS-272: renamed schema files to more consistent file naming convention....

TMSS-272: renamed schema files to more consistent file naming convention. Added populate script and config file. Deploy all schema files in share.

SAS/TMSS/client/bin/tmss_populate

@@ -18,8 +18,8 @@
 # with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
 
 import logging
-from lofar.sas.tmss.client.populate import populate_schemas
+from lofar.sas.tmss.client.populate import populate_schemas_main
 
 if __name__ == "__main__":
     logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO)
-    populate_schemas()
+    populate_schemas_main()

SAS/TMSS/client/lib/populate.py

+import logging
+logger = logging.getLogger(__name__)
+
 import json
-import argparse
-from pprint import pprint
 from lofar.sas.tmss.client.tmss_http_rest_client import TMSSsession
-from lofar.common.datetimeutils import parseDatetime
+import os
 
+def populate_schemas_main():
+    from optparse import OptionParser
 
-def populate_schemas():
-    parser = argparse.ArgumentParser()
-    args = parser.parse_args()
+    # Check the invocation arguments
+    parser = OptionParser('%prog [options]', description='upload the templates to TMSS')
+    parser.add_option('-d', '--dir', dest='schema_dir', type='string',
+                      default=os.path.expandvars('$LOFARROOT/share/tmss/schemas'),
+                      help='''directory path containing the schemas, default: '%default')''')
+    parser.add_option('-f', '--file', dest='templates_file', type='string',
+                      default='templates.json',
+                      help='''json file containing the list of templates with name, description, vesions, and template type. default: '%default')''')
+    (options, args) = parser.parse_args()
+    return populate_schemas(options.schema_dir, options.templates_file)
 
+def populate_schemas(schema_dir: str=None, templates_filename: str=None):
     try:
-        with TMSSsession.create_from_dbcreds_for_ldap() as session:
-            session.post_template(template_path='common_schema_template',
-                                  name="base",
-                                  description='email address schema',
-                                  version='1',
-                                  schema={
-                                      "$id": "http://127.0.0.1:8000/api/schemas/common/base/1/#",
-                                      "$schema": "http://json-schema.org/draft-06/schema#",
-                                      "definitions": {
-                                          "email": {
-                                              "type": "string",
-                                              "format": "email",
-                                              "pattern": "@example\\.com$",
-                                              "default": ""}
-                                      }})
+        if schema_dir is None:
+            schema_dir = os.path.expandvars('$LOFARROOT/share/tmss/schemas')
+
+        if templates_filename is None:
+            templates_filename = 'templates.json'
+
+        with TMSSsession.create_from_dbcreds_for_ldap() as client:
+            templates_filepath = os.path.join(schema_dir, templates_filename)
+            logger.info("Reading templates in: %s", templates_filepath)
+            with open(templates_filepath) as templates_file:
+                templates = json.loads(templates_file.read())
+
+                for template in templates:
+                    with open(os.path.join(schema_dir, template.pop('file_name'))) as schema_file:
+                        try:
+                            json_schema = json.loads(schema_file.read())
+
+                            template_path = template.pop('template')
+                            name = template.pop('name', json_schema.get('title', '<no name>'))
+                            description = template.pop('description', json_schema.get('description', '<no description>'))
+                            version = template.pop('version', '1')
+
+                            if template_path == 'subtask_template' and 'type' in template:
+                                # override plain-text type by its url
+                                template['type'] = client.get_path_as_json_object('subtask_type/'+template.pop('type'))['url']
+
+                            logger.info("Uploading template template='%s' name='%s' version='%s'", template_path, name, version)
 
+                            client.post_template(template_path=template_path,
+                                                  name=name,
+                                                  description=description,
+                                                  version=version,
+                                                  schema=json_schema,
+                                                  **template)
+                        except Exception as e:
+                            logger.error(e)
     except Exception as e:
-        print(e)
+        logger.error(e)
         exit(1)

SAS/TMSS/src/tmss/tmssapp/schemas/CMakeLists.txt

 
 include(PythonInstall)
 
-set(_json_schema_files
-    base.json
-    scheduling-unit.json
-    task-calibrator-addon.json
-    task-observation-with-stations.json
-    task-preprocessing.json
-    task-relation-sap.json
-    subtask-observation-control.json
-    subtask-pipeline-control.json
-    subtask-qa-files.json
-    subtask-qa-plots.json
-    empty-schema.json
-    )
+file(GLOB json_schema_files *.json)
+lofar_add_data_files(${json_schema_files} DESTINATION tmss/schemas)
 
-lofar_add_data_files(${_json_schema_files} DESTINATION tmss/schemas)
+lofar_add_data_files(Readme.txt DESTINATION tmss/schemas)

SAS/TMSS/src/tmss/tmssapp/schemas/Readme.txt

+For easy administration of the various templates, please use the following file naming convention: <snake_cased_template_name>_<name>_<version>.json
+
+These json files (should) contain a valid json schema, which can be uploaded to TMSS.
+
+Because there are various different types of Template models in TMSS, each with possible extra parameters,
+we've created this litte helper program tmss_populate which can upload all templates defined in the templates.json file.

SAS/TMSS/src/tmss/tmssapp/schemas/empty-schema.json

-{
-  "$id": "empty-schema.json",
-  "type": "object",
-  "$schema": "http://json-schema.org/draft-06/schema#",
-  "definitions": {},
-  "additionalProperties": false,
-  "properties": {}
-}
\ No newline at end of file

SAS/TMSS/src/tmss/tmssapp/schemas/templates.json

+[
+  {
+    "file_name": "common_schema_template-base-1.json",
+    "template": "common_schema_template",
+    "name": "base",
+    "version": 1
+  },
+  {
+    "file_name": "scheduling_unit_template-scheduling_unit-1.json",
+    "template": "scheduling_unit_template",
+    "name": "scheduling unit",
+    "version": 1,
+    "description": "This schema defines the structure of all tasks in a scheduling unit"
+  },
+  {
+    "file_name": "task_relation_selection_template-SAP-1.json",
+    "template": "task_relation_selection_template",
+    "name": "SAP",
+    "version": 1,
+    "description": "Select by SAP"
+  },
+  {
+    "file_name": "task_template-calibrator_observation-1.json",
+    "template": "task_template",
+    "name": "calibrator observation",
+    "version": 1,
+    "validation_code_js": "",
+    "description": "This schema defines the parameters for a calibrator observation."
+  },
+  {
+    "file_name": "task_template-target_observation-1.json",
+    "template": "task_template",
+    "name": "target observation",
+    "version": 1,
+    "validation_code_js": "",
+    "description": "This schema defines the parameters for a target observation."
+  },
+  {
+    "file_name": "task_template-preprocessing_pipeline-1.json",
+    "template": "task_template",
+    "name": "preprocessing pipeline",
+    "version": 1,
+    "validation_code_js": "",
+    "description": "This schema defines the parameters for a preprocessing pipeline."
+  },
+  {
+    "file_name": "subtask_template-observation-1.json",
+    "template": "subtask_template",
+    "name": "observation control",
+    "version": 1,
+    "description": "This schema defines the parameters to setup and control the observation subtask.",
+    "type": "observation",
+    "realtime": true,
+    "queue": false
+  },
+  {
+    "file_name": "subtask_template-pipeline-1.json",
+    "template": "subtask_template",
+    "name": "pipeline control",
+    "version": 1,
+    "description": "This schema defines the parameters to setup and control a pipeline.",
+    "type": "pipeline",
+    "realtime": true,
+    "queue": false
+  },
+  {
+    "file_name": "subtask_template-qa_files-1.json",
+    "template": "subtask_template",
+    "name": "QA file conversion",
+    "description": "This schema defines the parameters to do a file conversion from the observation/pipeline output data to a QA-h5 file.",
+    "version": 1,
+    "type": "qa_files",
+    "realtime": true,
+    "queue": false
+  },
+  {
+    "file_name": "subtask_template-qa_plots-1.json",
+    "template": "subtask_template",
+    "name": "QA plots",
+    "description": "This schema defines the parameters to create QA plots from a QA-h5 file.",
+    "version": 1,
+    "type": "qa_files",
+    "realtime": true,
+    "queue": false
+  }
+]
\ No newline at end of file

SAS/TMSS/test/test_utils.py

@@ -353,6 +353,9 @@ def main_test_environment():
     with RATestEnvironment(exchange=options.exchange, broker=options.broker):
         with TMSSTestEnvironment(host=options.host, preferred_django_port=options.port, exchange=options.exchange, broker=options.broker) as instance:
 
+            from lofar.sas.tmss.client.populate import populate_schemas
+            populate_schemas()
+
             from lofar.sas.tmss.tmss.tmssapp.populate import populate_test_data
             populate_test_data()