diff --git a/.DS_Store b/.DS_Store
new file mode 100644
index 0000000000000000000000000000000000000000..772182382fcc07c3b3b57de4515e7fbde30dc0ef
Binary files /dev/null and b/.DS_Store differ
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 02eb22467f56319e16296528badbd15a5a893a4b..e8e336c5fc789a9bc47574134599f41dea94eef5 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -54,7 +54,7 @@ build_TMSS:
- cmake -DBUILD_PACKAGES=$PACKAGE -DCASACORE_ROOT_DIR=/opt/casacore/ -DCASAREST_ROOT_DIR=/opt/casarest/ -DUSE_LOG4CPLUS=false ../..
- make -j 12
- make install
- dependencies:
+ needs:
- prepare_ci_sas_docker_image
artifacts:
expire_in: 6 hours
@@ -72,7 +72,7 @@ build_RAServices:
- cmake -DBUILD_PACKAGES=$PACKAGE -DCASACORE_ROOT_DIR=/opt/casacore/ -DCASAREST_ROOT_DIR=/opt/casarest/ -DUSE_LOG4CPLUS=false ../..
- make -j 12
- make install
- dependencies:
+ needs:
- prepare_ci_sas_docker_image
artifacts:
expire_in: 6 hours
@@ -90,7 +90,7 @@ build_LTAIngest:
- cmake -DBUILD_PACKAGES=$PACKAGE -DUSE_LOG4CPLUS=false ../..
- make -j 12
- make install
- dependencies:
+ needs:
- prepare_ci_lta_docker_image
artifacts:
expire_in: 6 hours
@@ -112,7 +112,7 @@ build_MCU_MAC:
- tar --ignore-failed-read --exclude=include --exclude="*.ztar" -czf MCU_MAC_${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}.ztar *
- curl --insecure --upload-file MCU_MAC_${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}.ztar -u upload:upload https://support.astron.nl/nexus/content/repositories/branches/nl/astron/lofar/${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}/MCU_MAC_${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}.x86_64.ztar
- dependencies:
+ needs:
- prepare_ci_mac_docker_image
artifacts:
expire_in: 6 hours
@@ -131,7 +131,7 @@ unit_test_TMSS:
- echo "Testing $PACKAGE..."
- cd build/gnucxx11_opt
- SKIP_INTEGRATION_TESTS=true ctest
- dependencies:
+ needs:
- build_TMSS
services:
- rabbitmq:latest
@@ -161,7 +161,7 @@ unit_test_RAServices:
RABBITMQ_DEFAULT_USER: guest
RABBITMQ_DEFAULT_PASS: guest
LOFAR_DEFAULT_BROKER: 'rabbitmq' # override default 'localhost' which does not work for CI service rabbitmq.
- dependencies:
+ needs:
- build_RAServices
artifacts:
name: unit-test-report
@@ -185,7 +185,7 @@ unit_test_LTAIngest:
RABBITMQ_DEFAULT_USER: guest
RABBITMQ_DEFAULT_PASS: guest
LOFAR_DEFAULT_BROKER: 'rabbitmq' # override default 'localhost' which does not work for CI service rabbitmq.
- dependencies:
+ needs:
- build_LTAIngest
artifacts:
name: unit-test-report
@@ -210,7 +210,7 @@ unit_test_MCU_MAC:
RABBITMQ_DEFAULT_USER: guest
RABBITMQ_DEFAULT_PASS: guest
LOFAR_DEFAULT_BROKER: 'rabbitmq' # override default 'localhost' which does not work for CI service rabbitmq.
- dependencies:
+ needs:
- build_MCU_MAC
artifacts:
name: unit-test-report
@@ -240,8 +240,11 @@ dockerize_TMSS:
- docker tag tmss_testprovider:$CI_COMMIT_SHORT_SHA nexus.cep4.control.lofar:18080/tmss_testprovider:$CI_COMMIT_SHORT_SHA
- docker push nexus.cep4.control.lofar:18080/tmss_testprovider:$CI_COMMIT_SHORT_SHA
- docker logout $CI_NEXUS_REGISTRY
- dependencies:
- - build_TMSS
+ needs:
+ - job: build_TMSS
+ artifacts: true
+ - job: integration_test_TMSS
+ artifacts: false
#
# INTEGRATION TEST STAGE
@@ -261,8 +264,8 @@ integration_test_TMSS:
RABBITMQ_DEFAULT_USER: guest
RABBITMQ_DEFAULT_PASS: guest
LOFAR_DEFAULT_BROKER: 'rabbitmq' # override default 'localhost' which does not work for CI service rabbitmq.
- dependencies:
- - build_TMSS
+ needs:
+ - unit_test_TMSS
artifacts:
name: integration-test-report
when: always
@@ -285,8 +288,8 @@ integration_test_RAServices:
- echo "Integration Testing $PACKAGE..."
- cd build/gnucxx11_opt
- SKIP_INTEGRATION_TESTS=false SKIP_UNIT_TESTS=true ctest
- dependencies:
- - build_RAServices
+ needs:
+ - unit_test_RAServices
artifacts:
name: integration-test-report
when: always
@@ -309,8 +312,8 @@ integration_test_LTAIngest:
RABBITMQ_DEFAULT_USER: guest
RABBITMQ_DEFAULT_PASS: guest
LOFAR_DEFAULT_BROKER: 'rabbitmq' # override default 'localhost' which does not work for CI service rabbitmq.
- dependencies:
- - build_LTAIngest
+ needs:
+ - unit_test_LTAIngest
artifacts:
name: integration-test-report
when: always
@@ -343,7 +346,7 @@ deploy-tmss-test:
- ssh lofarsys@scu199.control.lofar "docker tag ${CI_NEXUS_REGISTRY}/tmss_testprovider:$CI_COMMIT_SHORT_SHA ${CI_NEXUS_REGISTRY}/tmss_testprovider:latest"
- ssh lofarsys@scu199.control.lofar "docker tag ${CI_NEXUS_REGISTRY}/tmss_django:$CI_COMMIT_SHORT_SHA ${CI_NEXUS_REGISTRY}/tmss_django:latest"
- ssh lofarsys@scu199.control.lofar "docker-compose -f docker-compose-scu199.yml up -d"
- dependencies:
+ needs:
- dockerize_TMSS
when: manual
@@ -366,7 +369,7 @@ deploy-tmss-ua:
- ssh lofarsys@tmss-ua.control.lofar "docker tag ${CI_NEXUS_REGISTRY}/tmss_testprovider:$CI_COMMIT_SHORT_SHA ${CI_NEXUS_REGISTRY}/tmss_testprovider:latest"
- ssh lofarsys@tmss-ua.control.lofar "docker tag ${CI_NEXUS_REGISTRY}/tmss_django:$CI_COMMIT_SHORT_SHA ${CI_NEXUS_REGISTRY}/tmss_django:latest"
- ssh lofarsys@tmss-ua.control.lofar "docker-compose -f docker-compose-ua.yml up -d"
- dependencies:
+ needs:
- dockerize_TMSS
when: manual
only:
@@ -379,7 +382,7 @@ deploy-tmss-dockerhub:
- docker tag tmss_django:$CI_COMMIT_SHORT_SHA lofar/tmss:master-$CI_COMMIT_SHORT_SHA
- docker push lofar/tmss:master-$CI_COMMIT_SHORT_SHA
- docker logout
- dependencies:
+ needs:
- dockerize_TMSS
when: manual
only:
@@ -397,4 +400,6 @@ deploy-MCU_MAC-test:
- chmod 644 ~/.ssh/known_hosts
script:
- ssh lofarsys@mcu199.control.lofar "MAC_install -b ${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA} -v ${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}"
+ needs:
+ - unit_test_MCU_MAC
when: manual
\ No newline at end of file
diff --git a/Docker/lofar-ci/Dockerfile_ci_sas b/Docker/lofar-ci/Dockerfile_ci_sas
index d5d9b7ce6e4cb7cdc2eb0b4c5fead89d55ae6a5f..527639e256c50c98b1ef0550b41a7cbf69b3e1c3 100644
--- a/Docker/lofar-ci/Dockerfile_ci_sas
+++ b/Docker/lofar-ci/Dockerfile_ci_sas
@@ -16,7 +16,10 @@ RUN yum erase -y postgresql postgresql-server postgresql-devel && \
cd /bin && ln -s /usr/pgsql-9.6/bin/initdb && ln -s /usr/pgsql-9.6/bin/postgres
ENV PATH /usr/pgsql-9.6/bin:$PATH
-RUN pip3 install cython kombu lxml requests pygcn xmljson mysql-connector-python python-dateutil Django==3.0.9 djangorestframework djangorestframework-xml ldap==1.0.2 flask fabric coverage python-qpid-proton PyGreSQL numpy h5py psycopg2 testing.postgresql Flask-Testing scipy Markdown django-filter python-ldap python-ldap-test ldap3 djangorestframework django-jsonforms django-json-widget django-jsoneditor drf-yasg flex swagger-spec-validator django-auth-ldap mozilla-django-oidc jsonschema comet pyxb==1.2.5 graphviz isodate
+RUN pip3 install cython kombu lxml requests pygcn xmljson mysql-connector-python python-dateutil Django==3.0.9 djangorestframework==3.11.1 djangorestframework-xml ldap==1.0.2 flask fabric coverage python-qpid-proton PyGreSQL numpy h5py psycopg2 testing.postgresql Flask-Testing scipy Markdown django-filter python-ldap python-ldap-test ldap3 django-jsonforms django-json-widget django-jsoneditor drf-yasg flex swagger-spec-validator django-auth-ldap mozilla-django-oidc jsonschema comet pyxb==1.2.5 graphviz isodate astropy packaging
+
+#Viewflow package
+RUN pip3 install django-material django-viewflow
# Note: nodejs now comes with npm, do not install the npm package separately, since that will be taken from the epel repo and is conflicting.
RUN echo "Installing Nodejs packages..." && \
@@ -27,4 +30,4 @@ RUN echo "Installing Nodejs packages..." && \
npm install -g serve
-USER lofarsys
\ No newline at end of file
+USER lofarsys
diff --git a/LCS/PyCommon/json_utils.py b/LCS/PyCommon/json_utils.py
index e23b336d8fbb3bbda6ff477d5c88deec28b2b866..f1aee10a64206f6c4897c69868991fd724b92f72 100644
--- a/LCS/PyCommon/json_utils.py
+++ b/LCS/PyCommon/json_utils.py
@@ -15,9 +15,10 @@
# You should have received a copy of the GNU General Public License along
# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-from jsonschema import validators, Draft6Validator
+import json
+import jsonschema
from copy import deepcopy
-
+import requests
def _extend_with_default(validator_class):
"""
@@ -38,7 +39,7 @@ def _extend_with_default(validator_class):
):
yield error
- return validators.extend(
+ return jsonschema.validators.extend(
validator_class, {"properties" : set_defaults},
)
@@ -55,7 +56,7 @@ def _extend_with_required(validator_class):
def set_required_properties(validator, properties, instance, schema):
for property in properties:
- subschema = schema['properties'][property]
+ subschema = schema['properties'].get(property, {})
if "default" in subschema:
instance.setdefault(property, subschema["default"])
for error in validate_required(
@@ -63,20 +64,155 @@ def _extend_with_required(validator_class):
):
yield error
- return validators.extend(
+ return jsonschema.validators.extend(
validator_class, {"required" : set_required_properties},
)
# define a custom validator that fills in properties before validation
-_DefaultValidatingDraft6Validator = _extend_with_default(Draft6Validator)
+_DefaultValidatingDraft6Validator = _extend_with_default(jsonschema.Draft6Validator)
_DefaultValidatingDraft6Validator = _extend_with_required(_DefaultValidatingDraft6Validator)
+
def get_default_json_object_for_schema(schema: str) -> dict:
'''return a valid json object for the given schema with all properties with their default values'''
return add_defaults_to_json_object_for_schema({}, schema)
+
def add_defaults_to_json_object_for_schema(json_object: dict, schema: str) -> dict:
- '''return a copy of the json object with defaults filled in accoring to the schema for all the missing properties'''
+ '''return a copy of the json object with defaults filled in according to the schema for all the missing properties'''
copy_of_json_object = deepcopy(json_object)
+ #TODO: investigate if we want to use a 'common'/singleton validator and use (remote) schema caching for faster validation
_DefaultValidatingDraft6Validator(schema).validate(copy_of_json_object)
return copy_of_json_object
+
+def replace_host_in_urls(schema, new_base_url: str, keys=['$id', '$ref', '$schema']):
+ '''return the given schema with all fields in the given keys which start with the given old_base_url updated so they point to the given new_base_url'''
+ if isinstance(schema, dict):
+ updated_schema = {}
+ for key, value in schema.items():
+ if key in keys:
+ if isinstance(value,str) and (value.startswith('http://') or value.startswith('https://')) and 'json-schema.org' not in value:
+ try:
+ # deconstruct path from old url
+ head, anchor, tail = value.partition('#')
+ host, slash, path = head.lstrip('http://').lstrip('https://').partition('/')
+
+ # and reconstruct the proper new url
+ updated_schema[key] = (new_base_url.rstrip('/') + '/' + path + anchor + tail.rstrip('/')).replace(' ', '%20')
+ except:
+ # just accept the original value and assume that the user uploaded a proper schema
+ updated_schema[key] = value
+ else:
+ updated_schema[key] = value
+ else:
+ updated_schema[key] = replace_host_in_urls(value, new_base_url, keys)
+ return updated_schema
+
+ if isinstance(schema, list):
+ return [replace_host_in_urls(item, new_base_url, keys) for item in schema]
+
+ return schema
+
+def get_referenced_subschema(ref_url):
+ '''fetch the schema given by the ref_url, and get the sub-schema given by the #/ path in the ref_url'''
+ # deduct referred schema name and version from ref-value
+ head, anchor, tail = ref_url.partition('#')
+ # TODO: maybe use cache for requested urls?
+ referenced_schema = json.loads(requests.get(ref_url).text)
+
+ # extract sub-schema
+ tail = tail.strip('/')
+ if tail:
+ parts = tail.split('/')
+ for part in parts:
+ referenced_schema = referenced_schema[part]
+
+ return referenced_schema
+
+
+def resolved_refs(schema):
+ '''return the given schema with all $ref fields replaced by the referred json (sub)schema that they point to.'''
+ if isinstance(schema, dict):
+ updated_schema = {}
+ for key, value in schema.items():
+ if key in "$ref" and isinstance(value, str):
+ if value.startswith('#'):
+ # reference to local document, no need for http injection
+ updated_schema[key] = value
+ else:
+ try:
+ # by returning the referenced (sub)schema, the $ref-key and url-value are replaced from the caller's perspective.
+ # also, recursively resolve refs in referenced_subschema
+ referenced_subschema = get_referenced_subschema(value)
+ return resolved_refs(referenced_subschema)
+ except:
+ # can't get the referenced schema
+ # so, just accept the original value and assume that the user uploaded a proper schema
+ updated_schema[key] = value
+ else:
+ updated_schema[key] = resolved_refs(value)
+ return updated_schema
+
+ if isinstance(schema, list):
+ return [resolved_refs(item) for item in schema]
+
+ return schema
+
+def validate_json_against_its_schema(json_object: dict):
+ '''validate the give json object against its own schema (the URI/URL that its propery $schema points to)'''
+ schema_url = json_object['$schema']
+ response = requests.get(schema_url, headers={"Accept":"application/json"})
+ if response.status_code == 200:
+ return validate_json_against_schema(json_object, response.text)
+ raise jsonschema.exceptions.ValidationError("Could not get schema from '%s'\n%s" % (schema_url, str(response.text)))
+
+def validate_json_against_schema(json_string: str, schema: str):
+ '''validate the given json_string against the given schema.
+ If no exception if thrown, then the given json_string validates against the given schema.
+ :raises SchemaValidationException if the json_string does not validate against the schema
+ '''
+
+ # ensure the given arguments are strings
+ if type(json_string) != str:
+ json_string = json.dumps(json_string)
+ if type(schema) != str:
+ schema = json.dumps(schema)
+
+ # ensure the specification and schema are both valid json in the first place
+ try:
+ json_object = json.loads(json_string)
+ except json.decoder.JSONDecodeError as e:
+ raise jsonschema.exceptions.ValidationError("Invalid JSON: %s\n%s" % (str(e), json_string))
+
+ try:
+ schema_object = json.loads(schema)
+ except json.decoder.JSONDecodeError as e:
+ raise jsonschema.exceptions.ValidationError("Invalid JSON: %s\n%s" % (str(e), schema))
+
+ # now do the actual validation
+ try:
+ validate_json_object_with_schema(json_object, schema_object)
+ except jsonschema.ValidationError as e:
+ raise jsonschema.exceptions.ValidationError(str(e))
+
+
+def get_default_json_object_for_schema(schema: str) -> dict:
+ """
+ TMSS wrapper for TMSS 'add_defaults_to_json_object_for_schema'
+ :param schema:
+ :return: json_object with default values of the schema
+ """
+ data = add_defaults_to_json_object_for_schema({}, schema)
+ if '$id' in schema:
+ data['$schema'] = schema['$id']
+ return data
+
+
+def validate_json_object_with_schema(json_object, schema):
+ """
+ Validate the given json_object with schema
+ """
+ jsonschema.Draft6Validator(schema=schema).validate(json_object)
+
+
+
diff --git a/LCS/PyCommon/test/t_json_utils.py b/LCS/PyCommon/test/t_json_utils.py
index 831e7e610de4658c03266c764ae1acc702708956..2237f0f8d68717fe304b4babb301887b6bf89546 100755
--- a/LCS/PyCommon/test/t_json_utils.py
+++ b/LCS/PyCommon/test/t_json_utils.py
@@ -22,7 +22,9 @@ logger = logging.getLogger(__name__)
logging.basicConfig(format='%(asctime)s %(process)s %(threadName)s %(levelname)s %(message)s', level=logging.DEBUG)
import unittest
-from lofar.common.json_utils import get_default_json_object_for_schema
+import threading
+import json
+from lofar.common.json_utils import get_default_json_object_for_schema, replace_host_in_urls, resolved_refs
class TestJSONUtils(unittest.TestCase):
def test_empty_schema_yields_empty_object(self):
@@ -65,5 +67,112 @@ class TestJSONUtils(unittest.TestCase):
"prop_a": 42,
"prop_b": 3.14}, json)
+ def test_resolved_refs(self):
+ '''test if $refs to URL's are properly resolved'''
+ import http.server
+ import socketserver
+ from lofar.common.util import find_free_port
+
+ port = find_free_port(8000)
+ host = "127.0.0.1"
+ base_url = "http://%s:%s" % (host, port)
+
+ base_schema = { "$id": base_url + "/base_schema.json",
+ "$schema": "http://json-schema.org/draft-06/schema#",
+ "definitions": {
+ "email": {
+ "type": "string",
+ "format": "email",
+ "pattern": "@example\\.com$" }
+ } }
+
+ user_schema = {"$id": base_url + "/user_schema.json",
+ "$schema": "http://json-schema.org/draft-06/schema#",
+ "type": "object",
+ "default": {},
+ "properties": {
+ "name": {
+ "type": "string",
+ "minLength": 2 },
+ "email": {
+ "$ref": base_url + "/base_schema.json" + "#/definitions/email"
+ },
+ "other_emails": {
+ "type": "array",
+ "items": {
+ "$ref": base_url + "/base_schema.json" + "#/definitions/email"
+ }
+ } } }
+
+ class TestRequestHandler(http.server.BaseHTTPRequestHandler):
+ '''helper class to serve the schemas via http. Needed for resolving the $ref URLs'''
+ def send_json_response(self, json_object):
+ self.send_response(http.HTTPStatus.OK)
+ self.send_header("Content-type", "application/json")
+ self.end_headers()
+ self.wfile.write(json.dumps(json_object, indent=2).encode('utf-8'))
+
+ def do_GET(self):
+ try:
+ if self.path == "/base_schema.json":
+ self.send_json_response(base_schema)
+ elif self.path == "/user_schema.json":
+ self.send_json_response(user_schema)
+ else:
+ self.send_error(http.HTTPStatus.NOT_FOUND, "No such resource")
+ except Exception as e:
+ self.send_error(http.HTTPStatus.INTERNAL_SERVER_ERROR, str(e))
+
+ with socketserver.TCPServer((host, port), TestRequestHandler) as httpd:
+ thread = threading.Thread(target=httpd.serve_forever)
+ thread.start()
+
+ # the method-under-test
+ resolved_user_schema = resolved_refs(user_schema)
+
+ print('user_schema: ', json.dumps(user_schema, indent=2))
+ print('resolved_user_schema: ', json.dumps(resolved_user_schema, indent=2))
+
+ self.assertNotEqual(user_schema['properties']['email'], resolved_user_schema['properties']['email'])
+ self.assertEqual(base_schema['definitions']['email'], resolved_user_schema['properties']['email'])
+
+ httpd.shutdown()
+ thread.join(timeout=2)
+ self.assertFalse(thread.is_alive())
+
+ def test_replace_host_in_ref_urls(self):
+ base_host = "http://foo.bar.com"
+ path = "/my/path"
+
+ schema = {"$id": base_host + path + "/user_schema.json",
+ "$schema": "http://json-schema.org/draft-06/schema#",
+ "type": "object",
+ "default": {},
+ "properties": {
+ "name": {
+ "type": "string",
+ "minLength": 2 },
+ "email": {
+ "$ref": base_host + path + "/base_schema.json" + "#/definitions/email" },
+ "other_emails": {
+ "type": "array",
+ "items": {
+ "$ref": base_host + path + "/base_schema.json" + "#/definitions/email"
+ }
+ }
+ } }
+
+ new_base_host = 'http://127.0.0.1'
+ url_fixed_schema = replace_host_in_urls(schema, new_base_host)
+
+ print('schema: ', json.dumps(schema, indent=2))
+ print('url_fixed_schema: ', json.dumps(url_fixed_schema, indent=2))
+
+ self.assertEqual(new_base_host+path+"/user_schema.json", url_fixed_schema['$id'])
+ self.assertEqual(new_base_host+path+"/base_schema.json" + "#/definitions/email", url_fixed_schema['properties']['email']['$ref'])
+ self.assertEqual(new_base_host+path+"/base_schema.json" + "#/definitions/email", url_fixed_schema['properties']['other_emails']['items']['$ref'])
+ self.assertEqual("http://json-schema.org/draft-06/schema#", url_fixed_schema['$schema'])
+ self.assertEqual(json.dumps(schema, indent=2).replace(base_host, new_base_host), json.dumps(url_fixed_schema, indent=2))
+
if __name__ == '__main__':
unittest.main()
diff --git a/LTA/sip/lib/CMakeLists.txt b/LTA/sip/lib/CMakeLists.txt
index a9851a135a5f82c62c0110b5d802d191a95a2fe0..7cfd5468d9f8430570aeea7bb77a6e4c52b0495e 100644
--- a/LTA/sip/lib/CMakeLists.txt
+++ b/LTA/sip/lib/CMakeLists.txt
@@ -14,6 +14,7 @@ set(_py_files
constants.py
visualizer.py
query.py
+ station_coordinates.py
)
@@ -25,6 +26,7 @@ set(resource_files
python_install(${_py_files}
DESTINATION lofar/lta/sip)
+
install(FILES ${resource_files}
DESTINATION ${PYTHON_INSTALL_DIR}/lofar/lta/sip
COMPONENT ${lower_package_name})
diff --git a/LTA/sip/lib/siplib.py b/LTA/sip/lib/siplib.py
index bb3c6238a14a19117ee2d5379f838ddb6b04f66c..4f89a4fe91f5552972e9b43c0cc4afe903d9d9fc 100644
--- a/LTA/sip/lib/siplib.py
+++ b/LTA/sip/lib/siplib.py
@@ -28,6 +28,7 @@
from . import ltasip
import pyxb
from . import constants
+from . import station_coordinates
import os
import uuid
import xml.dom.minidom
@@ -38,8 +39,6 @@ import logging
logger = logging.getLogger(__name__)
VERSION = "SIPlib 0.4"
-d = os.path.dirname(os.path.realpath(__file__))
-STATION_CONFIG_PATH = d+'/station_coordinates.conf'
ltasip.Namespace.setPrefix('sip')
# todo: create docstrings for everything.
@@ -144,30 +143,28 @@ class Station():
__afield1=None
__afield2=None
- with open(STATION_CONFIG_PATH, 'r') as f:
- for line in f.readlines():
- if line.strip():
- field_coords = eval("dict("+line+")") # literal_eval does not accept dict definition via constructor. Make sure config file is not writable to prevent code execution!
- for type in antennafieldtypes:
- if field_coords["name"] == name+"_"+type:
- __afield=AntennafieldXYZ(
- type=type,
+ station_coords = station_coordinates.parse_station_coordinates()
+ for atype in antennafieldtypes:
+ if name+"_"+atype in station_coords.keys():
+ field_coords = station_coords[name+"_"+atype]
+ __afield=AntennafieldXYZ(
+ type=atype,
coordinate_system=field_coords["coordinate_system"],
coordinate_unit=constants.LENGTHUNIT_M, # Does this make sense? I have to give a lenght unit accoridng to the XSD, but ICRF should be decimal degrees?!
coordinate_x=field_coords["x"],
coordinate_y=field_coords["y"],
coordinate_z=field_coords["z"])
- if not __afield1:
- __afield1=__afield
- elif not __afield2:
- __afield2=__afield
+ if not __afield1:
+ __afield1=__afield
+ elif not __afield2:
+ __afield2=__afield
if not __afield1:
- raise Exception("no matching coordinates found for station:", name,"and fields",str(antennafieldtypes))
+ raise Exception("no matching coordinates found for station:", name, "and fields", str(antennafieldtypes))
- if name.startswith( 'CS' ):
+ if name.startswith('CS'):
sttype = "Core"
- elif name.startswith( "RS" ):
+ elif name.startswith("RS"):
sttype = "Remote"
else:
sttype = "International"
diff --git a/LTA/sip/lib/station_coordinates.conf b/LTA/sip/lib/station_coordinates.conf
index 07e488f9a72ccf960c6e6f30c9bc39823e3c7613..741cd1395f2a6a362e70335b7c97d0ac383eb746 100644
--- a/LTA/sip/lib/station_coordinates.conf
+++ b/LTA/sip/lib/station_coordinates.conf
@@ -154,3 +154,9 @@ coordinate_system='ITRF2005', x='3850973.9872', y='1439061.04111', z='4860478.99
coordinate_system='ITRF2005', x='3850980.8812', y='1438994.87911', z='4860498.993' , name='PL611_HBA'
coordinate_system='ITRF2005', x='3551478.64311', y='1334128.4928', z='5110179.160' , name='PL612_LBA'
coordinate_system='ITRF2005', x='3551481.8171', y='1334203.5728', z='5110157.410' , name='PL612_HBA'
+
+coordinate_system='ITRF2005', x='3801633.528060000', y='-529021.899396000', z='5076997.185' , name='IE613_LBA'
+coordinate_system='ITRF2005', x='3801691.943300000', y='-528983.966429000', z='5076957.924' , name='IE613_HBA'
+
+coordinate_system='ITRF2005', x='3183318.032280000', y='1276777.654760000', z='5359435.077' , name='LV614_LBA'
+coordinate_system='ITRF2005', x='3183249.285620000', y='1276801.742170000', z='5359469.949' , name='LV614_HBA'
\ No newline at end of file
diff --git a/LTA/sip/lib/station_coordinates.py b/LTA/sip/lib/station_coordinates.py
new file mode 100644
index 0000000000000000000000000000000000000000..f2952a203d2af5ee5578342eac1af1706c41662c
--- /dev/null
+++ b/LTA/sip/lib/station_coordinates.py
@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+
+# This module provides functions for easy creation of a Lofar LTA SIP document.
+# It builds upon a Pyxb-generated API from the schema definition, which is very clever but hard to use, since
+# the arguments in class constructors and functions definitions are not verbose and there is no intuitive way
+# to determine the mandatory and optional elements to create a valid SIP document. This module is designed to
+# provide easy-to-use functions that bridges this shortcoming of the Pyxb API.
+#
+# Usage: Import module. Create an instance of Sip.
+# Add elements through the Sip.add_X functions. Many require instances of other classes of the module.
+# call getprettyxml() and e.g. save to disk.
+#
+# Note on validation: From construction through every addition, the SIP should remain valid (or throw an error
+# that clearly points out where e.g. a given value does not meet the restrictions of the SIP schema.
+#
+# Note on code structure: This has to be seen as a compromise between elegant and maintainable code with well-
+# structured inheritance close to the schema definition on the one hand, and something more straightforward to use,
+# with flatter hierarchies on the other hand.
+#
+# Note on parameter maps: The ...Map objects are helper objects to create dictionaries for the commonly used
+# constructor arguments of several other objects. This could alternatively also be implemented via inheritance from
+# a supertype, and indeed is solved like this in the pyxb code. However, this then requires the use of an argument
+# list pointer, which hides the list of required and optional arguments from the user. Alternatively, all arguments
+# have to be mapped in all constructors repeatedly, creating lots of boilerplate code. This is the nicest approach
+# I could think of that keeps the whole thing reasonably maintainable AND usable.
+
+import os
+d = os.path.dirname(os.path.realpath(__file__))
+STATION_CONFIG_PATH = d+'/station_coordinates.conf'
+
+
+def parse_station_coordinates() -> dict:
+ """
+ :return: a dict mapping station field name, e.g. "CS002_LBA", to a dict containing ITRF coordinates
+ """
+ station_coordinates = {}
+ with open(STATION_CONFIG_PATH, 'r') as f:
+ for line in f.readlines():
+ if line.strip():
+ field_coords = eval("dict(" + line + ")") # literal_eval does not accept dict definition via constructor. Make sure config file is not writable to prevent code execution!
+ station_coordinates[field_coords.pop("name")] = field_coords
+ return station_coordinates
+
+
diff --git a/MAC/APL/MainCU/src/MACScheduler/MACScheduler.cc b/MAC/APL/MainCU/src/MACScheduler/MACScheduler.cc
index a402f3bef071ed0c3779e28203c083849a405293..d9b5a3953f718c732933231ab8d6e37cdbdf45f2 100644
--- a/MAC/APL/MainCU/src/MACScheduler/MACScheduler.cc
+++ b/MAC/APL/MainCU/src/MACScheduler/MACScheduler.cc
@@ -644,9 +644,10 @@ GCFEvent::TResult MACScheduler::active_state(GCFEvent& event, GCFPortInterface&
tm.setTreeState(theObs->second, tsc.get("finished"));
#endif
}
- else {
- itsTMSSconnection->setSubtaskState(theObs->second, "finished");
- }
+ //else {
+ // the TMSS subtask state is set to finished in TMSS after all feedback has been processed now
+ // itsTMSSconnection->setSubtaskState(theObs->second, "finished");
+ //}
}
else {
if(theObs->second < 2000000) {
diff --git a/MAC/Services/src/PipelineControl.py b/MAC/Services/src/PipelineControl.py
index eaf0693b8ae2e092b6c94a5d926b5a673a258417..0f6febfe8ce68168d2f93003444dfb52d90cb8ce 100755
--- a/MAC/Services/src/PipelineControl.py
+++ b/MAC/Services/src/PipelineControl.py
@@ -545,14 +545,12 @@ class PipelineControlTMSSHandler(TMSSSubTaskEventMessageHandler):
runcmd {setStatus_finishing}
if [ $RESULT -eq 0 ]; then
- # if we reached this point, the pipeline ran succesfully
- runcmd {setStatus_finished}
-
+ # if we reached this point, the pipeline ran succesfully, and TMSS will set it to finished once it processed the feedback
# notify ganglia
+ # !!! TODO Is TMSS supposed to inform Ganglia in future?
wget -O - -q "http://ganglia.control.lofar/ganglia/api/events.php?action=add&start_time=now&summary=Pipeline {obsid} FINISHED&host_regex="
else
# !!! TODO: How to set an "unsuccesfull" finished state in TMSS?
- runcmd {setStatus_finished}
fi
# report status back to SLURM
diff --git a/QA/QA_Service/test/t_qa_service.py b/QA/QA_Service/test/t_qa_service.py
index 2191bea8f073d28533de7233f011fb76e2718824..477ba9dc491fb229786c354b14ec0fc6e8fcd1fe 100755
--- a/QA/QA_Service/test/t_qa_service.py
+++ b/QA/QA_Service/test/t_qa_service.py
@@ -39,7 +39,6 @@ from lofar.messaging.messagebus import TemporaryExchange, BusListenerJanitor
from lofar.messaging.messages import EventMessage
from lofar.sas.otdb.config import DEFAULT_OTDB_NOTIFICATION_SUBJECT
from lofar.common.test_utils import unit_test, integration_test
-from lofar.common.json_utils import add_defaults_to_json_object_for_schema
# the tests below test is multi threaded (even multi process)
# define a SynchronizationQABusListener-derivative to handle synchronization (set the *_events)
@@ -100,6 +99,7 @@ class TestQAService(unittest.TestCase):
cls.tmss_test_env = TMSSTestEnvironment(exchange=cls.tmp_exchange.address)
cls.tmss_test_env.start()
+ cls.tmss_test_env.populate_schemas()
@classmethod
def tearDownClass(cls) -> None:
@@ -567,10 +567,10 @@ class TestQAService(unittest.TestCase):
qafile_subtask_template = tmss_client.get_subtask_template(name="QA file conversion")
- qafile_subtask_spec_doc = add_defaults_to_json_object_for_schema({}, qafile_subtask_template['schema'])
+ qafile_subtask_spec_doc = tmss_client.get_subtask_template_default_specification(name="QA file conversion")
subtask = tdc.post_data_and_get_response_as_json_object(tdc.Subtask(specifications_template_url=qafile_subtask_template['url'],
- specifications_doc=qafile_subtask_spec_doc), '/subtask/')
+ specifications_doc=qafile_subtask_spec_doc), '/subtask/')
subtask_id = subtask['id']
subtask_input = tdc.post_data_and_get_response_as_json_object(tdc.SubtaskInput(subtask['url'], dataproduct_urls=[uv_dataproduct['url']], subtask_output_url=uvdp_producer['url']), '/subtask_input/')
@@ -589,7 +589,7 @@ class TestQAService(unittest.TestCase):
qaservice.filtering_tmssbuslistener.start_listening()
qaplots_subtask_template = tmss_client.get_subtask_template(name="QA plots")
- qaplots_subtask_spec_doc = add_defaults_to_json_object_for_schema({}, qaplots_subtask_template['schema'])
+ qaplots_subtask_spec_doc = tmss_client.get_subtask_template_default_specification(name="QA plots")
# start waiting until ConvertedMS2Hdf5 event message received (or timeout)
qa_listener.converted_event.wait(30)
diff --git a/RTCP/Cobalt/GPUProc/src/scripts/stopBGL.sh b/RTCP/Cobalt/GPUProc/src/scripts/stopBGL.sh
index 938bc6d1bde108269554c60af11781d902d5a02c..64fe3c049ca409c8fd4c9423d41b91882b1adeba 100755
--- a/RTCP/Cobalt/GPUProc/src/scripts/stopBGL.sh
+++ b/RTCP/Cobalt/GPUProc/src/scripts/stopBGL.sh
@@ -19,7 +19,7 @@ JOB="$1"
OBSID="$2"
# The name of what will be our parset
-PARSET=$LOFARROOT/var/run/rtcp-$OBSID.parset
+PARSET=$LOFARROOT/nfs/parset/rtcp-$OBSID.parset
# The file to store the PID in
PIDFILE=$LOFARROOT/var/run/rtcp-$OBSID.pid
diff --git a/SAS/TMSS/client/bin/CMakeLists.txt b/SAS/TMSS/client/bin/CMakeLists.txt
index d2bd6170e887de5af55d2c6ec98eb1adfcf656bc..34d5fafe0d18747a3981c8e0491e1e01dc941600 100644
--- a/SAS/TMSS/client/bin/CMakeLists.txt
+++ b/SAS/TMSS/client/bin/CMakeLists.txt
@@ -7,3 +7,4 @@ lofar_add_bin_scripts(tmss_get_subtask_successors)
lofar_add_bin_scripts(tmss_schedule_subtask)
lofar_add_bin_scripts(tmss_get_setting)
lofar_add_bin_scripts(tmss_set_setting)
+lofar_add_bin_scripts(tmss_populate)
diff --git a/SAS/TMSS/client/bin/tmss_populate b/SAS/TMSS/client/bin/tmss_populate
new file mode 100755
index 0000000000000000000000000000000000000000..375f9112e408ce1aec45778ad16aef9230e429d0
--- /dev/null
+++ b/SAS/TMSS/client/bin/tmss_populate
@@ -0,0 +1,25 @@
+#!/usr/bin/python3
+
+# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+from lofar.sas.tmss.client.populate import populate_schemas_main
+
+if __name__ == "__main__":
+ logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO)
+ populate_schemas_main()
diff --git a/SAS/TMSS/client/lib/CMakeLists.txt b/SAS/TMSS/client/lib/CMakeLists.txt
index 94606c743637ebf74951b6d15efd87ec369737eb..2281da0ee8a6417e0bfb8969ab55e43deb93ee65 100644
--- a/SAS/TMSS/client/lib/CMakeLists.txt
+++ b/SAS/TMSS/client/lib/CMakeLists.txt
@@ -4,6 +4,7 @@ include(PythonInstall)
set(_py_files
tmssbuslistener.py
mains.py
+ populate.py
tmss_http_rest_client.py
)
diff --git a/SAS/TMSS/client/lib/populate.py b/SAS/TMSS/client/lib/populate.py
new file mode 100644
index 0000000000000000000000000000000000000000..ed05353790cb9db1ccdeeed71f0b5589201ca502
--- /dev/null
+++ b/SAS/TMSS/client/lib/populate.py
@@ -0,0 +1,83 @@
+import logging
+logger = logging.getLogger(__name__)
+
+import json
+from lofar.sas.tmss.client.tmss_http_rest_client import TMSSsession
+from lofar.common import json_utils
+import os
+
+def populate_schemas_main():
+ from optparse import OptionParser
+
+ # Check the invocation arguments
+ parser = OptionParser('%prog [options]', description='upload the templates to TMSS')
+ parser.add_option('-d', '--dir', dest='schema_dir', type='string',
+ default=os.path.expandvars('$LOFARROOT/share/tmss/schemas'),
+ help='''directory path containing the schemas, default: '%default')''')
+ parser.add_option('-f', '--file', dest='templates_file', type='string',
+ default='templates.json',
+ help='''json file containing the list of templates with name, description, vesions, and template type. default: '%default')''')
+ (options, args) = parser.parse_args()
+ return populate_schemas(options.schema_dir, options.templates_file)
+
+def populate_schemas(schema_dir: str=None, templates_filename: str=None):
+ if schema_dir is None:
+ schema_dir = os.path.expandvars('$LOFARROOT/share/tmss/schemas')
+
+ if templates_filename is None:
+ templates_filename = 'templates.json'
+
+ templates_filepath = os.path.join(schema_dir, templates_filename)
+ logger.info("Reading templates in: %s", templates_filepath)
+ with open(templates_filepath) as templates_file:
+ templates = json.loads(templates_file.read())
+
+ with TMSSsession.create_from_dbcreds_for_ldap() as client:
+ base_url = client.base_url.rstrip('/').rstrip('api').rstrip('/')
+ for template in templates:
+ try:
+ with open(os.path.join(schema_dir, template.pop('file_name'))) as schema_file:
+ try:
+ json_schema = json.loads(schema_file.read())
+
+ template_name = template.pop('template')
+ name = template.pop('name', json_schema.get('title', '<no name>'))
+ description = template.pop('description', json_schema.get('description', '<no description>'))
+ version = template.pop('version', '1')
+
+ if template_name == 'subtask_template' and 'type' in template:
+ # override plain-text type by its url
+ template['type'] = client.get_path_as_json_object('subtask_type/'+template.pop('type'))['url']
+
+ if template_name == 'task_template' and 'type' in template:
+ # override plain-text type by its url
+ template['type'] = client.get_path_as_json_object('task_type/'+template.pop('type'))['url']
+
+ if template_name == 'scheduling_unit_observing_strategy_template':
+ scheduling_unit_templates = client.get_path_as_json_object('scheduling_unit_template?name=' + template.pop('scheduling_unit_template_name') + '&version=' + template.pop('scheduling_unit_template_version'))
+ scheduling_unit_template = scheduling_unit_templates[0]
+ template['scheduling_unit_template'] = scheduling_unit_template['url']
+
+ # inject a unique id in the form of a unique URL to this schema
+ json_schema['$id'] = '%s/api/schemas/%s/%s/%s' % (base_url, template_name.replace('_',''), name, version)
+
+ # make sure that all urls point to the tmss base_url
+ json_schema = json_utils.replace_host_in_urls(json_schema, new_base_url=base_url)
+
+ if template_name == 'scheduling_unit_observing_strategy_template':
+ template['template'] = json_schema
+ else:
+ template['schema'] = json_schema
+
+ logger.info("Uploading template template='%s' name='%s' version='%s'", template, name, version)
+
+ client.post_template(template_path=template_name,
+ name=name,
+ description=description,
+ version=version,
+ **template)
+ except Exception as e:
+ logger.error(e)
+ except Exception as e:
+ logger.error(e)
+
diff --git a/SAS/TMSS/client/lib/tmss_http_rest_client.py b/SAS/TMSS/client/lib/tmss_http_rest_client.py
index a2f9534a64ad48df2fd515e69cc6138d8b550dd4..0fb13c71c657b88d82ad7bdd2dafdd6419cfdf99 100644
--- a/SAS/TMSS/client/lib/tmss_http_rest_client.py
+++ b/SAS/TMSS/client/lib/tmss_http_rest_client.py
@@ -1,5 +1,5 @@
import logging
-logger = logging.getLogger(__file__)
+logger = logging.getLogger(__name__)
import requests
from http.client import responses
@@ -63,6 +63,7 @@ class TMSSsession(object):
'''open the request session and login'''
self.session.__enter__()
self.session.verify = False
+ self.session.headers['Accept'] = 'application/json'
if self.authentication_method == self.OPENID:
# get authentication page of OIDC through TMSS redirect
@@ -157,21 +158,18 @@ class TMSSsession(object):
def get_path_as_json_object(self, path: str, params={}) -> object:
'''get resource at the given path, interpret it as json, and return it as as native object (usually a dict or a list of dicts)'''
- full_url = '%s/%s/' % (self.base_url, path.strip('/'))
+ full_url = '%s/%s' % (self.base_url, path.strip('/'))
return self.get_url_as_json_object(full_url, params=params)
def get_url_as_json_object(self, full_url: str, params={}) -> object:
'''get resource at the given full url (including http://<base_url>, interpret it as json, and return it as as native object (usually a dict or a list of dicts)'''
- if "format=json" not in full_url or params.get("format") != "json":
- params['format'] ='json'
-
- response = self.session.get(url=full_url, params=params)
+ response = self.session.get(url=full_url, params=params, timeout=100000)
logger.info("%s %s %s on %s", response.request.method.upper(), response.status_code, responses.get(response.status_code), response.request.url)
if response.status_code >= 200 and response.status_code < 300:
result = json.loads(response.content.decode('utf-8'))
if isinstance(result, dict):
- result_object = result.get('results', result) # return the 'results' list if any, or lese just the object itself
+ result_object = result.get('results', result) # return the 'results' list if any, or else just the object itself
if result.get('next'):
# recurse, get the 'next' url, and return a concatenation of the results
@@ -180,7 +178,7 @@ class TMSSsession(object):
return result
# ugly error message parsing
- content = response.content.decode('utf-8')
+ content = response.text
try:
error_msg = content.split('\n')[1] # magic! error message is at 2nd line of response...
except:
@@ -188,7 +186,7 @@ class TMSSsession(object):
raise Exception("Could not get %s - %s %s - %s" % (full_url, response.status_code, responses.get(response.status_code), error_msg))
- def _get_template(self, template_type_name: str, name: str, version: str=None) -> dict:
+ def _get_template(self, template_type_name: str, name: str, version: int=None) -> dict:
'''get the template of the given type as dict for the given name (and version)'''
clauses = {}
if name is not None:
@@ -204,18 +202,30 @@ class TMSSsession(object):
return None
return result
- def get_schedulingunit_template(self, name: str, version: str=None) -> dict:
+ def get_schedulingunit_template(self, name: str, version: int=None) -> dict:
'''get the schedulingunit_template as dict for the given name (and version)'''
return self._get_template('scheduling_unit_template', name, version)
- def get_task_template(self, name: str, version: str=None) -> dict:
+ def get_task_template(self, name: str, version: int=None) -> dict:
'''get the task_template as dict for the given name (and version)'''
return self._get_template('task_template', name, version)
- def get_subtask_template(self, name: str, version: str=None) -> dict:
+ def get_subtask_template(self, name: str, version: int=None) -> dict:
'''get the subtask_template as dict for the given name (and version)'''
return self._get_template('subtask_template', name, version)
+ def get_schedulingunit_template_default_specification(self, name: str, version: int=None) -> dict:
+ template = self.get_schedulingunit_template(name=name, version=version)
+ return self.get_url_as_json_object(template['url']+"/default")
+
+ def get_task_template_default_specification(self, name: str, version: int=None) -> dict:
+ template = self.get_task_template(name=name, version=version)
+ return self.get_url_as_json_object(template['url']+"/default")
+
+ def get_subtask_template_default_specification(self, name: str, version: int=None) -> dict:
+ template = self.get_subtask_template(name=name, version=version)
+ return self.get_url_as_json_object(template['url']+"/default")
+
def get_subtask_output_dataproducts(self, subtask_id: int) -> []:
'''get the output dataproducts of the subtask with the given subtask_id'''
return self.get_path_as_json_object('subtask/%s/output_dataproducts' % subtask_id)
@@ -265,3 +275,51 @@ class TMSSsession(object):
content = response.content.decode('utf-8')
raise Exception("Could not set status with url %s - %s %s - %s" % (response.request.url, response.status_code, responses.get(response.status_code), content))
+ def post_template(self, template_path:str, name: str, description: str, version: int, schema: str=None, template: str=None, **kwargs):
+ '''POST a template at <BASE_URL>/<template_path> with the given name, description and version'''
+ json_data = {'name': name,
+ 'description': description,
+ 'version': version}
+ if schema is not None:
+ json_data['schema'] = json.loads(schema) if isinstance(schema, str) else schema
+ if template is not None:
+ json_data['template'] = json.loads(template) if isinstance(template, str) else template
+ json_data.update(**kwargs)
+
+ response = self.session.post(url='%s/%s/' % (self.base_url, template_path), json=json_data)
+ if response.status_code == 201:
+ logger.info("created new template: %s", json.loads(response.text)['url'])
+ else:
+ raise Exception("Could not POST template: " + response.text)
+
+ def append_to_subtask_raw_feedback(self, subtask_id: int, feedback: str) -> {}:
+ '''append the raw_feedback for the given subtask, and return the subtask with its new state, or raise an error'''
+ existing_feedback = self.get_path_as_json_object('/subtask/%s/' % (subtask_id))['raw_feedback']
+ if existing_feedback is None or existing_feedback is "":
+ new_feedback = feedback
+ else:
+ new_feedback = "%s\n%s" % (existing_feedback, feedback)
+ response = self.session.patch(url='%s/subtask/%s/' % (self.base_url, subtask_id),
+ json={'raw_feedback': new_feedback},
+ params={'format': 'json'})
+
+ if response.status_code >= 200 and response.status_code < 300:
+ return json.loads(response.content.decode('utf-8'))
+
+ content = response.content.decode('utf-8')
+ raise Exception("Could not append feedback to subtask with url %s - %s %s - %s" % (
+ response.request.url, response.status_code, responses.get(response.status_code), content))
+
+ def process_subtask_feedback_and_set_finished(self, subtask_id: int) -> {}:
+ '''process the raw_feedback of a given subtask and set the subtask to finished on succes. Return the subtask
+ with its new state, or raise an error'''
+ response = self.session.post(url='%s/subtask/%s/process_feedback_and_set_finished' % (self.base_url, subtask_id),
+ params={'format': 'json'})
+
+ if response.status_code >= 200 and response.status_code < 300:
+ return json.loads(response.content.decode('utf-8'))
+
+ content = response.content.decode('utf-8')
+ raise Exception("Could not process feedback with url %s - %s %s - %s" % (
+ response.request.url, response.status_code, responses.get(response.status_code), content))
+
diff --git a/SAS/TMSS/docker-compose-scu199.yml b/SAS/TMSS/docker-compose-scu199.yml
index f2aa2b2c357fe81ae4724db31d79b31ed8702762..0778331fa0f4cbdbc15cf49c1c3c88273b98b4db 100644
--- a/SAS/TMSS/docker-compose-scu199.yml
+++ b/SAS/TMSS/docker-compose-scu199.yml
@@ -6,7 +6,8 @@ services:
restart: on-failure
env_file:
- ./.env
- command: bash -c 'source /opt/lofar/lofarinit.sh && ALLOWED_HOSTS=* tmss_test_environment -H 0.0.0.0 -p 8008'
+ network_mode: "host"
+ command: bash -c 'source /opt/lofar/lofarinit.sh && ALLOWED_HOSTS=* tmss_test_environment -H 0.0.0.0 -P `hostname -f` -p 8008 --data'
ports:
- "8008:8008"
testprovider:
diff --git a/SAS/TMSS/docker-compose-ua.yml b/SAS/TMSS/docker-compose-ua.yml
index 7d2f96c0c010b2c8614ad920c1977b78eaa33019..74752f8596f9daa35763a85b7f5e355288b38cbd 100644
--- a/SAS/TMSS/docker-compose-ua.yml
+++ b/SAS/TMSS/docker-compose-ua.yml
@@ -7,18 +7,19 @@ services:
ports:
- 5672:5672
- 15672:15672
+ oidc-provider:
+ image: nexus.cep4.control.lofar:18080/tmss_testprovider:latest
+ restart: unless-stopped
+ hostname: oidc-provider
+ env_file:
+ - ./.env
+ ports:
+ - "8088:8088"
web:
image: nexus.cep4.control.lofar:18080/tmss_django:latest
restart: on-failure
env_file:
- ./.env
- command: bash -c 'source /opt/lofar/lofarinit.sh && python3 lib/python3.6/site-packages/lofar/sas/tmss/manage.py runserver 0.0.0.0:8008'
+ command: bash -c 'source /opt/lofar/lofarinit.sh && python3 lib64/python3.6/site-packages/lofar/sas/tmss/manage.py runserver 0.0.0.0:8008'
ports:
- "8008:8008"
- testprovider:
- image: nexus.cep4.control.lofar:18080/tmss_testprovider:latest
- restart: unless-stopped
- env_file:
- - ./.env
- ports:
- - "8088:8088"
diff --git a/SAS/TMSS/frontend/tmss_webapp/package.json b/SAS/TMSS/frontend/tmss_webapp/package.json
index 98a358b8eab1a02049d514cb456e457f2e997b1d..0e86725052116a227147adefafb33d766db1d724 100644
--- a/SAS/TMSS/frontend/tmss_webapp/package.json
+++ b/SAS/TMSS/frontend/tmss_webapp/package.json
@@ -3,6 +3,7 @@
"version": "0.1.0",
"private": true,
"dependencies": {
+ "@apidevtools/json-schema-ref-parser": "^9.0.6",
"@fortawesome/fontawesome-free": "^5.13.1",
"@json-editor/json-editor": "^2.3.0",
"@testing-library/jest-dom": "^4.2.4",
@@ -14,6 +15,7 @@
"flatpickr": "^4.6.3",
"font-awesome": "^4.7.0",
"history": "^5.0.0",
+ "interactjs": "^1.9.22",
"lodash": "^4.17.19",
"match-sorter": "^4.1.0",
"moment": "^2.27.0",
@@ -26,12 +28,14 @@
"react-app-polyfill": "^1.0.6",
"react-bootstrap": "^1.0.1",
"react-bootstrap-datetimepicker": "0.0.22",
+ "react-calendar-timeline": "^0.27.0",
"react-dom": "^16.13.1",
"react-frame-component": "^4.1.2",
"react-json-view": "^1.19.1",
"react-loader-spinner": "^3.1.14",
"react-router-dom": "^5.2.0",
"react-scripts": "^3.4.2",
+ "react-split-pane": "^0.1.92",
"react-table": "^7.2.1",
"react-transition-group": "^2.5.1",
"reactstrap": "^8.5.1",
@@ -45,7 +49,7 @@
"test": "react-scripts test",
"eject": "react-scripts eject"
},
- "proxy": "http://192.168.99.100:8008/",
+ "proxy": "http://127.0.0.1:8008/",
"eslintConfig": {
"extends": "react-app"
},
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/App.css b/SAS/TMSS/frontend/tmss_webapp/src/App.css
index 766fff47baad6747a35c03a164125f8d181f5956..afca29b115546e020b56ad71b4a94fbe82d6c65d 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/App.css
+++ b/SAS/TMSS/frontend/tmss_webapp/src/App.css
@@ -53,6 +53,10 @@ a{
margin-bottom: 10px;
}
+.main-content {
+ margin-top:20px;
+}
+
.main-content span,a{
font-size: 14px;
}
@@ -90,6 +94,18 @@ p {
margin-bottom: 5px;
}
+.main-content .p-grid span {
+ margin-bottom: 10px;
+}
+
+.p-chips-token,.p-inputnumber span {
+ margin-bottom: 0px;
+}
+
+.p-chips-token .p-chips-token-label {
+ margin-bottom: 0px !important;
+}
+
.p-field {
margin-bottom: 0.5rem;
}
@@ -208,3 +224,11 @@ thead {
transform: rotate(360deg);
}
}
+
+div[data-schemapath='root.$schema'] {
+ display: none;
+}
+
+.app-header-menu ul li a span {
+ display: inline !important;
+}
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/App.js b/SAS/TMSS/frontend/tmss_webapp/src/App.js
index af0b8d760c3c17348d38f349362ee983a10dbc1b..28dc4939d93ef1dcc6a112bb1da40a4a8fa79067 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/App.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/App.js
@@ -23,20 +23,23 @@ class App extends Component {
layoutMode: 'static',
currentMenu: '',
currentPath: '/',
- staticMenuInactive: false,
- overlayMenuActive: false,
- mobileMenuActive: false,
+ PageTitle:'',
+ staticMenuInactive: localStorage.getItem('staticMenuInactive') === 'true' ? true : false,
+ overlayMenuActive: localStorage.getItem('overlayMenuActive') === 'true' ? true : false,
+ mobileMenuActive: localStorage.getItem('mobileMenuActive') === 'true' ? true : false,
};
this.onWrapperClick = this.onWrapperClick.bind(this);
this.onToggleMenu = this.onToggleMenu.bind(this);
this.onSidebarClick = this.onSidebarClick.bind(this);
this.onMenuItemClick = this.onMenuItemClick.bind(this);
+ this.setPageTitle = this.setPageTitle.bind(this);
this.menu = [
- {label: 'Dashboard', icon: 'pi pi-fw pi-home', to:'/dashboard'},
- {label: 'Cycle', icon:'pi pi-fw pi-spinner', to:'/cycle'},
- {label: 'Project', icon: 'fab fa-fw fa-wpexplorer', to:'/project'},
- {label: 'Scheduling Units', icon: 'pi pi-fw pi-calendar', to:'/schedulingunit'},
+ {label: 'Dashboard', icon: 'pi pi-fw pi-home', to:'/dashboard',section: 'dashboard'},
+ {label: 'Cycle', icon:'pi pi-fw pi-spinner', to:'/cycle',section: 'cycle'},
+ {label: 'Project', icon: 'fab fa-fw fa-wpexplorer', to:'/project',section: 'project'},
+ {label: 'Scheduling Units', icon: 'pi pi-fw pi-calendar', to:'/schedulingunit',section: 'schedulingunit'},
+ {label: 'Timeline', icon: 'pi pi-fw pi-clock', to:'/su/timelineview',section: 'su/timelineview'},
// {label: 'Tasks', icon: 'pi pi-fw pi-check-square', to:'/task'},
@@ -62,11 +65,16 @@ class App extends Component {
if (this.state.layoutMode === 'overlay') {
this.setState({
overlayMenuActive: !this.state.overlayMenuActive
- });
+ }, () => {
+ localStorage.setItem('overlayMenuActive', this.state.overlayMenuActive);
+ }
+ );
}
else if (this.state.layoutMode === 'static') {
this.setState({
staticMenuInactive: !this.state.staticMenuInactive
+ }, () => {
+ localStorage.setItem('staticMenuInactive', this.state.staticMenuInactive);
});
}
}
@@ -74,7 +82,10 @@ class App extends Component {
const mobileMenuActive = this.state.mobileMenuActive;
this.setState({
mobileMenuActive: !mobileMenuActive
- });
+ },() => {
+ localStorage.setItem('mobileMenuActive', this.state.mobileMenuActive);
+ }
+ );
}
event.preventDefault();
}
@@ -90,28 +101,34 @@ class App extends Component {
isDesktop() {
return window.innerWidth > 1024;
}
+
+ setPageTitle(PageTitle) {
+ if (PageTitle !== this.state.PageTitle) {
+ this.setState({ PageTitle })
+ }
+ }
render() {
- const wrapperClass = classNames('layout-wrapper', {
- 'layout-overlay': this.state.layoutMode === 'overlay',
- 'layout-static': this.state.layoutMode === 'static',
- 'layout-static-sidebar-inactive': this.state.staticMenuInactive && this.state.layoutMode === 'static',
- 'layout-overlay-sidebar-active': this.state.overlayMenuActive && this.state.layoutMode === 'overlay',
- 'layout-mobile-sidebar-active': this.state.mobileMenuActive
- });
- const AppBreadCrumbWithRouter = withRouter(AppBreadcrumb);
-
- return (
+ const wrapperClass = classNames('layout-wrapper', {
+ 'layout-overlay': this.state.layoutMode === 'overlay',
+ 'layout-static': this.state.layoutMode === 'static',
+ 'layout-static-sidebar-inactive': this.state.staticMenuInactive && this.state.layoutMode === 'static',
+ 'layout-overlay-sidebar-active': this.state.overlayMenuActive && this.state.layoutMode === 'overlay',
+ 'layout-mobile-sidebar-active': this.state.mobileMenuActive
+ });
+ const AppBreadCrumbWithRouter = withRouter(AppBreadcrumb);
+
+ return (
<React.Fragment>
<div className="App">
{/* <div className={wrapperClass} onClick={this.onWrapperClick}> */}
<div className={wrapperClass}>
<AppTopbar onToggleMenu={this.onToggleMenu}></AppTopbar>
<Router basename={ this.state.currentPath }>
- <AppMenu model={this.menu} onMenuItemClick={this.onMenuItemClick} />
+ <AppMenu model={this.menu} onMenuItemClick={this.onMenuItemClick} layoutMode={this.state.la} active={this.state.menuActive}/>
<div className="layout-main">
- <AppBreadCrumbWithRouter/>
- <RoutedContent />
+ <AppBreadCrumbWithRouter setPageTitle={this.setPageTitle} />
+ <RoutedContent />
</div>
</Router>
<AppFooter></AppFooter>
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/__mocks__/project.service.data.js b/SAS/TMSS/frontend/tmss_webapp/src/__mocks__/project.service.data.js
index 066fd339758c56e64cc51a2ae81142eb978a583d..114b35b7085b8cb2a3ab809696635554697f319d 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/__mocks__/project.service.data.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/__mocks__/project.service.data.js
@@ -1,4 +1,5 @@
+
const ProjectServiceMock= {
project_categories: [{url: "Regular", value: 'Regular'}, {url: "User Shared Support", value: 'User Shared Support'}],
period_categories: [{url: "Single Cycle", value: 'Single Cycle'}, {url: "Long Term", value: 'Long Term'}],
@@ -7,180 +8,90 @@ const ProjectServiceMock= {
"url": "http://localhost:3000/api/resource_type/LOFAR%20Observing%20Time/",
"created_at": "2020-07-29T07:31:21.708296",
"description": "LOFAR Observing Time",
- "resource_unit": "http://localhost:3000/api/resource_unit/second/",
- "resource_unit_id": "second",
"tags": [
],
"updated_at": "2020-07-29T07:31:21.708316",
- "resourceUnit": {
- "name": "second",
- "url": "http://localhost:3000/api/resource_unit/second/",
- "created_at": "2020-07-29T07:31:21.070088",
- "description": "Unit of time or duration",
- "tags": [
- ],
- "updated_at": "2020-07-29T07:31:21.070114"
- }
+ "quantity_value": "time"
},
{
"name": "LOFAR Observing Time prio A",
"url": "http://localhost:3000/api/resource_type/LOFAR%20Observing%20Time%20prio%20A/",
"created_at": "2020-07-29T07:31:21.827537",
"description": "LOFAR Observing Time prio A",
- "resource_unit": "http://localhost:3000/api/resource_unit/second/",
- "resource_unit_id": "second",
"tags": [
],
"updated_at": "2020-07-29T07:31:21.827675",
- "resourceUnit": {
- "name": "second",
- "url": "http://localhost:3000/api/resource_unit/second/",
- "created_at": "2020-07-29T07:31:21.070088",
- "description": "Unit of time or duration",
- "tags": [
- ],
- "updated_at": "2020-07-29T07:31:21.070114"
- }
+ "quantity_value": "time"
},
{
"name": "LOFAR Observing Time prio B",
"url": "http://localhost:3000/api/resource_type/LOFAR%20Observing%20Time%20prio%20B/",
"created_at": "2020-07-29T07:31:21.950948",
"description": "LOFAR Observing Time prio B",
- "resource_unit": "http://localhost:3000/api/resource_unit/second/",
- "resource_unit_id": "second",
"tags": [
],
"updated_at": "2020-07-29T07:31:21.950968",
- "resourceUnit": {
- "name": "second",
- "url": "http://localhost:3000/api/resource_unit/second/",
- "created_at": "2020-07-29T07:31:21.070088",
- "description": "Unit of time or duration",
- "tags": [
- ],
- "updated_at": "2020-07-29T07:31:21.070114"
- }
+ "quantity_value": "time"
},
{
"name": "CEP Processing Time",
"url": "http://localhost:3000/api/resource_type/CEP%20Processing%20Time/",
"created_at": "2020-07-29T07:31:22.097916",
"description": "CEP Processing Time",
- "resource_unit": "http://localhost:3000/api/resource_unit/second/",
- "resource_unit_id": "second",
"tags": [
],
"updated_at": "2020-07-29T07:31:22.097941",
- "resourceUnit": {
- "name": "second",
- "url": "http://localhost:3000/api/resource_unit/second/",
- "created_at": "2020-07-29T07:31:21.070088",
- "description": "Unit of time or duration",
- "tags": [
- ],
- "updated_at": "2020-07-29T07:31:21.070114"
- }
+ "quantity_value": "time"
},
{
"name": "LTA Storage",
"url": "http://localhost:3000/api/resource_type/LTA%20Storage/",
"created_at": "2020-07-29T07:31:22.210071",
"description": "LTA Storage",
- "resource_unit": "http://localhost:3000/api/resource_unit/byte/",
- "resource_unit_id": "byte",
"tags": [
],
"updated_at": "2020-07-29T07:31:22.210091",
- "resourceUnit": {
- "name": "byte",
- "url": "http://localhost:3000/api/resource_unit/byte/",
- "created_at": "2020-07-29T07:31:21.500997",
- "description": "Unit of data storage",
- "tags": [
- ],
- "updated_at": "2020-07-29T07:31:21.501028"
- }
+ "quantity_value": "bytes"
},
{
"name": "Number of triggers",
"url": "http://localhost:3000/api/resource_type/Number%20of%20triggers/",
"created_at": "2020-07-29T07:31:22.317313",
"description": "Number of triggers",
- "resource_unit": "http://localhost:3000/api/resource_unit/number/",
- "resource_unit_id": "number",
"tags": [
],
"updated_at": "2020-07-29T07:31:22.317341",
- "resourceUnit": {
- "name": "number",
- "url": "http://localhost:3000/api/resource_unit/number/",
- "created_at": "2020-07-29T07:31:21.596364",
- "description": "Unit of count",
- "tags": [
- ],
- "updated_at": "2020-07-29T07:31:21.596385"
- }
+ "quantity_value": "number"
},
{
"name": "LOFAR Support Time",
"url": "http://localhost:3000/api/resource_type/LOFAR%20Support%20Time/",
"created_at": "2020-07-29T07:31:22.437945",
"description": "LOFAR Support Time",
- "resource_unit": "http://localhost:3000/api/resource_unit/second/",
- "resource_unit_id": "second",
"tags": [
],
"updated_at": "2020-07-29T07:31:22.437964",
- "resourceUnit": {
- "name": "second",
- "url": "http://localhost:3000/api/resource_unit/second/",
- "created_at": "2020-07-29T07:31:21.070088",
- "description": "Unit of time or duration",
- "tags": [
- ],
- "updated_at": "2020-07-29T07:31:21.070114"
- }
+ "quantity_value": "time"
},
{
"name": "LOFAR Support hours",
"url": "http://localhost:3000/api/resource_type/LOFAR%20Support%20hours/",
"created_at": "2020-07-29T07:31:22.571850",
"description": "LOFAR Support hours",
- "resource_unit": "http://localhost:3000/api/resource_unit/second/",
- "resource_unit_id": "second",
"tags": [
],
"updated_at": "2020-07-29T07:31:22.571869",
- "resourceUnit": {
- "name": "second",
- "url": "http://localhost:3000/api/resource_unit/second/",
- "created_at": "2020-07-29T07:31:21.070088",
- "description": "Unit of time or duration",
- "tags": [
- ],
- "updated_at": "2020-07-29T07:31:21.070114"
- }
+ "quantity_value": "time"
},
{
"name": "Support hours",
"url": "http://localhost:3000/api/resource_type/Support%20hours/",
"created_at": "2020-07-29T07:31:22.694438",
"description": "Support hours",
- "resource_unit": "http://localhost:3000/api/resource_unit/second/",
- "resource_unit_id": "second",
"tags": [
],
"updated_at": "2020-07-29T07:31:22.694514",
- "resourceUnit": {
- "name": "second",
- "url": "http://localhost:3000/api/resource_unit/second/",
- "created_at": "2020-07-29T07:31:21.070088",
- "description": "Unit of time or duration",
- "tags": [
- ],
- "updated_at": "2020-07-29T07:31:21.070114"
- }
+ "quantity_value": "time"
}
],
projectResourceDefaults: {
@@ -212,7 +123,7 @@ const ProjectServiceMock= {
"private_data": true,
"project_category": "Regular",
"project_category_value": "Regular",
- "project_quota": [
+ "quota": [
"http://192.168.99.100:8008/api/project_quota/70/",
"http://192.168.99.100:8008/api/project_quota/71/",
"http://192.168.99.100:8008/api/project_quota/72/",
@@ -222,7 +133,7 @@ const ProjectServiceMock= {
"http://192.168.99.100:8008/api/project_quota/76/",
"http://192.168.99.100:8008/api/project_quota/77/"
],
- "project_quota_ids": [
+ "quota_ids": [
70,
71,
72,
@@ -307,9 +218,120 @@ const ProjectServiceMock= {
"project_id": "OSR-11",
"resource_type": "http://192.168.99.100:8008/api/resource_type/Support%20hours/",
"resource_type_id": "Support hours",
- "value": 8
+ "value": 32400
}
- ]
+ ],
+ projectList: [
+ {
+ "name": "OSR-01",
+ "url": "http://192.168.99.100:8008/api/project/OSR-01",
+ "can_trigger": false,
+ "created_at": "2020-08-25T14:29:04.881620",
+ "cycles": [
+ "http://192.168.99.100:8008/api/cycle/Cycle%2014"
+ ],
+ "cycles_ids": [
+ "Cycle 14"
+ ],
+ "description": "OSR-01",
+ "expert": false,
+ "filler": false,
+ "period_category": "http://192.168.99.100:8008/api/period_category/single_cycle",
+ "period_category_value": "single_cycle",
+ "priority_rank": 1,
+ "private_data": true,
+ "project_category": "http://192.168.99.100:8008/api/project_category/regular",
+ "project_category_value": "regular",
+ "quota": [
+ "http://192.168.99.100:8008/api/project_quota/1",
+ "http://192.168.99.100:8008/api/project_quota/2",
+ "http://192.168.99.100:8008/api/project_quota/3",
+ "http://192.168.99.100:8008/api/project_quota/4",
+ "http://192.168.99.100:8008/api/project_quota/5",
+ "http://192.168.99.100:8008/api/project_quota/6",
+ "http://192.168.99.100:8008/api/project_quota/7"
+ ],
+ "quota_ids": [
+ 1,
+ 2,
+ 3,
+ 4,
+ 5,
+ 6,
+ 7
+ ],
+ "tags": [],
+ "trigger_priority": 1000,
+ "updated_at": "2020-08-25T14:29:04.881640"
+ },
+ {
+ "name": "OSR-02",
+ "url": "http://192.168.99.100:8008/api/project/OSR-02",
+ "can_trigger": false,
+ "created_at": "2020-08-28T07:52:07.411136",
+ "cycles": [],
+ "cycles_ids": [],
+ "description": "OSR-02",
+ "expert": false,
+ "filler": false,
+ "period_category": null,
+ "period_category_value": null,
+ "priority_rank": 1,
+ "private_data": true,
+ "project_category": null,
+ "project_category_value": null,
+ "quota": [
+ "http://192.168.99.100:8008/api/project_quota/8",
+ "http://192.168.99.100:8008/api/project_quota/9",
+ "http://192.168.99.100:8008/api/project_quota/10",
+ "http://192.168.99.100:8008/api/project_quota/11",
+ "http://192.168.99.100:8008/api/project_quota/12",
+ "http://192.168.99.100:8008/api/project_quota/13",
+ "http://192.168.99.100:8008/api/project_quota/14"
+ ],
+ "quota_ids": [
+ 8,
+ 9,
+ 10,
+ 11,
+ 12,
+ 13,
+ 14
+ ],
+ "tags": [],
+ "trigger_priority": 1000,
+ "updated_at": "2020-08-28T07:52:07.411167"
+ },
+ {
+ "name": "TMSS-Commissioning",
+ "url": "http://192.168.99.100:8008/api/project/TMSS-Commissioning",
+ "can_trigger": false,
+ "created_at": "2020-08-25T13:28:34.760707",
+ "cycles": [
+ "http://192.168.99.100:8008/api/cycle/Cycle%2014"
+ ],
+ "cycles_ids": [
+ "Cycle 14"
+ ],
+ "description": "Project for all TMSS tests and commissioning",
+ "expert": true,
+ "filler": false,
+ "period_category": null,
+ "period_category_value": null,
+ "priority_rank": 1,
+ "private_data": true,
+ "project_category": null,
+ "project_category_value": null,
+ "quota": [],
+ "quota_ids": [],
+ "tags": [],
+ "trigger_priority": 1000,
+ "updated_at": "2020-08-25T13:28:34.760729"
+ }
+ ]
}
+
+
+
export default ProjectServiceMock;
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/__mocks__/scheduleunit.service.data.js b/SAS/TMSS/frontend/tmss_webapp/src/__mocks__/scheduleunit.service.data.js
new file mode 100644
index 0000000000000000000000000000000000000000..3dc1484cf61fa7e7f68250f0e986cbc794defd8f
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/src/__mocks__/scheduleunit.service.data.js
@@ -0,0 +1,753 @@
+const SUServiceMock= {
+ scheduleSetList: [
+ {
+ "id": 1,
+ "url": "http://192.168.99.100:8008/api/scheduling_set/1",
+ "created_at": "2020-08-25T13:28:42.045214",
+ "description": "",
+ "generator_doc": {},
+ "generator_source": null,
+ "generator_source_id": null,
+ "generator_template": "http://192.168.99.100:8008/api/generator_template/1",
+ "generator_template_id": 1,
+ "name": "Test Scheduling Set UC1 example 0",
+ "project": "http://192.168.99.100:8008/api/project/TMSS-Commissioning",
+ "project_id": "TMSS-Commissioning",
+ "scheduling_unit_drafts": [
+ "http://192.168.99.100:8008/api/scheduling_unit_draft/2",
+ "http://192.168.99.100:8008/api/scheduling_unit_draft/1"
+ ],
+ "scheduling_unit_drafts_ids": [
+ 2,
+ 1
+ ],
+ "tags": [
+ "TEST",
+ "UC1"
+ ],
+ "updated_at": "2020-08-25T13:28:42.047512"
+ },
+ {
+ "id": 2,
+ "url": "http://192.168.99.100:8008/api/scheduling_set/2",
+ "created_at": "2020-08-25T13:28:49.545042",
+ "description": "",
+ "generator_doc": {},
+ "generator_source": null,
+ "generator_source_id": null,
+ "generator_template": "http://192.168.99.100:8008/api/generator_template/2",
+ "generator_template_id": 2,
+ "name": "Test Scheduling Set UC1 example 1",
+ "project": "http://192.168.99.100:8008/api/project/TMSS-Commissioning",
+ "project_id": "TMSS-Commissioning",
+ "scheduling_unit_drafts": [
+ "http://192.168.99.100:8008/api/scheduling_unit_draft/4",
+ "http://192.168.99.100:8008/api/scheduling_unit_draft/3"
+ ],
+ "scheduling_unit_drafts_ids": [
+ 4,
+ 3
+ ],
+ "tags": [
+ "TEST",
+ "UC1"
+ ],
+ "updated_at": "2020-08-25T13:28:49.546151"
+ },
+ {
+ "id": 3,
+ "url": "http://192.168.99.100:8008/api/scheduling_set/3",
+ "created_at": "2020-08-25T13:28:57.025339",
+ "description": "",
+ "generator_doc": {},
+ "generator_source": null,
+ "generator_source_id": null,
+ "generator_template": "http://192.168.99.100:8008/api/generator_template/3",
+ "generator_template_id": 3,
+ "name": "Test Scheduling Set UC1 example 2",
+ "project": "http://192.168.99.100:8008/api/project/TMSS-Commissioning",
+ "project_id": "TMSS-Commissioning",
+ "scheduling_unit_drafts": [
+ "http://192.168.99.100:8008/api/scheduling_unit_draft/6",
+ "http://192.168.99.100:8008/api/scheduling_unit_draft/5"
+ ],
+ "scheduling_unit_drafts_ids": [
+ 6,
+ 5
+ ],
+ "tags": [
+ "TEST",
+ "UC1"
+ ],
+ "updated_at": "2020-08-25T13:28:57.026492"
+ }
+ ],
+ observStrategies: [
+ {
+ "id": 1,
+ "url": "http://192.168.99.100:8008/api/scheduling_unit_observing_strategy_template/1",
+ "created_at": "2020-08-25T13:28:33.974187",
+ "description": "UC1 observation strategy template",
+ "name": "UC1 observation strategy template",
+ "scheduling_unit_template": "http://192.168.99.100:8008/api/scheduling_unit_template/1",
+ "scheduling_unit_template_id": 1,
+ "tags": [
+ "UC1"
+ ],
+ "template": {
+ "tasks": {
+ "Pipeline 1": {
+ "tags": [],
+ "description": "Preprocessing Pipeline for Calibrator Observation 1",
+ "specifications_doc": {
+ "flag": {
+ "rfi_strategy": "auto",
+ "outerchannels": true,
+ "autocorrelations": true
+ },
+ "demix": {
+ "sources": {},
+ "time_steps": 10,
+ "ignore_target": false,
+ "frequency_steps": 64
+ },
+ "average": {
+ "time_steps": 1,
+ "frequency_steps": 4
+ },
+ "storagemanager": "dysco"
+ },
+ "specifications_template": "preprocessing schema"
+ },
+ "Pipeline 2": {
+ "tags": [],
+ "description": "Preprocessing Pipeline for Calibrator Observation 2",
+ "specifications_doc": {
+ "flag": {
+ "rfi_strategy": "auto",
+ "outerchannels": true,
+ "autocorrelations": true
+ },
+ "demix": {
+ "sources": {},
+ "time_steps": 10,
+ "ignore_target": false,
+ "frequency_steps": 64
+ },
+ "average": {
+ "time_steps": 1,
+ "frequency_steps": 4
+ },
+ "storagemanager": "dysco"
+ },
+ "specifications_template": "preprocessing schema"
+ },
+ "Pipeline SAP0": {
+ "tags": [],
+ "description": "Preprocessing Pipeline for Target Observation SAP0",
+ "specifications_doc": {
+ "flag": {
+ "rfi_strategy": "auto",
+ "outerchannels": true,
+ "autocorrelations": true
+ },
+ "demix": {
+ "sources": {},
+ "time_steps": 10,
+ "ignore_target": false,
+ "frequency_steps": 64
+ },
+ "average": {
+ "time_steps": 1,
+ "frequency_steps": 4
+ },
+ "storagemanager": "dysco"
+ },
+ "specifications_template": "preprocessing schema"
+ },
+ "Pipeline SAP1": {
+ "tags": [],
+ "description": "Preprocessing Pipeline for Target Observation SAP1",
+ "specifications_doc": {
+ "flag": {
+ "rfi_strategy": "auto",
+ "outerchannels": true,
+ "autocorrelations": true
+ },
+ "demix": {
+ "sources": {},
+ "time_steps": 10,
+ "ignore_target": false,
+ "frequency_steps": 64
+ },
+ "average": {
+ "time_steps": 1,
+ "frequency_steps": 4
+ },
+ "storagemanager": "dysco"
+ },
+ "specifications_template": "preprocessing schema"
+ },
+ "Target Observation": {
+ "tags": [],
+ "description": "Target Observation for UC1 HBA scheduling unit",
+ "specifications_doc": {
+ "QA": {
+ "plots": {
+ "enabled": true,
+ "autocorrelation": true,
+ "crosscorrelation": true
+ },
+ "file_conversion": {
+ "enabled": true,
+ "nr_of_subbands": -1,
+ "nr_of_timestamps": 256
+ }
+ },
+ "SAPs": [
+ {
+ "name": "target0",
+ "subbands": [
+ 349,
+ 372
+ ],
+ "digital_pointing": {
+ "angle1": 3.9935314947195253,
+ "angle2": 0.5324708659626034,
+ "angle3": 24,
+ "direction_type": "J2000"
+ }
+ },
+ {
+ "name": "target1",
+ "subbands": [
+ 349,
+ 372
+ ],
+ "digital_pointing": {
+ "angle1": 3.9935314947195253,
+ "angle2": 0.5324708659626034,
+ "angle3": 24,
+ "direction_type": "J2000"
+ }
+ }
+ ],
+ "filter": "HBA_110_190",
+ "duration": 28800,
+ "stations": [
+ {
+ "group": "ALL",
+ "min_stations": 1
+ }
+ ],
+ "tile_beam": {
+ "angle1": 5.324708659626033,
+ "angle2": 0.7099611546168045,
+ "angle3": 42,
+ "direction_type": "J2000"
+ },
+ "correlator": {
+ "storage_cluster": "CEP4",
+ "integration_time": 1,
+ "channels_per_subband": 64
+ },
+ "antenna_set": "HBA_DUAL_INNER"
+ },
+ "specifications_template": "observation schema"
+ },
+ "Calibrator Observation 1": {
+ "tags": [],
+ "description": "Calibrator Observation for UC1 HBA scheduling unit",
+ "specifications_doc": {
+ "duration": 600,
+ "pointing": {
+ "angle1": 0,
+ "angle2": 0,
+ "angle3": 0,
+ "direction_type": "J2000"
+ },
+ "autoselect": false
+ },
+ "specifications_template": "calibrator schema"
+ },
+ "Calibrator Observation 2": {
+ "tags": [],
+ "description": "Calibrator Observation for UC1 HBA scheduling unit",
+ "specifications_doc": {
+ "duration": 600,
+ "pointing": {
+ "angle1": 0,
+ "angle2": 0,
+ "angle3": 0,
+ "direction_type": "J2000"
+ },
+ "autoselect": false
+ },
+ "specifications_template": "calibrator schema"
+ }
+ },
+ "parameters": [
+ {
+ "name": "Target Pointing 0",
+ "refs": [
+ "#/tasks/Target Observation/specifications_doc/SAPs/0/digital_pointing"
+ ]
+ },
+ {
+ "name": "Target Pointing 1",
+ "refs": [
+ "#/tasks/Target Observation/specifications_doc/SAPs/1/digital_pointing"
+ ]
+ },
+ {
+ "name": "Tile Beam",
+ "refs": [
+ "#/tasks/Target Observation/specifications_doc/tile_beam"
+ ]
+ }
+ ],
+ "task_relations": [
+ {
+ "tags": [],
+ "input": {
+ "role": "input",
+ "datatype": "visibilities"
+ },
+ "output": {
+ "role": "correlator",
+ "datatype": "visibilities"
+ },
+ "consumer": "Pipeline 1",
+ "producer": "Calibrator Observation 1",
+ "dataformat": "MeasurementSet",
+ "selection_doc": {},
+ "selection_template": "All"
+ },
+ {
+ "tags": [],
+ "input": {
+ "role": "input",
+ "datatype": "visibilities"
+ },
+ "output": {
+ "role": "correlator",
+ "datatype": "visibilities"
+ },
+ "consumer": "Pipeline 2",
+ "producer": "Calibrator Observation 2",
+ "dataformat": "MeasurementSet",
+ "selection_doc": {},
+ "selection_template": "All"
+ },
+ {
+ "tags": [],
+ "input": {
+ "role": "input",
+ "datatype": "visibilities"
+ },
+ "output": {
+ "role": "correlator",
+ "datatype": "visibilities"
+ },
+ "consumer": "Pipeline SAP0",
+ "producer": "Target Observation",
+ "dataformat": "MeasurementSet",
+ "selection_doc": {
+ "sap": [
+ 0
+ ]
+ },
+ "selection_template": "SAP"
+ },
+ {
+ "tags": [],
+ "input": {
+ "role": "input",
+ "datatype": "visibilities"
+ },
+ "output": {
+ "role": "correlator",
+ "datatype": "visibilities"
+ },
+ "consumer": "Pipeline SAP1",
+ "producer": "Target Observation",
+ "dataformat": "MeasurementSet",
+ "selection_doc": {
+ "sap": [
+ 1
+ ]
+ },
+ "selection_template": "SAP"
+ }
+ ],
+ "task_scheduling_relations": [
+ {
+ "first": "Calibrator Observation 1",
+ "second": "Target Observation",
+ "placement": "before",
+ "time_offset": 60
+ },
+ {
+ "first": "Calibrator Observation 2",
+ "second": "Target Observation",
+ "placement": "after",
+ "time_offset": 60
+ }
+ ]
+ },
+ "updated_at": "2020-08-25T13:28:33.974209",
+ "version": "0.1"
+ }
+ ],
+ schedulingUnitFromObservStrategy: {
+ "id": 1,
+ "url": "http://192.168.99.100:8008/api/scheduling_unit_draft/1",
+ "copies": null,
+ "copies_id": null,
+ "copy_reason": null,
+ "copy_reason_value": null,
+ "created_at": "2020-08-25T13:28:42.092602",
+ "description": "",
+ "duration": 30120,
+ "generator_instance_doc": null,
+ "name": "UC1 test scheduling unit 1.1",
+ "observation_strategy_template": "http://192.168.99.100:8008/api/scheduling_unit_observing_strategy_template/1",
+ "observation_strategy_template_id": 1,
+ "requirements_doc": {
+ "tasks": {
+ "Pipeline 1": {
+ "tags": [],
+ "description": "Preprocessing Pipeline for Calibrator Observation 1",
+ "specifications_doc": {
+ "flag": {
+ "rfi_strategy": "auto",
+ "outerchannels": true,
+ "autocorrelations": true
+ },
+ "demix": {
+ "sources": {},
+ "time_steps": 10,
+ "ignore_target": false,
+ "frequency_steps": 64
+ },
+ "average": {
+ "time_steps": 1,
+ "frequency_steps": 4
+ },
+ "storagemanager": "dysco"
+ },
+ "specifications_template": "preprocessing schema"
+ },
+ "Pipeline 2": {
+ "tags": [],
+ "description": "Preprocessing Pipeline for Calibrator Observation 2",
+ "specifications_doc": {
+ "flag": {
+ "rfi_strategy": "auto",
+ "outerchannels": true,
+ "autocorrelations": true
+ },
+ "demix": {
+ "sources": {},
+ "time_steps": 10,
+ "ignore_target": false,
+ "frequency_steps": 64
+ },
+ "average": {
+ "time_steps": 1,
+ "frequency_steps": 4
+ },
+ "storagemanager": "dysco"
+ },
+ "specifications_template": "preprocessing schema"
+ },
+ "Pipeline SAP0": {
+ "tags": [],
+ "description": "Preprocessing Pipeline for Target Observation SAP0",
+ "specifications_doc": {
+ "flag": {
+ "rfi_strategy": "auto",
+ "outerchannels": true,
+ "autocorrelations": true
+ },
+ "demix": {
+ "sources": {},
+ "time_steps": 10,
+ "ignore_target": false,
+ "frequency_steps": 64
+ },
+ "average": {
+ "time_steps": 1,
+ "frequency_steps": 4
+ },
+ "storagemanager": "dysco"
+ },
+ "specifications_template": "preprocessing schema"
+ },
+ "Pipeline SAP1": {
+ "tags": [],
+ "description": "Preprocessing Pipeline for Target Observation SAP1",
+ "specifications_doc": {
+ "flag": {
+ "rfi_strategy": "auto",
+ "outerchannels": true,
+ "autocorrelations": true
+ },
+ "demix": {
+ "sources": {},
+ "time_steps": 10,
+ "ignore_target": false,
+ "frequency_steps": 64
+ },
+ "average": {
+ "time_steps": 1,
+ "frequency_steps": 4
+ },
+ "storagemanager": "dysco"
+ },
+ "specifications_template": "preprocessing schema"
+ },
+ "Target Observation": {
+ "tags": [],
+ "description": "Target Observation for UC1 HBA scheduling unit",
+ "specifications_doc": {
+ "QA": {
+ "plots": {
+ "enabled": true,
+ "autocorrelation": true,
+ "crosscorrelation": true
+ },
+ "file_conversion": {
+ "enabled": true,
+ "nr_of_subbands": -1,
+ "nr_of_timestamps": 256
+ }
+ },
+ "SAPs": [
+ {
+ "name": "target0",
+ "subbands": [
+ 349,
+ 372
+ ],
+ "digital_pointing": {
+ "angle1": 3.9935314947195253,
+ "angle2": 0.5324708659626034,
+ "angle3": 24,
+ "direction_type": "J2000"
+ }
+ },
+ {
+ "name": "target1",
+ "subbands": [
+ 349,
+ 372
+ ],
+ "digital_pointing": {
+ "angle1": 3.9935314947195253,
+ "angle2": 0.5324708659626034,
+ "angle3": 24,
+ "direction_type": "J2000"
+ }
+ }
+ ],
+ "filter": "HBA_110_190",
+ "duration": 28800,
+ "stations": [
+ {
+ "group": "ALL",
+ "min_stations": 1
+ }
+ ],
+ "tile_beam": {
+ "angle1": 5.324708659626033,
+ "angle2": 0.7099611546168045,
+ "angle3": 42,
+ "direction_type": "J2000"
+ },
+ "correlator": {
+ "storage_cluster": "CEP4",
+ "integration_time": 1,
+ "channels_per_subband": 64
+ },
+ "antenna_set": "HBA_DUAL_INNER"
+ },
+ "specifications_template": "observation schema"
+ },
+ "Calibrator Observation 1": {
+ "tags": [],
+ "description": "Calibrator Observation for UC1 HBA scheduling unit",
+ "specifications_doc": {
+ "duration": 600,
+ "pointing": {
+ "angle1": 0,
+ "angle2": 0,
+ "angle3": 0,
+ "direction_type": "J2000"
+ },
+ "autoselect": false
+ },
+ "specifications_template": "calibrator schema"
+ },
+ "Calibrator Observation 2": {
+ "tags": [],
+ "description": "Calibrator Observation for UC1 HBA scheduling unit",
+ "specifications_doc": {
+ "duration": 600,
+ "pointing": {
+ "angle1": 0,
+ "angle2": 0,
+ "angle3": 0,
+ "direction_type": "J2000"
+ },
+ "autoselect": false
+ },
+ "specifications_template": "calibrator schema"
+ }
+ },
+ "parameters": [
+ {
+ "name": "Target Pointing 0",
+ "refs": [
+ "#/tasks/Target Observation/specifications_doc/SAPs/0/digital_pointing"
+ ]
+ },
+ {
+ "name": "Target Pointing 1",
+ "refs": [
+ "#/tasks/Target Observation/specifications_doc/SAPs/1/digital_pointing"
+ ]
+ },
+ {
+ "name": "Tile Beam",
+ "refs": [
+ "#/tasks/Target Observation/specifications_doc/tile_beam"
+ ]
+ }
+ ],
+ "task_relations": [
+ {
+ "tags": [],
+ "input": {
+ "role": "input",
+ "datatype": "visibilities"
+ },
+ "output": {
+ "role": "correlator",
+ "datatype": "visibilities"
+ },
+ "consumer": "Pipeline 1",
+ "producer": "Calibrator Observation 1",
+ "dataformat": "MeasurementSet",
+ "selection_doc": {},
+ "selection_template": "All"
+ },
+ {
+ "tags": [],
+ "input": {
+ "role": "input",
+ "datatype": "visibilities"
+ },
+ "output": {
+ "role": "correlator",
+ "datatype": "visibilities"
+ },
+ "consumer": "Pipeline 2",
+ "producer": "Calibrator Observation 2",
+ "dataformat": "MeasurementSet",
+ "selection_doc": {},
+ "selection_template": "All"
+ },
+ {
+ "tags": [],
+ "input": {
+ "role": "input",
+ "datatype": "visibilities"
+ },
+ "output": {
+ "role": "correlator",
+ "datatype": "visibilities"
+ },
+ "consumer": "Pipeline SAP0",
+ "producer": "Target Observation",
+ "dataformat": "MeasurementSet",
+ "selection_doc": {
+ "sap": [
+ 0
+ ]
+ },
+ "selection_template": "SAP"
+ },
+ {
+ "tags": [],
+ "input": {
+ "role": "input",
+ "datatype": "visibilities"
+ },
+ "output": {
+ "role": "correlator",
+ "datatype": "visibilities"
+ },
+ "consumer": "Pipeline SAP1",
+ "producer": "Target Observation",
+ "dataformat": "MeasurementSet",
+ "selection_doc": {
+ "sap": [
+ 1
+ ]
+ },
+ "selection_template": "SAP"
+ }
+ ],
+ "task_scheduling_relations": [
+ {
+ "first": "Calibrator Observation 1",
+ "second": "Target Observation",
+ "placement": "before",
+ "time_offset": 60
+ },
+ {
+ "first": "Calibrator Observation 2",
+ "second": "Target Observation",
+ "placement": "after",
+ "time_offset": 60
+ }
+ ]
+ },
+ "requirements_template": "http://192.168.99.100:8008/api/scheduling_unit_template/1",
+ "requirements_template_id": 1,
+ "scheduling_set": "http://192.168.99.100:8008/api/scheduling_set/1",
+ "scheduling_set_id": 1,
+ "scheduling_unit_blueprints": [
+ "http://192.168.99.100:8008/api/scheduling_unit_blueprint/1"
+ ],
+ "scheduling_unit_blueprints_ids": [
+ 1
+ ],
+ "tags": [
+ "TEST",
+ "UC1"
+ ],
+ "task_drafts": [
+ "http://192.168.99.100:8008/api/task_draft/5",
+ "http://192.168.99.100:8008/api/task_draft/7",
+ "http://192.168.99.100:8008/api/task_draft/6",
+ "http://192.168.99.100:8008/api/task_draft/4",
+ "http://192.168.99.100:8008/api/task_draft/3",
+ "http://192.168.99.100:8008/api/task_draft/2",
+ "http://192.168.99.100:8008/api/task_draft/1"
+ ],
+ "task_drafts_ids": [
+ 5,
+ 7,
+ 6,
+ 4,
+ 3,
+ 2,
+ 1
+ ],
+ "updated_at": "2020-08-25T13:28:42.119417"
+ }
+};
+
+export default SUServiceMock;
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/__mocks__/task.service.data.js b/SAS/TMSS/frontend/tmss_webapp/src/__mocks__/task.service.data.js
new file mode 100644
index 0000000000000000000000000000000000000000..6b7abd1e336416472b6360461fdbd23345871d7e
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/src/__mocks__/task.service.data.js
@@ -0,0 +1,676 @@
+const TaskServiceMock= {
+ taskTemplates: [
+ {
+ "id": 1,
+ "url": "http://192.168.99.100:8008/api/task_template/1",
+ "created_at": "2020-08-25T13:28:33.979487",
+ "description": "preprocessing settings",
+ "name": "preprocessing schema",
+ "schema": {
+ "$id": "http://example.com/example.json",
+ "type": "object",
+ "$schema": "http://json-schema.org/draft-06/schema#",
+ "required": [
+ "storagemanager"
+ ],
+ "properties": {
+ "flag": {
+ "type": "object",
+ "title": "Flagging",
+ "default": {},
+ "required": [
+ "outerchannels",
+ "autocorrelations",
+ "rfi_strategy"
+ ],
+ "properties": {
+ "rfi_strategy": {
+ "enum": [
+ "none",
+ "auto",
+ "HBAdefault",
+ "LBAdefault"
+ ],
+ "type": "string",
+ "title": "RFI flagging strategy",
+ "default": "auto"
+ },
+ "outerchannels": {
+ "type": "boolean",
+ "title": "Flag outer channels",
+ "default": true
+ },
+ "autocorrelations": {
+ "type": "boolean",
+ "title": "Flag auto correlations",
+ "default": true
+ }
+ },
+ "additionalProperties": false
+ },
+ "demix": {
+ "type": "object",
+ "title": "Demixing",
+ "default": {},
+ "options": {
+ "dependencies": {
+ "demix": true
+ }
+ },
+ "required": [
+ "frequency_steps",
+ "time_steps",
+ "ignore_target",
+ "sources"
+ ],
+ "properties": {
+ "sources": {
+ "type": "object",
+ "title": "Sources",
+ "default": {},
+ "properties": {
+ "CasA": {
+ "$ref": "#/definitions/demix_strategy",
+ "title": "CasA"
+ },
+ "CygA": {
+ "$ref": "#/definitions/demix_strategy",
+ "title": "CygA"
+ },
+ "HerA": {
+ "$ref": "#/definitions/demix_strategy",
+ "title": "HerA"
+ },
+ "TauA": {
+ "$ref": "#/definitions/demix_strategy",
+ "title": "TauA"
+ },
+ "VirA": {
+ "$ref": "#/definitions/demix_strategy",
+ "title": "VirA"
+ },
+ "HydraA": {
+ "$ref": "#/definitions/demix_strategy",
+ "title": "HyrdraA"
+ }
+ },
+ "additionalProperties": false
+ },
+ "time_steps": {
+ "type": "integer",
+ "title": "Time steps",
+ "default": 10,
+ "minimum": 1,
+ "description": "Must be a multiple of the averaging time steps"
+ },
+ "ignore_target": {
+ "type": "boolean",
+ "title": "Ignore target",
+ "default": false
+ },
+ "frequency_steps": {
+ "type": "integer",
+ "title": "Frequency steps",
+ "default": 64,
+ "minimum": 1,
+ "description": "Must be a multiple of the averaging frequency steps"
+ }
+ },
+ "additionalProperties": false
+ },
+ "average": {
+ "type": "object",
+ "title": "Averaging",
+ "default": {},
+ "required": [
+ "frequency_steps",
+ "time_steps"
+ ],
+ "properties": {
+ "time_steps": {
+ "type": "integer",
+ "title": "Time steps",
+ "default": 1,
+ "minimum": 1
+ },
+ "frequency_steps": {
+ "type": "integer",
+ "title": "Frequency steps",
+ "default": 4,
+ "minimum": 1
+ }
+ },
+ "additionalProperties": false
+ },
+ "storagemanager": {
+ "enum": [
+ "basic",
+ "dysco"
+ ],
+ "type": "string",
+ "title": "Storage Manager",
+ "default": "dysco"
+ }
+ },
+ "definitions": {
+ "demix_strategy": {
+ "enum": [
+ "auto",
+ "yes",
+ "no"
+ ],
+ "type": "string",
+ "default": "auto"
+ }
+ },
+ "additionalProperties": false
+ },
+ "tags": [],
+ "type": "http://192.168.99.100:8008/api/task_type/pipeline",
+ "type_value": "pipeline",
+ "updated_at": "2020-08-25T13:28:33.979514",
+ "validation_code_js": "",
+ "version": "0.1"
+ },
+ {
+ "id": 2,
+ "url": "http://192.168.99.100:8008/api/task_template/2",
+ "created_at": "2020-08-25T13:28:33.983945",
+ "description": "schema for observations",
+ "name": "observation schema",
+ "schema": {
+ "$id": "http://example.com/example.json",
+ "type": "object",
+ "$schema": "http://json-schema.org/draft-06/schema#",
+ "required": [
+ "stations",
+ "antenna_set",
+ "filter",
+ "SAPs",
+ "duration",
+ "correlator"
+ ],
+ "properties": {
+ "QA": {
+ "type": "object",
+ "title": "Quality Assurance",
+ "default": {},
+ "properties": {
+ "plots": {
+ "type": "object",
+ "title": "Plots",
+ "default": {},
+ "properties": {
+ "enabled": {
+ "type": "boolean",
+ "title": "enabled",
+ "default": true,
+ "description": "Do/Don't create plots from the QA file from the observation"
+ },
+ "autocorrelation": {
+ "type": "boolean",
+ "title": "autocorrelation",
+ "default": true,
+ "description": "Create autocorrelation plots for all stations"
+ },
+ "crosscorrelation": {
+ "type": "boolean",
+ "title": "crosscorrelation",
+ "default": true,
+ "description": "Create crosscorrelation plots for all baselines"
+ }
+ },
+ "description": "Create dynamic spectrum plots",
+ "additionalProperties": false
+ },
+ "file_conversion": {
+ "type": "object",
+ "title": "File Conversion",
+ "default": {},
+ "properties": {
+ "enabled": {
+ "type": "boolean",
+ "title": "enabled",
+ "default": true,
+ "description": "Do/Don't create a QA file for the observation"
+ },
+ "nr_of_subbands": {
+ "type": "integer",
+ "title": "#subbands",
+ "default": -1,
+ "description": "Keep this number of subbands from the observation in the QA file, or all if -1"
+ },
+ "nr_of_timestamps": {
+ "type": "integer",
+ "title": "#timestamps",
+ "default": 256,
+ "minimum": 1,
+ "description": "Extract this number of timestamps from the observation in the QA file (equidistantanly sampled, no averaging/interpolation)"
+ }
+ },
+ "description": "Create a QA file for the observation",
+ "additionalProperties": false
+ }
+ },
+ "description": "Specify Quality Assurance steps for this observation",
+ "additionalProperties": false
+ },
+ "SAPs": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "title": "SAP",
+ "default": {},
+ "required": [
+ "digital_pointing",
+ "subbands"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "title": "Name/target",
+ "default": "",
+ "description": "Identifier for this beam"
+ },
+ "subbands": {
+ "type": "array",
+ "items": {
+ "type": "integer",
+ "title": "Subband",
+ "maximum": 511,
+ "minimum": 0
+ },
+ "title": "Subband list",
+ "default": [],
+ "additionalItems": false
+ },
+ "digital_pointing": {
+ "$ref": "#/definitions/pointing",
+ "title": "Digital pointing",
+ "default": {}
+ }
+ },
+ "headerTemplate": "{{ i0 }} - {{ self.name }}",
+ "additionalProperties": false
+ },
+ "title": "SAPs",
+ "default": [
+ {}
+ ],
+ "description": "Station beams",
+ "additionalItems": false
+ },
+ "filter": {
+ "enum": [
+ "LBA_10_70",
+ "LBA_30_70",
+ "LBA_10_90",
+ "LBA_30_90",
+ "HBA_110_190",
+ "HBA_210_250"
+ ],
+ "type": "string",
+ "title": "Band-pass filter",
+ "default": "HBA_110_190",
+ "description": "Must match antenna type"
+ },
+ "duration": {
+ "type": "number",
+ "title": "Duration (seconds)",
+ "default": 300,
+ "minimum": 1,
+ "description": "Duration of this observation"
+ },
+ "stations": {
+ "oneOf": [
+ {
+ "type": "array",
+ "items": {
+ "enum": [
+ "CS001",
+ "CS002",
+ "CS003",
+ "CS004",
+ "CS005",
+ "CS006",
+ "CS007",
+ "CS011",
+ "CS013",
+ "CS017",
+ "CS021",
+ "CS024",
+ "CS026",
+ "CS028",
+ "CS030",
+ "CS031",
+ "CS032",
+ "CS101",
+ "CS103",
+ "CS201",
+ "CS301",
+ "CS302",
+ "CS401",
+ "CS501",
+ "RS104",
+ "RS106",
+ "RS205",
+ "RS208",
+ "RS210",
+ "RS305",
+ "RS306",
+ "RS307",
+ "RS310",
+ "RS406",
+ "RS407",
+ "RS409",
+ "RS410",
+ "RS503",
+ "RS508",
+ "RS509",
+ "DE601",
+ "DE602",
+ "DE603",
+ "DE604",
+ "DE605",
+ "FR606",
+ "SE607",
+ "UK608",
+ "DE609",
+ "PL610",
+ "PL611",
+ "PL612",
+ "IE613",
+ "LV614"
+ ],
+ "type": "string",
+ "title": "Station",
+ "description": ""
+ },
+ "title": "Fixed list",
+ "default": [
+ "CS001"
+ ],
+ "minItems": 1,
+ "uniqueItems": true,
+ "additionalItems": false,
+ "additionalProperties": false
+ },
+ {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "title": "Station set",
+ "required": [
+ "group",
+ "min_stations"
+ ],
+ "properties": {
+ "group": {
+ "enum": [
+ "ALL",
+ "SUPERTERP",
+ "CORE",
+ "REMOTE",
+ "DUTCH",
+ "INTERNATIONAL"
+ ],
+ "type": "string",
+ "title": "Group/station",
+ "default": "ALL",
+ "description": "Which (group of) station(s) to select from"
+ },
+ "min_stations": {
+ "type": "integer",
+ "title": "Minimum nr of stations",
+ "default": 1,
+ "minimum": 0,
+ "description": "Number of stations to use within group/station"
+ }
+ },
+ "headerTemplate": "{{ self.group }}",
+ "additionalProperties": false
+ },
+ "title": "Dynamic list",
+ "default": [
+ {}
+ ],
+ "additionalItems": false
+ }
+ ],
+ "title": "Station list",
+ "default": [
+ "CS001"
+ ]
+ },
+ "tile_beam": {
+ "$ref": "#/definitions/pointing",
+ "title": "Tile beam",
+ "description": "HBA only"
+ },
+ "correlator": {
+ "type": "object",
+ "title": "Correlator Settings",
+ "default": {},
+ "required": [
+ "channels_per_subband",
+ "integration_time",
+ "storage_cluster"
+ ],
+ "properties": {
+ "storage_cluster": {
+ "enum": [
+ "CEP4",
+ "DragNet"
+ ],
+ "type": "string",
+ "title": "Storage cluster",
+ "default": "CEP4",
+ "description": "Cluster to write output to"
+ },
+ "integration_time": {
+ "type": "number",
+ "title": "Integration time (seconds)",
+ "default": 1,
+ "minimum": 0.1,
+ "description": "Desired integration period"
+ },
+ "channels_per_subband": {
+ "enum": [
+ 8,
+ 16,
+ 32,
+ 64,
+ 128,
+ 256,
+ 512,
+ 1024
+ ],
+ "type": "integer",
+ "title": "Channels/subband",
+ "default": 64,
+ "minimum": 8,
+ "description": "Number of frequency bands per subband"
+ }
+ },
+ "additionalProperties": false
+ },
+ "antenna_set": {
+ "enum": [
+ "HBA_DUAL",
+ "HBA_DUAL_INNER",
+ "HBA_ONE",
+ "HBA_ONE_INNER",
+ "HBA_ZERO",
+ "HBA_ZERO_INNER",
+ "LBA_INNER",
+ "LBA_OUTER",
+ "LBA_SPARSE_EVEN",
+ "LBA_SPARSE_ODD",
+ "LBA_ALL"
+ ],
+ "type": "string",
+ "title": "Antenna set",
+ "default": "HBA_DUAL",
+ "description": "Fields & antennas to use"
+ }
+ },
+ "definitions": {
+ "pointing": {
+ "type": "object",
+ "required": [
+ "angle1",
+ "angle2"
+ ],
+ "properties": {
+ "angle1": {
+ "type": "number",
+ "title": "Angle 1",
+ "default": 0,
+ "description": "First angle (e.g. RA)"
+ },
+ "angle2": {
+ "type": "number",
+ "title": "Angle 2",
+ "default": 0,
+ "description": "Second angle (e.g. DEC)"
+ },
+ "angle3": {
+ "type": "number",
+ "title": "Angle 3",
+ "default": 0,
+ "description": "Third angle (e.g. N in LMN)"
+ },
+ "direction_type": {
+ "enum": [
+ "J2000",
+ "AZELGEO",
+ "LMN",
+ "SUN",
+ "MOON",
+ "MERCURY",
+ "VENUS",
+ "MARS",
+ "JUPITER",
+ "SATURN",
+ "URANUS",
+ "NEPTUNE",
+ "PLUTO"
+ ],
+ "type": "string",
+ "title": "Reference frame",
+ "default": "J2000",
+ "description": ""
+ }
+ },
+ "additionalProperties": false
+ }
+ },
+ "additionalProperties": false
+ },
+ "tags": [],
+ "type": "http://192.168.99.100:8008/api/task_type/observation",
+ "type_value": "observation",
+ "updated_at": "2020-08-25T13:28:33.983964",
+ "validation_code_js": "",
+ "version": "0.1"
+ },
+ {
+ "id": 3,
+ "url": "http://192.168.99.100:8008/api/task_template/3",
+ "created_at": "2020-08-25T13:28:33.988294",
+ "description": "addon schema for calibrator observations",
+ "name": "calibrator schema",
+ "schema": {
+ "$id": "http://example.com/example.json",
+ "type": "object",
+ "$schema": "http://json-schema.org/draft-06/schema#",
+ "required": [
+ "autoselect",
+ "duration",
+ "pointing"
+ ],
+ "properties": {
+ "duration": {
+ "type": "number",
+ "title": "Duration (seconds)",
+ "default": 600,
+ "minimum": 1,
+ "description": "Duration of this observation"
+ },
+ "pointing": {
+ "$ref": "#/definitions/pointing",
+ "title": "Digital pointing",
+ "default": {},
+ "description": "Manually selected calibrator"
+ },
+ "autoselect": {
+ "type": "boolean",
+ "title": "Auto-select",
+ "default": true,
+ "description": "Auto-select calibrator based on elevation"
+ }
+ },
+ "definitions": {
+ "pointing": {
+ "type": "object",
+ "required": [
+ "angle1",
+ "angle2"
+ ],
+ "properties": {
+ "angle1": {
+ "type": "number",
+ "title": "Angle 1",
+ "default": 0,
+ "description": "First angle [rad] (e.g. RA)"
+ },
+ "angle2": {
+ "type": "number",
+ "title": "Angle 2",
+ "default": 0,
+ "description": "Second angle [rad] (e.g. DEC)"
+ },
+ "angle3": {
+ "type": "number",
+ "title": "Angle 3",
+ "default": 0,
+ "description": "Third angle [rad] (e.g. N in LMN)"
+ },
+ "direction_type": {
+ "enum": [
+ "J2000",
+ "AZELGEO",
+ "LMN",
+ "SUN",
+ "MOON",
+ "MERCURY",
+ "VENUS",
+ "MARS",
+ "JUPITER",
+ "SATURN",
+ "URANUS",
+ "NEPTUNE",
+ "PLUTO"
+ ],
+ "type": "string",
+ "title": "Reference frame",
+ "default": "J2000",
+ "description": ""
+ }
+ },
+ "additionalProperties": false
+ }
+ },
+ "additionalProperties": false
+ },
+ "tags": [],
+ "type": "http://192.168.99.100:8008/api/task_type/observation",
+ "type_value": "observation",
+ "updated_at": "2020-08-25T13:28:33.988312",
+ "validation_code_js": "",
+ "version": "0.1"
+ }
+ ]
+};
+
+export default TaskServiceMock;
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/components/JSONEditor/JEditor.js b/SAS/TMSS/frontend/tmss_webapp/src/components/JSONEditor/JEditor.js
index db176a3dec57157b173ab4491ecc03722c2a2200..06f9ead9e0cfceb602f0f859e4c249c958cbeb9f 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/components/JSONEditor/JEditor.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/components/JSONEditor/JEditor.js
@@ -5,20 +5,78 @@
import React, {useEffect, useRef} from 'react';
import _ from 'lodash';
import flatpickr from 'flatpickr';
-
+import $RefParser from "@apidevtools/json-schema-ref-parser";
import "@fortawesome/fontawesome-free/css/all.css";
import "flatpickr/dist/flatpickr.css";
const JSONEditor = require("@json-editor/json-editor").JSONEditor;
function Jeditor(props) {
- // console.log("In JEditor");
+ // console.log("In JEditor", props.schema);
const editorRef = useRef(null);
+ let pointingProps = useRef(null);
let editor = null;
- useEffect(() => {
- const element = document.getElementById('editor_holder');
+
+ /**
+ * Function to resolve external references
+ */
+ const resolveExternalRef = async () => {
let schema = {};
- Object.assign(schema, props.schema?props.schema:{});
-
+ Object.assign(schema, props.schema ? props.schema : {});
+ schema.definitions = schema.definitions?schema.definitions:{};
+ return (await resolveSchema(schema));
+ };
+
+ /**
+ * Function to resolve external reference in part based on the depth of schema iteration.
+ * @param {JSON Object} schema
+ */
+ const resolveSchema = async (schema) => {
+ let properties = schema.properties;
+ schema.definitions = schema.definitions?schema.definitions:{};
+ if (properties) {
+ for (const propertyKey in properties) {
+ let property = properties[propertyKey];
+ if (property["$ref"] && !property["$ref"].startsWith("#")) { // 1st level reference of the object
+ const refUrl = property["$ref"];
+ let newRef = refUrl.substring(refUrl.indexOf("#"));
+ if (refUrl.endsWith("/pointing")) { // For type pointing
+ schema.definitions["pointing"] = (await $RefParser.resolve(refUrl)).get(newRef);
+ property["$ref"] = newRef;
+ } else { // General object to resolve if any reference in child level
+ property = await resolveSchema((await $RefParser.resolve(refUrl)).get(newRef));
+ }
+ } else if(property["type"] === "array") { // reference in array items definition
+ let resolvedItems = await resolveSchema(property["items"]);
+ schema.definitions = {...schema.definitions, ...resolvedItems.definitions};
+ delete resolvedItems['definitions'];
+ property["items"] = resolvedItems;
+ }
+ properties[propertyKey] = property;
+ }
+ } else if (schema["oneOf"]) { // Reference in OneOf array
+ let resolvedOneOfList = []
+ for (const oneOfProperty of schema["oneOf"]) {
+ const resolvedOneOf = await resolveSchema(oneOfProperty);
+ resolvedOneOfList.push(resolvedOneOf);
+ }
+ schema["oneOf"] = resolvedOneOfList;
+ } else if (schema["$ref"] && !schema["$ref"].startsWith("#")) { //reference in oneOf list item
+ const refUrl = schema["$ref"];
+ let newRef = refUrl.substring(refUrl.indexOf("#"));
+ if (refUrl.endsWith("/pointing")) {
+ schema.definitions["pointing"] = (await $RefParser.resolve(refUrl)).get(newRef);
+ schema["$ref"] = newRef;
+ } else {
+ schema = await resolveSchema((await $RefParser.resolve(refUrl)).get(newRef));
+ }
+ }
+ return schema;
+ }
+
+ const init = async () => {
+ const element = document.getElementById('editor_holder');
+ let schema = await resolveExternalRef();
+ pointingProps = [];
// Customize the pointing property to capture angle1 and angle2 to specified format
for (const definitionKey in schema.definitions) {
if (definitionKey === 'pointing') {
@@ -42,7 +100,6 @@ function Jeditor(props) {
// Customize datatype of certain properties like subbands, duration, etc.,
getCustomProperties(schema.properties);
-
schema.title = props.title;
const subbandValidator = validateSubbandOutput;
const timeValidator = validateTime;
@@ -89,7 +146,9 @@ function Jeditor(props) {
disable_edit_json: true,
disable_properties: true,
disable_collapse: true,
- compact: true
+ show_errors: props.errorsOn?props.errorsOn:'change', // Can be 'interaction', 'change', 'always', 'never'
+ compact: true,
+ ajax: true
};
// Set Initial value to the editor
if (props.initValue) {
@@ -98,13 +157,19 @@ function Jeditor(props) {
editor = new JSONEditor(element, editorOptions);
// editor.getEditor('root').disable();
if (props.disabled) {
- editor.disable();
+ editor.on('ready',() => {
+ editor.disable();
+ });
}
if (props.parentFunction) {
props.parentFunction(editorFunction);
}
editorRef.current = editor;
editor.on('change', () => {setEditorOutput()});
+ };
+
+ useEffect(() => {
+ init();
}, [props.schema]);
/**
@@ -133,46 +198,6 @@ function Jeditor(props) {
* @param {Boolean} isDegree
*/
function getAngleProperty(defProperty, isDegree) {
- /*let newProperty = {
- "type": "object",
- "additionalProperties": false,
- "format": "grid",
- // "title": defProperty.title,
- // "description": defProperty.description};
- "title": "Duration",
- "description": "Duration of the observation"};
- let subProperties = {};
- if (isDegree) {
- subProperties["dd"] = { "type": "number",
- "title": "DD",
- "description": "Degrees",
- "default": 0,
- "minimum": 0,
- "maximum": 90 };
- } else {
- subProperties["hh"] = { "type": "number",
- "title": "HH",
- "description": "Hours",
- "default": 0,
- "minimum": 0,
- "maximum": 23 };
-
- }
- subProperties["mm"] = { "type": "number",
- "title": "MM",
- "description": "Minutes",
- "default": 0,
- "minimum": 0,
- "maximum": 59 };
- subProperties["ss"] = { "type": "number",
- "title": "SS",
- "description": "Seconds",
- "default": 0,
- "minimum": 0,
- "maximum": 59 };
-
- newProperty.properties = subProperties;
- newProperty.required = isDegree?["dd", "mm", "ss"]:["hh", "mm", "ss"];*/
let newProperty = {
type: "string",
title: defProperty.title,
@@ -209,39 +234,8 @@ function Jeditor(props) {
newProperty.default = '';
newProperty.description = "For Range enter Start and End seperated by 2 dots. Mulitple ranges can be separated by comma. Minimum should be 0 and maximum should be 511. For exmaple 11..20, 30..50";
newProperty.validationType = 'subband_list';
- // newProperty.options = {
- // grid_columns: 4
- // };
properties[propertyKey] = newProperty;
} else if (propertyKey.toLowerCase() === 'duration') {
- /*propertyValue.title = "Duration (minutes)";
- propertyValue.default = "1";
- propertyValue.description = "Duration of this observation. Enter in decimal for seconds. For example 0.5 for 30 seconds";
- propertyValue.minimum = 0.25;
- propertyValue.options = {
- grid_columns: 6
- };*/
- /*propertyValue.title = "Duration";
- propertyValue.default = "1H20M30S";
- propertyValue.type = "string";
- propertyValue.description = "Duration of the observation (H-hours,M-minutes,S-seconds & should be in the order of H, M and S respectively)";
- /*let newProperty = {
- type: "string",
- title: "Duration",
- description: `${propertyValue.description} (Hours:Minutes:Seconds)`,
- default: "00:00:00",
- "options": {
- "grid_columns": 5,
- "inputAttributes": {
- "placeholder": "HH:mm:ss"
- },
- "cleave": {
- date: true,
- datePattern: ['HH','mm','ss'],
- delimiter: ':'
- }
- }
- }*/
let newProperty = {
"type": "string",
"format": "time",
@@ -290,6 +284,9 @@ function Jeditor(props) {
options.grid_columns = 9;
propertyValue.options = options;
}
+ if (propertyValue['$ref'] && propertyValue['$ref'].endsWith("/pointing")) {
+ pointingProps.push(propertyKey);
+ }
getCustomProperties(propertyValue);
}
}
@@ -303,7 +300,7 @@ function Jeditor(props) {
for (const inputKey in editorInput) {
const inputValue = editorInput[inputKey];
if (inputValue instanceof Object) {
- if (inputKey.endsWith('pointing')) {
+ if (_.indexOf(pointingProps, inputKey) >= 0) {
inputValue.angle1 = getAngleInput(inputValue.angle1);
inputValue.angle2 = getAngleInput(inputValue.angle2, true);
} else if (inputKey === 'subbands') {
@@ -327,7 +324,7 @@ function Jeditor(props) {
for (const outputKey in editorOutput) {
let outputValue = editorOutput[outputKey];
if (outputValue instanceof Object) {
- if (outputKey.endsWith('pointing')) {
+ if (_.indexOf(pointingProps, outputKey) >= 0) {
outputValue.angle1 = getAngleOutput(outputValue.angle1, false);
outputValue.angle2 = getAngleOutput(outputValue.angle2, true);
} else {
@@ -355,21 +352,11 @@ function Jeditor(props) {
const dd = Math.floor(prpInput * 180 / Math.PI);
const mm = Math.floor((degrees-dd) * 60);
const ss = +((degrees-dd-(mm/60)) * 3600).toFixed(0);
- /*return {
- dd: dd,
- mm: mm,
- ss: ss
- }*/
return (dd<10?`0${dd}`:`${dd}`) + ':' + (mm<10?`0${mm}`:`${mm}`) + ':' + (ss<10?`0${ss}`:`${ss}`);
} else {
const hh = Math.floor(degrees/15);
const mm = Math.floor((degrees - (hh*15))/15 * 60 );
const ss = +((degrees -(hh*15)-(mm*15/60))/15 * 3600).toFixed(0);
- /*return {
- hh: hh,
- mm: mm,
- ss: ss
- }*/
return (hh<10?`0${hh}`:`${hh}`) + ':' + (mm<10?`0${mm}`:`${mm}`) + ':' + (ss<10?`0${ss}`:`${ss}`);
}
}
@@ -439,6 +426,9 @@ function Jeditor(props) {
if (splitOutput.length < 3) {
return false;
} else {
+ if (parseInt(splitOutput[0]) > 23 || parseInt(splitOutput[1])>59 || parseInt(splitOutput[2])>59) {
+ return false;
+ }
const timeValue = parseInt(splitOutput[0]*60*60) + parseInt(splitOutput[1]*60) + parseInt(splitOutput[2]);
if (timeValue >= 86400) {
return false;
@@ -456,6 +446,9 @@ function Jeditor(props) {
if (splitOutput.length < 3) {
return false;
} else {
+ if (parseInt(splitOutput[0]) > 90 || parseInt(splitOutput[1])>59 || parseInt(splitOutput[2])>59) {
+ return false;
+ }
const timeValue = parseInt(splitOutput[0]*60*60) + parseInt(splitOutput[1]*60) + parseInt(splitOutput[2]);
if (timeValue > 324000) {
return false;
@@ -522,7 +515,6 @@ function Jeditor(props) {
return (
<React.Fragment>
<div id='editor_holder'></div>
- {/* <div><input type="button" onClick={setEditorOutput} value="Show Output" /></div> */}
</React.Fragment>
);
};
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/components/Timeline/CalendarTimeline.js b/SAS/TMSS/frontend/tmss_webapp/src/components/Timeline/CalendarTimeline.js
new file mode 100644
index 0000000000000000000000000000000000000000..1825ef55a1a8191016e852cba1b9206e0b884c2b
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/src/components/Timeline/CalendarTimeline.js
@@ -0,0 +1,777 @@
+import React, {Component} from 'react';
+import Timeline, {
+ TimelineMarkers,
+ TimelineHeaders,
+ SidebarHeader,
+ DateHeader,
+ CustomMarker,
+ CursorMarker
+ } from 'react-calendar-timeline';
+import containerResizeDetector from 'react-calendar-timeline/lib/resize-detector/container';
+import moment from 'moment';
+import _ from 'lodash';
+
+import { Button } from 'primereact/button';
+import { Dropdown } from 'primereact/dropdown';
+
+import UtilService from '../../services/util.service';
+
+import 'react-calendar-timeline/lib/Timeline.css';
+import { Calendar } from 'primereact/calendar';
+
+// Label formats for day headers based on the interval label width
+const DAY_HEADER_FORMATS = [{ name: "longer", minWidth: 300, maxWidth: 50000, format: "DD dddd, MMMM YYYY"},
+ { name: "long", minWidth: 135, maxWidth: 300, format: "DD-MMMM-YYYY"},
+ {name: "mediumLong", minWidth: 100, maxWidth: 135, format: "DD-MMM-YYYY"},
+ {name: "medium", minWidth: 75, maxWidth: 100, format: "DD-MMM-YY"},
+ {name: "short", minWidth: 50, maxWidth: 75, format: "DD-MMM"},
+ {name: "mini", minWidth: 10, maxWidth: 50, format: "DD"},
+ {name: "micro", minWidth: 10, maxWidth: 25, format: "DD"},
+ {name: "nano", minWidth: 0, maxWidth: 0, format: ""}];
+
+//>>>>>> Constants for date/time formats, zoom level definition & defaults
+const UTC_DISPLAY_FORMAT = "YYYY-MM-DDTHH:mm:ss";
+const UTC_LST_KEY_FORMAT = "YYYY-MM-DDTHH:mm:00";
+const UTC_LST_HOUR_FORMAT = "YYYY-MM-DDTHH:00:00";
+const UTC_LST_DAY_FORMAT = "YYYY-MM-DDT00:00:00";
+const ZOOM_LEVELS = [{name: '30 Minutes', value: 30 * 60},
+ {name: '1 Hour', value: 1 * 60 * 60},
+ {name: '3 Hours', value: 3 * 60 * 60},
+ {name: '6 Hours', value: 6 * 60 * 60},
+ {name: '12 Hour', value: 12 * 60 * 60},
+ {name: '1 Day', value: 24 * 60 * 60},
+ {name: '2 Days', value: 2 * 24 * 60 * 60},
+ {name: '3 Days', value: 3 * 24 * 60 * 60},
+ {name: '5 Days', value: 5 * 24 * 60 * 60},
+ {name: '1 Week', value: 7 * 24 * 60 * 60},
+ {name: '2 Weeks', value: 14 * 24 * 60 * 60},
+ {name: '4 Weeks', value: 28 * 24 * 60 * 60},
+ {name: 'Custom', value: 24 * 60 * 60}];
+const DEFAULT_ZOOM_LEVEL = "2 Days";
+const DEFAULT_GROUP = [{'id': 0, 'title': ''}]; // 1st row is added purposefully to show cursor labels
+//<<<<<<
+
+/**
+ * Component to create a calendar timeline based out of react-calendar-timeline with UTC and LST date headers.
+ */
+export class CalendarTimeline extends Component {
+
+ constructor(props) {
+ super(props);
+ let group = DEFAULT_GROUP;
+ if (props.group) {
+ group = group.concat(props.group);
+ }
+ const defaultZoomLevel = _.find(ZOOM_LEVELS, {name: DEFAULT_ZOOM_LEVEL});
+ this.state = {
+ defaultStartTime: props.startTime || moment().utc().add(-1 * defaultZoomLevel.value/2, 'seconds'),
+ defaultEndTime: props.endTime || moment().utc().add(1 * defaultZoomLevel.value/2, 'seconds'),
+ group: group,
+ items: props.items || [],
+ //>>>>>> Properties to pass to react-calendar-timeline component
+ stackItems: props.stackItems || true,
+ zoomAllowed: props.zoomAllowed || true,
+ minZoom: props.minZoom || (1 * 60 * 1000), // One Minute
+ maxZoom: props.maxZoom || (32 * 24 * 60 * 60 * 1000), // 32 hours
+ zoomLevel: DEFAULT_ZOOM_LEVEL,
+ isTimelineZoom: true,
+ zoomRange: null,
+ prevZoomRange: null,
+ lineHeight: props.rowHeight || 50, // Row line height
+ sidebarWidth: props.sidebarWidth || 200,
+ timeSteps: props.timeSteps || {minute: 60},
+ canMove: props.itemsMovable || false,
+ canResize: props.itemsResizable || false,
+ canchangeGroup: props.itemGroupChangeable || true,
+ //<<<<<< Properties to pass to react-calendar-timeline component
+ showCursor: props.showCursor || true,
+ timeHeaderLabelVisibile: true,
+ currentUTC: props.currentUTC || moment().utc(), // Current UTC for clock display
+ currentLST: null, // Current LST for clock display
+ cursorLST: moment().format('HH:mm:ss'), // Holds the LST value for the cursot position in the timeline
+ lastCursorPosition: null, // To track the last cursor position and fetch the data from server if changed
+ utcLSTMap:{}, // JSON object to hold LST values fetched from server for UTC and show LST value in cursor label
+ lstDateHeaderMap: {}, // JSON object to hold header value for the LST axis in required format like 'HH' or 'MM' or others
+ lstDateHeaderUnit: 'hour', // Unit to be considered for the LST axis header based on the visible duration
+ isLSTDateHeaderLoading: true,
+ dayHeaderVisible: true, // To control the Day header visibility based on the zoom level
+ weekHeaderVisible: false // To control the Week header visibility based on the zoom level
+ }
+ this.itemClickCallback = props.itemClickCallback; // Pass timeline item click event back to parent
+
+ //>>>>>>> Override function of timeline component
+ this.onZoom = this.onZoom.bind(this);
+ this.onBoundsChange = this.onBoundsChange.bind(this);
+ this.onTimeChange = this.onTimeChange.bind(this);
+ //<<<<<< Override function of timeline component
+
+ //>>>>>> Custom Renderer Functions
+ this.renderSidebarHeader = this.renderSidebarHeader.bind(this);
+ this.renderDayHeader = this.renderDayHeader.bind(this);
+ this.renderUTCDateHeader = this.renderUTCDateHeader.bind(this);
+ this.renderLSTDateHeader = this.renderLSTDateHeader.bind(this);
+ this.renderCursor = this.renderCursor.bind(this);
+ this.renderItem = this.renderItem.bind(this);
+ //<<<<<<< Custom Renderer Functions
+
+ //>>>>>> Functions of this component
+ this.setCurrentUTC = this.setCurrentUTC.bind(this);
+ this.getLSTof = this.getLSTof.bind(this);
+ this.onItemClick = this.onItemClick.bind(this);
+ this.resetToCurrentTime = this.resetToCurrentTime.bind(this);
+ this.moveLeft = this.moveLeft.bind(this);
+ this.moveRight = this.moveRight.bind(this);
+ this.zoomIn = this.zoomIn.bind(this);
+ this.zoomOut = this.zoomOut.bind(this);
+ this.setZoomRange = this.setZoomRange.bind(this);
+ //<<<<<< Functions of this component
+ }
+
+ componentDidMount() {
+ const setCurrentUTC = this.setCurrentUTC;
+ // Load LST date header values
+ this.loadLSTDateHeaderMap(this.state.defaultStartTime, this.state.defaultEndTime, this.state.lstDateHeaderUnit);
+ // Set initial UTC clock time from server
+ setCurrentUTC(true);
+ // Update UTC clock periodically in sync with server
+ setInterval(function(){setCurrentUTC(true)}, 60000);
+ // Update UTC clock every second to keep the clock display live
+ setInterval(function(){setCurrentUTC()}, 1000);
+ }
+
+ shouldComponentUpdate() {
+ return true;
+ }
+
+ componentDidUpdate() {
+ // console.log("Component Updated");
+ }
+
+ /**
+ * Sets current UTC and LST time either from the server or locally.
+ * @param {boolean} systemClock - to differetiate whether tosync with server or local update
+ */
+ setCurrentUTC(systemClock) {
+ if(systemClock) {
+ UtilService.getUTC()
+ .then(async (utcString) => {
+ const currentUTC = moment.utc(utcString);
+ this.setState({currentUTC: currentUTC});
+ let currentLST = await UtilService.getLST(utcString);
+ this.setState({currentLST: moment(currentUTC.format('DD-MMM-YYYY ') + currentLST)})
+ } );
+ } else {
+ this.setState({currentUTC: this.state.currentUTC.add(1, 'second'),
+ currentLST: this.state.currentLST?this.state.currentLST.add(1, 'second'):null});
+ }
+ }
+
+ /**
+ * Loads LST header values from server and keeps in a state JSON object for respective UTC values in required format
+ * @param {moment} startTime
+ * @param {moment} endTime
+ * @param {string} lstDateHeaderUnit
+ */
+ async loadLSTDateHeaderMap(startTime, endTime, lstDateHeaderUnit) {
+ // let lstDateHeaderMap = this.state.lstDateHeaderMap;
+ let lstDateHeaderMap = {};
+ // const lstDateHeaderUnit = this.state.lstDateHeaderUnit;
+ const timeDiff = endTime.diff(startTime, lstDateHeaderUnit);
+ const range = _.range(timeDiff);
+ for (const value of range) {
+ const colUTC = startTime.clone().add(value, lstDateHeaderUnit).utc();
+ const formattedColUTC = colUTC.format(lstDateHeaderUnit==="hour"?UTC_LST_HOUR_FORMAT:UTC_LST_DAY_FORMAT);
+ // if (!lstDateHeaderMap[formattedColUTC]) {
+ const lst = await UtilService.getLST(formattedColUTC);
+ const lstDate = moment(colUTC.format(`DD-MMM-YYYY ${lst}`)).add(30, 'minutes');
+ lstDateHeaderMap[formattedColUTC] = lstDateHeaderUnit==="hour"?lstDate.format('HH'):lstDate.format('DD');
+ // }
+ }
+ this.setState({lstDateHeaderMap: lstDateHeaderMap, isLSTDateHeaderLoading: false});
+ }
+
+ /**
+ * Gets the LST value for the UTC passed.
+ * If no value present in the state JSON object, fetches from the server and update the state object
+ * @param {moment} utc
+ */
+ getLSTof(utc) {
+ utc = moment.utc(utc.format(UTC_LST_KEY_FORMAT));
+
+ // Condition to reduce or avoid repeated server request as the request is sent asynchronously if multiple calls are there for same value
+ if (!this.state.lastCursorPosition || this.state.lastCursorPosition.diff(utc, 'minutes')>0
+ || this.state.lastCursorPosition.diff(utc, 'minutes')<0) {
+ let utcLSTMap = this.state.utcLSTMap;
+ const formattedUTC = utc.format(UTC_LST_KEY_FORMAT);
+ if (utcLSTMap[formattedUTC]) {
+ this.setState({lastCursorPosition:utc, cursorLST: utcLSTMap[formattedUTC]});
+ } else {
+ if (_.keys(utcLSTMap).indexOf(formattedUTC)<0) {
+ UtilService.getLST(formattedUTC).then(lst => {
+ utcLSTMap[formattedUTC] = lst;
+ this.setState({utcLSTMap: utcLSTMap, lastCursorPosition:utc, cursorLST: lst});
+ });
+ utcLSTMap[formattedUTC] = null;
+ this.setState({utcLSTMap: utcLSTMap});
+ }
+ }
+ }
+ }
+
+ /** Custom Left Side Bar Header Render function that is passed to the timeline component */
+ renderSidebarHeader({ getRootProps }) {
+ let monthDuration = "";
+ const startMonth = this.state.defaultStartTime.format('MMM');
+ const endMonth = this.state.defaultEndTime.format('MMM');
+ if (startMonth !== endMonth) {
+ monthDuration = `(${startMonth}-${endMonth})`;
+ }
+ return (<div {...getRootProps()}
+ style={{color: '#ffffff', textAlign: "right", width: `${this.state.sidebarWidth}px`,
+ paddingRight: '10px', backgroundColor: '#8ba7d9'}}>
+ <div style={{height:'30px'}}>{this.state.dayHeaderVisible?`Day${monthDuration}`:`Week${monthDuration}`}</div>
+ <div style={{height:'30px'}}>{this.state.dayHeaderVisible?`UTC(Hr)`:`UTC(Day)`}</div>
+ <div style={{height:'30px'}}>{this.state.dayHeaderVisible?`LST(Hr)`:`LST(Day)`}</div>
+ </div>
+ );
+ }
+
+ /**
+ * Day header formatter based on the width of the header label
+ * @param {moment} time
+ * @param {number} labelWidth
+ */
+ formatDayHeader(time, labelWidth) {
+ const dayFormat = _.find(DAY_HEADER_FORMATS, (format) => { return (format.minWidth<labelWidth && format.maxWidth>labelWidth);});
+ return time.format(dayFormat?dayFormat.format:"DD-MM-YY");
+ }
+
+ /** Custom Render function for Day Header to pass to the DateHeader component of the Timeline compoent */
+ renderDayHeader({ getIntervalProps, intervalContext, data }) {
+ const currentZoomValue = _.find(ZOOM_LEVELS, {name: this.state.zoomLevel}).value;
+ const intervalStartTime = intervalContext.interval.startTime.utc();
+ const labelWidth = intervalContext.interval.labelWidth;
+ let displayValue = "";
+
+ // For zoom levels less than 1 day, header value is formatted and label width is re-calculated
+ let intervals = 86400 / currentZoomValue;
+ const formattedTime = intervalStartTime.format("HH");
+ intervals = intervals < 1? 1: intervals;
+ const newLabelWidth = labelWidth * 24 / intervals;
+ displayValue = parseInt(formattedTime)%(24/intervals)===0?this.formatDayHeader(intervalStartTime, newLabelWidth):""
+ let divStyle = getIntervalProps().style;
+ if (displayValue) {
+ divStyle.width = `${newLabelWidth}px`;
+ divStyle.fontSize = newLabelWidth < 20?"12px":"14px";
+ return (<div {...getIntervalProps()} className="rct-dateHeader" style={divStyle}>
+ <span>
+ {displayValue}
+ </span>
+ </div>);
+ } else {
+ return "";
+ }
+ }
+
+ /** Custom Render function for DateHeader component to display UTC time/date values in the date header */
+ renderUTCDateHeader({ getIntervalProps, intervalContext, data }) {
+ let showBorder = true;
+ let divStyle = getIntervalProps().style;
+ const labelWidth = intervalContext.interval.labelWidth;
+ let widthFactor = 1;
+ let displayValue = "";
+
+ // Display value decided based on the unit as per the zoom level
+ if (this.state.lstDateHeaderUnit === "hour") {
+ displayValue = intervalContext.interval.startTime.utc().format('HH');
+ widthFactor = 24;
+ } else if (this.state.lstDateHeaderUnit === "day") {
+ displayValue = intervalContext.interval.endTime.utc().format('DD');
+ widthFactor = 30;
+ }
+
+ // >>>>>>*** This code should be updated with reduced lines or by creating separate function
+ // Calculate width factor to adjust the label width based on the unit and interval length
+ if (labelWidth < 1) {
+ showBorder = false; // If the linewidth is very less, don't display border. Instead show a marker line below the label
+ } else if (labelWidth < 2) {
+ widthFactor = widthFactor/2;
+ showBorder = false;
+ } else if (labelWidth < 4) {
+ widthFactor = widthFactor/3;
+ } else if (labelWidth < 5) {
+ widthFactor = widthFactor/4;
+ } else if (labelWidth < 5.5) {
+ widthFactor = widthFactor/6;
+ } else if (labelWidth < 7) {
+ widthFactor = widthFactor/8;
+ } else if (labelWidth < 12) {
+ widthFactor = widthFactor/12;
+ } else {
+ widthFactor = 1;
+ }
+ // <<<<<<*** This code should be updated with reduced lines or by creating separate function
+ displayValue = parseInt(displayValue)%Math.floor(widthFactor)===0?displayValue:"";
+ divStyle.fontSize = labelWidth>16?"14px":(labelWidth>12?"12px":(labelWidth>10?"10px":"10px"));
+ divStyle.borderLeft = showBorder?divStyle.borderLeft:"0px dashed #bbb";
+
+ if (displayValue) {
+ divStyle.width = `${labelWidth * widthFactor}px`;
+ return <div {...getIntervalProps()} className="rct-dateHeader" style={divStyle}>
+ { (this.state.timeHeaderLabelVisibile)?
+ (showBorder)?
+ <span>
+ {displayValue}
+ </span>:
+ <>
+ <span style={{height: '30px', lineHeight:'15px', textAlign: 'center', transform:(labelWidth<12?"rotate(0deg)":"")}}>
+ { displayValue}<br/>
+ <span style={{color: '#bbb'}}>{"|"}</span>
+ </span>
+ </>
+ :""}
+ </div>
+ } else {
+ return "";
+ }
+ }
+
+ /** Custom Render function to be passed to DateHeader component to display LST values in date header */
+ renderLSTDateHeader({ getIntervalProps, intervalContext, data }) {
+ let showBorder = true;
+ const utc = moment(intervalContext.interval.endTime).utc();
+ // Decide the value to be displayed based on the unit set for the zoom level
+ let lstDisplayValue = this.state.lstDateHeaderMap[utc.format(this.state.lstDateHeaderUnit === "hour"?UTC_LST_HOUR_FORMAT:UTC_LST_DAY_FORMAT)];
+ let divStyle = getIntervalProps().style;
+ const labelWidth = intervalContext.interval.labelWidth;
+ let widthFactor = 1;
+ if (this.state.lstDateHeaderUnit === "hour") {
+ widthFactor = 24;
+ } else if (this.state.lstDateHeaderUnit === "day") {
+ widthFactor = 30;
+ }
+ // >>>>>>*** This code should be updated with reduced lines or by creating separate function
+ // Calculate width factor to adjust the label width based on the unit and interval length
+ if (labelWidth < 1) {
+ showBorder = false;
+ } else if (labelWidth < 2) {
+ widthFactor = widthFactor/2;
+ showBorder = false;
+ } else if (labelWidth < 4) {
+ widthFactor = widthFactor/3;
+ } else if (labelWidth < 5) {
+ widthFactor = widthFactor/4;
+ } else if (labelWidth < 5.5) {
+ widthFactor = widthFactor/6;
+ } else if (labelWidth < 7) {
+ widthFactor = widthFactor/8;
+ } else if (labelWidth < 12) {
+ widthFactor = widthFactor/12;
+ } else {
+ widthFactor = 1;
+ }
+ // <<<<<<*** This code should be updated with reduced lines or by creating separate function
+
+ // Values to be displayed at regular intervals only
+ if (widthFactor === 24) {
+ lstDisplayValue = lstDisplayValue==="12"?lstDisplayValue:"";
+ } else if (widthFactor === 30) {
+ lstDisplayValue = lstDisplayValue==="5"?lstDisplayValue:"";
+ } else {
+ lstDisplayValue = parseInt(lstDisplayValue)%widthFactor===0?lstDisplayValue:"";
+ }
+ divStyle.fontSize = labelWidth>16?"14px":(labelWidth>12?"12px":(labelWidth>10?"10px":"10px"));
+ divStyle.borderLeft = showBorder?divStyle.borderLeft:"0px dashed #bbb";
+ if (lstDisplayValue) {
+ divStyle.width = `${labelWidth * widthFactor}px`;
+ return <div {...getIntervalProps()} className="rct-dateHeader" style={divStyle}>
+ <span>
+ {/* {intervalContext.interval.startTime.format('HH')} */}
+ {lstDisplayValue}
+ </span>
+ </div>
+ } else {
+ return "";
+ }
+ }
+
+ /** Custom Render function to pass to the CursorMarker component to display cursor labels on cursor movement */
+ renderCursor({ styles, date }) {
+ const utc = moment(date).utc();
+ this.getLSTof(utc);
+ const cursorLST = this.state.cursorLST;
+ let cursorTextStyles = {};
+ cursorTextStyles.backgroundColor = '#c40719'
+ cursorTextStyles.width = `${this.state.lineHeight*4}px`;
+ cursorTextStyles.color = '#ffffff';
+ cursorTextStyles.zIndex = '9999';
+ cursorTextStyles.fontSize = `${this.state.lineHeight/30*8}px`;
+ cursorTextStyles.height = `${this.state.lineHeight - 2}px`;
+ cursorTextStyles.position = styles.position;
+ cursorTextStyles.left = styles.left-(this.state.lineHeight*2);
+ cursorTextStyles.top = '2px';
+ cursorTextStyles.paddingLeft = "5px";
+ cursorTextStyles.textAlign = "center";
+ styles.backgroundColor = '#c40719';
+ styles.display = "block !important";
+ return (
+ <>
+ <div style={styles} />
+ <div style={cursorTextStyles}>
+ <div>UTC: { utc.format('DD-MMM-YYYY HH:mm:00')}</div>
+ <div>LST: {cursorLST}</div>
+ </div>
+ </>
+ );
+ }
+
+ /** Custom function to pass to timeline component to render item */
+ renderItem({ item, timelineContext, itemContext, getItemProps, getResizeProps }) {
+ const { left: leftResizeProps, right: rightResizeProps } = getResizeProps();
+ const backgroundColor = itemContext.selected?item.bgColor:item.bgColor;
+ // const backgroundColor = itemContext.selected ? (itemContext.dragging ? "red" : item.selectedBgColor) : item.bgColor;
+ // const borderColor = itemContext.resizing ? "red" : item.color;
+ const itemContentStyle = {lineHeight: `${Math.floor(itemContext.dimensions.height)}px`,
+ fontSize: "14px",
+ overflow: "hidden", textOverflow: "ellipsis", whiteSpace: "nowrap",
+ textAlign: "center"};
+
+ return (
+ <div
+ {...getItemProps({
+ style: {
+ background: backgroundColor,
+ color: item.color,
+ // borderColor,
+ borderStyle: "solid",
+ borderWidth: 1,
+ borderRadius: 3,
+ borderLeftWidth: itemContext.selected ? 3 : 1,
+ borderRightWidth: itemContext.selected ? 3 : 1
+ },
+ onMouseDown: () => {
+ this.onItemClick(item);
+ }
+ })}
+ >
+ {itemContext.useResizeHandle ? <div {...leftResizeProps} /> : null}
+
+ <div
+ style={{
+ height: itemContext.dimensions.height,
+ //overflow: "hidden",
+ paddingLeft: 3,
+ //textOverflow: "ellipsis",
+ //whiteSpace: "nowrap"
+ }}
+ >
+ {/* <div style={itemContentStyle}><span>{item.project}</span></div>
+ <div style={itemContentStyle}><span>{item.name}</span></div>
+ <div style={itemContentStyle}><span>{item.duration}</span></div> */}
+ <div style={itemContentStyle}><span>{item.title}</span></div>
+ </div>
+ {itemContext.useResizeHandle ? <div {...rightResizeProps} /> : null}
+ </div>
+ );
+ };
+
+ /** Overriding function to pass to timeline component for zoom activities */
+ onZoom(timelineContext) {
+ // Update to the timeline values based on pre-defined zoom on mouse scroll
+ if (this.state.isTimelineZoom) {
+ if (!this.state.zoomLevel.startsWith("Custom")) {
+ let startTime = moment(timelineContext.visibleTimeStart);
+ let endTime = moment(timelineContext.visibleTimeEnd);
+ const zoomTimeDiff = endTime.diff(startTime, 'seconds');
+ const prevZoomLevel = _.find(ZOOM_LEVELS, {'name': this.state.zoomLevel});
+ let zoomIndex = ZOOM_LEVELS.indexOf(prevZoomLevel);
+ if (zoomTimeDiff < prevZoomLevel.value && zoomIndex>0) {
+ zoomIndex--;
+ } else if (endTime.diff(startTime, 'seconds') > prevZoomLevel.value && zoomIndex<ZOOM_LEVELS.length-2) {
+ zoomIndex++;
+ }
+ this.changeZoomLevel(ZOOM_LEVELS[zoomIndex].name, true)
+ }
+ } else {
+ this.setState({isTimelineZoom: true});
+ }
+ }
+
+ /** Override function to pass to timeline component for custom action when timeline boundary changes */
+ onBoundsChange(canvasTimeStart, canvasTimeEnd) {
+ // To be implemented for lazy loading items
+ }
+
+ /** Overriding function to pass to timeline component for custom actions when visible time changes */
+ onTimeChange(visibleTimeStart, visibleTimeEnd, updateScrollCanvas) {
+ this.loadLSTDateHeaderMap(moment(visibleTimeStart).utc(), moment(visibleTimeEnd).utc(), this.state.lstDateHeaderUnit);
+ updateScrollCanvas(visibleTimeStart, visibleTimeEnd);
+ this.setState({defaultStartTime: moment(visibleTimeStart), defaultEndTime: moment(visibleTimeEnd)})
+ }
+
+ /**
+ * Item Click event passed back to the parent.
+ * @param {Object} item
+ */
+ onItemClick(item) {
+ if (this.itemClickCallback) {
+ this.itemClickCallback(item);
+ }
+ }
+
+ /**
+ * Resets the timeline view to default zoom and move to the current timeline
+ */
+ resetToCurrentTime(){
+ this.setState({defaultStartTime: moment().utc().add(-24, 'hours'),
+ defaultEndTime: moment().utc().add(24, 'hours'), zoomLevel: DEFAULT_ZOOM_LEVEL,
+ dayHeaderVisible: true, weekHeaderVisible: false, lstDateHeaderUnit: "hour"});
+ }
+
+ /**
+ * Changes the zoom level and updates the timeline visible times, loads LST DateHeader values,
+ * callbacks the parent to fetch item and group for the changed visible timeline
+ * @param {String} zoomLevel
+ * @param {boolean} isTimelineZoom
+ */
+ changeZoomLevel(zoomLevel, isTimelineZoom) {
+ zoomLevel = zoomLevel?zoomLevel: DEFAULT_ZOOM_LEVEL;
+ const newZoomLevel = _.find(ZOOM_LEVELS, {'name': zoomLevel});
+ let startTime = this.state.defaultStartTime;
+ let endTime = this.state.defaultEndTime;
+ if (zoomLevel === 'Custom') {
+ if (this.state.prevZoomRange) {
+ this.setZoomRange(this.state.prevZoomRange);
+ }
+ } else {
+ const visibleDuration = endTime.diff(startTime, 'seconds');
+ if (newZoomLevel.value < visibleDuration) {
+ startTime = startTime.add(1 * (visibleDuration-newZoomLevel.value)/2, 'seconds');
+ endTime = endTime.add(-1 * (visibleDuration-newZoomLevel.value)/2, 'seconds');
+ } else {
+ startTime = startTime.add(-1 * (newZoomLevel.value-visibleDuration)/2, 'seconds');
+ endTime = endTime.add(1 * (newZoomLevel.value-visibleDuration)/2, 'seconds');
+ }
+ this.loadLSTDateHeaderMap(startTime, endTime, 'hour');
+ const result = this.props.dateRangeCallback(startTime, endTime);
+ let group = DEFAULT_GROUP.concat(result.group);
+ this.setState({zoomLevel: zoomLevel, defaultStartTime: startTime, defaultEndTime: endTime,
+ isTimelineZoom: isTimelineZoom, zoomRange: null,
+ dayHeaderVisible: true, weekHeaderVisible: false, lstDateHeaderUnit: 'hour',
+ group: group, items: result.items});
+ }
+ }
+
+ /**
+ * Moves the timeline left 1/10th of the visible timeline duration
+ */
+ moveLeft() {
+ let visibleTimeStart = this.state.defaultStartTime;
+ let visibleTimeEnd = this.state.defaultEndTime;
+ const visibleTimeDiff = visibleTimeEnd.valueOf()-visibleTimeStart.valueOf();
+ const secondsToMove = visibleTimeDiff / 1000 / 10 ;
+ this.setState({defaultStartTime: visibleTimeStart.add(-1 * secondsToMove, 'seconds'),
+ defaultEndTime: visibleTimeEnd.add(-1 * secondsToMove, 'seconds')});
+ }
+
+ /**
+ * Moves the timeline right 1/10th of the visible timeline length
+ */
+ moveRight() {
+ let visibleTimeStart = this.state.defaultStartTime;
+ let visibleTimeEnd = this.state.defaultEndTime;
+ const visibleTimeDiff = visibleTimeEnd.valueOf()-visibleTimeStart.valueOf();
+ const secondsToMove = visibleTimeDiff / 1000 / 10 ;
+ this.setState({defaultStartTime: visibleTimeStart.add(1 * secondsToMove, 'seconds'),
+ defaultEndTime: visibleTimeEnd.add(1 * secondsToMove, 'seconds')});
+ }
+
+ /**
+ * Zooms In to the next pre-defined zoom level
+ */
+ zoomIn() {
+ /*let visibleTimeStart = this.state.defaultStartTime;
+ let visibleTimeEnd = this.state.defaultEndTime;
+ const visibleTimeDiff = visibleTimeEnd.valueOf()-visibleTimeStart.valueOf();
+ if (visibleTimeDiff > this.state.minZoom) {
+ const secondsToZoom = visibleTimeDiff / 1000 / 2 / 4 * 3 ;
+ this.setState({defaultStartTime: visibleTimeStart.add(1*secondsToZoom, 'seconds'),
+ defaultEndTime: visibleTimeEnd.add(-1*secondsToZoom, 'seconds')});
+ }*/
+ let prevZoomLevel = this.state.zoomLevel;
+ const prevZoomObject = _.find(ZOOM_LEVELS, {'name': prevZoomLevel});
+ const prevZoomIndex = ZOOM_LEVELS.indexOf(prevZoomObject);
+ if (prevZoomIndex > 0) {
+ this.changeZoomLevel(ZOOM_LEVELS[prevZoomIndex-1].name, false);
+ }
+ }
+
+ /**
+ * Zooms out to the next pre-defined zoom level
+ */
+ zoomOut() {
+ /*let visibleTimeStart = this.state.defaultStartTime;
+ let visibleTimeEnd = this.state.defaultEndTime;
+ const visibleTimeDiff = visibleTimeEnd.valueOf()-visibleTimeStart.valueOf();
+ if (visibleTimeDiff < this.state.maxZoom) {
+ const secondsToZoom = visibleTimeDiff / 1000 * 3 / 2;
+ this.setState({defaultStartTime: visibleTimeStart.add(-1*secondsToZoom, 'seconds'),
+ defaultEndTime: visibleTimeEnd.add(1*secondsToZoom, 'seconds')});
+ }*/
+ let prevZoomLevel = this.state.zoomLevel;
+ const prevZoomObject = _.find(ZOOM_LEVELS, {'name': prevZoomLevel});
+ const prevZoomIndex = ZOOM_LEVELS.indexOf(prevZoomObject);
+ if (prevZoomIndex < ZOOM_LEVELS.length-2) {
+ this.changeZoomLevel(ZOOM_LEVELS[prevZoomIndex+1].name, false);
+ }
+ }
+
+ /**
+ * Function to call when the custom date range is changed. Updates visible timelines, date header unit,
+ * calls back parent to get updated group and item records, LST date header values
+ * @param {array} value - array of moment object
+ */
+ setZoomRange(value){
+ let startDate, endDate = null;
+ if (value) {
+ // Set all values only when both range values available in the array else just set the value to reflect in the date selection component
+ if (value[1]!==null) {
+ startDate = moment.utc(moment(value[0]).format("DD-MMM-YYYY"));
+ endDate = moment.utc(moment(value[1]).format("DD-MMM-YYYY 23:59:59"));
+ let dayHeaderVisible = this.state.dayHeaderVisible;
+ let weekHeaderVisible = this.state.weekHeaderVisible;
+ let lstDateHeaderUnit = this.state.lstDateHeaderUnit;
+ let rangeDays = endDate.diff(startDate, 'days');
+ dayHeaderVisible = rangeDays > 35?false: true;
+ weekHeaderVisible = rangeDays > 35?true: false;
+ lstDateHeaderUnit = rangeDays > 35?"day":"hour";
+ this.setState({zoomRange:value, prevZoomRange:value,
+ defaultStartTime: startDate, defaultEndTime: endDate,
+ zoomLevel: ZOOM_LEVELS[ZOOM_LEVELS.length-1].name, isTimelineZoom: false,
+ dayHeaderVisible: dayHeaderVisible, weekHeaderVisible: weekHeaderVisible,
+ lstDateHeaderUnit: lstDateHeaderUnit
+ });
+ const result = this.props.dateRangeCallback(startDate, endDate);
+ let group = DEFAULT_GROUP.concat(result.group);
+ this.setState({group: group, items: result.items});
+ this.loadLSTDateHeaderMap(startDate, endDate, lstDateHeaderUnit);
+ } else {
+ this.setState({zoomRange: value});
+ }
+ } else {
+ this.resetToCurrentTime();
+ }
+ }
+
+ render() {
+ return (
+ <React.Fragment>
+ {/* Toolbar for the timeline */}
+ <div className="p-fluid p-grid timeline-toolbar">
+ {/* Clock Display */}
+ <div className="p-col-3" style={{padding: '0px 0px 0px 10px'}}>
+ <div style={{marginTop: "0px"}}>
+ <label style={{marginBottom: "0px"}}>UTC:</label><span>{this.state.currentUTC.format(UTC_DISPLAY_FORMAT)}</span>
+ </div>
+ {this.state.currentLST &&
+ <div style={{marginTop: "0px"}}>
+ <label style={{marginBottom: "0px"}}>LST:</label><span>{this.state.currentLST.format("HH:mm:ss")}</span>
+ </div>
+ }
+ </div>
+ {/* Date Range Selection */}
+ <div className="p-col-4">
+ {/* <span className="p-float-label"> */}
+ <Calendar id="range" placeholder="Select Date Range" selectionMode="range" showIcon={!this.state.zoomRange}
+ value={this.state.zoomRange} onChange={(e) => this.setZoomRange( e.value )} readOnlyInput />
+ {/* <label htmlFor="range">Select Date Range</label>
+ </span> */}
+ {this.state.zoomRange && <i className="pi pi-times pi-primary" style={{position: 'relative', left:'90%', bottom:'20px', cursor:'pointer'}}
+ onClick={() => {this.setZoomRange( null)}}></i>}
+ </div>
+ {/* Reset to default zoom and current timeline */}
+ <div className="p-col-1" style={{padding: '5px 0px'}}>
+ <Button label="" icon="pi pi-arrow-down" className="p-button-rounded p-button-success" id="now-btn" onClick={this.resetToCurrentTime} title="Rest Zoom & Move to Current Time"/>
+ </div>
+ {/* Zoom Select */}
+ <div className="p-col-2" style={{paddingRight: '0px'}}>
+ <Dropdown optionLabel="name" optionValue="name"
+ style={{fontSize: '10px'}}
+ value={this.state.zoomLevel}
+ options={ZOOM_LEVELS}
+ filter showClear={false} filterBy="name"
+ onChange={(e) => {this.changeZoomLevel(e.value, false)}}
+ placeholder="Zoom"/>
+ </div>
+ {/* Zoom and Move Action */}
+ <div className="p-col-2 timeline-actionbar">
+ <button className="p-link" title="Move Left" onClick={e=> { this.moveLeft() }}><i className="pi pi-angle-left"></i></button>
+ <button className="p-link" title="Zoom Out" onClick={e=> { this.zoomOut() }} disabled={this.state.zoomLevel.startsWith('Custom')}><i className="pi pi-minus-circle"></i></button>
+ <button className="p-link" title="Zoom In" onClick={e=> { this.zoomIn() }} disabled={this.state.zoomLevel.startsWith('Custom')}><i className="pi pi-plus-circle"></i></button>
+ <button className="p-link" title="Move Right" onClick={e=> { this.moveRight() }} onMouseDown={e=> { this.moveRight() }}><i className="pi pi-angle-right"></i></button>
+ </div>
+ </div>
+ <Timeline
+ groups={this.state.group}
+ items={this.state.items}
+ // Use these below properties to stop zoom and move
+ // defaultTimeStart={this.props.defaultStartTime?this.props.defaultStartTime:this.state.defaultStartTime}
+ // defaultTimeStart={this.state.defaultStartTime}
+ // defaultTimeEnd={this.state.defaultEndTime}
+ visibleTimeStart={this.state.defaultStartTime.valueOf()}
+ visibleTimeEnd={this.state.defaultEndTime.valueOf()}
+ resizeDetector={containerResizeDetector}
+ stackItems={this.state.stackItems}
+ traditionalZoom={this.state.zoomAllowed}
+ minZoom={this.state.minZoom}
+ maxZoom={this.state.maxZoom}
+ lineHeight={this.state.lineHeight} itemHeightRatio={0.95}
+ sidebarWidth={this.state.sidebarWidth}
+ timeSteps={this.state.timeSteps}
+ onZoom={this.onZoom}
+ onBoundsChange={this.onBoundsChange}
+ onTimeChange={this.onTimeChange}
+ itemRenderer={this.renderItem}
+ canMove={this.state.canMove}
+ canResize={this.state.canResize}
+ canChangeGroup={this.state.canChangeGroup}>
+ <TimelineHeaders className="sticky">
+ <SidebarHeader>{({ getRootProps }) => {return this.renderSidebarHeader({ getRootProps })}}</SidebarHeader>
+ {this.state.weekHeaderVisible &&
+ <DateHeader unit="Week" labelFormat="w"></DateHeader> }
+ { this.state.dayHeaderVisible &&
+ <DateHeader unit="hour" intervalRenderer={this.renderDayHeader}></DateHeader> }
+ <DateHeader unit={this.state.lstDateHeaderUnit} intervalRenderer={this.renderUTCDateHeader} ></DateHeader>
+ {!this.state.isLSTDateHeaderLoading &&
+ // This method keeps updating the header labels, so that the LST values will be displayed after fetching from server
+ <DateHeader unit={this.state.lstDateHeaderUnit}
+ intervalRenderer={({ getIntervalProps, intervalContext, data })=>{return this.renderLSTDateHeader({ getIntervalProps, intervalContext, data })}}>
+ </DateHeader>
+ // This method will render once but will not update the values after fetching from server
+ // <DateHeader unit={this.state.lstDateHeaderUnit} intervalRenderer={this.renderLSTDateHeader}></DateHeader>
+ }
+ </TimelineHeaders>
+
+ <TimelineMarkers>
+ {/* Current time line marker */}
+ <CustomMarker date={this.state.currentUTC}>
+ {({ styles, date }) => {
+ const customStyles = {
+ ...styles,
+ backgroundColor: 'green',
+ width: '2px'
+ }
+ return <div style={customStyles} />
+ }}
+ </CustomMarker>
+ {this.state.showCursor?
+ <CursorMarker>
+ {this.renderCursor}
+ </CursorMarker>:""}
+ </TimelineMarkers>
+ </Timeline>
+ </React.Fragment>
+ );
+ }
+
+}
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/components/Timeline/index.js b/SAS/TMSS/frontend/tmss_webapp/src/components/Timeline/index.js
new file mode 100644
index 0000000000000000000000000000000000000000..ddf91940099685efc4e2541f2814f4f20acd4ed8
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/src/components/Timeline/index.js
@@ -0,0 +1,3 @@
+import { CalendarTimeline } from './CalendarTimeline';
+
+export default CalendarTimeline;
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/components/ViewTable.js b/SAS/TMSS/frontend/tmss_webapp/src/components/ViewTable.js
index 7647b3bac915ae86e717f04d5a69089e8fdd054b..21f9326233262c6e69a655eb877a7868d4ca4d9b 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/components/ViewTable.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/components/ViewTable.js
@@ -1,22 +1,27 @@
-import React, {useRef } from "react";
+import React, {useRef, useState } from "react";
import { useSortBy, useTable, useFilters, useGlobalFilter, useAsyncDebounce, usePagination } from 'react-table'
import matchSorter from 'match-sorter'
import _ from 'lodash';
import moment from 'moment';
import { useHistory } from "react-router-dom";
import {OverlayPanel} from 'primereact/overlaypanel';
-
+import {InputSwitch} from 'primereact/inputswitch';
+import { Calendar } from 'primereact/calendar';
+import {Paginator} from 'primereact/paginator';
+import { Button } from "react-bootstrap";
+import { InputNumber } from "primereact/inputnumber";
let tbldata =[];
let isunittest = false;
+let showTopTotal = true;
let columnclassname =[];
+
// Define a default UI for filtering
function GlobalFilter({
preGlobalFilteredRows,
globalFilter,
setGlobalFilter,
}) {
-
const [value, setValue] = React.useState(globalFilter)
const onChange = useAsyncDebounce(value => {setGlobalFilter(value || undefined)}, 200)
return (
@@ -46,10 +51,173 @@ function DefaultColumnFilter({
)
}
+
+// This is a custom filter UI for selecting
+// a unique option from a list
+function SelectColumnFilter({
+ column: { filterValue, setFilter, preFilteredRows, id },
+}) {
+ // Calculate the options for filtering
+ // using the preFilteredRows
+ const options = React.useMemo(() => {
+ const options = new Set()
+ preFilteredRows.forEach(row => {
+ options.add(row.values[id])
+ })
+ return [...options.values()]
+ }, [id, preFilteredRows])
+
+ // Render a multi-select box
+ return (
+ <select
+ value={filterValue}
+ onChange={e => {
+ setFilter(e.target.value || undefined)
+ }}
+ >
+ <option value="">All</option>
+ {options.map((option, i) => (
+ <option key={i} value={option}>
+ {option}
+ </option>
+ ))}
+ </select>
+ )
+}
+
+// This is a custom filter UI that uses a
+// slider to set the filter value between a column's
+// min and max values
+function SliderColumnFilter({
+ column: { filterValue, setFilter, preFilteredRows, id },
+}) {
+ // Calculate the min and max
+ // using the preFilteredRows
+
+ const [min, max] = React.useMemo(() => {
+ let min = preFilteredRows.length ? preFilteredRows[0].values[id] : 0
+ let max = preFilteredRows.length ? preFilteredRows[0].values[id] : 0
+ preFilteredRows.forEach(row => {
+ min = Math.min(row.values[id], min)
+ max = Math.max(row.values[id], max)
+ })
+ return [min, max]
+ }, [id, preFilteredRows])
+
+ return (
+ <>
+ <input
+ type="range"
+ min={min}
+ max={max}
+ value={filterValue || min}
+ onChange={e => {
+ setFilter(parseInt(e.target.value, 10))
+ }}
+ />
+ <button onClick={() => setFilter(undefined)}>Off</button>
+ </>
+ )
+}
+
+// This is a custom filter UI that uses a
+// switch to set the value
+function BooleanColumnFilter({
+ column: { setFilter},
+}) {
+ const [value, setValue] = useState(true);
+ return (
+ <>
+ <InputSwitch checked={value} onChange={() => { setValue(!value); setFilter(!value); }} />
+ <button onClick={() => setFilter(undefined)}>Off</button>
+ </>
+ )
+}
+
+// This is a custom filter UI that uses a
+// calendar to set the value
+function CalendarColumnFilter({
+ column: { setFilter},
+}) {
+ const [value, setValue] = useState('');
+ return (
+ <>
+ <Calendar value={value} onChange={(e) => {
+ const value = moment(e.value, moment.ISO_8601).format("YYYY-MMM-DD")
+ setValue(value); setFilter(value);
+ }} showIcon></Calendar>
+ <button onClick={() => setFilter(undefined)}>Off</button>
+ </>
+ )
+}
+
+
+// This is a custom UI for our 'between' or number range
+// filter. It uses two number boxes and filters rows to
+// ones that have values between the two
+function NumberRangeColumnFilter({
+ column: { filterValue = [], preFilteredRows, setFilter, id },
+}) {
+ const [min, max] = React.useMemo(() => {
+ let min = preFilteredRows.length ? preFilteredRows[0].values[id] : 0
+ let max = preFilteredRows.length ? preFilteredRows[0].values[id] : 0
+ preFilteredRows.forEach(row => {
+ min = Math.min(row.values[id], min)
+ max = Math.max(row.values[id], max)
+ })
+ return [min, max]
+ }, [id, preFilteredRows])
+
+ return (
+ <div
+ style={{
+ display: 'flex',
+ }}
+ >
+ <input
+ value={filterValue[0] || ''}
+ type="number"
+ onChange={e => {
+ const val = e.target.value
+ setFilter((old = []) => [val ? parseInt(val, 10) : undefined, old[1]])
+ }}
+ placeholder={`Min (${min})`}
+ style={{
+ width: '70px',
+ marginRight: '0.5rem',
+ }}
+ />
+ to
+ <input
+ value={filterValue[1] || ''}
+ type="number"
+ onChange={e => {
+ const val = e.target.value
+ setFilter((old = []) => [old[0], val ? parseInt(val, 10) : undefined])
+ }}
+ placeholder={`Max (${max})`}
+ style={{
+ width: '70px',
+ marginLeft: '0.5rem',
+ }}
+ />
+ </div>
+ )
+}
+
+
function fuzzyTextFilterFn(rows, id, filterValue) {
return matchSorter(rows, filterValue, { keys: [row => row.values[id]] })
}
+const filterTypes = {
+ 'select': SelectColumnFilter,
+ 'switch': BooleanColumnFilter,
+ 'slider': SliderColumnFilter,
+ 'date': CalendarColumnFilter,
+ 'range': NumberRangeColumnFilter
+};
+
// Let the table remove the filter if the string is empty
fuzzyTextFilterFn.autoRemove = val => !val
@@ -65,7 +233,7 @@ const IndeterminateCheckbox = React.forwardRef(
)
// Our table component
-function Table({ columns, data, defaultheader, optionalheader }) {
+function Table({ columns, data, defaultheader, optionalheader, defaultSortColumn, tablename, defaultpagesize }) {
const filterTypes = React.useMemo(
() => ({
// Add a new fuzzyTextFilterFn filter type.
@@ -93,7 +261,7 @@ function Table({ columns, data, defaultheader, optionalheader }) {
}),
[]
)
-
+
const {
getTableProps,
getTableBodyProps,
@@ -103,33 +271,27 @@ function Table({ columns, data, defaultheader, optionalheader }) {
allColumns,
getToggleHideAllColumnsProps,
state,
+ page,
preGlobalFilteredRows,
setGlobalFilter,
setHiddenColumns,
- page,
- canPreviousPage,
- canNextPage,
- pageOptions,
- pageCount,
gotoPage,
- nextPage,
- previousPage,
setPageSize,
- state: { pageIndex, pageSize },
} = useTable(
{
columns,
data,
defaultColumn,
filterTypes,
- initialState: { pageIndex: 0 }
+ initialState: { pageIndex: 0,
+ pageSize: (defaultpagesize && defaultpagesize>0)?defaultpagesize:10,
+ sortBy: defaultSortColumn }
},
useFilters,
useGlobalFilter,
useSortBy,
usePagination
)
-
React.useEffect(() => {
setHiddenColumns(
columns.filter(column => !column.isVisible).map(column => column.accessor)
@@ -138,9 +300,54 @@ function Table({ columns, data, defaultheader, optionalheader }) {
let op = useRef(null);
+ const [currentpage, setcurrentPage] = React.useState(0);
+ const [currentrows, setcurrentRows] = React.useState(defaultpagesize);
+ const [custompagevalue,setcustompagevalue] = React.useState();
+
+ const onPagination = (e) => {
+ gotoPage(e.page);
+ setcurrentPage(e.first);
+ setcurrentRows(e.rows);
+ setPageSize(e.rows)
+ if([10,25,50,100].includes(e.rows)){
+ setcustompagevalue();
+ }
+ };
+
+ const onCustomPage = (e) => {
+ if(typeof custompagevalue === 'undefined' || custompagevalue == null) return;
+ gotoPage(0);
+ setcurrentPage(0);
+ setcurrentRows(custompagevalue);
+ setPageSize(custompagevalue)
+ };
+
+ const onChangeCustompagevalue = (e) => {
+ setcustompagevalue(e.target.value);
+ }
+
+ const onShowAllPage = (e) => {
+ gotoPage(e.page);
+ setcurrentPage(e.first);
+ setcurrentRows(e.rows);
+ setPageSize(tbldata.length)
+ setcustompagevalue();
+ };
+
+ const onToggleChange = (e) =>{
+ let lsToggleColumns = [];
+ allColumns.forEach( acolumn =>{
+ let jsonobj = {};
+ let visible = (acolumn.Header === e.target.id) ? ((acolumn.isVisible)?false:true) :acolumn.isVisible
+ jsonobj['Header'] = acolumn.Header;
+ jsonobj['isVisible'] = visible;
+ lsToggleColumns.push(jsonobj)
+ })
+ localStorage.setItem(tablename,JSON.stringify(lsToggleColumns))
+ }
return (
<>
- <div id="block_container" style={{ display: 'flex', verticalAlign: 'middle', marginTop:'20px'}}>
+ <div id="block_container">
<div style={{textAlign:'left', marginRight:'30px'}}>
<i className="fa fa-columns col-filter-btn" label="Toggle Columns" onClick={(e) => op.current.toggle(e)} />
<OverlayPanel ref={op} id="overlay_panel" showCloseIcon={false} >
@@ -148,7 +355,7 @@ function Table({ columns, data, defaultheader, optionalheader }) {
<div style={{textAlign: 'center'}}>
<label>Select column(s) to view</label>
</div>
- <div style={{float: 'left', backgroundColor: '#d1cdd936', width: '250px', height: '400px', overflow: 'auto', marginBottom:'10px', padding:'5px'}}>
+ <div style={{float: 'left', backgroundColor: '#d1cdd936', width: '250px', minHeight: '100px', maxHeight: '300px' , overflow: 'auto', marginBottom:'10px', padding:'5px'}}>
<div id="tagleid" >
<div >
<div style={{marginBottom:'5px'}}>
@@ -156,7 +363,11 @@ function Table({ columns, data, defaultheader, optionalheader }) {
</div>
{allColumns.map(column => (
<div key={column.id} style={{'display':column.id !== 'actionpath'?'block':'none'}}>
- <input type="checkbox" {...column.getToggleHiddenProps()} /> {(defaultheader[column.id])?defaultheader[column.id]:(optionalheader[column.id]?optionalheader[column.id]:column.id)}
+ <input type="checkbox" {...column.getToggleHiddenProps()}
+ id={(defaultheader[column.id])?defaultheader[column.id]:(optionalheader[column.id]?optionalheader[column.id]:column.id)}
+ onClick={onToggleChange}
+ /> {
+ (defaultheader[column.id]) ? defaultheader[column.id] : (optionalheader[column.id] ? optionalheader[column.id] : column.id)}
</div>
))}
<br />
@@ -176,28 +387,33 @@ function Table({ columns, data, defaultheader, optionalheader }) {
/>
}
</div>
-</div>
+ { showTopTotal &&
+ <div className="total_records_top_label"> <label >Total records ({data.length})</label></div>
+ }
+ </div>
- <div style={{overflow: 'auto', padding: '0.75em',}}>
- <table {...getTableProps()} style={{width:'100%'}} data-testid="viewtable" className="viewtable" >
+ <div className="table_container">
+ <table {...getTableProps()} data-testid="viewtable" className="viewtable" >
<thead>
{headerGroups.map(headerGroup => (
<tr {...headerGroup.getHeaderGroupProps()}>
{headerGroup.headers.map(column => (
- <th {...column.getHeaderProps(column.getSortByToggleProps())} >
+ <th>
+ <div {...column.getHeaderProps(column.getSortByToggleProps())}>
{column.Header !== 'actionpath' && column.render('Header')}
{column.Header !== 'Action'?
- column.isSorted ? (column.isSortedDesc ? <i className="pi pi-sort-down" aria-hidden="true"></i> : <i className="pi pi-sort-up" aria-hidden="true"></i>) : <i className="pi pi-sor" aria-hidden="true"></i>
+ column.isSorted ? (column.isSortedDesc ? <i className="pi pi-sort-down" aria-hidden="true"></i> : <i className="pi pi-sort-up" aria-hidden="true"></i>) : ""
: ""
}
- {/* Render the columns filter UI */}
+ </div>
+
+ {/* Render the columns filter UI */}
{column.Header !== 'actionpath' &&
<div className={columnclassname[0][column.Header]} >
{column.canFilter && column.Header !== 'Action' ? column.render('Filter') : null}
-
</div>
}
- </th>
+ </th>
))}
</tr>
))}
@@ -210,52 +426,32 @@ function Table({ columns, data, defaultheader, optionalheader }) {
<tr {...row.getRowProps()}>
{row.cells.map(cell => {
if(cell.column.id !== 'actionpath')
- return <td {...cell.getCellProps()}>{cell.render('Cell')}</td>
+ return <td {...cell.getCellProps()}>{cell.render('Cell')}</td>
+ else
+ return "";
})}
</tr>
)
})}
</tbody>
</table>
-
</div>
- <div className="pagination" style={{marginTop:"10px"}}>
- <button onClick={() => gotoPage(0)} disabled={!canPreviousPage} title="Go to first page">
- {'<<'}
- </button>{' '}
- <button onClick={() => previousPage()} disabled={!canPreviousPage} title="Go to previous page">
- {'<'}
- </button>{' '}
- <span style={{marginLeft:"5px"}}>
- Page{' '}
- <strong>
- {pageIndex + 1} of {pageOptions.length}
- </strong>{' '}
- </span>
- <button onClick={() => nextPage()} disabled={!canNextPage} title="Go to next page">
- {'>'}
- </button>{' '}
- <button onClick={() => gotoPage(pageCount - 1)} disabled={!canNextPage} title="Go to last page">
- {'>>'}
- </button>{' '}
- <select
- style={{marginLeft:"3px"}}
- value={pageSize}
- onChange={e => {
- setPageSize(Number(e.target.value))
- }}
- >
- {[10, 20, 30, 40, 50].map(pageSize => (
- <option key={pageSize} value={pageSize}>
- Show {pageSize}
- </option>
- ))}
- </select>
+ <div className="pagination p-grid">
+ <div className="total_records_bottom_label" ><label >Total records ({data.length})</label></div>
+ <div>
+ <Paginator rowsPerPageOptions={[10,25,50,100]} first={currentpage} rows={currentrows} totalRecords={rows.length} onPageChange={onPagination} > </Paginator>
+ </div>
+ <div>
+ <InputNumber id="custompage" value={custompagevalue} onChange ={onChangeCustompagevalue}
+ min={0} />
+ <label >Records/Page</label>
+ <Button onClick={onCustomPage} style={{width: "50px"}}> Show </Button>
+ <Button onClick={onShowAllPage} style={{marginLeft: "1em", width: "80px"}}> Show All </Button>
+ </div>
</div>
</>
)
}
-
// Define a custom filter filter function!
function filterGreaterThan(rows, id, filterValue) {
@@ -277,16 +473,22 @@ function ViewTable(props) {
tbldata = props.data;
isunittest = props.unittest;
columnclassname = props.columnclassname;
-
+ showTopTotal = props.showTopTotal==='false'? false:true;
// Default Header to show in table and other columns header will not show until user action on UI
let defaultheader = props.defaultcolumns;
let optionalheader = props.optionalcolumns;
-
+ let defaultSortColumn = props.defaultSortColumn;
+ let tablename = (props.tablename)?props.tablename:window.location.pathname;
+
+ if(!defaultSortColumn){
+ defaultSortColumn =[{}];
+ }
+ let defaultpagesize = (typeof props.defaultpagesize === 'undefined' || props.defaultpagesize == null)?10:props.defaultpagesize;
let columns = [];
let defaultdataheader = Object.keys(defaultheader[0]);
let optionaldataheader = Object.keys(optionalheader[0]);
- if(props.showaction === 'true'){
+ if(props.showaction === 'true') {
columns.push({
Header: 'Action',
id:'Action',
@@ -310,30 +512,46 @@ function ViewTable(props) {
// Object.entries(props.paths[0]).map(([key,value]) =>{})
}
- //Default Columns
- defaultdataheader.forEach(header =>{
- columns.push({
- Header: defaultheader[0][header],
- id: defaultheader[0][header],
- accessor: header,
- filter: 'fuzzyText',
- isVisible: true,
- Cell: props => <div> {updatedCellvalue(header, props.value)} </div>,
- })
- })
+ //Default Columns
+ defaultdataheader.forEach(header =>{
+ const isString = typeof defaultheader[0][header] === 'string';
+ columns.push({
+ Header: isString ? defaultheader[0][header] : defaultheader[0][header].name,
+ id: header,
+ accessor: header,
+ filter: (!isString && defaultheader[0][header].filter=== 'date') ? 'includes' : 'fuzzyText',
+ Filter: isString ? DefaultColumnFilter : (filterTypes[defaultheader[0][header].filter] ? filterTypes[defaultheader[0][header].filter] : DefaultColumnFilter),
+ isVisible: true,
+ Cell: props => <div> {updatedCellvalue(header, props.value)} </div>,
+ })
+})
+
+//Optional Columns
- //Optional Columns
- optionaldataheader.forEach(header => {
- columns.push({
- Header: optionalheader[0][header],
- id: header,
+optionaldataheader.forEach(header => {
+ const isString = typeof optionalheader[0][header] === 'string';
+ columns.push({
+ Header: isString ? optionalheader[0][header] : optionalheader[0][header].name,
+ id: isString ? header : optionalheader[0][header].name,
accessor: header,
- filter: 'fuzzyText',
+ filter: (!isString && optionalheader[0][header].filter=== 'date') ? 'includes' : 'fuzzyText',
+ Filter: isString ? DefaultColumnFilter : (filterTypes[optionalheader[0][header].filter] ? filterTypes[optionalheader[0][header].filter] : DefaultColumnFilter),
isVisible: false,
Cell: props => <div> {updatedCellvalue(header, props.value)} </div>,
- })
- });
-
+ })
+});
+
+ let togglecolumns = localStorage.getItem(tablename);
+ if(togglecolumns){
+ togglecolumns = JSON.parse(togglecolumns)
+ columns.forEach(column =>{
+ togglecolumns.filter(tcol => {
+ column.isVisible = (tcol.Header === column.Header)?tcol.isVisible:column.isVisible;
+ return tcol;
+ })
+ })
+ }
+
function updatedCellvalue(key, value){
try{
if(key === 'blueprint_draft' && _.includes(value,'/task_draft/')){
@@ -362,11 +580,10 @@ function ViewTable(props) {
return value;
}
-
-
return (
<div>
- <Table columns={columns} data={tbldata} defaultheader={defaultheader[0]} optionalheader={optionalheader[0]} />
+ <Table columns={columns} data={tbldata} defaultheader={defaultheader[0]} optionalheader={optionalheader[0]}
+ defaultSortColumn={defaultSortColumn} tablename={tablename} defaultpagesize={defaultpagesize}/>
</div>
)
}
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/layout/_overrides.scss b/SAS/TMSS/frontend/tmss_webapp/src/layout/_overrides.scss
index ddf77d3e4b228914fd7fc97e74cb11820d821755..03b42a5aeb25770e4dc82919a4ee85d33cec416d 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/layout/_overrides.scss
+++ b/SAS/TMSS/frontend/tmss_webapp/src/layout/_overrides.scss
@@ -22,4 +22,32 @@
.layout-sidebar-dark .layout-menu li a {
border-top: none;
}
-}
\ No newline at end of file
+}
+.disable-action {
+ pointer-events: none;
+ opacity: 0.5;
+}
+.nav-btn {
+ position: absolute;
+ height: 100%;
+ width: 100%;
+ top: 0;
+ left: 0;
+ background-color: transparent !important;
+ border: 0 !important;
+ box-shadow: none !important;
+ :focus {
+ background-color: transparent !important;
+ border: 0 !important;
+ box-shadow: none !important;
+ }
+ :hover {
+ background-color: transparent !important;
+ border: 0 !important;
+ box-shadow: none !important;
+ }
+ span {
+ display: none !important;
+ }
+}
+
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/layout/components/AppBreadcrumb.js b/SAS/TMSS/frontend/tmss_webapp/src/layout/components/AppBreadcrumb.js
index 0abba394b1c1190aa1c84bf7ec20af19f97c720c..7c966d34bb8190efdab77a9c5211691f48ce2a77 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/layout/components/AppBreadcrumb.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/layout/components/AppBreadcrumb.js
@@ -27,11 +27,13 @@ export class AppBreadcrumb extends Component {
onRoute() {
const { breadcrumbs } = this.state;
+ const { setPageTitle } = this.props;
const currentRoute = routes.find(route => matchPath(this.props.location.pathname, {path: route.path, exact: true, strict: true}));
//for intial route ,there wont be any route object so it failed
if(!currentRoute){
return;
- }
+ }
+ setPageTitle(currentRoute.pageTitle);
if (!breadcrumbs.length) {
this.setState({ breadcrumbs: [{...this.props.location, name: currentRoute.name}] });
return;
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/layout/components/AppMenu.js b/SAS/TMSS/frontend/tmss_webapp/src/layout/components/AppMenu.js
index c46698462cfea5aff95a48cbf167c56a0dfd736e..9c0760c8e1a6dcd90fccae8e80939a172dbfb521 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/layout/components/AppMenu.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/layout/components/AppMenu.js
@@ -2,6 +2,7 @@
import {NavLink} from 'react-router-dom'
import PropTypes from 'prop-types';
import classNames from 'classnames';
+import { Button } from 'primereact/button';
class AppSubmenu extends Component {
@@ -51,18 +52,35 @@ class AppSubmenu extends Component {
});
}
}
+
+ componentDidMount() {
+ if (!this.props.items) {
+ return;
+ }
+ const pathname = window.location.pathname;
+ console.log(pathname);
+ for (let i = 0; i < this.props.items.length; i++) {
+
+ if (pathname.indexOf(this.props.items[i].section) > -1) {
+ this.setState({activeIndex: i});
+ break
+ }
+ }
+ }
renderLinkContent(item) {
let submenuIcon = item.items && <i className="pi pi-fw pi-angle-down menuitem-toggle-icon"></i>;
let badge = item.badge && <span className="menuitem-badge">{item.badge}</span>;
-
+
return (
<React.Fragment>
- <i className={item.icon}></i>
- <span>{item.label}</span>
- {submenuIcon}
- {badge}
+ <i className={item.icon}></i>
+ <Button className="nav-btn nav-btn-tooltip" tooltip={item.label}></Button>
+ <Button className="nav-btn nav-btn-notooltip"></Button>
+ <span>{item.label}</span>
+ {submenuIcon}
+ {badge}
</React.Fragment>
);
}
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/layout/components/PageHeader.js b/SAS/TMSS/frontend/tmss_webapp/src/layout/components/PageHeader.js
new file mode 100644
index 0000000000000000000000000000000000000000..fb95ec75a094fc8a2d86bdf74ba78fab8c885a39
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/src/layout/components/PageHeader.js
@@ -0,0 +1,62 @@
+import React, { useEffect, useState } from 'react';
+import { routes } from '../../routes';
+import {matchPath, Link} from 'react-router-dom';
+
+export default ({ title, subTitle, actions, ...props}) => {
+ const [page, setPage] = useState({});
+
+ useEffect(() => {
+ const currentRoute = routes.find(route => matchPath(props.location.pathname, {path: route.path, exact: true, strict: true}));
+ //for intial route ,there wont be any route object so it failed
+ if(!currentRoute){
+ return;
+ }
+ setPage(currentRoute);
+ }, [props.location.pathname]);
+
+ const onClickLink = (action) => {
+ if (action.link) {
+ action.link();
+ }
+ };
+
+ const onButtonClick = (e, action) => {
+ if (action.actOn && action.actOn === 'click') {
+ action.props.callback(e);
+ }
+ };
+
+ const onButtonMouseOver = (e, action) => {
+ if (action.actOn && action.actOn === 'mouseOver') {
+ action.props.callback(e);
+ }
+ }
+
+ return (
+ <div className="page-header">
+ <div className="title">
+ <h2 className="page-title">{title || page.title}</h2>
+ {(page.subTitle || subTitle) && <h6 className="page-subtitle">{subTitle || page.subTitle}</h6>}
+ </div>
+ <div className="page-action-menu">
+ {(actions || []).map((action, index) =>{
+ if (action.type === 'button') {
+ return (
+ <button className="p-link" key={index}>
+ <i className={`fa ${action.icon}`}
+ onMouseOver={(e) => onButtonMouseOver(e, action)}
+ onClick={(e) => onButtonClick(e, action)} />
+ </button>
+ );
+ } else {
+ return (
+ <Link key={index} className={action.classname} to={{ ...action.props }} title={action.title || ''} onClick={() => onClickLink(action)}>
+ <i className={`fa ${action.icon}`}></i>
+ </Link>
+ );
+ }
+ })}
+ </div>
+ </div>
+ );
+}
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_layout.scss b/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_layout.scss
index 79f22ea107106c4e26e10b2cc375414ea77b293d..6b44aeea7f7b30d89026fc885b4c9e6c8d4d1a5c 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_layout.scss
+++ b/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_layout.scss
@@ -12,3 +12,6 @@
@import "./_dashboard";
@import "./_breadcrumb";
@import "./_viewtable";
+@import "./_pageheader";
+@import "./timeline";
+// @import "./splitpane";
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_menu.scss b/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_menu.scss
index 9fea8e94f390367c23a1874b1d0e19060a43e5b5..6942fe6836f93ff1cb4b2b285f8136ff2c3189a2 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_menu.scss
+++ b/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_menu.scss
@@ -181,4 +181,20 @@
}
}
}
+}
+
+.layout-static .nav-btn-tooltip {
+ display: none;
+}
+
+.layout-static .nav-btn-notooltip {
+ display: block;
+}
+
+.layout-static-sidebar-inactive .nav-btn-tooltip {
+ display: block;
+}
+
+.layout-static-sidebar-inactive .nav-btn-notooltip {
+ display: none;
}
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_pageheader.scss b/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_pageheader.scss
new file mode 100644
index 0000000000000000000000000000000000000000..32a00c556353bf9a2324cc7d421b1d627525f637
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_pageheader.scss
@@ -0,0 +1,24 @@
+.page-header {
+ display: flex;
+ justify-content: space-between;
+ align-items: baseline;
+ border-bottom: 1px solid #e0e0e0;
+ margin-bottom: 10px;
+ padding-bottom: 5px;
+}
+.page-title {
+ margin-bottom: 0;
+}
+.page-subtitle {
+ color: #b4b2b2;
+ font-size: 100%;
+ font-weight: 400;
+ margin-bottom: 0px;
+}
+.page-action-menu i {
+ margin-left: 5px;
+}
+
+.page-header .fa {
+ font-size: 25px !important;
+}
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_splitpane.scss b/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_splitpane.scss
new file mode 100644
index 0000000000000000000000000000000000000000..867f96752ebeb301571b008af505e2e7de2dbad7
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_splitpane.scss
@@ -0,0 +1,120 @@
+*,
+*:before,
+*:after {
+ -moz-box-sizing: border-box;
+ -webkit-box-sizing: border-box;
+ box-sizing: border-box;
+ position: relative;
+}
+
+.Resizer {
+ box-sizing: border-box;
+ background: #000;
+ opacity: 0.5;
+ z-index: 1;
+ background-clip: padding-box;
+ }
+
+ .Resizer:hover {
+ -webkit-transition: all 2s ease;
+ transition: all 2s ease;
+ }
+
+ .Resizer.horizontal {
+ height: 11px;
+ margin: -5px 0;
+ border-top: 5px solid rgba(255, 255, 255, 0);
+ border-bottom: 5px solid rgba(255, 255, 255, 0);
+ cursor: row-resize;
+ width: 100%;
+ }
+
+ .Resizer.horizontal:hover {
+ border-top: 5px solid rgba(0, 0, 0, 0.5);
+ border-bottom: 5px solid rgba(0, 0, 0, 0.5);
+ }
+
+ .Resizer.vertical {
+ width: 11px;
+ margin: 0 -5px;
+ border-left: 5px solid rgba(255, 255, 255, 0);
+ border-right: 5px solid rgba(255, 255, 255, 0);
+ cursor: col-resize;
+ }
+
+ .Resizer.vertical:hover {
+ border-left: 5px solid rgba(0, 0, 0, 0.5);
+ border-right: 5px solid rgba(0, 0, 0, 0.5);
+ }
+
+ .vertical section {
+ width: 100vh;
+ height: 100vh;
+ display: -webkit-box;
+ display: -webkit-flex;
+ display: -ms-flexbox;
+ display: flex;
+ -webkit-box-orient: vertical;
+ -webkit-box-direction: normal;
+ -webkit-flex-direction: column;
+ -ms-flex-direction: column;
+ flex-direction: column;
+ }
+
+ .vertical header {
+ padding: 1rem;
+ background: #eee;
+ }
+
+ .vertical footer {
+ padding: 1rem;
+ background: #eee;
+ }
+
+ .horizontal section {
+ width: 100vh;
+ height: 100vh;
+ display: flex;
+ flex-direction: column;
+ }
+
+ .horizontal header {
+ padding: 1rem;
+ background: #eee;
+ }
+
+ .horizontal footer {
+ padding: 1rem;
+ background: #eee;
+ }
+
+ .parent {
+ width: 100%;
+ height: 100%;
+ -webkit-box-flex: 1;
+ -webkit-flex: 1;
+ -ms-flex: 1;
+ flex: 1;
+ display: -webkit-box;
+ display: -webkit-flex;
+ display: -ms-flexbox;
+ display: flex;
+ -webkit-box-orient: vertical;
+ -webkit-box-direction: normal;
+ -webkit-flex-direction: column;
+ -ms-flex-direction: column;
+ flex-direction: column;
+ }
+ .header {
+ background: #aaa;
+ height: 3rem;
+ line-height: 3rem;
+ }
+ .wrapper {
+ background: #ffa;
+ margin: 5rem;
+ -webkit-box-flex: 1;
+ -webkit-flex: 1;
+ -ms-flex: 1;
+ flex: 1;
+ }
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_timeline.scss b/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_timeline.scss
new file mode 100644
index 0000000000000000000000000000000000000000..c2b32f1d6f45d477613e4d68d7257c3fedaff4ab
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_timeline.scss
@@ -0,0 +1,117 @@
+.rct-sidebar-row {
+ font-size: 14px;
+}
+
+.react-calendar-timeline .rct-header-root {
+ background-color: #f0f0f0;
+}
+
+.timeline-toolbar {
+ margin-top: 25px;
+ margin-bottom: 2px;
+ font-size: 12px;
+}
+
+.timeline-actionbar {
+ padding-right: 0px;
+}
+
+.timeline-actionbar button {
+ padding-top: 3px;
+ font-size: 1.0rem;
+ // float: right;
+}
+
+.timeline-toolbar .p-dropdown {
+ float: right;
+}
+
+.timeline-toolbar .p-dropdown,.timeline-toolbar .p-dropdown-panel {
+ font-size: 12px !important;
+ white-space: nowrap;
+}
+
+#now-btn {
+ margin-left: 20px;
+}
+
+.resize-div,
+.resize-div-min,
+.resize-div-avg,
+.resize-div-max {
+ position: fixed !important;
+ // top: 50%;
+ // transition: all 0.2s ease-in 0s;//this is the key attribute
+ z-index: 9999;
+ //cursor: pointer;
+ font-size: 8px;
+ color: #ffffff;
+ margin-left: -39px;
+}
+
+.resize-div-min,
+.resize-div-avg,
+.resize-div-max button {
+ // height: 50px;
+ // width: 10px;
+}
+
+.resize-div-min,
+.resize-div-avg,
+.resize-div-max pi {
+ // font-size: 10px;
+}
+
+.layout-static-sidebar-inactive .resize-div-min {
+ // padding-right: 0.4% !important;
+}
+
+.layout-static .resize-div-min {
+ // padding-right: 5.8% ;
+}
+
+.layout-static-sidebar-inactive .resize-div-avg {
+ // padding-right: 1% !important;
+}
+
+.layout-static .resize-div-avg {
+ // padding-right: 7.7% ;
+}
+.layout-static-sidebar-inactive .resize-div-max {
+ // padding-right: 1.4% !important;
+}
+
+.layout-static .resize-div-max {
+ // padding-right: 9.6% ;
+}
+
+.resize-btn {
+ // float: right;
+ height: 25px;
+ width: 20px;
+ font-size: 10px !important;
+ background-color: #007ad9;
+ color: #faf6f6 !important;
+ margin-left: 1px;
+ padding-left: 5px;
+}
+
+.resize-btn i {
+ font-size: 12px;
+ cursor: pointer;
+ color: #faf6f6 !important;
+}
+
+.resize-btn:disabled {
+ background-color: #e4e4e4;
+ cursor: not-allowed;
+ color: #928f8f !important;
+}
+
+.resize-btn:disabled i {
+ cursor: not-allowed;
+ color: #928f8f !important;
+}
+// .float-button:hover {
+// right: -7px;//hide it by pushing it off the screen
+// }
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_viewtable.scss b/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_viewtable.scss
index d0fbd873bc26c3e701607d8181d3fb04e8ef1ad5..47803d3ad81264ab7634da3e4b9194d89af2b6f8 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_viewtable.scss
+++ b/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_viewtable.scss
@@ -1,7 +1,19 @@
- .viewtable{
+#block_container {
+ display: flex;
+ vertical-align: middle;
+ margin-top: 20px
+}
+
+.table_container {
+ overflow: auto;
+ padding: 0.75em 0;
+}
+
+.viewtable{
overflow: auto !important;
padding: 0.75em;
- }
+ width: 100%;
+}
.viewtable th {
color: #7e8286;
@@ -21,8 +33,71 @@
padding: .65rem;
border-bottom: 1px solid lightgray;
overflow-wrap: anywhere;
-}
-
+}
+
+.pagination {
+ margin-top: .25em;
+ display: flex;
+ justify-content: center;
+ background-color: #ebeaea;
+ border: none;
+ border-bottom: 1px solid lightgray;
+ border-top: 1px solid lightgray;
+}
+
+body .p-paginator {
+ margin-top: .25em;
+ margin-bottom: .15em;
+ background-color: #ebeaea;
+ border: none;
+}
+
+.p-paginator .p-paginator-icon {
+ display: block;
+ position: absolute;
+ left: 50%;
+ top: 50%;
+ width: 1em;
+ height: 1em;
+ margin-top: -.5em;
+ margin-left: -.5em;
+ border-color: black;
+ color: #007ad9;
+}
+
+.p-dropdown .p-dropdown-trigger .p-dropdown-trigger-icon {
+ top: 50%;
+ left: 50%;
+ margin-top: -.5em;
+ margin-left: -.5em;
+ position: absolute;
+ border-color: black;
+ border: black;
+}
+
+.pagination span {
+ margin-bottom: 0px;
+}
+
+.pagination input {
+ margin-top: .25em;
+ margin-bottom: .15em;
+ margin-left: 1em;
+ margin-right: 0.75em;
+ width: 5em;
+ height: 2.25em;
+ border: none;
+ text-align: center;
+}
+
+.pagination button {
+ margin-left: 5px;
+ height: 35px;
+ margin-bottom: .15em;
+ font-size: 14px;
+ padding-left:10px;
+
+}
.filter-input input{
max-width: 175px;
}
@@ -51,20 +126,15 @@
width: 175px;
}
-.pagination button {
- margin-left: 3px;
- background-color: #005b9f;
- border: 1px solid #005b9f;
- border-radius: 4px;
- color: white;
- font-weight: 900;
+.table_container .pi {
+ padding-left: 3px;
}
-.pagination button:disabled {
- margin-left: 3px;
- background-color: #c8c9c9;
- border: 1px solid #c8c9c9;
- border-radius: 4px;
- color: white;
+.total_records_bottom_label {
+ text-align: left;
+ margin-right: 20px;
+ margin-top: 7px;
+}
+.total_records_top_label {
+ margin-left: 15px;
}
-
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Cycle/create.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Cycle/create.js
index 79ddd60b6b80b15e064ae8a60df296ff6b51674b..8e11645ca22f57e71bf20c836ac45bf76423c39a 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Cycle/create.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Cycle/create.js
@@ -1,5 +1,5 @@
import React, {Component} from 'react';
-import { Link, Redirect } from 'react-router-dom';
+import { Redirect } from 'react-router-dom';
import {InputText} from 'primereact/inputtext';
import {Calendar} from 'primereact/calendar';
import {InputTextarea} from 'primereact/inputtextarea';
@@ -12,6 +12,7 @@ import moment from 'moment'
import _ from 'lodash';
import AppLoader from '../../layout/components/AppLoader';
+import PageHeader from '../../layout/components/PageHeader';
import CycleService from '../../services/cycle.service';
import UnitConverter from '../../utils/unit.converter';
import UIConstants from '../../utils/ui.constants';
@@ -326,9 +327,8 @@ export class CycleCreate extends Component {
return (
<React.Fragment>
- <div className="p-grid">
- <Growl ref={(el) => this.growl = el} />
-
+ <Growl ref={(el) => this.growl = el} />
+ { /*<div className="p-grid">
<div className="p-col-10 p-lg-10 p-md-10">
<h2>Cycle - Add</h2>
</div>
@@ -337,7 +337,11 @@ export class CycleCreate extends Component {
<i className="fa fa-window-close" style={{marginTop: "10px"}}></i>
</Link>
</div>
- </div>
+ </div> */ }
+
+ <PageHeader location={this.props.location} title={'Cycle - Add'} actions={[{icon:'fa-window-close',
+ title:'Click to Close Add Cycle',
+ props:{pathname: '/cycle' }}]}/>
{ this.state.isLoading ? <AppLoader /> :
<>
<div>
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Cycle/edit.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Cycle/edit.js
index 4d599c33ef64cd790084b5f0012a3c2ad3fde1e4..d04ac553d56581a384a458044384e5b64a349845 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Cycle/edit.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Cycle/edit.js
@@ -1,5 +1,5 @@
import React, {Component} from 'react';
-import { Link, Redirect } from 'react-router-dom';
+import { Redirect } from 'react-router-dom';
import _ from 'lodash';
import moment from 'moment'
@@ -14,6 +14,7 @@ import {Growl} from 'primereact/components/growl/Growl';
import {ResourceInputList} from './ResourceInputList';
import AppLoader from '../../layout/components/AppLoader';
+import PageHeader from '../../layout/components/PageHeader';
import CycleService from '../../services/cycle.service';
import UnitConverter from '../../utils/unit.converter';
import UIConstants from '../../utils/ui.constants';
@@ -351,7 +352,7 @@ export class CycleEdit extends Component {
* Cancel edit and redirect to Cycle View page
*/
cancelEdit() {
- this.setState({redirect: `/cycle/view/${this.state.cycle.name}`});
+ this.props.history.goBack();
}
render() {
@@ -360,9 +361,9 @@ export class CycleEdit extends Component {
}
return (
<React.Fragment>
- <div className="p-grid">
- <Growl ref={(el) => this.growl = el} />
-
+ <Growl ref={(el) => this.growl = el} />
+ {/*} <div className="p-grid">
+
<div className="p-col-10 p-lg-10 p-md-10">
<h2>Cycle - Edit</h2>
</div>
@@ -371,7 +372,10 @@ export class CycleEdit extends Component {
<i className="fa fa-window-close" style={{marginTop: "10px"}}></i>
</Link>
</div>
- </div>
+ </div> */}
+ <PageHeader location={this.props.location} title={'Cycle - Edit'} actions={[{icon:'fa-window-close',
+ link: this.props.history.goBack,title:'Click to Close Cycle-Edit',
+ props:{ pathname: `/cycle/view/${this.state.cycle.name}`}}]}/>
{ this.state.isLoading ? <AppLoader/> :
<>
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Cycle/list.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Cycle/list.js
index 8ecbf062eccfb70a68b53adfb27e3e8fbe81a115..4ce2ebeb1cdfd4cfa707af03503f99c7fdcbcee6 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Cycle/list.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Cycle/list.js
@@ -1,12 +1,13 @@
import React, { Component } from 'react'
import 'primeflex/primeflex.css';
-import { Link } from 'react-router-dom/cjs/react-router-dom.min';
+// import { Link } from 'react-router-dom/cjs/react-router-dom.min';
import _ from 'lodash';
import ViewTable from '../../components/ViewTable';
import CycleService from '../../services/cycle.service';
import UnitConversion from '../../utils/unit.converter';
import AppLoader from '../../layout/components/AppLoader';
+import PageHeader from '../../layout/components/PageHeader';
class CycleList extends Component{
constructor(props){
@@ -48,8 +49,9 @@ class CycleList extends Component{
"Lofar Observing Time Commissioning (Hrs)" : "filter-input-75",
"Lofar Observing Time Prio A (Hrs)" : "filter-input-75",
"Lofar Observing Time Prio B (Hrs)" : "filter-input-75" }];
+
+ this.defaultSortColumn = [{id: "Cycle Code", desc: false}];
}
-
getUnitConvertedQuotaValue(cycle, cycleQuota, resourceName) {
const quota = _.find(cycleQuota, {'cycle_id': cycle.name, 'resource_type_id': resourceName});
const unitQuantity = this.state.resources.find(i => i.name === resourceName).quantity_value;
@@ -58,7 +60,7 @@ class CycleList extends Component{
getCycles(cycles = [], cycleQuota) {
const promises = [];
- cycles.map(cycle => promises.push(CycleService.getCycleById(cycle.name)));
+ cycles.map(cycle => promises.push(CycleService.getProjectsByCycle(cycle.name)));
Promise.all(promises).then(responses => {
const results = cycles;
results.map(async (cycle, index) => {
@@ -109,7 +111,7 @@ class CycleList extends Component{
render(){
return (
<>
- <div className="p-grid">
+ { /*<div className="p-grid">
<div className="p-col-10 p-lg-10 p-md-10">
<h2>Cycle - List </h2>
</div>
@@ -118,7 +120,15 @@ class CycleList extends Component{
<i className="fa fa-plus-square" style={{marginTop: "10px"}}></i>
</Link>
</div>
- </div>
+ </div> */}
+ {/*
+ * Call View table to show table data, the parameters are,
+ data - Pass API data
+ defaultcolumns - This colum will be populate by default in table with header mentioned
+ showaction - {true/false} -> to show the action column
+ paths - specify the path for navigation - Table will set "id" value for each row in action button
+ */}
+ <PageHeader location={this.props.location} title={'Cycle - List'} actions={[{icon:'fa-plus-square',title:'Click to Add Cycle', props:{ pathname: '/cycle/create'}}]}/>
{/*
* Call View table to show table data, the parameters are,
data - Pass API data
@@ -134,8 +144,10 @@ class CycleList extends Component{
defaultcolumns={this.defaultcolumns}
optionalcolumns={this.optionalcolumns}
columnclassname = {this.columnclassname}
+ defaultSortColumn= {this.defaultSortColumn}
showaction="true"
paths={this.state.paths}
+ tablename="cycle_list"
/> : <></>
}
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Cycle/view.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Cycle/view.js
index 55430cc1fd440589988e30d2a1e13aa1e91c4d51..2d2b1cc5c039afd5b4759bb0391a6d2094825f4d 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Cycle/view.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Cycle/view.js
@@ -3,13 +3,15 @@ import {Link, Redirect} from 'react-router-dom'
import moment from 'moment';
import _ from 'lodash';
-import { Chips } from 'primereact/chips';
+// import { Chips } from 'primereact/chips';
import ResourceDisplayList from './ResourceDisplayList';
import AppLoader from '../../layout/components/AppLoader';
+import PageHeader from '../../layout/components/PageHeader';
import CycleService from '../../services/cycle.service';
import UnitConverter from '../../utils/unit.converter';
+import {ProjectList} from './../Project/list';
/**
* Component to view the details of a cycle
@@ -20,6 +22,7 @@ export class CycleView extends Component {
super(props);
this.state = {
isLoading: true,
+ cycle:'',
};
if (this.props.match.params.id) {
this.state.cycleId = this.props.match.params.id;
@@ -75,7 +78,7 @@ export class CycleView extends Component {
return (
<React.Fragment>
- <div className="p-grid">
+ {/* <div className="p-grid">
<div className="p-col-10 p-lg-10 p-md-10">
<h2>Cycle - Details </h2>
</div>
@@ -90,7 +93,11 @@ export class CycleView extends Component {
</Link>
</div>
}
- </div>
+ </div> */ }
+ <PageHeader location={this.props.location} title={'Cycle - Details'}
+ actions={[ {icon:'fa-edit', title:'Click to Edit Cycle', props:{ pathname: `/cycle/edit/${this.state.cycle.name}`,
+ state: {id: this.state.cycle?this.state.cycle.name:''}}},
+ {icon: 'fa-window-close',link: this.props.history.goBack}]}/>
{ this.state.isLoading && <AppLoader /> }
{ this.state.cycle &&
<React.Fragment>
@@ -108,10 +115,10 @@ export class CycleView extends Component {
<span className="col-lg-4 col-md-4 col-sm-12">{moment.utc(this.state.cycle.updated_at).format(this.DATE_FORMAT)}</span>
</div>
- <div className="p-grid">
+ {/* <div className="p-grid">
<label className="col-lg-2 col-md-2 col-sm-12">Projects</label>
<Chips className="col-lg-4 col-md-4 col-sm-12 chips-readonly" disabled value={this.state.cycle.projects_ids}></Chips>
- </div>
+ </div> */}
<div className="p-fluid">
<div className="p-field p-grid">
<div className="col-lg-3 col-md-3 col-sm-12">
@@ -131,10 +138,19 @@ export class CycleView extends Component {
<div className="p-field p-grid resource-input-grid">
<ResourceDisplayList cycleQuota={this.state.cycleQuota} unitMap={this.resourceUnitMap} />
</div>
+ {/* Show Project list for this Cycle */}
+ <div className="p-fluid">
+ <div className="p-field p-grid">
+ <div className="col-lg-3 col-md-3 col-sm-12">
+ <h5 data-testid="project-list">Projects</h5>
+ </div>
+ </div>
+ </div>
+ <ProjectList cycle={this.state.cycle.name}/>
</div>
</React.Fragment>
}
</React.Fragment>
);
}
-}
\ No newline at end of file
+}
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Dashboard/index.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Dashboard/index.js
index a0a798161d65edf83901902a88e3957569741bd8..409176dce304825d2cc252ec3482e0cec2ee2291 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Dashboard/index.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Dashboard/index.js
@@ -1,16 +1,14 @@
import React, {Component} from 'react';
+import PageHeader from '../../layout/components/PageHeader';
export class Dashboard extends Component {
- constructor(props){
- super(props)
- console.log(this.props)
- }
render() {
+
return (
- <h1>Dashboard</h1>
- );
+ <PageHeader location={this.props.location} title={'Dashboard'} />
+ )
}
}
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/ResourceInputList.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/ResourceInputList.js
index c1d9019421ff16c5570e3371e8ff338342b404f7..3d6ead61a870c889a801b9093f6a19fa13e01d70 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/ResourceInputList.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/ResourceInputList.js
@@ -40,7 +40,8 @@ export class ResourceInputList extends Component {
<div key={'div1-'+ index} className="col-lg-3 col-md-3 col-sm-12">
<InputNumber key={'item1-'+ index} id={'item1-'+ index} name={'item1-'+ index}
suffix={` ${this.props.unitMap[item.quantity_value]?this.props.unitMap[item.quantity_value].display:''}`}
- placeholder={` ${this.props.unitMap[item.quantity_value]?this.props.unitMap[item.quantity_value].display:item.name}`} min={0} useGrouping={false}
+ placeholder={` ${this.props.unitMap[item.quantity_value]?this.props.unitMap[item.quantity_value].display:item.name}`}
+ inputId={`${item.name}`} min={0} useGrouping={false}
value={this.state.projectQuota[item.name]}
onChange={(e) => this.onInputChange(item.name, e)}
onBlur={(e) => this.onInputChange(item.name, e)}
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/create.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/create.js
index 64a96b3f0a9b6ba7ff8a99cf52d416e0e49a23a0..b6dd8fadf65b41523c301e7845f09991132242dc 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/create.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/create.js
@@ -1,7 +1,6 @@
import React, {Component} from 'react';
-import { Link, Redirect } from 'react-router-dom';
+import { Redirect } from 'react-router-dom';
import _ from 'lodash';
-
import {InputText} from 'primereact/inputtext';
import {InputNumber} from 'primereact/inputnumber';
import {InputTextarea} from 'primereact/inputtextarea';
@@ -15,11 +14,13 @@ import {Growl} from 'primereact/components/growl/Growl';
import {ResourceInputList} from './ResourceInputList';
import AppLoader from '../../layout/components/AppLoader';
+import PageHeader from '../../layout/components/PageHeader';
import CycleService from '../../services/cycle.service';
import ProjectService from '../../services/project.service';
import UnitConverter from '../../utils/unit.converter';
import UIConstants from '../../utils/ui.constants';
+
/**
* Component to create a new Project
*/
@@ -27,6 +28,7 @@ export class ProjectCreate extends Component {
constructor(props) {
super(props);
this.state = {
+ ltaStorage: [],
isLoading: true,
dialog: { header: '', detail: ''},
project: {
@@ -43,13 +45,16 @@ export class ProjectCreate extends Component {
projectCategories: [],
resources: [], // Selected Resources for Allocation
resourceList: [], // Available Resources for Allocation
- cycles: []
+ cycles: [],
+ archive_location:[],
+ cluster:[]
}
// Validateion Rules
this.formRules = {
name: {required: true, message: "Name can not be empty"},
description: {required: true, message: "Description can not be empty"},
- priority_rank: {required: true, message: "Enter Project Rank"}
+ priority_rank: {required: true, message: "Enter Project Rank"},
+ archive_subdirectory: {required:true, message:"Enter Storage Path"}
};
this.defaultResourcesEnabled = true; // This property and functionality to be concluded based on PO input
this.defaultResources = [{name:'LOFAR Observing Time'},
@@ -91,6 +96,18 @@ export class ProjectCreate extends Component {
.then(categories => {
this.setState({periodCategories: categories});
});
+ Promise.all([ProjectService.getFileSystem(), ProjectService.getCluster()]).then(response => {
+ const options = [];
+ response[0].map(fileSystem => {
+ const cluster = response[1].find(clusterObj => clusterObj.id === fileSystem.cluster_id && clusterObj.archive_site);
+ if (cluster) {
+ fileSystem.label =`${cluster.name} - ${fileSystem.name}`
+ options.push(fileSystem);
+ }
+ return fileSystem;
+ });
+ this.setState({archive_location: response[0], ltaStorage: options, cluster: response[1] });
+ });
ProjectService.getResources()
.then(resourceList => {
const defaultResources = this.defaultResources;
@@ -169,12 +186,31 @@ export class ProjectCreate extends Component {
project[key] = value?parseInt(value):0;
break;
}
+ case 'SUB-DIRECTORY': {
+ const directory = value.split(' ').join('_').toLowerCase();
+ project[key] = (directory!=="" && !directory.endsWith('/'))? `${directory}/`: `${directory}`;
+ break;
+ }
+ case 'PROJECT_NAME': {
+ let directory = project[key]?project[key].split(' ').join('_').toLowerCase():"";
+ if (!project['archive_subdirectory'] || project['archive_subdirectory'] === "" ||
+ project['archive_subdirectory'] === `${directory}/`) {
+ directory = value.split(' ').join('_').toLowerCase();
+ project['archive_subdirectory'] = `${directory}/`;
+ }
+ project[key] = value;
+ break;
+ }
default: {
project[key] = value;
break;
}
}
- this.setState({project: project, validForm: this.validateForm(key)});
+ let validForm = this.validateForm(key);
+ if (type==='PROJECT_NAME' & value!=="") {
+ validForm = this.validateForm('archive_subdirectory');
+ }
+ this.setState({project: project, validForm: validForm});
}
/**
@@ -284,7 +320,7 @@ export class ProjectCreate extends Component {
* Function to cancel form creation and navigate to other page/component
*/
cancelCreate() {
- this.setState({redirect: '/project'});
+ this.props.history.goBack();
}
/**
@@ -313,7 +349,9 @@ export class ProjectCreate extends Component {
description: '',
trigger_priority: 1000,
priority_rank: null,
- quota: []
+ quota: [],
+ archive_location: null,
+ archive_subdirectory:""
},
projectQuota: projectQuota,
validFields: {},
@@ -332,21 +370,10 @@ export class ProjectCreate extends Component {
if (this.state.redirect) {
return <Redirect to={ {pathname: this.state.redirect} }></Redirect>
}
-
return (
<React.Fragment>
- <div className="p-grid">
- <Growl ref={(el) => this.growl = el} />
-
- <div className="p-col-10 p-lg-10 p-md-10">
- <h2>Project - Add</h2>
- </div>
- <div className="p-col-2 p-lg-2 p-md-2">
- <Link to={{ pathname: '/project'}} tite="Close Edit" style={{float: "right"}}>
- <i className="fa fa-window-close" style={{marginTop: "10px"}}></i>
- </Link>
- </div>
- </div>
+ <Growl ref={(el) => this.growl = el} />
+ <PageHeader location={this.props.location} title={'Project - Add'} actions={[{icon:'fa-window-close',link:this.props.history.goBack, title:'Click to Close Project', props:{ pathname: '/project'}}]}/>
{ this.state.isLoading ? <AppLoader /> :
<>
<div>
@@ -363,8 +390,8 @@ export class ProjectCreate extends Component {
<InputText className={this.state.errors.name ?'input-error':''} id="projectName" data-testid="name"
tooltip="Enter name of the project" tooltipOptions={this.tooltipOptions} maxLength="128"
value={this.state.project.name}
- onChange={(e) => this.setProjectParams('name', e.target.value)}
- onBlur={(e) => this.setProjectParams('name', e.target.value)}/>
+ onChange={(e) => this.setProjectParams('name', e.target.value,'PROJECT_NAME')}
+ onBlur={(e) => this.setProjectParams('name', e.target.value,'PROJECT_NAME')}/>
<label className={this.state.errors.name?"error":"info"}>
{this.state.errors.name ? this.state.errors.name : "Max 128 characters"}
</label>
@@ -449,9 +476,34 @@ export class ProjectCreate extends Component {
{this.state.errors.priority_rank ? this.state.errors.priority_rank : ""}
</label>
</div>
- </div>
-
- {this.defaultResourcesEnabled && this.state.resourceList &&
+ </div>
+ <div className="p-field p-grid">
+ <label htmlFor="ltaStorage" className="col-lg-2 col-md-2 col-sm-12">LTA Storage Location</label>
+ <div className="col-lg-3 col-md-3 col-sm-12" >
+ <Dropdown inputId="ltaStore"
+ optionValue="url"
+ tooltip="LTA Storage" tooltipOptions={this.tooltipOptions}
+ value={this.state.project.archive_location}
+ options={this.state.ltaStorage}
+ onChange={(e) => this.setProjectParams('archive_location', e.target.value)}
+ placeholder="Select LTA Storage" />
+ </div>
+
+ <div className="col-lg-1 col-md-1 col-sm-12"></div>
+ <label htmlFor="ltastoragepath" className="col-lg-2 col-md-2 col-sm-12">LTA Storage Path <span style={{color:'red'}}>*</span> </label>
+ <div className="col-lg-3 col-md-3 col-sm-12">
+ <InputText className={this.state.errors.archive_subdirectory ?'input-error':''} id="StoragePath" data-testid="name"
+ tooltip="Enter storage relative path" tooltipOptions={this.tooltipOptions} maxLength="1024"
+ value={this.state.project.archive_subdirectory}
+ onChange={(e) => this.setProjectParams('archive_subdirectory', e.target.value)}
+ onBlur={(e) => this.setProjectParams('archive_subdirectory', e.target.value,'SUB-DIRECTORY')}/>
+ <label className={this.state.errors.archive_subdirectory ?"error":"info"}>
+ {this.state.errors.archive_subdirectory ? this.state.errors.archive_subdirectory : "Max 1024 characters"}
+ </label>
+ </div>
+
+ </div>
+ {this.defaultResourcesEnabled && this.state.resourceList &&
<div className="p-fluid">
<div className="p-field p-grid">
<div className="col-lg-2 col-md-2 col-sm-112">
@@ -509,8 +561,7 @@ export class ProjectCreate extends Component {
</div>
</Dialog>
</div>
-
</React.Fragment>
);
}
-}
\ No newline at end of file
+}
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/create.test.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/create.test.js
index 84c2b338449d107059e43fa4396dead622a042fe..7eccc6d8e8814da4fb5788fbb43c47527697d628 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/create.test.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/create.test.js
@@ -3,6 +3,7 @@ import { BrowserRouter as Router } from 'react-router-dom';
import { act } from "react-dom/test-utils";
import { render, cleanup, fireEvent } from '@testing-library/react';
import '@testing-library/jest-dom/extend-expect';
+import _ from 'lodash';
import {ProjectCreate} from './create';
import ProjectService from '../../services/project.service';
@@ -65,6 +66,7 @@ it("renders without crashing with all back-end data loaded", async () => {
await act(async () => {
content = render(<Router><ProjectCreate /></Router>);
});
+ const spinButtons = content.queryAllByRole("spinbutton");
expect(content.queryByText('Project - Add')).not.toBe(null); // Page loaded successfully
expect(projectCategoriesSpy).toHaveBeenCalled(); // Mock Spy called successfully
@@ -73,8 +75,8 @@ it("renders without crashing with all back-end data loaded", async () => {
expect(content.queryByText('Cycle-0')).toBeInTheDocument(); // Cycle multi-select loaded successfully
expect(content.queryAllByText('Add Resources').length).toBe(2); // Resource Dropdown loaded successfully
expect(content.queryByText('Support hours')).toBeInTheDocument(); // Resources other than Default Resources listed in dropdown
- expect(content.queryByPlaceholderText('Support Hours')).toBe(null); // No resources other than Default Resources listed to get input
- expect(content.queryByPlaceholderText('LOFAR Observing Time').value).toBe('1 Hours'); // Default Resource Listed with default value
+ expect(_.filter(spinButtons, {"id": "Support hours"}).length).toBe(0); // No resources other than Default Resources listed to get input
+ expect(_.filter(spinButtons, {"id": "LOFAR Observing Time"})[0].value).toBe('1 Hours'); // Default Resource Listed with default value
});
it("Save button disabled initially when no data entered", async () => {
@@ -222,31 +224,31 @@ it("save project with default resources", async () => {
expect(content.queryByTestId('projectId').value).toBe("");
expect(content.queryByText("Success")).toBe(null);
- const lofarObsTimeInput = content.queryByPlaceholderText('LOFAR Observing Time');
+ const lofarObsTimeInput = _.filter(spinButtons, {"id": "LOFAR Observing Time"})[0];
fireEvent.change(lofarObsTimeInput, { target: { value: 10 } });
expect(lofarObsTimeInput.value).toBe('10');
- const lofarObsTimeAInput = content.queryByPlaceholderText('LOFAR Observing Time prio A');
+ const lofarObsTimeAInput = _.filter(spinButtons, {"id": "LOFAR Observing Time prio A"})[0];
fireEvent.change(lofarObsTimeAInput, { target: { value: 15 } });
expect(lofarObsTimeAInput.value).toBe('15');
- const lofarObsTimeBInput = content.queryByPlaceholderText('LOFAR Observing Time prio B');
+ const lofarObsTimeBInput = _.filter(spinButtons, {"id": "LOFAR Observing Time prio B"})[0];
fireEvent.change(lofarObsTimeBInput, { target: { value: 20 } });
expect(lofarObsTimeBInput.value).toBe('20');
- const cepProcTimeInput = content.queryByPlaceholderText('CEP Processing Time');
+ const cepProcTimeInput = _.filter(spinButtons, {"id": "CEP Processing Time"})[0];
fireEvent.change(cepProcTimeInput, { target: { value: 5 } });
expect(cepProcTimeInput.value).toBe('5');
- const ltaStorageInput = content.queryByPlaceholderText('LTA Storage');
+ const ltaStorageInput = _.filter(spinButtons, {"id": "LTA Storage"})[0];
fireEvent.change(ltaStorageInput, { target: { value: 2 } });
expect(ltaStorageInput.value).toBe('2');
- const noOfTriggerInput = content.queryByPlaceholderText('Number of triggers');
+ const noOfTriggerInput = _.filter(spinButtons, {"id": "Number of triggers"})[0];
fireEvent.change(noOfTriggerInput, { target: { value: 3 } });
expect(noOfTriggerInput.value).toBe('3');
- const lofarSupTimeInput = content.queryByPlaceholderText('LOFAR Support Time');
+ const lofarSupTimeInput = _.filter(spinButtons, {"id": "LOFAR Support Time"})[0];
fireEvent.change(lofarSupTimeInput, { target: { value: 25 } });
expect(lofarSupTimeInput.value).toBe('25');
@@ -268,7 +270,7 @@ it("save project with added resources", async () => {
const nameInput = content.queryByTestId('name');
const descInput = content.queryByTestId('description');
- const spinButtons = content.queryAllByRole("spinbutton");
+ let spinButtons = content.queryAllByRole("spinbutton");
const rankInput = spinButtons.filter(function(element) { return element.id==="proj-rank"})[0];
fireEvent.change(nameInput, { target: { value: 'OSR' } });
@@ -281,31 +283,31 @@ it("save project with added resources", async () => {
expect(content.queryByTestId('projectId').value).toBe("");
expect(content.queryByText("Success")).toBe(null);
- const lofarObsTimeInput = content.queryByPlaceholderText('LOFAR Observing Time');
+ const lofarObsTimeInput = _.filter(spinButtons, {"id": "LOFAR Observing Time"})[0];
fireEvent.change(lofarObsTimeInput, { target: { value: 10 } });
expect(lofarObsTimeInput.value).toBe('10');
- const lofarObsTimeAInput = content.queryByPlaceholderText('LOFAR Observing Time prio A');
+ const lofarObsTimeAInput = _.filter(spinButtons, {"id": "LOFAR Observing Time prio A"})[0];
fireEvent.change(lofarObsTimeAInput, { target: { value: 15 } });
expect(lofarObsTimeAInput.value).toBe('15');
- const lofarObsTimeBInput = content.queryByPlaceholderText('LOFAR Observing Time prio B');
+ const lofarObsTimeBInput = _.filter(spinButtons, {"id": "LOFAR Observing Time prio B"})[0];
fireEvent.change(lofarObsTimeBInput, { target: { value: 20 } });
expect(lofarObsTimeBInput.value).toBe('20');
- const cepProcTimeInput = content.queryByPlaceholderText('CEP Processing Time');
+ const cepProcTimeInput = _.filter(spinButtons, {"id": "CEP Processing Time"})[0];
fireEvent.change(cepProcTimeInput, { target: { value: 5 } });
expect(cepProcTimeInput.value).toBe('5');
- const ltaStorageInput = content.queryByPlaceholderText('LTA Storage');
+ const ltaStorageInput = _.filter(spinButtons, {"id": "LTA Storage"})[0];
fireEvent.change(ltaStorageInput, { target: { value: 2 } });
expect(ltaStorageInput.value).toBe('2');
- const noOfTriggerInput = content.queryByPlaceholderText('Number of triggers');
+ const noOfTriggerInput = _.filter(spinButtons, {"id": "Number of triggers"})[0];
fireEvent.change(noOfTriggerInput, { target: { value: 3 } });
expect(noOfTriggerInput.value).toBe('3');
- const lofarSupTimeInput = content.queryByPlaceholderText('LOFAR Support Time');
+ const lofarSupTimeInput = _.filter(spinButtons, {"id": "LOFAR Support Time"})[0];
fireEvent.change(lofarSupTimeInput, { target: { value: 25 } });
expect(lofarSupTimeInput.value).toBe('25');
@@ -313,7 +315,7 @@ it("save project with added resources", async () => {
expect(content.queryAllByText('Add Resources').length).toBe(2);
expect(content.queryAllByText('Support hours').length).toBe(1);
expect(content.getAllByRole("listbox")[3].children.length).toBe(2);
- expect(content.queryByPlaceholderText('Support hours')).toBe(null);
+ expect(_.filter(spinButtons, {"id": "Support hours"}).length).toBe(0);
const addResourceInput = content.getAllByRole("listbox")[3].children[1] ;
fireEvent.click(addResourceInput);
// After selecting New Resource
@@ -323,9 +325,10 @@ it("save project with added resources", async () => {
const addResourceBtn = content.queryByTestId('add_res_btn');
fireEvent.click(addResourceBtn);
expect(content.queryAllByText('Add Resources').length).toBe(2);
- expect(content.queryByPlaceholderText('Support hours')).not.toBe(null);
+ spinButtons = content.queryAllByRole("spinbutton");
- const newResourceInput = content.queryByPlaceholderText('Support hours');
+ const newResourceInput = _.filter(spinButtons, {"id": "Support hours"})[0];
+ expect(newResourceInput).not.toBe(null);
fireEvent.change(newResourceInput, { target: { value: 30 } });
expect(newResourceInput.value).toBe('30');
@@ -360,15 +363,18 @@ it("remove default resource and added resource", async () => {
const addResourceBtn = content.queryByTestId('add_res_btn');
fireEvent.click(addResourceBtn);
expect(content.queryAllByText('Add Resources').length).toBe(2);
- expect(content.queryByPlaceholderText('Support hours')).not.toBe(null);
- expect(content.queryByPlaceholderText('CEP Processing Time')).not.toBe(null);
+ const spinButtons = content.queryAllByRole("spinbutton");
+
+ expect(_.filter(spinButtons, {"id": "Support hours"})[0]).not.toBe(null);
+
+ expect(_.filter(spinButtons, {"id": "CEP Processing Time"})[0]).not.toBe(null);
expect(content.queryByTestId('CEP Processing Time-btn')).not.toBe(null);
const removeDefResBtn = content.queryByTestId('CEP Processing Time-btn');
await act(async () => {
fireEvent.click(content.queryByTestId('CEP Processing Time-btn'));
});
- expect(content.queryByPlaceholderText('CEP Processing Time')).toBe(null);
+ expect(_.filter(spinButtons, {"id": "CEP Processing Time"}).length).toBe(0);
expect(content.queryByTestId('CEP Processing Time-btn')).toBe(null);
const removeResourceBtn = content.queryByTestId('Support hours-btn');
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/edit.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/edit.js
index 78b443a5c2d1eb457b3806a0cc4fe89e3fd2ab99..8d4ec839e244c148a7b601c04b9af0603cc502e7 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/edit.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/edit.js
@@ -1,5 +1,5 @@
import React, {Component} from 'react';
-import { Link, Redirect } from 'react-router-dom';
+import { Redirect } from 'react-router-dom';
import _ from 'lodash';
import {InputText} from 'primereact/inputtext';
@@ -15,6 +15,7 @@ import {Growl} from 'primereact/components/growl/Growl';
import {ResourceInputList} from './ResourceInputList';
import AppLoader from '../../layout/components/AppLoader';
+import PageHeader from '../../layout/components/PageHeader';
import CycleService from '../../services/cycle.service';
import ProjectService from '../../services/project.service';
import UnitConverter from '../../utils/unit.converter';
@@ -25,6 +26,7 @@ export class ProjectEdit extends Component {
super(props);
this.state = {
isLoading: true,
+ ltaStorage: [],
dialog: { header: '', detail: ''},
project: {
trigger_priority: 1000,
@@ -47,7 +49,8 @@ export class ProjectEdit extends Component {
this.formRules = {
name: {required: true, message: "Name can not be empty"},
description: {required: true, message: "Description can not be empty"},
- priority_rank: {required: true, message: "Enter Project Rank"}
+ priority_rank: {required: true, message: "Enter Project Rank"},
+ archive_subdirectory: {required:true, message: "Enter Storage Path"}
};
this.defaultResources = [{name:'LOFAR Observing Time'},
{name:'LOFAR Observing Time prio A'},
@@ -89,6 +92,18 @@ export class ProjectEdit extends Component {
.then(categories => {
this.setState({periodCategories: categories});
});
+ Promise.all([ProjectService.getFileSystem(), ProjectService.getCluster()]).then(response => {
+ const options = [];
+ response[0].map(fileSystem => {
+ const cluster = response[1].filter(clusterObj => clusterObj.id === fileSystem.cluster_id && clusterObj.archive_site);
+ if (cluster.length) {
+ fileSystem.label =`${cluster[0].name} - ${fileSystem.name}`
+ options.push(fileSystem);
+ }
+ return fileSystem;
+ });
+ this.setState({archive_location: response[0], ltaStorage: options, cluster: response[1] });
+ });
ProjectService.getResources()
.then(resourceList => {
this.setState({resourceList: resourceList});
@@ -192,12 +207,31 @@ export class ProjectEdit extends Component {
project[key] = value?parseInt(value):0;
break;
}
+ case 'SUB-DIRECTORY': {
+ const directory = value.split(' ').join('_').toLowerCase();
+ project[key] = (directory!=="" && !directory.endsWith('/'))? `${directory}/`: `${directory}`;
+ break;
+ }
+ case 'PROJECT_NAME': {
+ let directory = project[key]?project[key].split(' ').join('_').toLowerCase():"";
+ if (!project['archive_subdirectory'] || project['archive_subdirectory'] === "" ||
+ project['archive_subdirectory'] === `${directory}/`) {
+ directory = value.split(' ').join('_').toLowerCase();
+ project['archive_subdirectory'] = `${directory}/`;
+ }
+ project[key] = value;
+ break;
+ }
default: {
project[key] = value;
break;
}
}
- this.setState({project: project, validForm: this.validateForm(key)});
+ let validForm = this.validateForm(key);
+ if (type==='PROJECT_NAME' & value!=="") {
+ validForm = this.validateForm('archive_subdirectory');
+ }
+ this.setState({project: project, validForm: validForm});
}
/**
@@ -276,7 +310,9 @@ export class ProjectEdit extends Component {
*/
saveProject() {
if (this.validateForm) {
- ProjectService.updateProject(this.props.match.params.id, this.state.project)
+ const project = { ...this.state.project };
+ // project['archive_subdirectory'] = (project['archive_subdirectory'].substr(-1) === '/' ? project['archive_subdirectory'] : `${project['archive_subdirectory']}/`).toLowerCase();
+ ProjectService.updateProject(this.props.match.params.id, project)
.then(async (project) => {
if (project && this.state.project.updated_at !== project.updated_at) {
this.saveProjectQuota(project);
@@ -346,7 +382,7 @@ export class ProjectEdit extends Component {
* Cancel edit and redirect to Project View page
*/
cancelEdit() {
- this.setState({redirect: `/project/view/${this.state.project.name}`});
+ this.props.history.goBack();
}
render() {
@@ -356,18 +392,8 @@ export class ProjectEdit extends Component {
return (
<React.Fragment>
- <div className="p-grid">
- <Growl ref={(el) => this.growl = el} />
-
- <div className="p-col-10 p-lg-10 p-md-10">
- <h2>Project - Edit</h2>
- </div>
- <div className="p-col-2 p-lg-2 p-md-2">
- <Link to={{ pathname: `/project/view/${this.state.project.name}`}} title="Close Edit" style={{float: "right"}}>
- <i className="fa fa-window-close" style={{marginTop: "10px"}}></i>
- </Link>
- </div>
- </div>
+ <Growl ref={(el) => this.growl = el} />
+ <PageHeader location={this.props.location} title={'Project - Edit'} actions={[{icon:'fa-window-close',link: this.props.history.goBack,title:'Click to Close Project Edit Page', props : { pathname: `/project/view/${this.state.project.name}`}}]}/>
{ this.state.isLoading ? <AppLoader/> :
<>
@@ -379,8 +405,8 @@ export class ProjectEdit extends Component {
<InputText className={this.state.errors.name ?'input-error':''} id="projectName" data-testid="name"
tooltip="Enter name of the project" tooltipOptions={this.tooltipOptions} maxLength="128"
value={this.state.project.name}
- onChange={(e) => this.setProjectParams('name', e.target.value)}
- onBlur={(e) => this.setProjectParams('name', e.target.value)}/>
+ onChange={(e) => this.setProjectParams('name', e.target.value, 'PROJECT_NAME')}
+ onBlur={(e) => this.setProjectParams('name', e.target.value, 'PROJECT_NAME')}/>
<label className={this.state.errors.name?"error":"info"}>
{this.state.errors.name ? this.state.errors.name : "Max 128 characters"}
</label>
@@ -466,6 +492,30 @@ export class ProjectEdit extends Component {
</label>
</div>
</div>
+ <div className="p-field p-grid">
+ <label htmlFor="ltaStorage" className="col-lg-2 col-md-2 col-sm-12">LTA Storage Location</label>
+ <div className="col-lg-3 col-md-3 col-sm-12" >
+ <Dropdown inputId="ltaStore" optionValue="url"
+ tooltip="LTA Storage" tooltipOptions={this.tooltipOptions}
+ value={this.state.project.archive_location}
+ options={this.state.ltaStorage}
+ onChange={(e) => {this.setProjectParams('archive_location', e.value)}}
+ placeholder="Select LTA Storage" />
+ </div>
+
+ <div className="col-lg-1 col-md-1 col-sm-12"></div>
+ <label htmlFor="ltastoragepath" className="col-lg-2 col-md-2 col-sm-12">LTA Storage Path <span style={{color:'red'}}>*</span> </label>
+ <div className="col-lg-3 col-md-3 col-sm-12">
+ <InputText className={this.state.errors.archive_subdirectory ?'input-error':''} id="StoragePath" data-testid="name"
+ tooltip="Enter storage relative path" tooltipOptions={this.tooltipOptions} maxLength="1024"
+ value={this.state.project.archive_subdirectory}
+ onChange={(e) => this.setProjectParams('archive_subdirectory', e.target.value)}
+ onBlur={(e) => this.setProjectParams('archive_subdirectory', e.target.value,'SUB-DIRECTORY')}/>
+ <label className={this.state.errors.archive_subdirectory?"error":"info"}>
+ {this.state.errors.archive_subdirectory? this.state.errors.archive_subdirectory : "Max 1024 characters"}
+ </label>
+ </div>
+ </div>
{this.state.resourceList &&
<div className="p-fluid">
<div className="p-field p-grid">
@@ -529,4 +579,4 @@ export class ProjectEdit extends Component {
</React.Fragment>
);
}
-}
\ No newline at end of file
+}
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/edit.test.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/edit.test.js
index dd13177fac9e2e1321d5b873783a691d4b96e769..eae81b7db6746b68d5239aa7232026fed08aa4a5 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/edit.test.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/edit.test.js
@@ -112,10 +112,10 @@ it("renders input fields with Project details if found", async () => {
expect(content.queryByTestId("name").value).toBe('OSR-11');
const spinButtons = content.queryAllByRole("spinbutton");
- const trigPrioInput = spinButtons.filter(function(element) { return element.id==="trig_prio"})[0];
+ const trigPrioInput = _.filter(spinButtons, {"id": "trig_prio"})[0];
expect(trigPrioInput.value).toBe("990");
- const rankInput = spinButtons.filter(function(element) { return element.id==="proj-rank"})[0];
+ const rankInput = _.filter(spinButtons, {"id": "proj-rank"})[0];
expect(rankInput.value).toBe("5");
const trigger = content.getAllByLabelText(/trigger/i).filter((element) => { return element.id==="trigger"})[0];
@@ -132,15 +132,15 @@ it("renders input fields with Project details if found", async () => {
const cycleInput = content.getAllByRole("listbox")[2] ;
expect(content.queryAllByText('Cycle 0').length).toBe(2);
- expect(content.queryByPlaceholderText("CEP Processing Time").value).toBe("10 Hours");
- expect(content.queryByPlaceholderText("LOFAR Observing Time").value).toBe("20 Hours");
- expect(content.queryByPlaceholderText("LOFAR Observing Time prio A").value).toBe("30 Hours");
- expect(content.queryByPlaceholderText("LOFAR Observing Time prio B").value).toBe("40 Hours");
- expect(content.queryByPlaceholderText("LOFAR Support Time").value).toBe("50 Hours");
- expect(content.queryByPlaceholderText("LTA Storage").value).toBe("6 TB");
- expect(content.queryByPlaceholderText("Number of triggers").value).toBe("7 Numbers");
- expect(content.queryByPlaceholderText("Support hours").value).toBe("8 ");
-
+ expect(_.filter(spinButtons, {"id": "CEP Processing Time"})[0].value).toBe("10 Hours");
+ expect(_.filter(spinButtons, {"id": "LOFAR Observing Time"})[0].value).toBe("20 Hours");
+ expect(_.filter(spinButtons, {"id": "LOFAR Observing Time prio A"})[0].value).toBe("30 Hours");
+ expect(_.filter(spinButtons, {"id": "LOFAR Observing Time prio B"})[0].value).toBe("40 Hours");
+ expect(_.filter(spinButtons, {"id": "LOFAR Support Time"})[0].value).toBe("50 Hours");
+ expect(_.filter(spinButtons, {"id": "LTA Storage"})[0].value).toBe("6 TB");
+ expect(_.filter(spinButtons, {"id": "Number of triggers"})[0].value).toBe("7 Numbers");
+ expect(_.filter(spinButtons, {"id": "Support hours"})[0].value).toBe("9 Hours");
+
expect(content.queryByTestId('save-btn').hasAttribute("disabled")).toBeFalsy();
});
@@ -157,11 +157,11 @@ it("save Project after editing fields", async () => {
expect(content.queryByTestId("name").value).toBe('OSR-11');
const spinButtons = content.queryAllByRole("spinbutton");
- const trigPrioInput = spinButtons.filter(function(element) { return element.id==="trig_prio"})[0];
+ const trigPrioInput = _.filter(spinButtons, {"id": "trig_prio"})[0];
fireEvent.blur(trigPrioInput, { target: { value: 900 } });
expect(trigPrioInput.value).toBe("900");
- const rankInput = spinButtons.filter(function(element) { return element.id==="proj-rank"})[0];
+ const rankInput = _.filter(spinButtons, {"id": "proj-rank"})[0];
fireEvent.blur(rankInput, { target: { value: 2 } });
expect(rankInput.value).toBe("2");
@@ -190,19 +190,19 @@ it("save Project after editing fields", async () => {
expect(content.queryAllByText('Cycle-0').length).toBe(2);
expect(content.queryAllByText('Cycle 0').length).toBe(1);
- const lofarObsTimeInput = content.queryByPlaceholderText('LOFAR Observing Time');
+ const lofarObsTimeInput = _.filter(spinButtons, {"id": "LOFAR Observing Time"})[0];
fireEvent.blur(lofarObsTimeInput, { target: { value: 10 } });
expect(lofarObsTimeInput.value).toBe('10 Hours');
- const cepProcTimeInput = content.queryByPlaceholderText('CEP Processing Time');
+ const cepProcTimeInput = _.filter(spinButtons, {"id": "CEP Processing Time"})[0];
fireEvent.blur(cepProcTimeInput, { target: { value: 5 } });
expect(cepProcTimeInput.value).toBe('5 Hours');
- const ltaStorageInput = content.queryByPlaceholderText('LTA Storage');
+ const ltaStorageInput = _.filter(spinButtons, {"id": "LTA Storage"})[0];
fireEvent.blur(ltaStorageInput, { target: { value: 2 } });
expect(ltaStorageInput.value).toBe('2 TB');
- const noOfTriggerInput = content.queryByPlaceholderText('Number of triggers');
+ const noOfTriggerInput = _.filter(spinButtons, {"id": "Number of triggers"})[0];
fireEvent.blur(noOfTriggerInput, { target: { value: 3 } });
expect(noOfTriggerInput.value).toBe('3 Numbers');
@@ -238,13 +238,15 @@ it("save Project after adding, modifying and deleting resources", async () => {
const addResourceBtn = content.queryByTestId('add_res_btn');
fireEvent.click(addResourceBtn);
expect(content.queryAllByText('Add Resources').length).toBe(2);
- expect(content.queryByPlaceholderText('LOFAR Support hours')).not.toBe(null);
- const lofarSupHrsInput = content.queryByPlaceholderText('LOFAR Support hours');
+
+ const spinButtons = content.queryAllByRole("spinbutton");
+ const lofarSupHrsInput = _.filter(spinButtons, {"id": "LOFAR Support hours"})[0];
+ expect(lofarSupHrsInput).not.toBe(null);
fireEvent.blur(lofarSupHrsInput, { target: { value: 100 } });
- expect(lofarSupHrsInput.value).toBe('100 ');
+ expect(lofarSupHrsInput.value).toBe('100 Hours');
// Editing existing resource
- const lofarObsTimeInput = content.queryByPlaceholderText('LOFAR Observing Time');
+ const lofarObsTimeInput = _.filter(spinButtons, {"id": "LOFAR Observing Time"})[0];
fireEvent.blur(lofarObsTimeInput, { target: { value: 10 } });
expect(lofarObsTimeInput.value).toBe('10 Hours');
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/list.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/list.js
index af3869b13dd7c31251f4dec33f350ee3f5d59100..2072bf74706c387a9b84682cd48ed59cb53850a8 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/list.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/list.js
@@ -1,22 +1,22 @@
import React, {Component} from 'react';
import ProjectService from '../../services/project.service';
import ViewTable from '../../components/ViewTable';
-import { Link } from 'react-router-dom/cjs/react-router-dom.min';
import AppLoader from '../../layout/components/AppLoader';
+import PageHeader from '../../layout/components/PageHeader';
+import CycleService from '../../services/cycle.service';
export class ProjectList extends Component{
constructor(props){
super(props)
this.state = {
projectlist: [],
- paths: [{
- "View": "/project/view",
- }],
defaultcolumns: [ {
"name":"Name / Project Code",
"status":"Status" ,
"project_category_value":"Category of Project",
- "description":"Description"
+ "description":"Description",
+ "archive_location_label":"LTA Storage Location",
+ "archive_subdirectory":"LTA Storage Path"
}],
optionalcolumns: [{
"priority_rank":"Project Priority",
@@ -31,6 +31,7 @@ export class ProjectList extends Component{
"LTA Storage":"LTA storage (TB)",
"Number of triggers":"Number of Triggers",
"actionpath":"actionpath",
+
}],
columnclassname: [{
"Observing time (Hrs)":"filter-input-50",
@@ -45,32 +46,57 @@ export class ProjectList extends Component{
"Trigger Priority":"filter-input-50",
"Category of Period":"filter-input-50",
"Cycles":"filter-input-100",
+ "LTA Storage Location":"filter-input-100",
+ "LTA Storage Path":"filter-input-100"
}],
+ defaultSortColumn: [{id: "Name / Project Code", desc: false}],
isprocessed: false,
isLoading: true
}
+ this.getPopulatedProjectList = this.getPopulatedProjectList.bind(this);
}
- componentDidMount(){
- // for Unit test, Table data
- this.unittestDataProvider();
- ProjectService.getProjectList()
- .then(async (projects) => {
- await ProjectService.getUpdatedProjectQuota(projects)
- .then( async projlist => {
- this.setState({
- projectlist: projlist,
- isprocessed: true,
- isLoading: false
- })
+ getPopulatedProjectList(cycleId) {
+ Promise.all([ProjectService.getFileSystem(), ProjectService.getCluster()]).then(async(response) => {
+ const options = {};
+ response[0].map(fileSystem => {
+ const cluster = response[1].filter(clusterObj => { return (clusterObj.id === fileSystem.cluster_id && clusterObj.archive_site);});
+ if (cluster.length) {
+ fileSystem.label =`${cluster[0].name} - ${fileSystem.name}`
+ options[fileSystem.url] = fileSystem;
+ }
+ return fileSystem;
+ });
+ let projects = [];
+ if (cycleId) {
+ projects = await CycleService.getProjectsByCycle(cycleId);
+ } else {
+ projects = await ProjectService.getProjectList();
+ }
+ projects = await ProjectService.getUpdatedProjectQuota(projects);
+ let list = projects.map(project => {
+ project.archive_location_label = (options[project.archive_location] || {}).label;
+ return project;
+ });
+ this.setState({
+ projectlist: list,
+ isprocessed: true,
+ isLoading: false,
+ ltaStorage: options
})
});
}
+
+ componentDidMount(){
+ // Show Project for the Cycle, This request will be coming from Cycle View. Otherwise it is consider as normal Project List.
+ let cycle = this.props.cycle;
+ this.getPopulatedProjectList(cycle);
+ }
render(){
return(
<>
- <div className="p-grid">
+ {/*<div className="p-grid">
<div className="p-col-10 p-lg-10 p-md-10">
<h2>Project - List </h2>
</div>
@@ -79,18 +105,30 @@ export class ProjectList extends Component{
<i className="fa fa-plus-square" style={{marginTop: "10px"}}></i>
</Link>
</div>
- </div>
- {this.state.isLoading? <AppLoader /> : this.state.isprocessed &&
+ </div> */}
+ { (this.props.cycle) ?
+ <>
+ </>
+ :
+ <PageHeader location={this.props.location} title={'Project - List'}
+ actions={[{icon: 'fa-plus-square',title:'Click to Add Project', props:{pathname: '/project/create' }}]}
+ />
+
+ }
+ {this.state.isLoading? <AppLoader /> : (this.state.isprocessed && this.state.projectlist.length>0) ?
<ViewTable
data={this.state.projectlist}
defaultcolumns={this.state.defaultcolumns}
optionalcolumns={this.state.optionalcolumns}
columnclassname={this.state.columnclassname}
+ defaultSortColumn={this.state.defaultSortColumn}
showaction="true"
paths={this.state.paths}
keyaccessor="name"
unittest={this.state.unittest}
- />
+ tablename="project_list"
+ />
+ : <div>No project found </div>
}
</>
)
@@ -125,4 +163,3 @@ export class ProjectList extends Component{
}
}
}
-
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/view.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/view.js
index 2ede26034deb218bf3ba36f0049ada765236d2a9..1dce9c77522247d114a91f6181ce3f05a00e1861 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/view.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/view.js
@@ -4,13 +4,14 @@ import moment from 'moment';
import _ from 'lodash';
import { Chips } from 'primereact/chips';
+import { TieredMenu } from 'primereact/tieredmenu';
import ResourceDisplayList from './ResourceDisplayList';
-
import AppLoader from '../../layout/components/AppLoader';
+import PageHeader from '../../layout/components/PageHeader';
import ProjectService from '../../services/project.service';
import UnitConverter from '../../utils/unit.converter';
-
+import SchedulingUnitList from './../Scheduling/SchedulingUnitList';
/**
* Component to view the details of a project
*/
@@ -19,7 +20,9 @@ export class ProjectView extends Component {
constructor(props) {
super(props);
this.state = {
+ ltaStorage: [],
isLoading: true,
+ project:'',
};
if (this.props.match.params.id) {
this.state.projectId = this.props.match.params.id;
@@ -28,6 +31,11 @@ export class ProjectView extends Component {
}
this.state.redirect = this.state.projectId?"":'/project' // If no project id is passed, redirect to Project list page
this.resourceUnitMap = UnitConverter.resourceUnitMap; // Resource unit conversion factor and constraints
+ this.optionsMenu = React.createRef();
+ this.menuOptions = [ {label:'Add Scheduling Unit', icon: "fa fa-", command: () => {this.selectOptionMenu('Add SU')}} ];
+
+ this.showOptionMenu = this.showOptionMenu.bind(this);
+ this.selectOptionMenu = this.selectOptionMenu.bind(this);
}
componentDidMount() {
@@ -37,6 +45,17 @@ export class ProjectView extends Component {
} else {
this.setState({redirect: "/not-found"});
}
+ Promise.all([ProjectService.getFileSystem(), ProjectService.getCluster()]).then(response => {
+ const options = {};
+ response[0].map(fileSystem => {
+ const cluster = response[1].filter(clusterObj => clusterObj.id === fileSystem.cluster_id && clusterObj.archive_site);
+ if (cluster.length) {
+ options[fileSystem.url] = `${cluster[0].name} - ${fileSystem.name}`
+ }
+ return fileSystem;
+ });
+ this.setState({archive_location: response[0], ltaStorage: options, cluster: response[1] });
+ });
}
/**
@@ -46,9 +65,10 @@ export class ProjectView extends Component {
async getProjectDetails() {
let project = await ProjectService.getProjectDetails(this.state.projectId);
let projectQuota = [];
- let resources = [];
+ let resources = [];
if (project) {
+
// If resources are allocated for the project quota fetch the resources master from the API
if (project.quota) {
resources = await ProjectService.getResources();
@@ -68,6 +88,22 @@ export class ProjectView extends Component {
}
+ showOptionMenu(event) {
+ this.optionsMenu.toggle(event);
+ }
+
+ selectOptionMenu(menuName) {
+ switch(menuName) {
+ case 'Add SU': {
+ this.setState({redirect: `/project/${this.state.project.name}/schedulingunit/create`});
+ break;
+ }
+ default: {
+ break;
+ }
+ }
+ }
+
render() {
if (this.state.redirect) {
return <Redirect to={ {pathname: this.state.redirect} }></Redirect>
@@ -75,22 +111,15 @@ export class ProjectView extends Component {
return (
<React.Fragment>
- <div className="p-grid">
- <div className="p-col-10 p-lg-10 p-md-10">
- <h2>Project - Details </h2>
- </div>
- { this.state.project &&
- <div className="p-col-2 p-lg-2 p-md-2">
- <Link to={{ pathname: `/project`}} title="Close View" style={{float: "right"}}>
- <i className="fa fa-times" style={{marginTop: "10px", marginLeft: "5px"}}></i>
- </Link>
- <Link to={{ pathname: `/project/edit/${this.state.project.name}`, state: {id: this.state.project?this.state.project.name:''}}} title="Edit Project"
- style={{float: "right"}}>
- <i className="fa fa-edit" style={{marginTop: "10px"}}></i>
- </Link>
- </div>
- }
- </div>
+ <TieredMenu className="app-header-menu" model={this.menuOptions} popup ref={el => this.optionsMenu = el} />
+ <PageHeader location={this.props.location} title={'Project - View'}
+ actions={[ {icon:'fa-bars',title: '', type:'button',
+ actOn:'mouseOver', props : { callback: this.showOptionMenu},
+ },
+ {icon: 'fa-edit',title:'Click to Edit Project', type:'link',
+ props : { pathname: `/project/edit/${this.state.project.name}`,
+ state: {id: this.state.project?this.state.project.name:''&& this.state.project}}},
+ {icon:'fa-window-close',title: 'Click to Close Project View', link: this.props.history.goBack}]}/>
{ this.state.isLoading && <AppLoader /> }
{ this.state.project &&
<React.Fragment>
@@ -125,6 +154,12 @@ export class ProjectView extends Component {
<label className="col-lg-2 col-md-2 col-sm-12">Project Rank</label>
<span className="col-lg-4 col-md-4 col-sm-12">{this.state.project.priority_rank}</span>
</div>
+ <div className="p-grid">
+ <label className="col-lg-2 col-md-2 col-sm-12">LTA Storage Location</label>
+ <span className="col-lg-4 col-md-4 col-sm-12">{this.state.ltaStorage[this.state.project.archive_location]}</span>
+ <label className="col-lg-2 col-md-2 col-sm-12">LTA Storage Path</label>
+ <span className="col-lg-4 col-md-4 col-sm-12">{this.state.project.archive_subdirectory }</span>
+ </div>
<div className="p-fluid">
<div className="p-field p-grid">
<div className="col-lg-3 col-md-3 col-sm-12">
@@ -144,6 +179,15 @@ export class ProjectView extends Component {
<div className="p-field p-grid resource-input-grid">
<ResourceDisplayList projectQuota={this.state.projectQuota} unitMap={this.resourceUnitMap} />
</div>
+ {/* Show Schedule Unit blongest to Project */}
+ <div className="p-fluid">
+ <div className="p-field p-grid">
+ <div className="col-lg-3 col-md-3 col-sm-12">
+ <h5 data-testid="resource_alloc">Scheduling Unit - List</h5>
+ </div>
+ </div>
+ </div>
+ <SchedulingUnitList project={this.state.project.name}/>
</div>
</React.Fragment>
}
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/view.test.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/view.test.js
index aaf9327d0308d4076806ba99897dc7a345df0e23..3847248a84ccbf4d50b6d39a0498b19a7d267d84 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/view.test.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/view.test.js
@@ -66,7 +66,7 @@ it("renders Project details if found", async () => {
expect(content.queryByText("50 Hours")).not.toBe(null);
expect(content.queryByText("6 TB")).not.toBe(null);
expect(content.queryByText("7 Numbers")).not.toBe(null);
- expect(content.queryByText("8")).not.toBe(null);
+ expect(content.queryByText("9 Hours")).not.toBe(null);
});
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/SchedulingUnitList.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/SchedulingUnitList.js
index a3ec244ea830589b7cc5b830d3415aee43c046af..6cec1f3d60c3f628370e79557f3566549969dc22 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/SchedulingUnitList.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/SchedulingUnitList.js
@@ -5,7 +5,6 @@ import AppLoader from "./../../layout/components/AppLoader";
import ViewTable from './../../components/ViewTable';
import ScheduleService from '../../services/schedule.service';
-
class SchedulingUnitList extends Component{
@@ -35,35 +34,45 @@ class SchedulingUnitList extends Component{
"Template":"filter-input-50",
"Duration":"filter-input-50",
"Type": "filter-input-75"
- }]
+ }],
+ defaultSortColumn: [{id: "Name", desc: false}],
}
}
-
-
async getSchedulingUnitList () {
- const bluePrint = await ScheduleService.getSchedulingUnitBlueprint();
- ScheduleService.getSchedulingUnitDraft().then(scheduleunit =>{
- const output = [];
- var scheduleunits = scheduleunit.data.results;
- for( const scheduleunit of scheduleunits){
- const blueprintdata = bluePrint.data.results.filter(i => i.draft_id === scheduleunit.id);
- blueprintdata.map(blueP => {
- blueP.duration = moment.utc(blueP.duration*1000).format('HH:mm:ss');
- blueP.type="Blueprint";
- blueP['actionpath'] = '/task/view/type/id';
- return blueP;
+ //Get SU Draft/Blueprints for the Project ID. This request is coming from view Project page. Otherwise it will show all SU
+ let project = this.props.project;
+ if(project){
+ let scheduleunits = await ScheduleService.getSchedulingListByProject(project);
+ if(scheduleunits){
+ this.setState({
+ scheduleunit: scheduleunits, isLoading: false
});
- output.push(...blueprintdata);
- scheduleunit['actionpath']='/schedulingunit/view';
- scheduleunit['type'] = 'Draft';
- scheduleunit['duration'] = moment.utc(scheduleunit.duration*1000).format('HH:mm:ss');
- output.push(scheduleunit);
}
- this.setState({
- scheduleunit: output, isLoading:false
- });
- })
+ }else{
+ const bluePrint = await ScheduleService.getSchedulingUnitBlueprint();
+ ScheduleService.getSchedulingUnitDraft().then(scheduleunit =>{
+ const output = [];
+ var scheduleunits = scheduleunit.data.results;
+ for( const scheduleunit of scheduleunits){
+ const blueprintdata = bluePrint.data.results.filter(i => i.draft_id === scheduleunit.id);
+ blueprintdata.map(blueP => {
+ blueP.duration = moment.utc(blueP.duration*1000).format('HH:mm:ss');
+ blueP.type="Blueprint";
+ blueP['actionpath'] ='/schedulingunit/view/blueprint/'+blueP.id;
+ return blueP;
+ });
+ output.push(...blueprintdata);
+ scheduleunit['actionpath']='/schedulingunit/view/draft/'+scheduleunit.id;
+ scheduleunit['type'] = 'Draft';
+ scheduleunit['duration'] = moment.utc(scheduleunit.duration*1000).format('HH:mm:ss');
+ output.push(scheduleunit);
+ }
+ this.setState({
+ scheduleunit: output, isLoading: false
+ });
+ })
+ }
}
componentDidMount(){
@@ -87,17 +96,21 @@ class SchedulingUnitList extends Component{
paths - specify the path for navigation - Table will set "id" value for each row in action button
*/}
- {this.state.scheduleunit &&
+
+ { (this.state.scheduleunit && this.state.scheduleunit.length>0)?
<ViewTable
data={this.state.scheduleunit}
defaultcolumns={this.state.defaultcolumns}
optionalcolumns={this.state.optionalcolumns}
columnclassname={this.state.columnclassname}
+ defaultSortColumn={this.state.defaultSortColumn}
showaction="true"
keyaccessor="id"
paths={this.state.paths}
unittest={this.state.unittest}
+ tablename="scheduleunit_list"
/>
+ :<div>No scheduling unit found </div>
}
</>
)
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/ViewSchedulingUnit.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/ViewSchedulingUnit.js
index 2a33a72b8c5c54a7f283b1363120ab270779226b..c57445b61784420eceb8a9f538d5c113fea84909 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/ViewSchedulingUnit.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/ViewSchedulingUnit.js
@@ -1,9 +1,10 @@
import React, { Component } from 'react'
-import {Link} from 'react-router-dom'
+// import {Link} from 'react-router-dom'
import 'primeflex/primeflex.css';
import { Chips } from 'primereact/chips';
import AppLoader from "./../../layout/components/AppLoader";
+import PageHeader from '../../layout/components/PageHeader';
import ViewTable from './../../components/ViewTable';
import ScheduleService from '../../services/schedule.service';
@@ -40,7 +41,6 @@ class ViewSchedulingUnit extends Component{
"url":"URL",
"actionpath":"actionpath",
}],
-
columnclassname: [{
"Type":"filter-input-75",
"ID":"filter-input-50",
@@ -52,35 +52,70 @@ class ViewSchedulingUnit extends Component{
"Relative End Time (HH:mm:ss)": "filter-input-75",
}]
}
+ this.actions = [
+ {icon: 'fa-window-close',title:'Click to Close Scheduling Unit View', link: this.props.history.goBack}
+ ];
+ if (this.props.match.params.type === 'draft') {
+ this.actions.unshift({icon: 'fa-edit', title: 'Click to edit', props : { pathname:`/schedulingunit/edit/${ this.props.match.params.id}`
+ } });
+ } else {
+ this.actions.unshift({icon: 'fa-lock', title: 'Cannot edit blueprint'});
+ }
+ if (this.props.match.params.id) {
+ this.state.scheduleunitId = this.props.match.params.id;
+ }
+ if (this.props.match.params.type) {
+ this.state.scheduleunitType = this.props.match.params.type;
+ }
}
componentDidMount(){
- let schedule_id = this.props.location.state.id
- if (schedule_id) {
- ScheduleService.getSchedulingUnitDraftById(schedule_id)
- .then(scheduleunit =>{
- ScheduleService.getScheduleTasksBySchedulingUnitId(scheduleunit.data.id)
- .then(tasks =>{
- tasks.map(task => {
- task.duration = moment.utc(task.duration*1000).format('HH:mm:ss');
- task.relative_start_time = moment.utc(task.relative_start_time*1000).format('HH:mm:ss');
- task.relative_stop_time = moment.utc(task.relative_stop_time*1000).format('HH:mm:ss');
- return task;
+ let schedule_id = this.state.scheduleunitId;
+ let schedule_type = this.state.scheduleunitType;
+ if (schedule_type && schedule_id) {
+ this.getScheduleUnit(schedule_type, schedule_id)
+ .then(schedulingUnit =>{
+ if (schedulingUnit) {
+ this.getScheduleUnitTasks(schedule_type, schedulingUnit)
+ .then(tasks =>{
+ /* tasks.map(task => {
+ task.duration = moment.utc(task.duration*1000).format('HH:mm:ss');
+ task.relative_start_time = moment.utc(task.relative_start_time*1000).format('HH:mm:ss');
+ task.relative_stop_time = moment.utc(task.relative_stop_time*1000).format('HH:mm:ss');
+ return task;
+ });*/
+ this.setState({
+ scheduleunit : schedulingUnit,
+ schedule_unit_task : tasks,
+ isLoading: false,
+ });
});
+ } else {
this.setState({
- scheduleunit : scheduleunit.data,
- schedule_unit_task : tasks,
- isLoading: false
+ isLoading: false,
});
- });
- })
+ }
+ });
}
}
-
+
+ getScheduleUnitTasks(type, scheduleunit){
+ if(type === 'draft')
+ return ScheduleService.getTasksBySchedulingUnit(scheduleunit.id);
+ else
+ return ScheduleService.getTaskBlueprintsBySchedulingUnit(scheduleunit);
+ }
+ getScheduleUnit(type, id){
+ if(type === 'draft')
+ return ScheduleService.getSchedulingUnitDraftById(id)
+ else
+ return ScheduleService.getSchedulingUnitBlueprintById(id)
+ }
+
render(){
return(
<>
- <div className="p-grid">
+ {/*} <div className="p-grid">
<div className="p-col-10">
<h2>Scheduling Unit - Details </h2>
</div>
@@ -89,15 +124,19 @@ class ViewSchedulingUnit extends Component{
style={{float:'right'}}>
<i className="fa fa-times" style={{marginTop: "10px", marginLeft: '5px'}}></i>
</Link>
- {/* <Link to={{ pathname: '/schedulingunit/edit', state: {id: this.state.scheduleunit?this.state.scheduleunit.id:''}}} title="Edit"
+ <Link to={{ pathname: '/schedulingunit/edit', state: {id: this.state.scheduleunit?this.state.scheduleunit.id:''}}} title="Edit"
style={{float:'right'}}>
<i className="fa fa-edit" style={{marginTop: "10px"}}></i>
- </Link> */}
- </div>
+ </Link>
</div>
+ </div> */
+ /*TMSS-363 Blueprint icon changes */}
+ <PageHeader location={this.props.location} title={'Scheduling Unit - Details'}
+ actions={this.actions}/>
{ this.state.isLoading ? <AppLoader/> :this.state.scheduleunit &&
<>
- <div className="p-grid">
+ <div className="main-content">
+ <div className="p-grid">
<label className="col-lg-2 col-md-2 col-sm-12">Name</label>
<span className="p-col-lg-4 col-md-4 col-sm-12">{this.state.scheduleunit.name}</span>
<label className="col-lg-2 col-md-2 col-sm-12">Description</label>
@@ -119,14 +158,14 @@ class ViewSchedulingUnit extends Component{
<label className="col-lg-2 col-md-2 col-sm-12">Template ID</label>
<span className="col-lg-4 col-md-4 col-sm-12">{this.state.scheduleunit.requirements_template_id}</span>
<label className="col-lg-2 col-md-2 col-sm-12">Scheduling set</label>
- <span className="col-lg-4 col-md-4 col-sm-12">{this.state.scheduleunit.scheduling_set_id}</span>
+ <span className="col-lg-4 col-md-4 col-sm-12">{this.state.scheduleunit.scheduling_set_object.name}</span>
</div>
<div className="p-grid">
<label className="col-lg-2 col-md-2 col-sm-12">Duration (HH:mm:ss)</label>
<span className="col-lg-4 col-md-4 col-sm-12">{moment.utc(this.state.scheduleunit.duration*1000).format('HH:mm:ss')}</span>
<label className="col-lg-2 col-md-2 col-sm-12">Tags</label>
<Chips className="p-col-4 chips-readonly" disabled value={this.state.scheduleunit.tags}></Chips>
- <span className="col-lg-4 col-md-4 col-sm-12">{this.state.scheduleunit.tags}</span>
+ </div>
</div>
</>
@@ -150,10 +189,12 @@ class ViewSchedulingUnit extends Component{
defaultcolumns={this.state.defaultcolumns}
optionalcolumns={this.state.optionalcolumns}
columnclassname={this.state.columnclassname}
+ defaultSortColumn={this.state.defaultSortColumn}
showaction="true"
keyaccessor="id"
paths={this.state.paths}
unittest={this.state.unittest}
+ tablename="scheduleunit_task_list"
/>
}
</>
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/create.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/create.js
new file mode 100644
index 0000000000000000000000000000000000000000..62db24996287f74d5e98fc1c816d7d519f944898
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/create.js
@@ -0,0 +1,449 @@
+import React, {Component} from 'react';
+import { Link, Redirect } from 'react-router-dom';
+import _ from 'lodash';
+import $RefParser from "@apidevtools/json-schema-ref-parser";
+
+import {InputText} from 'primereact/inputtext';
+import {InputTextarea} from 'primereact/inputtextarea';
+import {Dropdown} from 'primereact/dropdown';
+import { Button } from 'primereact/button';
+import {Dialog} from 'primereact/components/dialog/Dialog';
+import {Growl} from 'primereact/components/growl/Growl';
+
+import AppLoader from '../../layout/components/AppLoader';
+import Jeditor from '../../components/JSONEditor/JEditor';
+
+import ProjectService from '../../services/project.service';
+import ScheduleService from '../../services/schedule.service';
+import TaskService from '../../services/task.service';
+import UIConstants from '../../utils/ui.constants';
+import PageHeader from '../../layout/components/PageHeader';
+
+/**
+ * Component to create a new SchedulingUnit from Observation strategy template
+ */
+export class SchedulingUnitCreate extends Component {
+ constructor(props) {
+ super(props);
+ this.state = {
+ isLoading: true, // Flag for loading spinner
+ dialog: { header: '', detail: ''}, // Dialog properties
+ redirect: null, // URL to redirect
+ errors: [], // Form Validation errors
+ schedulingSets: [], // Scheduling set of the selected project
+ schedulingUnit: {
+ project: (props.match?props.match.params.project:null) || null,
+ },
+ projectDisabled: (props.match?(props.match.params.project? true:false):false), // Disable project selection if
+ observStrategy: {}, // Selected strategy to create SU
+ paramsSchema: null, // JSON Schema to be generated from strategy template to pass to JSOn editor
+ validEditor: false, // For JSON editor validation
+ validFields: {}, // For Form Validation
+ }
+ this.projects = []; // All projects to load project dropdown
+ this.schedulingSets = []; // All scheduling sets to be filtered for project
+ this.observStrategies = []; // All Observing strategy templates
+ this.taskTemplates = []; // All task templates to be filtered based on tasks in selected strategy template
+ this.tooltipOptions = UIConstants.tooltipOptions;
+ this.nameInput = React.createRef(); // Ref to Name field for auto focus
+ this.formRules = { // Form validation rules
+ name: {required: true, message: "Name can not be empty"},
+ description: {required: true, message: "Description can not be empty"},
+ project: {required: true, message: "Select project to get Scheduling Sets"},
+ scheduling_set_id: {required: true, message: "Select the Scheduling Set"},
+ };
+
+ this.setEditorOutput = this.setEditorOutput.bind(this);
+ this.changeProject = this.changeProject.bind(this);
+ this.changeStrategy = this.changeStrategy.bind(this);
+ this.setSchedUnitParams = this.setSchedUnitParams.bind(this);
+ this.validateForm = this.validateForm.bind(this);
+ this.validateEditor = this.validateEditor.bind(this);
+ this.setEditorFunction = this.setEditorFunction.bind(this);
+ this.saveSchedulingUnit = this.saveSchedulingUnit.bind(this);
+ this.cancelCreate = this.cancelCreate.bind(this);
+ this.reset = this.reset.bind(this);
+ }
+
+ componentDidMount() {
+ const promises = [ ProjectService.getProjectList(),
+ ScheduleService.getSchedulingSets(),
+ ScheduleService.getObservationStrategies(),
+ TaskService.getTaskTemplates()]
+ Promise.all(promises).then(responses => {
+ this.projects = responses[0];
+ this.schedulingSets = responses[1];
+ this.observStrategies = responses[2];
+ this.taskTemplates = responses[3];
+ if (this.state.schedulingUnit.project) {
+ const projectSchedSets = _.filter(this.schedulingSets, {'project_id': this.state.schedulingUnit.project});
+ this.setState({isLoading: false, schedulingSets: projectSchedSets});
+ } else {
+ this.setState({isLoading: false});
+ }
+ });
+ }
+
+ /**
+ * Function to call on change of project and reload scheduling set dropdown
+ * @param {string} projectName
+ */
+ changeProject(projectName) {
+ const projectSchedSets = _.filter(this.schedulingSets, {'project_id': projectName});
+ let schedulingUnit = this.state.schedulingUnit;
+ schedulingUnit.project = projectName;
+ this.setState({schedulingUnit: schedulingUnit, schedulingSets: projectSchedSets, validForm: this.validateForm('project')});
+ }
+
+ /**
+ * Function called when observation strategy template is changed.
+ * It generates the JSON schema for JSON editor and defult vales for the parameters to be captured
+ * @param {number} strategyId
+ */
+ async changeStrategy (strategyId) {
+ const observStrategy = _.find(this.observStrategies, {'id': strategyId});
+ const tasks = observStrategy.template.tasks;
+ let paramsOutput = {};
+ let schema = { type: 'object', additionalProperties: false,
+ properties: {}, definitions:{}
+ };
+
+ for (const taskName of _.keys(tasks)) {
+ const task = tasks[taskName];
+ //Resolve task from the strategy template
+ const $taskRefs = await $RefParser.resolve(task);
+
+ // Identify the task specification template of every task in the strategy template
+ const taskTemplate = _.find(this.taskTemplates, {'name': task['specifications_template']});
+ schema['$id'] = taskTemplate.schema['$id'];
+ schema['$schema'] = taskTemplate.schema['$schema'];
+ let index = 0;
+ for (const param of observStrategy.template.parameters) {
+ if (param.refs[0].indexOf(`/tasks/${taskName}`) > 0) {
+ // Resolve the identified template
+ const $templateRefs = await $RefParser.resolve(taskTemplate);
+ let property = { };
+ let tempProperty = null;
+ const taskPaths = param.refs[0].split("/");
+ // Get the property type from the template and create new property in the schema for the parameters
+ try {
+ const parameterRef = param.refs[0];//.replace(`#/tasks/${taskName}/specifications_doc`, '#/schema/properties');
+ tempProperty = $templateRefs.get(parameterRef);
+ // property = _.cloneDeep(taskTemplate.schema.properties[taskPaths[4]]);
+
+ } catch(error) {
+ tempProperty = _.cloneDeep(taskTemplate.schema.properties[taskPaths[4]]);
+ if (tempProperty.type === 'array') {
+ tempProperty = tempProperty.items.properties[taskPaths[6]];
+ }
+ property = tempProperty;
+ }
+ /* if (property['$ref'] && !property['$ref'].startsWith("#")) {
+ const $propDefinition = await $RefParser.resolve(property['$ref']);
+ const propDefinitions = $propDefinition.get("#/definitions");
+ for (const propDefinition in propDefinitions) {
+ schema.definitions[propDefinition] = propDefinitions[propDefinition];
+ property['$ref'] = "#/definitions/"+ propDefinition ;
+ }
+ } */
+ property.title = param.name;
+ property.default = $taskRefs.get(param.refs[0].replace(`#/tasks/${taskName}`, '#'));
+ paramsOutput[`param_${index}`] = property.default;
+ schema.properties[`param_${index}`] = property;
+ // Set property defintions taken from the task template in new schema
+ for (const definitionName in taskTemplate.schema.definitions) {
+ schema.definitions[definitionName] = taskTemplate.schema.definitions[definitionName];
+
+ }
+ }
+ index++;
+
+ }
+
+ }
+ this.setState({observStrategy: observStrategy, paramsSchema: schema, paramsOutput: paramsOutput});
+
+ // Function called to clear the JSON Editor fields and reload with new schema
+ if (this.state.editorFunction) {
+ this.state.editorFunction();
+ }
+ }
+
+ /**
+ * This is the callback method to be passed to the JSON editor.
+ * JEditor will call this function when there is change in the editor.
+ * @param {Object} jsonOutput
+ * @param {Array} errors
+ */
+ setEditorOutput(jsonOutput, errors) {
+ this.paramsOutput = jsonOutput;
+ this.validEditor = errors.length === 0;
+ this.setState({ paramsOutput: jsonOutput,
+ validEditor: errors.length === 0,
+ validForm: this.validateForm()});
+ }
+
+ /**
+ * This function is mainly added for Unit Tests. If this function is removed Unit Tests will fail.
+ */
+ validateEditor() {
+ return this.validEditor?true:false;
+ }
+
+ /**
+ * Function to set form values to the SU object
+ * @param {string} key
+ * @param {object} value
+ */
+ setSchedUnitParams(key, value) {
+ let schedulingUnit = this.state.schedulingUnit;
+ schedulingUnit[key] = value;
+ this.setState({schedulingUnit: schedulingUnit, validForm: this.validateForm(key), validEditor: this.validateEditor()});
+ this.validateEditor();
+ }
+
+ /**
+ * JEditor's function that to be called when parent wants to trigger change in the JSON Editor
+ * @param {Function} editorFunction
+ */
+ setEditorFunction(editorFunction) {
+ this.setState({editorFunction: editorFunction});
+ }
+
+ /**
+ * Validation function to validate the form or field based on the form rules.
+ * If no argument passed for fieldName, validates all fields in the form.
+ * @param {string} fieldName
+ */
+ validateForm(fieldName) {
+ let validForm = false;
+ let errors = this.state.errors;
+ let validFields = this.state.validFields;
+ if (fieldName) {
+ delete errors[fieldName];
+ delete validFields[fieldName];
+ if (this.formRules[fieldName]) {
+ const rule = this.formRules[fieldName];
+ const fieldValue = this.state.schedulingUnit[fieldName];
+ if (rule.required) {
+ if (!fieldValue) {
+ errors[fieldName] = rule.message?rule.message:`${fieldName} is required`;
+ } else {
+ validFields[fieldName] = true;
+ }
+ }
+ }
+ } else {
+ errors = {};
+ validFields = {};
+ for (const fieldName in this.formRules) {
+ const rule = this.formRules[fieldName];
+ const fieldValue = this.state.schedulingUnit[fieldName];
+ if (rule.required) {
+ if (!fieldValue) {
+ errors[fieldName] = rule.message?rule.message:`${fieldName} is required`;
+ } else {
+ validFields[fieldName] = true;
+ }
+ }
+ }
+ }
+ this.setState({errors: errors, validFields: validFields});
+ if (Object.keys(validFields).length === Object.keys(this.formRules).length) {
+ validForm = true;
+ }
+ return validForm;
+ }
+
+ /**
+ * Function to create Scheduling unit
+ */
+ async saveSchedulingUnit() {
+ let observStrategy = _.cloneDeep(this.state.observStrategy);
+ const $refs = await $RefParser.resolve(observStrategy.template);
+ observStrategy.template.parameters.forEach(async(param, index) => {
+ $refs.set(observStrategy.template.parameters[index]['refs'][0], this.state.paramsOutput['param_' + index]);
+ });
+
+ const schedulingUnit = await ScheduleService.saveSUDraftFromObservStrategy(observStrategy, this.state.schedulingUnit);
+ if (schedulingUnit) {
+ // this.growl.show({severity: 'success', summary: 'Success', detail: 'Scheduling Unit and tasks created successfully!'});
+ const dialog = {header: 'Success', detail: 'Scheduling Unit and Tasks are created successfully. Do you want to create another Scheduling Unit?'};
+ this.setState({schedulingUnit: schedulingUnit, dialogVisible: true, dialog: dialog})
+ } else {
+ this.growl.show({severity: 'error', summary: 'Error Occured', detail: 'Unable to save Scheduling Unit/Tasks'});
+ }
+ }
+
+ /**
+ * Cancel SU creation and redirect
+ */
+ cancelCreate() {
+ this.props.history.goBack();
+ }
+
+ /**
+ * Reset function to be called when user wants to create new SU
+ */
+ reset() {
+ const schedulingSets = this.state.schedulingSets;
+ this.nameInput.element.focus();
+ this.setState({
+ dialogVisible: false,
+ dialog: { header: '', detail: ''},
+ errors: [],
+ schedulingSets: this.props.match.params.project?schedulingSets:[],
+ schedulingUnit: {
+ name: '',
+ description: '',
+ project: this.props.match.params.project || null,
+ },
+ projectDisabled: (this.props.match.params.project? true:false),
+ observStrategy: {},
+ paramsOutput: null,
+ validEditor: false,
+ validFields: {}
+ });
+ this.state.editorFunction();
+ }
+
+ render() {
+ if (this.state.redirect) {
+ return <Redirect to={ {pathname: this.state.redirect} }></Redirect>
+ }
+
+ const schema = this.state.paramsSchema;
+
+ let jeditor = null;
+ if (schema) {
+
+ jeditor = React.createElement(Jeditor, {title: "Task Parameters",
+ schema: schema,
+ initValue: this.state.paramsOutput,
+ callback: this.setEditorOutput,
+ parentFunction: this.setEditorFunction
+ });
+ }
+ return (
+ <React.Fragment>
+ <Growl ref={(el) => this.growl = el} />
+ <PageHeader location={this.props.location} title={'Scheduling Unit - Add'}
+ actions={[{icon: 'fa-window-close',link: this.props.history.goBack,title:'Click to close Scheduling Unit creation', props : { pathname: `/schedulingunit`}}]}/>
+ { this.state.isLoading ? <AppLoader /> :
+ <>
+ <div>
+ <div className="p-fluid">
+ <div className="p-field p-grid">
+ <label htmlFor="schedUnitName" className="col-lg-2 col-md-2 col-sm-12">Name <span style={{color:'red'}}>*</span></label>
+ <div className="col-lg-3 col-md-3 col-sm-12">
+ <InputText className={this.state.errors.name ?'input-error':''} id="schedUnitName" data-testid="name"
+ tooltip="Enter name of the Scheduling Unit" tooltipOptions={this.tooltipOptions} maxLength="128"
+ ref={input => {this.nameInput = input;}}
+ value={this.state.schedulingUnit.name} autoFocus
+ onChange={(e) => this.setSchedUnitParams('name', e.target.value)}
+ onBlur={(e) => this.setSchedUnitParams('name', e.target.value)}/>
+ <label className={this.state.errors.name?"error":"info"}>
+ {this.state.errors.name ? this.state.errors.name : "Max 128 characters"}
+ </label>
+ </div>
+ <div className="col-lg-1 col-md-1 col-sm-12"></div>
+ <label htmlFor="description" className="col-lg-2 col-md-2 col-sm-12">Description <span style={{color:'red'}}>*</span></label>
+ <div className="col-lg-3 col-md-3 col-sm-12">
+ <InputTextarea className={this.state.errors.description ?'input-error':''} rows={3} cols={30}
+ tooltip="Longer description of the scheduling unit" tooltipOptions={this.tooltipOptions} maxLength="128"
+ data-testid="description" value={this.state.schedulingUnit.description}
+ onChange={(e) => this.setSchedUnitParams('description', e.target.value)}
+ onBlur={(e) => this.setSchedUnitParams('description', e.target.value)}/>
+ <label className={this.state.errors.description ?"error":"info"}>
+ {this.state.errors.description ? this.state.errors.description : "Max 255 characters"}
+ </label>
+ </div>
+ </div>
+ <div className="p-field p-grid">
+ <label htmlFor="project" className="col-lg-2 col-md-2 col-sm-12">Project <span style={{color:'red'}}>*</span></label>
+ <div className="col-lg-3 col-md-3 col-sm-12" data-testid="project" >
+ <Dropdown inputId="project" optionLabel="name" optionValue="name"
+ tooltip="Project" tooltipOptions={this.tooltipOptions}
+ value={this.state.schedulingUnit.project} disabled={this.state.projectDisabled}
+ options={this.projects}
+ onChange={(e) => {this.changeProject(e.value)}}
+ placeholder="Select Project" />
+ <label className={this.state.errors.project ?"error":"info"}>
+ {this.state.errors.project ? this.state.errors.project : "Select Project to get Scheduling Sets"}
+ </label>
+ </div>
+ <div className="col-lg-1 col-md-1 col-sm-12"></div>
+ <label htmlFor="schedSet" className="col-lg-2 col-md-2 col-sm-12">Scheduling Set <span style={{color:'red'}}>*</span></label>
+ <div className="col-lg-3 col-md-3 col-sm-12">
+ <Dropdown data-testid="schedSet" id="schedSet" optionLabel="name" optionValue="id"
+ tooltip="Scheduling set of the project" tooltipOptions={this.tooltipOptions}
+ value={this.state.schedulingUnit.scheduling_set_id}
+ options={this.state.schedulingSets}
+ onChange={(e) => {this.setSchedUnitParams('scheduling_set_id',e.value)}}
+ placeholder="Select Scheduling Set" />
+ <label className={this.state.errors.scheduling_set_id ?"error":"info"}>
+ {this.state.errors.scheduling_set_id ? this.state.errors.scheduling_set_id : "Scheduling Set of the Project"}
+ </label>
+ </div>
+ </div>
+ <div className="p-field p-grid">
+ <label htmlFor="observStrategy" className="col-lg-2 col-md-2 col-sm-12">Observation Strategy <span style={{color:'red'}}>*</span></label>
+ <div className="col-lg-3 col-md-3 col-sm-12" data-testid="observStrategy" >
+ <Dropdown inputId="observStrategy" optionLabel="name" optionValue="id"
+ tooltip="Observation Strategy Template to be used to create the Scheduling Unit and Tasks" tooltipOptions={this.tooltipOptions}
+ value={this.state.observStrategy.id}
+ options={this.observStrategies}
+ onChange={(e) => {this.changeStrategy(e.value)}}
+ placeholder="Select Strategy" />
+ </div>
+ <div className="col-lg-1 col-md-1 col-sm-12"></div>
+ </div>
+
+ </div>
+ <div className="p-fluid">
+ <div className="p-grid">
+ <div className="p-col-12">
+ {this.state.paramsSchema?jeditor:""}
+ </div>
+ </div>
+ </div>
+
+ <div className="p-grid p-justify-start">
+ <div className="p-col-1">
+ <Button label="Save" className="p-button-primary" icon="pi pi-check" onClick={this.saveSchedulingUnit}
+ disabled={!this.state.validEditor || !this.state.validForm} data-testid="save-btn" />
+ </div>
+ <div className="p-col-1">
+ <Button label="Cancel" className="p-button-danger" icon="pi pi-times" onClick={this.cancelCreate} />
+ </div>
+ </div>
+ </div>
+
+ </>
+ }
+
+ {/* Dialog component to show messages and get input */}
+ <div className="p-grid" data-testid="confirm_dialog">
+ <Dialog header={this.state.dialog.header} visible={this.state.dialogVisible} style={{width: '25vw'}} inputId="confirm_dialog"
+ modal={true} onHide={() => {this.setState({dialogVisible: false})}}
+ footer={<div>
+ <Button key="back" onClick={() => {this.setState({dialogVisible: false, redirect: `/schedulingunit/view/draft/${this.state.schedulingUnit.id}`});}} label="No" />
+ <Button key="submit" type="primary" onClick={this.reset} label="Yes" />
+ </div>
+ } >
+ <div className="p-grid">
+ <div className="col-lg-2 col-md-2 col-sm-2" style={{margin: 'auto'}}>
+ <i className="pi pi-check-circle pi-large pi-success"></i>
+ </div>
+ <div className="col-lg-10 col-md-10 col-sm-10">
+ {this.state.dialog.detail}
+ </div>
+ </div>
+ </Dialog>
+ </div>
+ </React.Fragment>
+ );
+ }
+}
+
+export default SchedulingUnitCreate;
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/create.test.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/create.test.js
new file mode 100644
index 0000000000000000000000000000000000000000..ccdaf6f98f69ceb13f23508624b911590cd9f148
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/create.test.js
@@ -0,0 +1,140 @@
+import React from 'react';
+import { BrowserRouter as Router } from 'react-router-dom';
+import { act } from "react-dom/test-utils";
+import { render, cleanup, fireEvent } from '@testing-library/react';
+import '@testing-library/jest-dom/extend-expect';
+
+import {SchedulingUnitCreate} from './create';
+
+import ScheduleService from '../../services/schedule.service';
+import ProjectService from '../../services/project.service';
+import TaskService from '../../services/task.service';
+
+import SUServiceMock from '../../__mocks__/scheduleunit.service.data';
+import ProjectServiceMock from '../../__mocks__/project.service.data';
+import TaskServiceMock from '../../__mocks__/task.service.data';
+
+
+
+let projectListSpy, scheduleSetListSpy, observStrategiesSpy, taskTemplatesSpy, saveSUFromStrategySpy, updateSUSpy, createSUTasksSpy;
+
+beforeEach(() => {
+ setMockSpy();
+});
+
+afterEach(() => {
+ // cleanup on exiting
+ clearMockSpy();
+ cleanup();
+});
+
+const setMockSpy = () => {
+ projectListSpy = jest.spyOn(ProjectService, 'getProjectList');
+ projectListSpy.mockImplementation(() => { return Promise.resolve(ProjectServiceMock.projectList)});
+ scheduleSetListSpy = jest.spyOn(ScheduleService, 'getSchedulingSets');
+ scheduleSetListSpy.mockImplementation(() => { return Promise.resolve(SUServiceMock.scheduleSetList)});
+ observStrategiesSpy = jest.spyOn(ScheduleService, 'getObservationStrategies');
+ observStrategiesSpy.mockImplementation(() => { return Promise.resolve(SUServiceMock.observStrategies)});
+ taskTemplatesSpy = jest.spyOn(TaskService, 'getTaskTemplates');
+ taskTemplatesSpy.mockImplementation(() => { return Promise.resolve(TaskServiceMock.taskTemplates)});
+ saveSUFromStrategySpy = jest.spyOn(ScheduleService, 'saveSUDraftFromObservStrategy');
+ saveSUFromStrategySpy.mockImplementation((observStrategy, schedulingUnit) => {
+ return Promise.resolve(SUServiceMock.schedulingUnitFromObservStrategy);
+ });
+ updateSUSpy = jest.spyOn(ScheduleService, 'updateSchedulingUnitDraft');
+ updateSUSpy.mockImplementation((schedulingUnit) => {
+ return Promise.resolve(SUServiceMock.schedulingUnitFromObservStrategy);
+ });
+ createSUTasksSpy = jest.spyOn(ScheduleService, 'createSUTaskDrafts');
+ createSUTasksSpy.mockImplementation((schedulingUnit) => {
+ return Promise.resolve(SUServiceMock.schedulingUnitFromObservStrategy);
+ });
+
+}
+
+const clearMockSpy = () => {
+ projectListSpy.mockRestore();
+ scheduleSetListSpy.mockRestore();
+ observStrategiesSpy.mockRestore();
+ taskTemplatesSpy.mockRestore();
+ saveSUFromStrategySpy.mockRestore();
+ updateSUSpy.mockRestore();
+ createSUTasksSpy.mockRestore();
+}
+
+it("renders create page with all fields and default values", async() => {
+ console.log("renders create page with all fields and default values ------------------------");
+
+ let content;
+ await act(async () => {
+ content = render(<Router><SchedulingUnitCreate /></Router>);
+ });
+
+ expect(content.queryByText('Scheduling Unit - Add')).not.toBe(null); // Page loaded successfully
+ expect(projectListSpy).toHaveBeenCalled(); // Mock Spy called successfully
+ expect(observStrategiesSpy).toHaveBeenCalled(); // Mock Spy called successfully
+ expect(scheduleSetListSpy).toHaveBeenCalled(); // Mock Spy called successfully
+ expect(taskTemplatesSpy).toHaveBeenCalled(); // Mock Spy called successfully
+ expect(content.queryByText('TMSS-Commissioning')).toBeInTheDocument(); // Project Dropdown loaded successfully
+ expect(content.queryByText('UC1 observation strategy template')).toBeInTheDocument(); // Observation Strategy Dropdown loaded successfully
+ expect(content.queryByText('Task Parameters')).not.toBeInTheDocument(); // JSON Editor not rendered
+ expect(content.queryByTestId('save-btn')).toHaveAttribute("disabled");
+});
+
+it("creates new Scheduling Unit with default values", async() => {
+ console.log("creates new Scheduling Unit with default values ------------------------");
+
+ let content;
+ await act(async () => {
+ content = render(<Router><SchedulingUnitCreate /></Router>);
+ });
+
+ const nameInput = content.queryByTestId('name');
+ const descInput = content.queryByTestId('description');
+ const projInput = content.getAllByRole("listbox")[0].children[2] ;
+ const observStrategyInput = content.getAllByRole("listbox")[2].children[0] ;
+
+ // Set values for all mandatory input and test if save button is enabled
+ fireEvent.change(nameInput, { target: { value: 'UC1 test scheduling unit 1.1' } });
+ expect(nameInput.value).toBe("UC1 test scheduling unit 1.1");
+ fireEvent.change(descInput, { target: { value: 'UC1 test scheduling unit 1.1' } });
+ expect(descInput.value).toBe("UC1 test scheduling unit 1.1");
+
+ // After selecting values for all dropdowns
+ await act(async () => {
+ fireEvent.click(projInput);
+ });
+ const schedulingSetInput = content.getAllByRole("listbox")[1].children[0] ;
+ expect(content.queryAllByText('Select Project').length).toBe(1);
+ expect(content.queryAllByText('TMSS-Commissioning').length).toBe(3);
+
+ await act(async () => {
+ fireEvent.click(schedulingSetInput);
+ });
+ expect(content.queryAllByText('Select Scheduling Set').length).toBe(1);
+ expect(content.queryAllByText('Test Scheduling Set UC1 example 0').length).toBe(3);
+
+ await act( async() => {
+ fireEvent.click(observStrategyInput);
+ });
+ expect(content.queryAllByText('Select Strategy').length).toBe(1);
+ expect(content.queryAllByText('UC1 observation strategy template').length).toBe(3);
+ expect(content.queryByText('Task Parameters')).toBeInTheDocument();
+ expect(content.queryByText('Target Pointing 0')).toBeInTheDocument();
+ expect(content.queryByText('Not a valid input. Mimimum: 00:00:00, Maximum:23:59:59.')).not.toBeInTheDocument();
+ expect(content.queryByText('Not a valid input. Mimimum: 00:00:00, Maximum:90:00:00.')).not.toBeInTheDocument();
+
+ /* This is set again to call the validateEditor function in the component.
+ If this is removed, the editor validation will not occur in the test but works in browser.*/
+ await act( async() => {
+ fireEvent.change(nameInput, { target: { value: 'UC1 test scheduling unit 1.1' } });
+ });
+
+ expect(content.queryByTestId('save-btn').hasAttribute("disabled")).toBeFalsy();
+
+ await act(async () => {
+ fireEvent.click(content.queryByTestId('save-btn'));
+ });
+ expect(saveSUFromStrategySpy).toHaveBeenCalled();
+
+});
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/edit.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/edit.js
new file mode 100644
index 0000000000000000000000000000000000000000..23c61c0f6fb0d6893913e3275699ea54224b2acb
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/edit.js
@@ -0,0 +1,395 @@
+import React, {Component} from 'react';
+import { Redirect } from 'react-router-dom';
+import _ from 'lodash';
+import $RefParser from "@apidevtools/json-schema-ref-parser";
+
+import {InputText} from 'primereact/inputtext';
+import {InputTextarea} from 'primereact/inputtextarea';
+import {Dropdown} from 'primereact/dropdown';
+import { Button } from 'primereact/button';
+import {Growl} from 'primereact/components/growl/Growl';
+
+
+import AppLoader from '../../layout/components/AppLoader';
+import PageHeader from '../../layout/components/PageHeader';
+import Jeditor from '../../components/JSONEditor/JEditor';
+
+import ProjectService from '../../services/project.service';
+import ScheduleService from '../../services/schedule.service';
+import TaskService from '../../services/task.service';
+import UIConstants from '../../utils/ui.constants';
+
+/**
+ * Compoenent to edit scheduling unit draft
+ */
+export class EditSchedulingUnit extends Component {
+ constructor(props) {
+ super(props);
+ this.state = {
+ isLoading: true,
+ dialog: { header: '', detail: ''},
+ redirect: null,
+ errors: [],
+ schedulingSets: [],
+ schedulingUnit: {
+ },
+ projectDisabled: (props.match?(props.match.params.project? true:false):false),
+ observStrategy: {},
+ paramsSchema: null,
+ validEditor: false,
+ validFields: {},
+ observStrategyVisible: false
+ }
+ this.projects = [];
+ this.schedulingSets = [];
+ this.observStrategies = [];
+ this.taskTemplates = [];
+ this.tooltipOptions = UIConstants.tooltipOptions;
+ this.nameInput = React.createRef();
+ this.formRules = {
+ name: {required: true, message: "Name can not be empty"},
+ description: {required: true, message: "Description can not be empty"},
+ };
+
+ this.setEditorOutput = this.setEditorOutput.bind(this);
+ this.changeStrategy = this.changeStrategy.bind(this);
+ this.setSchedUnitParams = this.setSchedUnitParams.bind(this);
+ this.validateForm = this.validateForm.bind(this);
+ this.validateEditor = this.validateEditor.bind(this);
+ this.setEditorFunction = this.setEditorFunction.bind(this);
+ this.saveSchedulingUnit = this.saveSchedulingUnit.bind(this);
+ this.cancelCreate = this.cancelCreate.bind(this);
+
+ }
+
+
+
+ /**
+ * Function called when observation strategy template is changed.
+ * It generates the JSON schema for JSON editor and defult vales for the parameters to be captured
+ * @param {number} strategyId
+ */
+ async changeStrategy (strategyId) {
+ let tasksToUpdate = {};
+ const observStrategy = _.find(this.observStrategies, {'id': strategyId});
+ const tasks = observStrategy.template.tasks;
+ let paramsOutput = {};
+ let schema = { type: 'object', additionalProperties: false,
+ properties: {}, definitions:{}
+ };
+ for (const taskName in tasks) {
+ const task = tasks[taskName];
+ const taskDraft = this.state.taskDrafts.find(taskD => taskD.name === taskName);
+ if (taskDraft) {
+ task.specifications_doc = taskDraft.specifications_doc;
+ }
+ //Resolve task from the strategy template
+ const $taskRefs = await $RefParser.resolve(task);
+
+ // Identify the task specification template of every task in the strategy template
+ const taskTemplate = _.find(this.taskTemplates, {'name': task['specifications_template']});
+ schema['$id'] = taskTemplate.schema['$id'];
+ schema['$schema'] = taskTemplate.schema['$schema'];
+ let index = 0;
+ for (const param of observStrategy.template.parameters) {
+ if (param.refs[0].indexOf(`/tasks/${taskName}`) > 0) {
+ tasksToUpdate[taskName] = taskName;
+ // Resolve the identified template
+ const $templateRefs = await $RefParser.resolve(taskTemplate);
+ let property = { };
+ let tempProperty = null;
+ const taskPaths = param.refs[0].split("/");
+ // Get the property type from the template and create new property in the schema for the parameters
+ try {
+ const parameterRef = param.refs[0];//.replace(`#/tasks/${taskName}/specifications_doc`, '#/schema/properties');
+ tempProperty = $templateRefs.get(parameterRef);
+ } catch(error) {
+ tempProperty = _.cloneDeep(taskTemplate.schema.properties[taskPaths[4]]);
+ if (tempProperty.type === 'array') {
+ tempProperty = tempProperty.items.properties[taskPaths[6]];
+ }
+ property = tempProperty;
+ }
+ property.title = param.name;
+ property.default = $taskRefs.get(param.refs[0].replace(`#/tasks/${taskName}`, '#'));
+ paramsOutput[`param_${index}`] = property.default;
+ schema.properties[`param_${index}`] = property;
+ // Set property defintions taken from the task template in new schema
+ for (const definitionName in taskTemplate.schema.definitions) {
+ schema.definitions[definitionName] = taskTemplate.schema.definitions[definitionName];
+ }
+ }
+ index++;
+ }
+ }
+ this.setState({observStrategy: observStrategy, paramsSchema: schema, paramsOutput: paramsOutput, tasksToUpdate: tasksToUpdate});
+
+ // Function called to clear the JSON Editor fields and reload with new schema
+ if (this.state.editorFunction) {
+ this.state.editorFunction();
+ }
+ }
+
+ componentDidMount() {
+ const promises = [ ProjectService.getProjectList(),
+ ScheduleService.getSchedulingSets(),
+ ScheduleService.getObservationStrategies(),
+ TaskService.getTaskTemplates(),
+ ScheduleService.getSchedulingUnitDraftById(this.props.match.params.id),
+ ScheduleService.getTasksDraftBySchedulingUnitId(this.props.match.params.id)
+ ];
+ Promise.all(promises).then(responses => {
+ this.projects = responses[0];
+ this.schedulingSets = responses[1];
+ this.observStrategies = responses[2];
+ this.taskTemplates = responses[3];
+ responses[4].project = this.schedulingSets.find(i => i.id === responses[4].scheduling_set_id).project_id;
+ this.setState({ schedulingUnit: responses[4], taskDrafts: responses[5].data.results,
+ observStrategyVisible: responses[4].observation_strategy_template_id?true:false });
+ if (responses[4].observation_strategy_template_id) {
+ this.changeStrategy(responses[4].observation_strategy_template_id);
+ }
+ if (this.state.schedulingUnit.project) {
+ const projectSchedSets = _.filter(this.schedulingSets, {'project_id': this.state.schedulingUnit.project});
+ this.setState({isLoading: false, schedulingSets: projectSchedSets});
+ } else {
+ this.setState({isLoading: false});
+ }
+ });
+ }
+
+ /**
+ * This is the callback method to be passed to the JSON editor.
+ * JEditor will call this function when there is change in the editor.
+ * @param {Object} jsonOutput
+ * @param {Array} errors
+ */
+ setEditorOutput(jsonOutput, errors) {
+ this.paramsOutput = jsonOutput;
+ this.validEditor = errors.length === 0;
+ this.setState({ paramsOutput: jsonOutput,
+ validEditor: errors.length === 0,
+ validForm: this.validateForm()});
+ }
+
+ /**
+ * This function is mainly added for Unit Tests. If this function is removed Unit Tests will fail.
+ */
+ validateEditor() {
+ return this.validEditor?true:false;
+ }
+
+ /**
+ * Function to set form values to the SU object
+ * @param {string} key
+ * @param {object} value
+ */
+ setSchedUnitParams(key, value) {
+ let schedulingUnit = this.state.schedulingUnit;
+ schedulingUnit[key] = value;
+ this.setState({schedulingUnit: schedulingUnit, validForm: this.validateForm(key), validEditor: this.validateEditor()});
+ this.validateEditor();
+ }
+
+ /**
+ * JEditor's function that to be called when parent wants to trigger change in the JSON Editor
+ * @param {Function} editorFunction
+ */
+ setEditorFunction(editorFunction) {
+ this.setState({editorFunction: editorFunction});
+ }
+
+ /**
+ * Validation function to validate the form or field based on the form rules.
+ * If no argument passed for fieldName, validates all fields in the form.
+ * @param {string} fieldName
+ */
+ validateForm(fieldName) {
+ let validForm = false;
+ let errors = this.state.errors;
+ let validFields = this.state.validFields;
+ if (fieldName) {
+ delete errors[fieldName];
+ delete validFields[fieldName];
+ if (this.formRules[fieldName]) {
+ const rule = this.formRules[fieldName];
+ const fieldValue = this.state.schedulingUnit[fieldName];
+ if (rule.required) {
+ if (!fieldValue) {
+ errors[fieldName] = rule.message?rule.message:`${fieldName} is required`;
+ } else {
+ validFields[fieldName] = true;
+ }
+ }
+ }
+ } else {
+ errors = {};
+ validFields = {};
+ for (const fieldName in this.formRules) {
+ const rule = this.formRules[fieldName];
+ const fieldValue = this.state.schedulingUnit[fieldName];
+ if (rule.required) {
+ if (!fieldValue) {
+ errors[fieldName] = rule.message?rule.message:`${fieldName} is required`;
+ } else {
+ validFields[fieldName] = true;
+ }
+ }
+ }
+ }
+ this.setState({errors: errors, validFields: validFields});
+ if (Object.keys(validFields).length === Object.keys(this.formRules).length) {
+ validForm = true;
+ }
+ return validForm;
+ }
+
+ /**
+ * Function to create Scheduling unit
+ */
+ async saveSchedulingUnit() {
+ if (this.state.schedulingUnit.observation_strategy_template_id) {
+ let observStrategy = _.cloneDeep(this.state.observStrategy);
+ const $refs = await $RefParser.resolve(observStrategy.template);
+ observStrategy.template.parameters.forEach(async(param, index) => {
+ $refs.set(observStrategy.template.parameters[index]['refs'][0], this.state.paramsOutput['param_' + index]);
+ });
+ const schedulingUnit = await ScheduleService.updateSUDraftFromObservStrategy(observStrategy, this.state.schedulingUnit, this.state.taskDrafts, this.state.tasksToUpdate);
+ if (schedulingUnit) {
+ // this.growl.show({severity: 'success', summary: 'Success', detail: 'Scheduling Unit and tasks edited successfully!'});
+ this.props.history.push({
+ pathname: `/schedulingunit/view/draft/${this.props.match.params.id}`,
+ });
+ } else {
+ this.growl.show({severity: 'error', summary: 'Error Occured', detail: 'Unable to update Scheduling Unit/Tasks'});
+ }
+ } else {
+ this.growl.show({severity: 'error', summary: 'Error Occured', detail: 'Template Missing.'});
+ }
+ }
+
+ /**
+ * Cancel SU creation and redirect
+ */
+ cancelCreate() {
+ this.props.history.goBack();
+ }
+
+ render() {
+ if (this.state.redirect) {
+ return <Redirect to={ {pathname: this.state.redirect} }></Redirect>
+ }
+
+ const schema = this.state.paramsSchema;
+
+ let jeditor = null;
+ if (schema) {
+ jeditor = React.createElement(Jeditor, {title: "Task Parameters",
+ schema: schema,
+ initValue: this.state.paramsOutput,
+ callback: this.setEditorOutput,
+ parentFunction: this.setEditorFunction
+ });
+ }
+ return (
+ <React.Fragment>
+ <Growl ref={el => (this.growl = el)} />
+ <PageHeader location={this.props.location} title={'Scheduling Unit - Edit'}
+ actions={[{icon: 'fa-window-close',link: this.props.history.goBack,title:'Click to Close Scheduling Unit View', props : { pathname: `/schedulingunit/view/draft/${this.props.match.params.id}`}}]}/>
+ { this.state.isLoading ? <AppLoader /> :
+ <>
+ <div>
+ <div className="p-fluid">
+ <div className="p-field p-grid">
+ <label htmlFor="schedUnitName" className="col-lg-2 col-md-2 col-sm-12">Name <span style={{color:'red'}}>*</span></label>
+ <div className="col-lg-3 col-md-3 col-sm-12">
+ <InputText className={this.state.errors.name ?'input-error':''} id="schedUnitName" data-testid="name"
+ tooltip="Enter name of the Scheduling Unit" tooltipOptions={this.tooltipOptions} maxLength="128"
+ ref={input => {this.nameInput = input;}}
+ value={this.state.schedulingUnit.name} autoFocus
+ onChange={(e) => this.setSchedUnitParams('name', e.target.value)}
+ onBlur={(e) => this.setSchedUnitParams('name', e.target.value)}/>
+ <label className={this.state.errors.name?"error":"info"}>
+ {this.state.errors.name ? this.state.errors.name : "Max 128 characters"}
+ </label>
+ </div>
+ <div className="col-lg-1 col-md-1 col-sm-12"></div>
+ <label htmlFor="description" className="col-lg-2 col-md-2 col-sm-12">Description <span style={{color:'red'}}>*</span></label>
+ <div className="col-lg-3 col-md-3 col-sm-12">
+ <InputTextarea className={this.state.errors.description ?'input-error':''} rows={3} cols={30}
+ tooltip="Longer description of the scheduling unit" tooltipOptions={this.tooltipOptions} maxLength="128"
+ data-testid="description" value={this.state.schedulingUnit.description}
+ onChange={(e) => this.setSchedUnitParams('description', e.target.value)}
+ onBlur={(e) => this.setSchedUnitParams('description', e.target.value)}/>
+ <label className={this.state.errors.description ?"error":"info"}>
+ {this.state.errors.description ? this.state.errors.description : "Max 255 characters"}
+ </label>
+ </div>
+ </div>
+ <div className="p-field p-grid">
+ <label htmlFor="project" className="col-lg-2 col-md-2 col-sm-12">Project </label>
+ <div className="col-lg-3 col-md-3 col-sm-12" data-testid="project" >
+ <Dropdown inputId="project" optionLabel="name" optionValue="name"
+ tooltip="Project" tooltipOptions={this.tooltipOptions}
+ value={this.state.schedulingUnit.project} disabled={this.state.schedulingUnit.project?true:false}
+ options={this.projects}
+ placeholder="Select Project" />
+ </div>
+ <div className="col-lg-1 col-md-1 col-sm-12"></div>
+ <label htmlFor="schedSet" className="col-lg-2 col-md-2 col-sm-12">Scheduling Set </label>
+ <div className="col-lg-3 col-md-3 col-sm-12">
+ <Dropdown data-testid="schedSet" id="schedSet" optionLabel="name" optionValue="id"
+ tooltip="Scheduling set of the project" tooltipOptions={this.tooltipOptions}
+ value={this.state.schedulingUnit.scheduling_set_id}
+ options={this.state.schedulingSets}
+ disabled={this.state.schedulingUnit.scheduling_set_id?true:false}
+ placeholder="Select Scheduling Set" />
+ </div>
+ </div>
+ <div className="p-field p-grid">
+ { this.state.observStrategyVisible &&
+ <>
+ <label htmlFor="observStrategy" className="col-lg-2 col-md-2 col-sm-12">Observation Strategy </label>
+ <div className="col-lg-3 col-md-3 col-sm-12" data-testid="observStrategy" >
+ <Dropdown inputId="observStrategy" optionLabel="name" optionValue="id"
+ tooltip="Observation Strategy Template to be used to create the Scheduling Unit and Tasks" tooltipOptions={this.tooltipOptions}
+ value={this.state.schedulingUnit.observation_strategy_template_id}
+ disabled={this.state.schedulingUnit.observation_strategy_template_id?true:false}
+ options={this.observStrategies}
+ onChange={(e) => {this.changeStrategy(e.value)}}
+ placeholder="Select Strategy" />
+ </div>
+ </>
+ }
+ <div className="col-lg-1 col-md-1 col-sm-12"></div>
+ </div>
+
+ </div>
+ <div className="p-fluid">
+ <div className="p-grid">
+ <div className="p-col-12">
+ {this.state.paramsSchema?jeditor:""}
+ </div>
+ </div>
+ </div>
+
+ <div className="p-grid p-justify-start">
+ <div className="p-col-1">
+ <Button label="Save" className="p-button-primary" icon="pi pi-check" onClick={this.saveSchedulingUnit}
+ disabled={!this.state.validEditor || !this.state.validForm} data-testid="save-btn" />
+ </div>
+ <div className="p-col-1">
+ <Button label="Cancel" className="p-button-danger" icon="pi pi-times" onClick={this.cancelCreate} />
+ </div>
+ </div>
+ </div>
+
+ </>
+ }
+
+ </React.Fragment>
+ );
+ }
+}
+
+export default EditSchedulingUnit;
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/index.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/index.js
index 062317b2127f42ca3115e39fba3964121f83f3c4..6d202556c02b04be61955e04696017798ce623ae 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/index.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/index.js
@@ -1,5 +1,6 @@
import React, {Component} from 'react';
import SchedulingUnitList from './SchedulingUnitList';
+import PageHeader from '../../layout/components/PageHeader';
export class Scheduling extends Component {
constructor(props){
@@ -14,7 +15,9 @@ export class Scheduling extends Component {
render() {
return (
<>
- <h2>Scheduling Unit - List</h2>
+ <PageHeader location={this.props.location} title={'Scheduling Unit - List'}
+ actions={[{icon: 'fa fa-plus-square', title: 'Add New Scheduling Unit',
+ props: {pathname: '/schedulingunit/create'}}]} />
{this.state.scheduleunit &&
<SchedulingUnitList /> }
</>
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Task/dataproduct.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Task/dataproduct.js
new file mode 100644
index 0000000000000000000000000000000000000000..62996534798c14247bf60e8e9064baea919f83a3
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Task/dataproduct.js
@@ -0,0 +1,133 @@
+import React, {Component} from 'react';
+import {Link} from 'react-router-dom'
+import AppLoader from '../../layout/components/AppLoader';
+import DataProductService from '../../services/data.product.service';
+import TaskService from '../../services/task.service';
+import ViewTable from './../../components/ViewTable';
+import UnitConverter from './../../utils/unit.converter'
+
+export class DataProduct extends Component{
+ constructor(props){
+ super(props);
+ this.state = {
+ isLoading: true,
+ dataproduct:[],
+ defaultcolumns: [ {
+ "type":"Type",
+ "filename":"File Name",
+ "fullpath":"File Path",
+ "storagelocation":"Storage Location",
+ "size":"Size (TB)",
+ "completed":"Completed %",
+ "deleted_since":"Deleted at",
+ }],
+ optionalcolumns: [{
+ }],
+ columnclassname: [{
+ "Type" : "filter-input-50", "Completed %" : "filter-input-50", "Size (TB)": "filter-input-50",
+ "Deleted at" : "filter-input-150","Storage Location" : "filter-input-125"
+ }],
+ defaultSortColumn: [{id: "File Name", desc: true}],
+ }
+
+ if (this.props.match.params.id) {
+ this.state.taskId = this.props.match.params.id;
+ this.state.taskType = 'blueprint';
+ }
+ }
+
+ componentDidMount(){
+ this.getDataProduct(this.state.taskId, this.state.taskType);
+ }
+
+ /*
+ Fetch Data Product for the Task, data are getting like below
+ */
+ async getDataProduct(id, type){
+ // Task type = blueprint
+ await TaskService.getTaskDetails(type, id).then(async taskBlueprint =>{
+ let subTaskIds = taskBlueprint['subtasks_ids'];
+ if(subTaskIds){
+ let dataproducts = [];
+ for(const id of subTaskIds){
+ let storageLocation = '';
+ await DataProductService.getSubtask(id).then( subtask =>{
+ storageLocation = subtask.data.cluster_value;
+ })
+ //Fetch data product for Input Subtask and Output Subtask
+ await DataProductService.getSubtaskInputDataproduct(id).then(async inputdata =>{
+ for(const dataproduct of inputdata.data){
+ dataproduct['type'] = 'Input';
+ dataproduct['size'] = UnitConverter.getUIResourceUnit('bytes', dataproduct['size']);
+ dataproduct['fullpath'] = dataproduct['directory'];
+ dataproduct['storagelocation'] = storageLocation;
+ dataproducts.push(dataproduct);
+ }
+ }).then(
+ await DataProductService.getSubtaskOutputDataproduct(id).then(outputdata =>{
+ for(const dataproduct of outputdata.data){
+ dataproduct['type'] = 'Output';
+ dataproduct['size'] = UnitConverter.getUIResourceUnit('bytes', dataproduct['size']);
+ dataproduct['fullpath'] = dataproduct['directory'];
+ dataproduct['storagelocation'] = storageLocation;
+ dataproducts.push(dataproduct);
+ }
+ })
+ )
+ }
+ this.setState({
+ dataproduct: dataproducts,
+ task: taskBlueprint,
+ isLoading: false,
+ })
+
+ }
+ })
+ }
+
+ render(){
+ return(
+ <React.Fragment>
+ <div className="p-grid">
+ <div className="p-col-10 p-lg-10 p-md-10">
+ <h2> Data Product - {this.state.task &&
+ <Link to={ { pathname:`/task/view/blueprint/${this.state.taskId}`}}> {this.state.task.name}</Link>
+ } </h2>
+ </div>
+ <div className="p-col-2">
+ {this.state.task &&
+ <Link to={{ pathname:`/task/view/blueprint/${this.state.taskId}`}} title="Close"
+ style={{float:'right'}}>
+ <i className="fa fa-times" style={{marginTop: "10px", marginLeft: '5px'}}></i>
+ </Link>
+ }
+ </div>
+ </div>
+
+ {this.state.isLoading? <AppLoader /> :
+ <React.Fragment>
+ {(!this.state.dataproduct || this.state.dataproduct.length === 0) &&
+ <div > No data found!</div>
+ }
+ {this.state.dataproduct.length>0 &&
+ <ViewTable
+ data={this.state.dataproduct}
+ defaultcolumns={this.state.defaultcolumns}
+ optionalcolumns={this.state.optionalcolumns}
+ columnclassname={this.state.columnclassname}
+ defaultSortColumn={this.state.defaultSortColumn}
+ showaction="false"
+ keyaccessor="id"
+ paths={this.state.paths}
+ defaultpagesize={this.state.dataproduct.length}
+ unittest={this.state.unittest}
+ />
+ }
+ </React.Fragment>
+
+ }
+ </React.Fragment>
+ )
+ }
+}
+
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Task/edit.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Task/edit.js
index 90ad6359847a5a18718b005ddb73608d9188a08a..0b3f2d234e0e0272736f2a234d91819bf5695137 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Task/edit.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Task/edit.js
@@ -12,6 +12,7 @@ import Jeditor from '../../components/JSONEditor/JEditor';
import TaskService from '../../services/task.service';
import AppLoader from "./../../layout/components/AppLoader";
+import PageHeader from '../../layout/components/PageHeader';
export class TaskEdit extends Component {
@@ -38,6 +39,7 @@ export class TaskEdit extends Component {
name: {required: true, message: "Name can not be empty"},
description: {required: true, message: "Description can not be empty"}
};
+ this.readOnlyProperties = ['duration', 'relative_start_time', 'relative_stop_time'];
this.setEditorOutput = this.setEditorOutput.bind(this);
this.setTaskParams = this.setTaskParams.bind(this);
this.changeTaskTemplate = this.changeTaskTemplate.bind(this);
@@ -126,16 +128,18 @@ export class TaskEdit extends Component {
saveTask() {
let task = this.state.task;
task.specifications_doc = this.templateOutput[task.specifications_template_id];
+ // Remove read only properties from the object before sending to API
+ this.readOnlyProperties.forEach(property => { delete task[property]});
TaskService.updateTask("draft", task)
.then( (taskDraft) => {
if (taskDraft) {
- this.setState({redirect: '/task/view'});
+ this.setState({redirect: '/task/view/draft/' + task.id});
}
});
}
cancelEdit() {
- this.setState({redirect: '/task/view'});
+ this.props.history.goBack();
}
componentDidMount() {
@@ -187,7 +191,7 @@ export class TaskEdit extends Component {
return (
<React.Fragment>
- <div className="p-grid">
+ {/*} <div className="p-grid">
<div className="p-col-10 p-lg-10 p-md-10">
<h2>Task - Edit</h2>
</div>
@@ -197,8 +201,8 @@ export class TaskEdit extends Component {
<i className="fa fa-window-close" style={{marginTop: "10px"}}></i>
</Link>
</div>
- </div>
-
+ </div> */}
+ <PageHeader location={this.props.location} title={'Task - Edit'} actions={[{icon: 'fa-window-close',link: this.props.history.goBack,title:'Click to Close Task Edit Page' ,props : { pathname: `/task/view/draft/${this.state.task?this.state.task.id:''}`}}]}/>
{isLoading ? <AppLoader/> :
<div>
<div className="p-fluid">
@@ -276,4 +280,4 @@ export class TaskEdit extends Component {
</React.Fragment>
);
}
-}
\ No newline at end of file
+}
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Task/index.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Task/index.js
index d7e2c03a6532dbb07064d9ca603f5796bbedd986..91955b294875ad02e7bba6314ccadeac920920f1 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Task/index.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Task/index.js
@@ -1,4 +1,5 @@
import {TaskEdit} from './edit';
import {TaskView} from './view';
+import {DataProduct} from './dataproduct';
-export {TaskEdit, TaskView} ;
+export {TaskEdit, TaskView, DataProduct} ;
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Task/view.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Task/view.js
index a8d612e25fed0eb865b57cd798a5ad8134e0d3a3..c89becf141598037f681283cfdf53c8291380b7f 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Task/view.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Task/view.js
@@ -1,12 +1,13 @@
import React, {Component} from 'react';
import {Link, Redirect} from 'react-router-dom'
import moment from 'moment';
-
+import _ from 'lodash';
import Jeditor from '../../components/JSONEditor/JEditor';
import TaskService from '../../services/task.service';
import { Chips } from 'primereact/chips';
import AppLoader from '../../layout/components/AppLoader';
+import PageHeader from '../../layout/components/PageHeader';
export class TaskView extends Component {
DATE_FORMAT = 'YYYY-MMM-DD HH:mm:ss';
@@ -81,7 +82,8 @@ export class TaskView extends Component {
if (task) {
TaskService.getSchedulingUnit(taskType, (taskType==='draft'?task.scheduling_unit_draft_id:task.scheduling_unit_blueprint_id))
.then((schedulingUnit) => {
- this.setState({schedulingUnit: schedulingUnit});
+ let path = _.join(['/schedulingunit','view',((this.state.taskType === "draft")?'draft':'blueprint'),schedulingUnit.id], '/');
+ this.setState({schedulingUnit: schedulingUnit, supath:path});
});
TaskService.getTaskTemplate(task.specifications_template_id)
.then((taskTemplate) => {
@@ -113,6 +115,21 @@ export class TaskView extends Component {
});
}
+ let actions = [ ];
+ if (this.state.taskType === 'draft') {
+ actions = [{ icon: 'fa-edit',
+ title:'Click to Edit Task',
+ props : { pathname:'/task/edit',
+ state: {taskId: this.state.task?this.state.task.id:''}
+ }
+ }];
+ } else {
+ actions = [{ icon: 'fa-lock',
+ title: 'Cannot edit blueprint'}];
+ }
+ actions.push({ icon: 'fa-window-close', link: this.props.history.goBack,
+ title:'Click to Close Task', props : { pathname:'/schedulingunit' }});
+
// Child component to render predecessors and successors list
const TaskRelationList = ({ list }) => (
<ul className="task-list">
@@ -126,7 +143,7 @@ export class TaskView extends Component {
);
return (
<React.Fragment>
- <div className="p-grid">
+ {/* <div className="p-grid">
<div className="p-col-10 p-lg-10 p-md-10">
<h2>Task - Details </h2>
</div>
@@ -147,7 +164,9 @@ export class TaskView extends Component {
<i className="fa fa-lock" style={{float:"right", marginTop: "10px"}}></i>
}
</div>
- </div>
+ </div> */}
+ <PageHeader location={this.props.location} title={'Task - View'}
+ actions={actions}/>
{ this.state.isLoading? <AppLoader /> : this.state.task &&
<React.Fragment>
<div className="main-content">
@@ -181,7 +200,7 @@ export class TaskView extends Component {
{this.state.schedulingUnit &&
<>
<label className="col-lg-2 col-md-2 col-sm-12">Scheduling Unit</label>
- <Link className="col-lg-4 col-md-4 col-sm-12" to={ { pathname:'/schedulingunit/view', state: {id: this.state.schedulingUnit.id}}}>{this.state.schedulingUnit?this.state.schedulingUnit.name:''}</Link>
+ <Link className="col-lg-4 col-md-4 col-sm-12" to={ { pathname:this.state.supath, state: {id: this.state.schedulingUnit.id}}}>{this.state.schedulingUnit?this.state.schedulingUnit.name:''}</Link>
</>}
</div>
<div className="p-grid">
@@ -204,10 +223,20 @@ export class TaskView extends Component {
}
{this.state.taskType === 'blueprint' &&
// <Link className="col-lg-4 col-md-4 col-sm-12" to={ { pathname:'/task/view', state: {id: this.state.task.draft_id, type: 'draft'}}}>{this.state.task.draftObject.name}</Link>
- <Link className="col-lg-4 col-md-4 col-sm-12" to={ { pathname:`/task/view/draft/${this.state.task.draft_id}`}}>{this.state.task.draftObject.name}</Link>
+ <Link to={ { pathname:`/task/view/draft/${this.state.task.draft_id}`}}>{this.state.task.draftObject.name}</Link>
}
</div>
</div>
+ {this.state.taskType === 'blueprint' &&
+ <div className="p-grid">
+ <label className="col-lg-2 col-md-2 col-sm-12">Data Product</label>
+ <div className="col-lg-4 col-md-4 col-sm-12">
+
+ <Link to={ { pathname:`/task/view/blueprint/${this.state.taskId}/dataproducts`}}> View Data Product</Link>
+ </div>
+
+ </div>
+ }
<div className="p-fluid">
<div className="p-grid"><div className="p-col-12">
{this.state.taskTemplate?jeditor:""}
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Timeline/index.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Timeline/index.js
new file mode 100644
index 0000000000000000000000000000000000000000..cbffd8667fca689b0585140a41d58b38985be123
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Timeline/index.js
@@ -0,0 +1,3 @@
+import {TimelineView} from './view';
+
+export {TimelineView} ;
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Timeline/view.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Timeline/view.js
new file mode 100644
index 0000000000000000000000000000000000000000..7b1577973c9b081ff5534cb129c013846e890b5b
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Timeline/view.js
@@ -0,0 +1,273 @@
+import React, {Component} from 'react';
+import { Redirect } from 'react-router-dom/cjs/react-router-dom.min';
+import moment from 'moment';
+import _ from 'lodash';
+
+// import SplitPane, { Pane } from 'react-split-pane';
+
+import AppLoader from '../../layout/components/AppLoader';
+import PageHeader from '../../layout/components/PageHeader';
+import Timeline from '../../components/Timeline';
+import ViewTable from '../../components/ViewTable';
+
+import ProjectService from '../../services/project.service';
+import ScheduleService from '../../services/schedule.service';
+import UtilService from '../../services/util.service';
+
+import UnitConverter from '../../utils/unit.converter';
+
+// Color constant for status
+const STATUS_COLORS = { "ERROR": "FF0000", "CANCELLED": "#00FF00", "DEFINED": "#00BCD4",
+ "SCHEDULABLE":"#0000FF", "SCHEDULED": "#abc", "OBSERVING": "#bcd",
+ "OBSERVED": "#cde", "PROCESSING": "#cddc39", "PROCESSED": "#fed",
+ "INGESTING": "#edc", "FINISHED": "#47d53d"};
+
+/**
+ * Scheduling Unit timeline view component to view SU List and timeline
+ */
+export class TimelineView extends Component {
+
+ constructor(props) {
+ super(props);
+ this.state = {
+ isLoading: true,
+ suBlueprints: [], // Scheduling Unit Blueprints
+ suDrafts: [], // Scheduling Unit Drafts
+ suBlueprintList: [], // SU Blueprints filtered to view
+ group:[], // Timeline group from scheduling unit draft name
+ items:[], // Timeline items from scheduling unit blueprints grouped by scheduling unit draft
+ isSUDetsVisible: false,
+ canExtendSUList: true,
+ canShrinkSUList: false,
+ selectedItem: null
+ }
+
+ this.onItemClick = this.onItemClick.bind(this);
+ this.closeSUDets = this.closeSUDets.bind(this);
+ this.dateRangeCallback = this.dateRangeCallback.bind(this);
+ this.resizeSUList = this.resizeSUList.bind(this);
+ }
+
+ async componentDidMount() {
+ // Fetch all details from server and prepare data to pass to timeline and table components
+ const promises = [ ProjectService.getProjectList(),
+ ScheduleService.getSchedulingUnitBlueprint(),
+ ScheduleService.getSchedulingUnitDraft(),
+ ScheduleService.getSchedulingSets(),
+ UtilService.getUTC()] ;
+ Promise.all(promises).then(responses => {
+ const projects = responses[0];
+ const suBlueprints = _.sortBy(responses[1].data.results, 'name');
+ const suDrafts = responses[2].data.results;
+ const suSets = responses[3]
+ const group = [], items = [];
+ const currentUTC = moment.utc(responses[4]);
+ const defaultStartTime = currentUTC.clone().add(-24, 'hours'); // Default start time, this should be updated if default view is changed.
+ const defaultEndTime = currentUTC.clone().add(24, 'hours'); // Default end time, this should be updated if default view is changed.
+ let suList = [];
+ for (const suDraft of suDrafts) {
+ const suSet = suSets.find((suSet) => { return suDraft.scheduling_set_id===suSet.id});
+ const project = projects.find((project) => { return suSet.project_id===project.name});
+ if (suDraft.scheduling_unit_blueprints.length > 0) {
+ for (const suBlueprintId of suDraft.scheduling_unit_blueprints_ids) {
+ const suBlueprint = _.find(suBlueprints, {'id': suBlueprintId});
+ suBlueprint['actionpath'] = `/schedulingunit/view/blueprint/${suBlueprintId}`;
+ suBlueprint.suDraft = suDraft;
+ suBlueprint.project = project;
+ suBlueprint.suSet = suSet;
+ suBlueprint.durationInSec = suBlueprint.duration;
+ suBlueprint.duration = UnitConverter.getSecsToHHmmss(suBlueprint.duration);
+ // Select only blueprints with start_time and stop_time in the default time limit
+ if (suBlueprint.start_time &&
+ (moment.utc(suBlueprint.start_time).isBetween(defaultStartTime, defaultEndTime) ||
+ moment.utc(suBlueprint.stop_time).isBetween(defaultStartTime, defaultEndTime))) {
+ items.push(this.getTimelineItem(suBlueprint));
+ if (!_.find(group, {'id': suDraft.id})) {
+ group.push({'id': suDraft.id, title: suDraft.name});
+ }
+ suList.push(suBlueprint);
+ }
+ }
+ }
+ }
+
+ this.setState({suBlueprints: suBlueprints, suDrafts: suDrafts, group: group, suSets: suSets,
+ projects: projects, suBlueprintList: suList,
+ items: items, currentUTC: currentUTC, isLoading: false});
+ });
+ }
+
+ /**
+ * Function to get/prepare Item object to be passed to Timeline component
+ * @param {Object} suBlueprint
+ */
+ getTimelineItem(suBlueprint) {
+ // Temporary for testing
+ const diffOfCurrAndStart = moment().diff(moment(suBlueprint.stop_time), 'seconds');
+ suBlueprint.status = diffOfCurrAndStart>=0?"FINISHED":"DEFINED";
+ let item = { id: suBlueprint.id,
+ group: suBlueprint.suDraft.id,
+ title: `${suBlueprint.project.name} - ${suBlueprint.suDraft.name} - ${(suBlueprint.durationInSec/3600).toFixed(2)}Hrs`,
+ project: suBlueprint.project.name,
+ name: suBlueprint.suDraft.name,
+ duration: suBlueprint.durationInSec?`${(suBlueprint.durationInSec/3600).toFixed(2)}Hrs`:"",
+ start_time: moment.utc(suBlueprint.start_time),
+ end_time: moment.utc(suBlueprint.stop_time),
+ bgColor: suBlueprint.status? STATUS_COLORS[suBlueprint.status.toUpperCase()]:"#2196f3",
+ selectedBgColor: suBlueprint.status? STATUS_COLORS[suBlueprint.status.toUpperCase()]:"#2196f3"};
+ return item;
+ }
+
+ /**
+ * Callback function to pass to Timeline component for item click.
+ * @param {Object} item
+ */
+ onItemClick(item) {
+ if (this.state.isSUDetsVisible && item.id===this.state.selectedItem.id) {
+ this.closeSUDets();
+ } else {
+ this.setState({selectedItem: item, isSUDetsVisible: true, canExtendSUList: false, canShrinkSUList:false});
+ }
+ }
+
+ /**
+ * Closes the SU details section
+ */
+ closeSUDets() {
+ this.setState({isSUDetsVisible: false, canExtendSUList: true, canShrinkSUList: false});
+ }
+
+ /**
+ * Callback function to pass to timeline component which is called on date range change to fetch new item and group records
+ * @param {moment} startTime
+ * @param {moment} endTime
+ */
+ dateRangeCallback(startTime, endTime) {
+ let suBlueprintList = [], group=[], items = [];
+ if (startTime && endTime) {
+ for (const suBlueprint of this.state.suBlueprints) {
+ if (moment.utc(suBlueprint.start_time).isBetween(startTime, endTime)
+ || moment.utc(suBlueprint.stop_time).isBetween(startTime, endTime)) {
+ suBlueprintList.push(suBlueprint);
+ items.push(this.getTimelineItem(suBlueprint));
+ if (!_.find(group, {'id': suBlueprint.suDraft.id})) {
+ group.push({'id': suBlueprint.suDraft.id, title: suBlueprint.suDraft.name});
+ }
+ }
+ }
+ } else {
+ suBlueprintList = _.clone(this.state.suBlueprints);
+ group = this.state.group;
+ items = this.state.items;
+ }
+ this.setState({suBlueprintList: _.filter(suBlueprintList, (suBlueprint) => {return suBlueprint.start_time!=null})});
+ // On range change close the Details pane
+ this.closeSUDets();
+ return {group: group, items: items};
+ }
+
+ /**
+ * Function called to shrink or expand the SU list section width
+ * @param {number} step - (-1) to shrink and (+1) to expand
+ */
+ resizeSUList(step) {
+ let canExtendSUList = this.state.canExtendSUList;
+ let canShrinkSUList = this.state.canShrinkSUList;
+ if (step === 1) {
+ // Can Extend when fully shrunk and still extendable
+ canExtendSUList = (!canShrinkSUList && canExtendSUList)?true:false;
+ canShrinkSUList = true;
+ } else {
+ // Can Shrink when fully extended and still shrinkable
+ canShrinkSUList = (canShrinkSUList && !canExtendSUList)?true:false;
+ canExtendSUList = true;
+ }
+ this.setState({canExtendSUList: canExtendSUList, canShrinkSUList: canShrinkSUList});
+ }
+
+ render() {
+ if (this.state.redirect) {
+ return <Redirect to={ {pathname: this.state.redirect} }></Redirect>
+ }
+ const isSUDetsVisible = this.state.isSUDetsVisible;
+ const canExtendSUList = this.state.canExtendSUList;
+ const canShrinkSUList = this.state.canShrinkSUList;
+ return (
+ <React.Fragment>
+ <PageHeader location={this.props.location} title={'Scheduling Units - Timeline View'} />
+ { this.state.isLoading ? <AppLoader /> :
+ <div className="p-grid">
+ {/* <SplitPane split="vertical" defaultSize={600} style={{height: 'auto'}} primary="second"> */}
+ {/* <div className={isSUDetsVisible || (canExtendSUList && !canShrinkSUList)?"resize-div-min col-lg-4 col-md-4 col-sm-12":((canExtendSUList && canShrinkSUList)?"resize-div-avg col-lg-5 col-md-5 col-sm-12":"resize-div-max col-lg-6 col-md-6 col-sm-12")}>
+ <button className="p-link resize-btn" disabled={!this.state.canExtendSUList}
+ onClick={(e)=> { this.resizeSUList(1)}}>
+ <i className="pi pi-step-forward"></i>
+ </button>
+ <button className="p-link resize-btn" disabled={!this.state.canShrinkSUList}
+ onClick={(e)=> { this.resizeSUList(-1)}}>
+ <i className="pi pi-step-backward"></i>
+ </button></div> */}
+ {/* SU List Panel */}
+ <div className={isSUDetsVisible || (canExtendSUList && !canShrinkSUList)?"col-lg-4 col-md-4 col-sm-12":((canExtendSUList && canShrinkSUList)?"col-lg-5 col-md-5 col-sm-12":"col-lg-6 col-md-6 col-sm-12")}
+ style={{position: "inherit", borderRight: "5px solid #efefef", paddingTop: "10px"}}>
+ <ViewTable
+ data={this.state.suBlueprintList}
+ defaultcolumns={[{name: "Name",
+ start_time:"Start Time", stop_time:"End Time"}]}
+ optionalcolumns={[{description: "Description", duration:"Duration (HH:mm:ss)", actionpath: "actionpath"}]}
+ columnclassname={[{"Start Time":"filter-input-50", "End Time":"filter-input-50",
+ "Duration (HH:mm:ss)" : "filter-input-50",}]}
+ defaultSortColumn= {[{id: "Name", desc: false}]}
+ showaction="true"
+ tablename="timeline_scheduleunit_list"
+ showTopTotal="false"
+ />
+ </div>
+ {/* Timeline Panel */}
+ <div className={isSUDetsVisible || (!canExtendSUList && canShrinkSUList)?"col-lg-6 col-md-6 col-sm-12":((canExtendSUList && canShrinkSUList)?"col-lg-7 col-md-7 col-sm-12":"col-lg-8 col-md-8 col-sm-12")}>
+ {/* Panel Resize buttons */}
+ <div className="resize-div">
+ <button className="p-link resize-btn" disabled={!this.state.canShrinkSUList}
+ title="Shrink List/Expand Timeline"
+ onClick={(e)=> { this.resizeSUList(-1)}}>
+ <i className="pi pi-step-backward"></i>
+ </button>
+ <button className="p-link resize-btn" disabled={!this.state.canExtendSUList}
+ title="Expandd List/Shrink Timeline"
+ onClick={(e)=> { this.resizeSUList(1)}}>
+ <i className="pi pi-step-forward"></i>
+ </button>
+ </div>
+ <Timeline ref={(tl)=>{this.timeline=tl}}
+ group={this.state.group}
+ items={this.state.items}
+ currentUTC={this.state.currentUTC}
+ rowHeight={30} itemClickCallback={this.onItemClick}
+ dateRangeCallback={this.dateRangeCallback}></Timeline>
+ </div>
+ {/* </SplitPane> */}
+ {/* Details Panel */}
+ {this.state.isSUDetsVisible &&
+ <div className="col-lg-2 col-md-2 col-sm-12"
+ style={{borderLeft: "1px solid #efefef", marginTop: "0px", backgroundColor: "#f2f2f2"}}>
+ <div className="p-grid" style={{marginTop: '10px'}}>
+ <h6 className="col-lg-10 col-sm-10">Details</h6>
+ <button className="p-link" onClick={this.closeSUDets}><i className="fa fa-times"></i></button>
+
+ <div className="col-12">
+ {this.state.selectedItem.title}
+ </div>
+
+ <div className="col-12">Still In Development</div>
+ </div>
+ </div>
+ }
+
+ </div>
+
+ }
+ </React.Fragment>
+ );
+ }
+
+}
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/index.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/index.js
index 0739e7824a404d7e1a21de18298cd809ae0dd79d..f13587989bebe2c3dbc2f796fcbf0c443d9b54fb 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/index.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/index.js
@@ -9,84 +9,136 @@ import {NotFound} from '../layout/components/NotFound';
import {ProjectList, ProjectCreate, ProjectView, ProjectEdit} from './Project';
import {Dashboard} from './Dashboard';
import {Scheduling} from './Scheduling';
-import {TaskEdit, TaskView} from './Task';
+import {TaskEdit, TaskView, DataProduct} from './Task';
import ViewSchedulingUnit from './Scheduling/ViewSchedulingUnit'
+import SchedulingUnitCreate from './Scheduling/create';
+import EditSchedulingUnit from './Scheduling/edit';
import { CycleList, CycleCreate, CycleView, CycleEdit } from './Cycle';
+import {TimelineView} from './Timeline';
export const routes = [
{
path: "/not-found",
- component: NotFound
+ component: NotFound,
+
},{
path: "/dashboard",
component: Dashboard,
- name: 'Dashboard'
+ name: 'Dashboard',
+ title: 'Dashboard'
},{
path: "/schedulingunit",
component: Scheduling,
- name: 'Scheduling Unit'
+ name: 'Scheduling Unit',
+ title: 'Scheduling Unit - List'
+ },{
+ path: "/schedulingunit/create",
+ component: SchedulingUnitCreate,
+ name: 'Scheduling Unit Add',
+ title: 'Scheduling Unit - Add'
},{
path: "/task",
component: TaskView,
- name: 'Task'
+ name: 'Task',
+ title: 'Task-View'
},{
path: "/task/view",
component: TaskView,
- name: 'Task'
+ name: 'Task',
+ title: 'Task View'
},{
path: "/task/view/:type/:id",
component: TaskView,
- name: 'Task Details'
+ name: 'Task Details',
+ title: 'Task Details'
},{
path: "/task/edit",
component: TaskEdit,
- name: 'Task Edit'
+ name: 'Task Edit',
+ title: 'Task-Edit'
},{
path: "/schedulingunit/view",
component: ViewSchedulingUnit,
+ name: 'Scheduling View',
+ title: 'Scheduling Unit - Details'
+ },{
+ path: "/schedulingunit/edit/:id",
+ component: EditSchedulingUnit,
+ name: 'Scheduling Edit',
+ title: 'Scheduling Unit - Edit'
+ },{
+ path: "/schedulingunit/view/:type/:id",
+ component: ViewSchedulingUnit,
name: 'Scheduling View'
},{
path: "/project",
component: ProjectList,
- name: 'Project List'
+ name: 'Project List',
+ title: 'Project - List'
},{
path: "/project/create",
component: ProjectCreate,
- name: 'Project Add'
+ name: 'Project Add',
+ title: 'Project - Add'
},{
path: "/project/view",
component: ProjectView,
- name: 'Project View'
+ name: 'Project View',
+ title: 'Project - Details '
},{
path: "/project/view/:id",
component: ProjectView,
- name: 'Project View'
- },{
+ name: 'Project View',
+ title: 'Project - View'
+ },
+ {
path: "/project/edit/:id",
component: ProjectEdit,
- name: 'Project Edit'
+ name: 'Project Edit',
+ title: 'Project Edit'
+ },{
+ path: "/project/:project/schedulingunit/create",
+ component: SchedulingUnitCreate,
+ name: 'Scheduling Unit Add',
+ title: 'Scheduling Unit - Add'
},{
path: "/cycle/edit/:id",
component: CycleEdit,
- name: 'Cycle Edit'
+ name: 'Cycle Edit',
+ title:'Cycle-Edit'
},{
path: "/cycle/view",
component: CycleView,
- name: 'Cycle View'
+ name: 'Cycle View',
+ title:'Cycle-View'
},{
path: "/cycle/view/:id",
component: CycleView,
- name: 'Cycle View'
+ name: 'Cycle View',
+ title:'Cycle-View'
}, {
path: "/cycle/create",
component: CycleCreate,
- name: 'Cycle Add'
+ name: 'Cycle Add',
+ title:'Cycle-Add'
},
{
path: "/cycle",
component: CycleList,
- name: 'Cycle List'
+ name: 'Cycle List',
+ title:'Cycle-List'
},
+ {
+ path: "/su/timelineview",
+ component: TimelineView,
+ name: 'Scheduling Unit Timeline',
+ title:'SU Timeline View'
+ },
+ {
+ path: "/task/view/blueprint/:id/dataproducts",
+ component: DataProduct,
+ name: 'Data Product'
+ }
];
export const RoutedContent = () => {
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/services/cycle.service.js b/SAS/TMSS/frontend/tmss_webapp/src/services/cycle.service.js
index f758c543f5a72a8096175dee6484822dc4a2fb48..4a8d7be0068238c5142a5eadb4f18daf81ebbd78 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/services/cycle.service.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/services/cycle.service.js
@@ -14,7 +14,8 @@ const CycleService = {
}
},
// Duplicate
- getCycleById: async function (id) {
+ // now renamed getCycleById to getProjectsByCycle
+ getProjectsByCycle: async function (id) {
try {
const url = `/api/cycle/${id}/project`;
const response = await axios.get(url);
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/services/data.product.service.js b/SAS/TMSS/frontend/tmss_webapp/src/services/data.product.service.js
new file mode 100644
index 0000000000000000000000000000000000000000..bfae26441c89541577dd08bd64ba82dfdf049d66
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/src/services/data.product.service.js
@@ -0,0 +1,48 @@
+const axios = require('axios');
+
+//axios.defaults.baseURL = 'http://192.168.99.100:8008/api';
+axios.defaults.headers.common['Authorization'] = 'Basic dGVzdDp0ZXN0';
+
+const DataProductService = {
+
+ getSubtaskInputDataproduct: async function(id){
+ try {
+ const url = `/api/subtask/${id}/input_dataproducts/`;
+ const response = axios.get(url);
+ return response;
+ } catch (error) {
+ console.error('[data.product.getSubtaskInputDataproduct]',error);
+ }
+ },
+ getSubtaskOutputDataproduct: async function(id){
+ try {
+ const url = `/api/subtask/${id}/output_dataproducts/`;
+ const response = axios.get(url);
+ return response;
+ } catch (error) {
+ console.error('[data.product.getSubtaskOutputDataproduct]',error);
+ }
+ },
+ getSubTaskTypes: async function(id){
+ try {
+ const url = `/api/subtask_template/${id}`;
+ const response = axios.get(url);
+ return response;
+ } catch (error) {
+ console.error('[data.product.getSubTaskTypes]',error);
+ }
+ },
+ getSubtask: async function(id){
+ try {
+ const url = `/api/subtask/${id}`;
+ const response = axios.get(url);
+ return response;
+ } catch (error) {
+ console.error('[data.product.getSubtask]',error);
+ }
+
+ }
+
+}
+
+export default DataProductService;
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/services/project.service.js b/SAS/TMSS/frontend/tmss_webapp/src/services/project.service.js
index 18598cc1c90e304931d4f4e55c1a4c39a4d6c69e..76ccb93de8442f9ea043f0d41f142f3a95b7e3c2 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/services/project.service.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/services/project.service.js
@@ -25,6 +25,24 @@ const ProjectService = {
console.error(error);
}
},
+ getFileSystem: async function(){
+ try {
+ const url = `/api/filesystem/`;
+ const response = await axios.get(url);
+ return response.data.results;
+ } catch (error) {
+ console.error(error);
+ }
+ },
+ getCluster:async function(){
+ try {
+ const url = `/api/cluster/`;
+ const response = await axios.get(url);
+ return response.data.results;
+ } catch (error) {
+ console.error(error);
+ }
+ },
getResources: async function() {
try {
// const url = `/api/resource_type/?ordering=name`;
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/services/project.services.js b/SAS/TMSS/frontend/tmss_webapp/src/services/project.services.js
new file mode 100644
index 0000000000000000000000000000000000000000..77483cb57de2f45e0c8c304c69c03a3146da534c
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/src/services/project.services.js
@@ -0,0 +1,162 @@
+import _ from 'lodash';
+
+
+const axios = require('axios');
+
+
+// axios.defaults.baseURL = 'http://localhost:3000/api';
+
+const ProjectServices = {
+
+ getFileSystem: async function(){
+ try {
+ const url = `/filesystem/`;
+ const response = await axios.get(url);
+ return [
+ {
+ "id": 1,
+ "url": "http://localhost:3000/api/filesystem/1",
+ "capacity": 3600000000000000,
+ "cluster": "http://localhost:3000/api/cluster/1",
+ "cluster_id": 1,
+ "created_at": "2020-08-26T08:40:24.880194",
+ "description": "",
+ "name": "LustreFS",
+ "tags": [],
+ "updated_at": "2020-08-26T08:40:24.880239"
+ },
+ {
+ "id": 2,
+ "url": "http://localhost:3000/api/filesystem/2",
+ "capacity": 360000000000000,
+ "cluster": "http://localhost:3000/api/cluster/1",
+ "cluster_id": 1,
+ "created_at": "2020-08-26T13:06:20.442543",
+ "description": "new storage",
+ "name": "Lofar Storage (SARA)",
+ "tags": [],
+ "updated_at": "2020-08-26T13:06:20.442591"
+ },
+ {
+ "id": 3,
+ "url": "http://localhost:3000/api/filesystem/3",
+ "capacity": 360000000000000,
+ "cluster": "http://localhost:3000/api/cluster/1",
+ "cluster_id": 1,
+ "created_at": "2020-08-26T13:06:21.060517",
+ "description": "new storage",
+ "name": "Lofar Test Storage (SARA)",
+ "tags": [],
+ "updated_at": "2020-08-26T13:06:21.060545"
+ },
+ {
+ "id": 4,
+ "url": "http://localhost:3000/api/filesystem/4",
+ "capacity": 360000000000000,
+ "cluster": "http://localhost:3000/api/cluster/1",
+ "cluster_id": 1,
+ "created_at": "2020-08-26T13:06:22.776714",
+ "description": "new storage",
+ "name": "Sara",
+ "tags": [],
+ "updated_at": "2020-08-26T13:06:22.776760"
+ },
+ {
+ "id": 5,
+ "url": "http://localhost:3000/api/filesystem/5",
+ "capacity": 360000000000000,
+ "cluster": "http://localhost:3000/api/cluster/1",
+ "cluster_id": 1,
+ "created_at": "2020-08-26T13:12:22.651907",
+ "description": "new storage",
+ "name": "Lofar Storage (Jülich)",
+ "tags": [],
+ "updated_at": "2020-08-26T13:12:22.651953"
+ },
+ {
+ "id": 6,
+ "url": "http://localhost:3000/api/filesystem/6",
+ "capacity": 360000000000000,
+ "cluster": "http://localhost:3000/api/cluster/1",
+ "cluster_id": 1,
+ "created_at": "2020-08-26T13:12:24.505652",
+ "description": "new storage",
+ "name": "Lofar User Disk Storage (SARA)",
+ "tags": [],
+ "updated_at": "2020-08-26T13:12:24.505701"
+ },
+ {
+ "id": 7,
+ "url": "http://localhost:3000/api/filesystem/1",
+ "capacity": 360000000000000,
+ "cluster": "http://localhost:3000/api/cluster/3",
+ "cluster_id": 3,
+ "created_at": "2020-08-26T13:12:24.505652",
+ "description": "new storage",
+ "name": "Lofar Storage (Poznan)",
+ "tags": [],
+ "updated_at": "2020-08-26T13:12:24.505701"
+ },
+ {
+ "id": 8,
+ "url": "http://localhost:3000/api/filesystem/1",
+ "capacity": 360000000000000,
+ "cluster": "http://localhost:3000/api/cluster/3",
+ "cluster_id": 3,
+ "created_at": "2020-08-26T13:12:24.505652",
+ "description": "new storage",
+ "name": "Lofar (Poznan)",
+ "tags": [],
+ "updated_at": "2020-08-26T13:12:24.505701"
+ }
+ ];
+ } catch (error) {
+ console.error(error);
+ }
+ },
+ getCluster:async function(){
+ try {
+ const url = `/cluster/`;
+ const response = await axios.get(url);
+ // return response.data.results;
+ return [{
+ "id": 1,
+ "url": "http://localhost:3000/api/cluster/1",
+ "created_at": "2020-08-26T08:40:24.876529",
+ "description": "",
+ "location": "CIT",
+ "name": "CEP4",
+ "tags": [],
+ "archieve_site":false,
+ "updated_at": "2020-08-26T08:40:24.876560"
+ },
+ {
+ "id": 2,
+ "url": "http://localhost:3000/api/cluster/2",
+ "created_at": "2020-08-26T08:40:24.876529",
+ "description": "",
+ "location": "CSK",
+ "name": "CEP4",
+ "tags": [],
+ "archieve_site":false,
+ "updated_at": "2020-08-26T08:40:24.876560"
+ },
+ {
+ "id": 3,
+ "url": "http://localhost:3000/api/cluster/3",
+ "created_at": "2020-08-26T08:40:24.876529",
+ "description": "",
+ "location": "CSK",
+ "name": "ABC",
+ "tags": [],
+ "archive_site":true,
+ "updated_at": "2020-08-26T08:40:24.876560"
+ }
+
+ ]
+ } catch (error) {
+ console.error(error);
+ }
+ },
+}
+export default ProjectServices;
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/services/schedule.service.js b/SAS/TMSS/frontend/tmss_webapp/src/services/schedule.service.js
index f09e1da9e877f7867ce4f2614cabadccf3a9f6aa..30a2c0db09bf506a44b47f156ba4f8b26ec2c3f6 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/services/schedule.service.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/services/schedule.service.js
@@ -1,5 +1,6 @@
import axios from 'axios'
-import _ from 'lodash';
+import moment from 'moment';
+import TaskService from './task.service';
axios.defaults.headers.common['Authorization'] = 'Basic dGVzdDp0ZXN0';
@@ -24,26 +25,72 @@ const ScheduleService = {
});
return res;
},
+ getSchedulingUnitBlueprintById: async function (id){
+ try {
+ const response = await axios.get('/api/scheduling_unit_blueprint/'+id);
+ let schedulingUnit = response.data;
+ if (schedulingUnit) {
+ const schedulingUnitDraft = await this.getSchedulingUnitDraftById(schedulingUnit.draft_id);
+ schedulingUnit.scheduling_set_id = schedulingUnitDraft.scheduling_set_id;
+ schedulingUnit.scheduling_set = schedulingUnitDraft.scheduling_set;
+ schedulingUnit.scheduling_set_object = schedulingUnitDraft.scheduling_set_object;
+ }
+ return schedulingUnit;
+ } catch(error) {
+ console.error(error);
+ return null;
+ }
+ },
getSchedulingUnitDraftById: async function (id){
+ try {
+ const schedulingUnit = (await axios.get('/api/scheduling_unit_draft/'+id)).data;
+ const schedulingSet = (await axios.get(`/api/scheduling_set/${schedulingUnit.scheduling_set_id}`)).data;
+ schedulingUnit.scheduling_set_object = schedulingSet;
+ return schedulingUnit;
+ } catch(error){
+ console.error('[schedule.services.getSchedulingUnitDraftById]',error);
+ return null;
+ }
+ },
+ getTaskBlueprintById: async function(id){
let res = [];
- await axios.get('/api/scheduling_unit_draft/'+id)
+ await axios.get('/api/task_blueprint/'+id)
.then(response => {
res= response;
}).catch(function(error) {
- console.error('[schedule.services.getSchedulingUnitDraftById]',error);
+ console.error('[schedule.services.getTaskBlueprintById]',error);
});
return res;
},
- getScheduleTasksBySchedulingUnitId: async function(id){
+ getTaskBlueprintsBySchedulingUnit: async function(scheduleunit){
+ // there no single api to fetch associated task_blueprint, so iteare the task_blueprint id to fetch associated task_blueprint
+ let taskblueprintsList = [];
+ if(scheduleunit.task_blueprints_ids){
+ for(const id of scheduleunit.task_blueprints_ids){
+ await this.getTaskBlueprintById(id).then(response =>{
+ let taskblueprint = response.data;
+ taskblueprint['tasktype'] = 'Blueprint';
+ taskblueprint['actionpath'] = '/task/view/blueprint/'+taskblueprint['id'];
+ taskblueprint['blueprint_draft'] = taskblueprint['draft'];
+ taskblueprint['relative_start_time'] = 0;
+ taskblueprint['relative_stop_time'] = 0;
+ taskblueprint.duration = moment.utc(taskblueprint.duration*1000).format('HH:mm:ss');
+ taskblueprintsList.push(taskblueprint);
+ })
+ }
+ }
+ return taskblueprintsList;
+ },
+ getTasksBySchedulingUnit: async function(id){
let scheduletasklist=[];
- let taskblueprints = [];
+ // let taskblueprints = [];
// Common keys for Task and Blueprint
let commonkeys = ['id','created_at','description','name','tags','updated_at','url','do_cancel','relative_start_time','relative_stop_time','start_time','stop_time','duration'];
- await this.getTaskBlueprints().then( blueprints =>{
- taskblueprints = blueprints.data.results;
- })
+ // await this.getTaskBlueprints().then( blueprints =>{
+ // taskblueprints = blueprints.data.results;
+ // });
await this.getTasksDraftBySchedulingUnitId(id)
- .then(response =>{
+ .then(async(response) =>{
for(const task of response.data.results){
let scheduletask = [];
scheduletask['tasktype'] = 'Draft';
@@ -54,13 +101,16 @@ const ScheduleService = {
for(const key of commonkeys){
scheduletask[key] = task[key];
}
-
+ scheduletask.duration = moment.utc(scheduletask.duration*1000).format('HH:mm:ss');
+ scheduletask.relative_start_time = moment.utc(scheduletask.relative_start_time*1000).format('HH:mm:ss');
+ scheduletask.relative_stop_time = moment.utc(scheduletask.relative_stop_time*1000).format('HH:mm:ss');
//Fetch blueprint details for Task Draft
- let filteredblueprints = _.filter(taskblueprints, function(o) {
- if (o.draft_id === task['id']) return o;
- });
+ const draftBlueprints = await TaskService.getDraftsTaskBlueprints(task.id);
+ // let filteredblueprints = _.filter(taskblueprints, function(o) {
+ // if (o.draft_id === task['id']) return o;
+ // });
- for(const blueprint of filteredblueprints){
+ for(const blueprint of draftBlueprints){
let taskblueprint = [];
taskblueprint['tasktype'] = 'Blueprint';
taskblueprint['actionpath'] = '/task/view/blueprint/'+blueprint['id'];
@@ -68,6 +118,10 @@ const ScheduleService = {
for(const key of commonkeys){
taskblueprint[key] = blueprint[key];
}
+ taskblueprint.duration = moment.utc(taskblueprint.duration*1000).format('HH:mm:ss');
+ taskblueprint.relative_start_time = moment.utc(taskblueprint.relative_start_time*1000).format('HH:mm:ss');
+ taskblueprint.relative_stop_time = moment.utc(taskblueprint.relative_stop_time*1000).format('HH:mm:ss');
+
//Add Blue print details to array
scheduletasklist.push(taskblueprint);
}
@@ -119,6 +173,134 @@ const ScheduleService = {
});
return res;
},
+ getSchedulingSets: async function() {
+ try {
+ const response = await axios.get('/api/scheduling_set/');
+ return response.data.results;
+ } catch(error) {
+ console.error(error);
+ return [];
+ };
+ },
+ getObservationStrategies: async function() {
+ try {
+ const response = await axios.get('/api/scheduling_unit_observing_strategy_template/');
+ return response.data.results;
+ } catch(error) {
+ console.error(error);
+ return [];
+ };
+ },
+ saveSUDraftFromObservStrategy: async function(observStrategy, schedulingUnit) {
+ try {
+ // Create the scheduling unit draft with observation strategy and scheduling set
+ const url = `/api/scheduling_unit_observing_strategy_template/${observStrategy.id}/create_scheduling_unit/?scheduling_set_id=${schedulingUnit.scheduling_set_id}&name=${schedulingUnit.name}&description=${schedulingUnit.description}`
+ const suObsResponse = await axios.get(url);
+ schedulingUnit = suObsResponse.data;
+ if (schedulingUnit && schedulingUnit.id) {
+ // Update the newly created SU draft requirement_doc with captured parameter values
+ schedulingUnit.requirements_doc = observStrategy.template;
+ delete schedulingUnit['duration'];
+ schedulingUnit = await this.updateSchedulingUnitDraft(schedulingUnit);
+ if (!schedulingUnit || !schedulingUnit.id) {
+ return null;
+ }
+ // Create task drafts with updated requirement_doc
+ schedulingUnit = await this.createSUTaskDrafts(schedulingUnit);
+ if (schedulingUnit && schedulingUnit.task_drafts.length > 0) {
+ return schedulingUnit;
+ }
+ }
+ return null;
+ } catch(error) {
+ console.error(error);
+ return null;
+ };
+ },
+
+ updateSUDraftFromObservStrategy: async function(observStrategy, schedulingUnit,tasks,tasksToUpdate) {
+ try {
+ delete schedulingUnit['duration'];
+ schedulingUnit = await this.updateSchedulingUnitDraft(schedulingUnit);
+ for (const taskToUpdate in tasksToUpdate) {
+ let task = tasks.find(task => { return task.name === taskToUpdate});
+ task.specifications_doc = observStrategy.template.tasks[taskToUpdate].specifications_doc;
+ delete task['duration'];
+ delete task['relative_start_time'];
+ delete task['relative_stop_time'];
+ task = await TaskService.updateTask('draft', task);
+ }
+ return schedulingUnit;
+ } catch(error) {
+ console.error(error);
+ return null;
+ };
+ },
+ updateSchedulingUnitDraft: async function(schedulingUnit) {
+ try {
+ console.log(schedulingUnit);
+ const suUpdateResponse = await axios.put(`/api/scheduling_unit_draft/${schedulingUnit.id}/`, schedulingUnit);
+ return suUpdateResponse.data;
+ } catch(error) {
+ console.error("Mistake",error);
+ return null
+ }
+ },
+ createSUTaskDrafts: async (schedulingUnit) => {
+ try {
+ const suCreateTaskResponse = await axios.get(`/api/scheduling_unit_draft/${schedulingUnit.id}/create_task_drafts/`);
+ return suCreateTaskResponse.data;
+ } catch(error) {
+ console.error(error);
+ return null;
+ }
+ },
+ getSchedulingListByProject: async function(project){
+ /*
+ SU - Schedulign Unit
+ Get Scheduling Unit Draft and It's Blueprints using Project ID. there is no direct API to get SU form project (API request -TMSS-349)
+ Use Fetch all Scheduling Set and filter Scheduling Set with Project ID => Get SU Draft list and SU Blueprints
+ */
+ try {
+ let schedulingunitlist = [];
+ //Fetch all Scheduling Set as there is no API to fetch Scheduling Set for a Project
+ await this.getSchedulingSets().then(async schedulingsetlist =>{
+ let schedulingsets = schedulingsetlist.filter(scheduingset => scheduingset.project_id === project)
+ for(const scheduleset of schedulingsets){
+ //Fecth SU Drafts for the Scheduling Set
+ await this.getSchedulingBySet(scheduleset.id).then(async suDraftList =>{
+ for(const suDraft of suDraftList){
+ suDraft['actionpath']='/schedulingunit/view/draft/'+suDraft.id;
+ suDraft['type'] = 'Draft';
+ suDraft['duration'] = moment.utc(suDraft.duration*1000).format('HH:mm:ss');
+ schedulingunitlist = schedulingunitlist.concat(suDraft);
+ //Fetch SU Blue prints for the SU Draft
+ await this.getBlueprintsByschedulingUnitId(suDraft.id).then(suBlueprintList =>{
+ for(const suBlueprint of suBlueprintList.data.results){
+ suBlueprint.duration = moment.utc(suBlueprint.duration*1000).format('HH:mm:ss');
+ suBlueprint.type="Blueprint";
+ suBlueprint['actionpath'] = '/schedulingunit/view/blueprint/'+suBlueprint.id;
+ schedulingunitlist = schedulingunitlist.concat(suBlueprint);
+ }
+ })
+ }
+ })
+ }
+ })
+ return schedulingunitlist;
+ } catch (error) {
+ console.error('[project.services.getSchedulingListByProject]',error);
+ }
+ },
+ getSchedulingBySet: async function(id){
+ try{
+ const response = await axios.get(`/api/scheduling_set/${id}/scheduling_unit_draft/?ordering=id`);
+ return response.data.results;
+ } catch (error) {
+ console.error('[project.services.getSchedulingUnitBySet]',error);
+ }
+ }
}
-export default ScheduleService;
\ No newline at end of file
+
+export default ScheduleService;
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/services/task.service.js b/SAS/TMSS/frontend/tmss_webapp/src/services/task.service.js
index e55f34d594e5c81fd4aca1d819e9def4820155ac..9acbdb40c6adc69fab4c24e2c1221ebdb0a71b8f 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/services/task.service.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/services/task.service.js
@@ -49,7 +49,8 @@ const TaskService = {
},
getSchedulingUnit: async function(type, id) {
try {
- const response = await axios.get('/api/scheduling_unit_draft/' + id);
+ const url = `/api/scheduling_unit_${type}/${id}`;
+ const response = await axios.get(url);
return response.data;
} catch (error) {
console.error(error);
@@ -119,6 +120,14 @@ const TaskService = {
console.error(error);
}
},
+ getTaskTemplateSchemaResolved: async function(templateId) {
+ try {
+ const response = await axios.get('/api/task_template/' + templateId + '/ref_resolved_schema' );
+ return response.data;
+ } catch (error) {
+ console.log(error);
+ }
+ },
getDraftsTaskBlueprints: async function(id) {
try {
const url = `/api/task_draft/${id}/task_blueprint`;
@@ -140,4 +149,4 @@ const TaskService = {
}
-export default TaskService;
\ No newline at end of file
+export default TaskService;
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/services/util.service.js b/SAS/TMSS/frontend/tmss_webapp/src/services/util.service.js
new file mode 100644
index 0000000000000000000000000000000000000000..4d0d81cba4207dafe52925d3e049a92c1e946426
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/src/services/util.service.js
@@ -0,0 +1,43 @@
+const axios = require('axios');
+
+axios.defaults.headers.common['Authorization'] = 'Basic dGVzdDp0ZXN0';
+/**
+ * Utility Service to fetch miscellaneous data from the server
+ */
+const UtilService = {
+ /** Function to fetch current UTC time from the server which is the System Clock */
+ getUTC: async function() {
+ try {
+ const url = `/api/util/utc`;
+ const response = await axios.get(url);
+ return response.data;
+ } catch (error) {
+ console.error(error);
+ }
+ },
+ /** Function to fetch the equivalent LST (Local Sidereal Time) for a UTC timestamp.
+ * 'timestamp' should be a string in "YYYY-MM-DDTHH:mm:ss" format.
+ * Data fetched is stored in local storage to avoid repeated request to server for the same value.*/
+ getLST: async (timestamp) => {
+ try {
+ let localUtcLstMap = localStorage.getItem('UTC_LST_MAP');
+ if (localUtcLstMap) {
+ localUtcLstMap = JSON.parse(localUtcLstMap);
+ if (localUtcLstMap[timestamp]) {
+ return Promise.resolve(localUtcLstMap[timestamp]);
+ }
+ }
+ localUtcLstMap = localUtcLstMap?localUtcLstMap:{};
+ const url = `/api/util/lst?timestamp=${timestamp}`;
+ const response = await axios.get(url);
+ const utcToLST = response.data.replace('h',':').replace('m',':').replace('s','');
+ localUtcLstMap[timestamp] = utcToLST;
+ localStorage.setItem('UTC_LST_MAP', JSON.stringify(localUtcLstMap));
+ return utcToLST;
+ } catch(error) {
+ console.error(error);
+ }
+ }
+}
+
+export default UtilService;
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/utils/unit.converter.js b/SAS/TMSS/frontend/tmss_webapp/src/utils/unit.converter.js
index 4a4e3cee4bb33b08618894fb8611bf0b99989b5e..0d1d9e440c37a654c51966d0a702e59dbf33a462 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/utils/unit.converter.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/utils/unit.converter.js
@@ -18,6 +18,15 @@ const UnitConverter = {
console.error('[unit.converter.getUIResourceUnit]',error);
}
return value;
+ },
+ getSecsToHHmmss: function(seconds) {
+ if (seconds) {
+ const hh = Math.floor(seconds/3600);
+ const mm = Math.floor((seconds - hh*3600) / 60 );
+ const ss = +((seconds -(hh*3600)-(mm*60)) / 1);
+ return (hh<10?`0${hh}`:`${hh}`) + ':' + (mm<10?`0${mm}`:`${mm}`) + ':' + (ss<10?`0${ss}`:`${ss}`);
+ }
+ return seconds;
}
};
diff --git a/SAS/TMSS/services/CMakeLists.txt b/SAS/TMSS/services/CMakeLists.txt
index 5530d7230557be342223c4cc311dcbd1f9ee68e7..2a0779696572e29b2f0ebc6186221b165de654de 100644
--- a/SAS/TMSS/services/CMakeLists.txt
+++ b/SAS/TMSS/services/CMakeLists.txt
@@ -1,2 +1,2 @@
lofar_add_package(TMSSSchedulingService scheduling)
-
+lofar_add_package(TMSSFeedbackHandlingService feedback_handling)
diff --git a/SAS/TMSS/services/feedback_handling/CMakeLists.txt b/SAS/TMSS/services/feedback_handling/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..af48000ed59e13275a9709b0a9ce4bc2c423fd2c
--- /dev/null
+++ b/SAS/TMSS/services/feedback_handling/CMakeLists.txt
@@ -0,0 +1,8 @@
+lofar_package(TMSSFeedbackHandlingService 0.1 DEPENDS TMSSClient PyCommon pyparameterset PyMessaging)
+
+lofar_find_package(PythonInterp 3.4 REQUIRED)
+
+add_subdirectory(lib)
+add_subdirectory(bin)
+add_subdirectory(test)
+
diff --git a/SAS/TMSS/services/feedback_handling/bin/CMakeLists.txt b/SAS/TMSS/services/feedback_handling/bin/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..bdadbae5ab943b9aa965cae462d214c59ca40062
--- /dev/null
+++ b/SAS/TMSS/services/feedback_handling/bin/CMakeLists.txt
@@ -0,0 +1,4 @@
+lofar_add_bin_scripts(tmss_feedback_handling_service)
+
+# supervisord config files
+lofar_add_sysconf_files(tmss_feedback_handling_service.ini DESTINATION supervisord.d)
diff --git a/SAS/TMSS/services/feedback_handling/bin/tmss_feedback_handling_service b/SAS/TMSS/services/feedback_handling/bin/tmss_feedback_handling_service
new file mode 100755
index 0000000000000000000000000000000000000000..2ecd686a25fd88e45094bf4cda143e41de1fb61d
--- /dev/null
+++ b/SAS/TMSS/services/feedback_handling/bin/tmss_feedback_handling_service
@@ -0,0 +1,24 @@
+#!/usr/bin/python3
+
+# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+
+
+from lofar.sas.tmss.services.subtask_scheduling import main
+
+if __name__ == "__main__":
+ main()
diff --git a/SAS/TMSS/services/feedback_handling/bin/tmss_feedback_handling_service.ini b/SAS/TMSS/services/feedback_handling/bin/tmss_feedback_handling_service.ini
new file mode 100644
index 0000000000000000000000000000000000000000..e43c0d3e66f4534b32c6d6129397a0309a2b95e7
--- /dev/null
+++ b/SAS/TMSS/services/feedback_handling/bin/tmss_feedback_handling_service.ini
@@ -0,0 +1,9 @@
+[program:tmss_subtask_scheduling_service]
+command=/bin/bash -c 'source $LOFARROOT/lofarinit.sh;exec tmss_subtask_scheduling_service'
+user=lofarsys
+stopsignal=INT ; KeyboardInterrupt
+stopasgroup=true ; bash does not propagate signals
+stdout_logfile=%(program_name)s.log
+redirect_stderr=true
+stderr_logfile=NONE
+stdout_logfile_maxbytes=0
diff --git a/SAS/TMSS/services/feedback_handling/lib/CMakeLists.txt b/SAS/TMSS/services/feedback_handling/lib/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..e909ec0c8249d955afb078bf514786548047ba19
--- /dev/null
+++ b/SAS/TMSS/services/feedback_handling/lib/CMakeLists.txt
@@ -0,0 +1,10 @@
+lofar_find_package(PythonInterp 3.4 REQUIRED)
+include(PythonInstall)
+
+set(_py_files
+ feedback_handling.py
+ )
+
+python_install(${_py_files}
+ DESTINATION lofar/sas/tmss/services)
+
diff --git a/SAS/TMSS/services/feedback_handling/lib/feedback_handling.py b/SAS/TMSS/services/feedback_handling/lib/feedback_handling.py
new file mode 100644
index 0000000000000000000000000000000000000000..b5d3ccb180607fccf24738f8d5515665c782819d
--- /dev/null
+++ b/SAS/TMSS/services/feedback_handling/lib/feedback_handling.py
@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+
+# subtask_scheduling.py
+#
+# Copyright (C) 2015
+# ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it
+# and/or modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be
+# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+#
+# $Id: subtask_scheduling.py 1580 2015-09-30 14:18:57Z loose $
+
+"""
+The subtask_scheduling service schedules TMSS subtasks.
+It listens on the lofar notification message bus for state changes of TMSS subtasks; when a task finished,
+it schedules (rest action) all successors that are in state 'defined'.
+"""
+
+import os
+import logging
+import threading
+
+logger = logging.getLogger(__name__)
+
+from lofar.messagebus.messagebus import FromBus, broker_feedback
+from lofar.sas.tmss.client.tmss_http_rest_client import TMSSsession
+
+class TMSSFeedbackListener:
+ stop_listening = False
+ # Note we can't use fancy bus listeners since we have to use the really old messagebus package for feedback
+ # todo: get rid of old-style messaging or improve this service stub
+ threads = []
+ exchanges = ["otdb.task.feedback.processing", "otdb.task.feedback.dataproducts"]
+
+ def append_feedback_to_tmss_subtask_raw_feedback(self, subtask_id: int, raw_feedback: str):
+ logger.info('Appending feedback to TMSS subtask %s' % subtask_id)
+ with TMSSsession.create_from_dbcreds_for_ldap() as session:
+ session.append_to_subtask_raw_feedback(subtask_id, raw_feedback)
+
+ def process_subtask_feedback_and_set_finished(self, subtask_id: int):
+ logger.info('Calling TMSS to process feedback of subtask %s' % subtask_id)
+ with TMSSsession.create_from_dbcreds_for_ldap() as session:
+ session.process_subtask_feedback_and_set_finished(subtask_id)
+
+ def start_handling(self):
+
+ def listen(ex):
+ fbus = FromBus(ex, broker=broker_feedback)
+ logger.info('Start listening on exchange=%s broker=%s' % (ex, broker_feedback))
+ while not self.stop_listening:
+ try:
+ # get message from messagebus
+ msg = fbus.get(1)
+ # add contained feedback to TMSS
+ self.append_feedback_to_tmss_subtask_raw_feedback(msg.momid, msg.payload)
+ # try processing it, which will will fail until feedback of the subtask is complete.
+ self.process_subtask_feedback_and_set_finished(msg.momid)
+ except TimeoutError:
+ pass
+ logger.info('Stopped listening on exchange=%s broker=%s' % (ex, broker_feedback))
+
+ for exchange in self.exchanges:
+ thread = threading.Thread(target=listen, name=exchange, args=(exchange,))
+ thread.start()
+ self.threads.append(thread)
+
+ def stop_handling(self):
+ self.stop_listening = True
+ while self.threads:
+ thread = self.threads.pop()
+ thread.join(5)
+
+if __name__ == '__main__':
+ TMSSFeedbackListener().start_handling()
diff --git a/SAS/TMSS/services/feedback_handling/test/CMakeLists.txt b/SAS/TMSS/services/feedback_handling/test/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1ae9bd6d30c48c5382e82966f486f8a914e5930c
--- /dev/null
+++ b/SAS/TMSS/services/feedback_handling/test/CMakeLists.txt
@@ -0,0 +1,7 @@
+# $Id: CMakeLists.txt 32679 2015-10-26 09:31:56Z schaap $
+
+if(BUILD_TESTING)
+ include(LofarCTest)
+
+ lofar_add_test(t_feedback_handling_service)
+endif()
diff --git a/SAS/TMSS/services/feedback_handling/test/t_feedback_handling_service.py b/SAS/TMSS/services/feedback_handling/test/t_feedback_handling_service.py
new file mode 100755
index 0000000000000000000000000000000000000000..b3fe3a8361596c851e50b5a364d663532c2ef03b
--- /dev/null
+++ b/SAS/TMSS/services/feedback_handling/test/t_feedback_handling_service.py
@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+
+# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+
+import unittest
+from time import sleep
+import datetime
+
+import logging
+logger = logging.getLogger(__name__)
+
+from lofar.sas.tmss.test.test_utils import TMSSTestEnvironment
+from lofar.sas.tmss.test.tmss_test_environment_unittest_setup import *
+from lofar.sas.tmss.test.tmss_test_data_rest import TMSSRESTTestDataCreator
+
+from lofar.sas.tmss.services.feedback_handling import TMSSFeedbackListener
+from lofar.common.test_utils import integration_test
+from lofar.messagebus.messagebus import broker_feedback, ToBus
+from lofar.messagebus.protocols import TaskFeedbackProcessing, TaskFeedbackDataproducts
+
+@integration_test
+class TestFeedbackHandlingService(unittest.TestCase):
+ '''
+ Tests for the FeedbackHandlingService
+ '''
+
+ feedback_1 = """feedback_version=03.01.00
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[0].centralFrequency=33593750.0
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[0].channelsPerSubband=32
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[0].channelWidth=6103.515625"""
+
+ feedback_2 = """Observation.Correlator.channelWidth=3051.7578125
+Observation.Correlator.channelsPerSubband=64
+Observation.Correlator.integrationInterval=1.00663296
+Observation.DataProducts.Output_Correlated_[0].SAP=0
+Observation.DataProducts.Output_Correlated_[0].centralFrequency=30468750.000000
+Observation.DataProducts.Output_Correlated_[0].channelWidth=3051.757812"""
+
+ @classmethod
+ def setUpClass(cls) -> None:
+ cls.feedback_listener = TMSSFeedbackListener()
+ cls.feedback_listener.start_handling()
+ cls.tmss_test_env = TMSSTestEnvironment()
+ cls.tmss_test_env.start()
+ cls.test_data_creator = TMSSRESTTestDataCreator(cls.tmss_test_env.django_server.url,
+ (cls.tmss_test_env.ldap_server.dbcreds.user,
+ cls.tmss_test_env.ldap_server.dbcreds.password))
+
+ @classmethod
+ def tearDownClass(cls) -> None:
+ cls.feedback_listener.stop_handling()
+ cls.tmss_test_env.stop()
+
+
+ @integration_test
+ @unittest.skip('requires old Qpid environment')
+ def test_feedback_arriving_on_messagebus_is_added_to_tmss_subtask(self):
+ """
+ ! This does not work yet, unfortunately, messages are sent, but for some reason not received.
+ ! I assume that this is some exchange/queue/routing issue with Qpid and that it should work against a proper broker setup....
+
+ Note that this test only works against an old Qpid broker, not RabbitMQ, because the feedback messages are legacy
+ for MoM compatibility and have not been converted to the new messaging library we use nowadays.
+
+ In the SAS CI container, I stopped rabbitmq and ran this instead:
+
+ > yum install qpid-cpp-server
+ > yum install qpid-tools
+ > qpid-config add queue devel.otdb.task.feedback.processing
+ > qpid-config add queue devel.otdb.task.feedback.dataproducts
+ > qpidd &
+
+ Not sure how to best run rabbitmq and qpid in parallel...
+ """
+ subtask = self.test_data_creator.post_data_and_get_response_as_json_object(self.test_data_creator.Subtask(), '/subtask/')
+ subtask_id = subtask['id']
+
+ with self.tmss_test_env.create_tmss_client() as tmss_client:
+ def send_feedback_to_exchange_and_assert_in_subtask(exchange, feedback, subtask_id):
+ subtask = tmss_client.get_subtask(subtask_id)
+
+ # send feedback on messagebus
+ bus = ToBus(exchange, broker=broker_feedback)
+ msg = TaskFeedbackProcessing(
+ "tmss.test",
+ "",
+ "Test feedback emerging from the tombs of LOFAR",
+ subtask_id,
+ subtask_id,
+ self.feedback_1)
+ bus.send(msg)
+
+ # wait for service to update subtask
+ start = datetime.datetime.utcnow()
+ subtask_updated_at = subtask['updated_at']
+ while subtask_updated_at == subtask["updated_at"]:
+ subtask = tmss_client.get_subtask(subtask_id)
+ sleep(0.5)
+ if datetime.datetime.utcnow() - start > datetime.timedelta(seconds=2):
+ raise TimeoutError()
+
+ # assert feedback is on the subtask
+ self.assertIsNotNone(subtask['raw_feedback'])
+ self.assertTrue(feedback in subtask['raw_feedback'])
+
+ # send and assert two feedback snippets
+ send_feedback_to_exchange_and_assert_in_subtask("otdb.task.feedback.dataproducts", self.feedback_1, subtask_id)
+ send_feedback_to_exchange_and_assert_in_subtask("otdb.task.feedback.processing", self.feedback_2, subtask_id)
+
+ # assert once more that BOTH feedbacks are present to make sure it gets appended and not replaced
+ subtask = tmss_client.get_subtask(subtask_id)
+ logger.warning(subtask)
+ self.assertTrue(self.feedback_1 in subtask["raw_feedback"] and self.feedback_2 in subtask["raw_feedback"])
+
+ @integration_test
+ def test_append_feedback_to_tmss_subtask_raw_feedback_updates_subtask(self):
+
+ # create subtask
+ subtask = self.test_data_creator.post_data_and_get_response_as_json_object(self.test_data_creator.Subtask(), '/subtask/')
+ subtask_id = subtask['id']
+
+ with self.tmss_test_env.create_tmss_client() as tmss_client:
+
+ # append bits of feedback
+ self.feedback_listener.append_feedback_to_tmss_subtask_raw_feedback(subtask_id, self.feedback_1)
+ self.feedback_listener.append_feedback_to_tmss_subtask_raw_feedback(subtask_id, self.feedback_2)
+
+ # assert all feedback is there
+ subtask = tmss_client.get_subtask(subtask_id)
+ self.assertIsNotNone(subtask['raw_feedback'])
+ self.maxDiff = None
+ self.assertEqual(self.feedback_1 + '\n' + self.feedback_2, subtask['raw_feedback'])
+
+
+logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO)
+
+if __name__ == '__main__':
+ #run the unit tests
+ unittest.main()
diff --git a/SAS/TMSS/services/feedback_handling/test/t_feedback_handling_service.run b/SAS/TMSS/services/feedback_handling/test/t_feedback_handling_service.run
new file mode 100755
index 0000000000000000000000000000000000000000..c5099fab09bc439e3e2c7c4bd406af4e26a81729
--- /dev/null
+++ b/SAS/TMSS/services/feedback_handling/test/t_feedback_handling_service.run
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# Run the unit test
+source python-coverage.sh
+python_coverage_test "*tmss*" t_feedback_handling_service.py
+
diff --git a/SAS/TMSS/services/feedback_handling/test/t_feedback_handling_service.sh b/SAS/TMSS/services/feedback_handling/test/t_feedback_handling_service.sh
new file mode 100755
index 0000000000000000000000000000000000000000..c7e18e21d2578c39ddac1503b3a00650344c5854
--- /dev/null
+++ b/SAS/TMSS/services/feedback_handling/test/t_feedback_handling_service.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+./runctest.sh t_feedback_handling_service
\ No newline at end of file
diff --git a/SAS/TMSS/src/CMakeCache.txt b/SAS/TMSS/src/CMakeCache.txt
new file mode 100644
index 0000000000000000000000000000000000000000..0b2dc14cb11f159cf34cbf5f5ad840ce0aaab7d0
--- /dev/null
+++ b/SAS/TMSS/src/CMakeCache.txt
@@ -0,0 +1,326 @@
+# This is the CMakeCache file.
+# For build in directory: /lofar/SAS/TMSS/src
+# It was generated by CMake: /usr/bin/cmake
+# You can edit this file to change values found and used by cmake.
+# If you do not want to change any of the values, simply exit the editor.
+# If you do want to change a value, simply edit, save, and exit the editor.
+# The syntax for the file is as follows:
+# KEY:TYPE=VALUE
+# KEY is the name of a variable in the cache.
+# TYPE is a hint to GUIs for the type of VALUE, DO NOT EDIT TYPE!.
+# VALUE is the current value for the KEY.
+
+########################
+# EXTERNAL cache entries
+########################
+
+//No help, variable specified on the command line.
+BUILD_PACKAGES:UNINITIALIZED=TMSS
+
+//Path to a program.
+CMAKE_AR:FILEPATH=/usr/bin/ar
+
+//For backwards compatibility, what version of CMake commands and
+// syntax should this version of CMake try to support.
+CMAKE_BACKWARDS_COMPATIBILITY:STRING=2.4
+
+//Choose the type of build, options are: None(CMAKE_CXX_FLAGS or
+// CMAKE_C_FLAGS used) Debug Release RelWithDebInfo MinSizeRel.
+CMAKE_BUILD_TYPE:STRING=
+
+//Enable/Disable color output during build.
+CMAKE_COLOR_MAKEFILE:BOOL=ON
+
+//CXX compiler.
+CMAKE_CXX_COMPILER:FILEPATH=/usr/bin/c++
+
+//Flags used by the compiler during all build types.
+CMAKE_CXX_FLAGS:STRING=
+
+//Flags used by the compiler during debug builds.
+CMAKE_CXX_FLAGS_DEBUG:STRING=-g
+
+//Flags used by the compiler during release minsize builds.
+CMAKE_CXX_FLAGS_MINSIZEREL:STRING=-Os -DNDEBUG
+
+//Flags used by the compiler during release builds (/MD /Ob1 /Oi
+// /Ot /Oy /Gs will produce slightly less optimized but smaller
+// files).
+CMAKE_CXX_FLAGS_RELEASE:STRING=-O3 -DNDEBUG
+
+//Flags used by the compiler during Release with Debug Info builds.
+CMAKE_CXX_FLAGS_RELWITHDEBINFO:STRING=-O2 -g -DNDEBUG
+
+//C compiler.
+CMAKE_C_COMPILER:FILEPATH=/usr/bin/cc
+
+//Flags used by the compiler during all build types.
+CMAKE_C_FLAGS:STRING=
+
+//Flags used by the compiler during debug builds.
+CMAKE_C_FLAGS_DEBUG:STRING=-g
+
+//Flags used by the compiler during release minsize builds.
+CMAKE_C_FLAGS_MINSIZEREL:STRING=-Os -DNDEBUG
+
+//Flags used by the compiler during release builds (/MD /Ob1 /Oi
+// /Ot /Oy /Gs will produce slightly less optimized but smaller
+// files).
+CMAKE_C_FLAGS_RELEASE:STRING=-O3 -DNDEBUG
+
+//Flags used by the compiler during Release with Debug Info builds.
+CMAKE_C_FLAGS_RELWITHDEBINFO:STRING=-O2 -g -DNDEBUG
+
+//Flags used by the linker.
+CMAKE_EXE_LINKER_FLAGS:STRING=' '
+
+//Flags used by the linker during debug builds.
+CMAKE_EXE_LINKER_FLAGS_DEBUG:STRING=
+
+//Flags used by the linker during release minsize builds.
+CMAKE_EXE_LINKER_FLAGS_MINSIZEREL:STRING=
+
+//Flags used by the linker during release builds.
+CMAKE_EXE_LINKER_FLAGS_RELEASE:STRING=
+
+//Flags used by the linker during Release with Debug Info builds.
+CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO:STRING=
+
+//Enable/Disable output of compile commands during generation.
+CMAKE_EXPORT_COMPILE_COMMANDS:BOOL=OFF
+
+//Install path prefix, prepended onto install directories.
+CMAKE_INSTALL_PREFIX:PATH=/usr/local
+
+//Path to a program.
+CMAKE_LINKER:FILEPATH=/usr/bin/ld
+
+//Path to a program.
+CMAKE_MAKE_PROGRAM:FILEPATH=/usr/bin/gmake
+
+//Flags used by the linker during the creation of modules.
+CMAKE_MODULE_LINKER_FLAGS:STRING=' '
+
+//Flags used by the linker during debug builds.
+CMAKE_MODULE_LINKER_FLAGS_DEBUG:STRING=
+
+//Flags used by the linker during release minsize builds.
+CMAKE_MODULE_LINKER_FLAGS_MINSIZEREL:STRING=
+
+//Flags used by the linker during release builds.
+CMAKE_MODULE_LINKER_FLAGS_RELEASE:STRING=
+
+//Flags used by the linker during Release with Debug Info builds.
+CMAKE_MODULE_LINKER_FLAGS_RELWITHDEBINFO:STRING=
+
+//Path to a program.
+CMAKE_NM:FILEPATH=/usr/bin/nm
+
+//Path to a program.
+CMAKE_OBJCOPY:FILEPATH=/usr/bin/objcopy
+
+//Path to a program.
+CMAKE_OBJDUMP:FILEPATH=/usr/bin/objdump
+
+//Value Computed by CMake
+CMAKE_PROJECT_NAME:STATIC=Project
+
+//Path to a program.
+CMAKE_RANLIB:FILEPATH=/usr/bin/ranlib
+
+//Flags used by the linker during the creation of dll's.
+CMAKE_SHARED_LINKER_FLAGS:STRING=' '
+
+//Flags used by the linker during debug builds.
+CMAKE_SHARED_LINKER_FLAGS_DEBUG:STRING=
+
+//Flags used by the linker during release minsize builds.
+CMAKE_SHARED_LINKER_FLAGS_MINSIZEREL:STRING=
+
+//Flags used by the linker during release builds.
+CMAKE_SHARED_LINKER_FLAGS_RELEASE:STRING=
+
+//Flags used by the linker during Release with Debug Info builds.
+CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO:STRING=
+
+//If set, runtime paths are not added when installing shared libraries,
+// but are added when building.
+CMAKE_SKIP_INSTALL_RPATH:BOOL=NO
+
+//If set, runtime paths are not added when using shared libraries.
+CMAKE_SKIP_RPATH:BOOL=NO
+
+//Flags used by the linker during the creation of static libraries.
+CMAKE_STATIC_LINKER_FLAGS:STRING=
+
+//Flags used by the linker during debug builds.
+CMAKE_STATIC_LINKER_FLAGS_DEBUG:STRING=
+
+//Flags used by the linker during release minsize builds.
+CMAKE_STATIC_LINKER_FLAGS_MINSIZEREL:STRING=
+
+//Flags used by the linker during release builds.
+CMAKE_STATIC_LINKER_FLAGS_RELEASE:STRING=
+
+//Flags used by the linker during Release with Debug Info builds.
+CMAKE_STATIC_LINKER_FLAGS_RELWITHDEBINFO:STRING=
+
+//Path to a program.
+CMAKE_STRIP:FILEPATH=/usr/bin/strip
+
+//If true, cmake will use relative paths in makefiles and projects.
+CMAKE_USE_RELATIVE_PATHS:BOOL=OFF
+
+//If this value is on, makefiles will be generated without the
+// .SILENT directive, and all commands will be echoed to the console
+// during the make. This is useful for debugging only. With Visual
+// Studio IDE projects all commands are done without /nologo.
+CMAKE_VERBOSE_MAKEFILE:BOOL=FALSE
+
+//Single output directory for building all executables.
+EXECUTABLE_OUTPUT_PATH:PATH=
+
+//Single output directory for building all libraries.
+LIBRARY_OUTPUT_PATH:PATH=
+
+//Value Computed by CMake
+Project_BINARY_DIR:STATIC=/lofar/SAS/TMSS/src
+
+//Value Computed by CMake
+Project_SOURCE_DIR:STATIC=/lofar/SAS
+
+
+########################
+# INTERNAL cache entries
+########################
+
+//ADVANCED property for variable: CMAKE_AR
+CMAKE_AR-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_BUILD_TOOL
+CMAKE_BUILD_TOOL-ADVANCED:INTERNAL=1
+//What is the target build tool cmake is generating for.
+CMAKE_BUILD_TOOL:INTERNAL=/usr/bin/gmake
+//This is the directory where this CMakeCache.txt was created
+CMAKE_CACHEFILE_DIR:INTERNAL=/lofar/SAS/TMSS/src
+//Major version of cmake used to create the current loaded cache
+CMAKE_CACHE_MAJOR_VERSION:INTERNAL=2
+//Minor version of cmake used to create the current loaded cache
+CMAKE_CACHE_MINOR_VERSION:INTERNAL=8
+//Patch version of cmake used to create the current loaded cache
+CMAKE_CACHE_PATCH_VERSION:INTERNAL=12
+//ADVANCED property for variable: CMAKE_COLOR_MAKEFILE
+CMAKE_COLOR_MAKEFILE-ADVANCED:INTERNAL=1
+//Path to CMake executable.
+CMAKE_COMMAND:INTERNAL=/usr/bin/cmake
+//Path to cpack program executable.
+CMAKE_CPACK_COMMAND:INTERNAL=/usr/bin/cpack
+//Path to ctest program executable.
+CMAKE_CTEST_COMMAND:INTERNAL=/usr/bin/ctest
+//ADVANCED property for variable: CMAKE_CXX_COMPILER
+CMAKE_CXX_COMPILER-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_CXX_FLAGS
+CMAKE_CXX_FLAGS-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_CXX_FLAGS_DEBUG
+CMAKE_CXX_FLAGS_DEBUG-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_CXX_FLAGS_MINSIZEREL
+CMAKE_CXX_FLAGS_MINSIZEREL-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_CXX_FLAGS_RELEASE
+CMAKE_CXX_FLAGS_RELEASE-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_CXX_FLAGS_RELWITHDEBINFO
+CMAKE_CXX_FLAGS_RELWITHDEBINFO-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_C_COMPILER
+CMAKE_C_COMPILER-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_C_FLAGS
+CMAKE_C_FLAGS-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_C_FLAGS_DEBUG
+CMAKE_C_FLAGS_DEBUG-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_C_FLAGS_MINSIZEREL
+CMAKE_C_FLAGS_MINSIZEREL-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_C_FLAGS_RELEASE
+CMAKE_C_FLAGS_RELEASE-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_C_FLAGS_RELWITHDEBINFO
+CMAKE_C_FLAGS_RELWITHDEBINFO-ADVANCED:INTERNAL=1
+//Path to cache edit program executable.
+CMAKE_EDIT_COMMAND:INTERNAL=/usr/bin/ccmake
+//Executable file format
+CMAKE_EXECUTABLE_FORMAT:INTERNAL=ELF
+//ADVANCED property for variable: CMAKE_EXE_LINKER_FLAGS
+CMAKE_EXE_LINKER_FLAGS-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_EXE_LINKER_FLAGS_DEBUG
+CMAKE_EXE_LINKER_FLAGS_DEBUG-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_EXE_LINKER_FLAGS_MINSIZEREL
+CMAKE_EXE_LINKER_FLAGS_MINSIZEREL-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_EXE_LINKER_FLAGS_RELEASE
+CMAKE_EXE_LINKER_FLAGS_RELEASE-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO
+CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_EXPORT_COMPILE_COMMANDS
+CMAKE_EXPORT_COMPILE_COMMANDS-ADVANCED:INTERNAL=1
+//Name of generator.
+CMAKE_GENERATOR:INTERNAL=Unix Makefiles
+//Name of generator toolset.
+CMAKE_GENERATOR_TOOLSET:INTERNAL=
+//Start directory with the top level CMakeLists.txt file for this
+// project
+CMAKE_HOME_DIRECTORY:INTERNAL=/lofar/SAS
+//Install .so files without execute permission.
+CMAKE_INSTALL_SO_NO_EXE:INTERNAL=0
+//ADVANCED property for variable: CMAKE_LINKER
+CMAKE_LINKER-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_MAKE_PROGRAM
+CMAKE_MAKE_PROGRAM-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_MODULE_LINKER_FLAGS
+CMAKE_MODULE_LINKER_FLAGS-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_MODULE_LINKER_FLAGS_DEBUG
+CMAKE_MODULE_LINKER_FLAGS_DEBUG-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_MODULE_LINKER_FLAGS_MINSIZEREL
+CMAKE_MODULE_LINKER_FLAGS_MINSIZEREL-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_MODULE_LINKER_FLAGS_RELEASE
+CMAKE_MODULE_LINKER_FLAGS_RELEASE-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_MODULE_LINKER_FLAGS_RELWITHDEBINFO
+CMAKE_MODULE_LINKER_FLAGS_RELWITHDEBINFO-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_NM
+CMAKE_NM-ADVANCED:INTERNAL=1
+//number of local generators
+CMAKE_NUMBER_OF_LOCAL_GENERATORS:INTERNAL=1
+//ADVANCED property for variable: CMAKE_OBJCOPY
+CMAKE_OBJCOPY-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_OBJDUMP
+CMAKE_OBJDUMP-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_RANLIB
+CMAKE_RANLIB-ADVANCED:INTERNAL=1
+//Path to CMake installation.
+CMAKE_ROOT:INTERNAL=/usr/share/cmake
+//ADVANCED property for variable: CMAKE_SHARED_LINKER_FLAGS
+CMAKE_SHARED_LINKER_FLAGS-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_SHARED_LINKER_FLAGS_DEBUG
+CMAKE_SHARED_LINKER_FLAGS_DEBUG-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_SHARED_LINKER_FLAGS_MINSIZEREL
+CMAKE_SHARED_LINKER_FLAGS_MINSIZEREL-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_SHARED_LINKER_FLAGS_RELEASE
+CMAKE_SHARED_LINKER_FLAGS_RELEASE-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO
+CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_SKIP_INSTALL_RPATH
+CMAKE_SKIP_INSTALL_RPATH-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_SKIP_RPATH
+CMAKE_SKIP_RPATH-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_STATIC_LINKER_FLAGS
+CMAKE_STATIC_LINKER_FLAGS-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_STATIC_LINKER_FLAGS_DEBUG
+CMAKE_STATIC_LINKER_FLAGS_DEBUG-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_STATIC_LINKER_FLAGS_MINSIZEREL
+CMAKE_STATIC_LINKER_FLAGS_MINSIZEREL-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_STATIC_LINKER_FLAGS_RELEASE
+CMAKE_STATIC_LINKER_FLAGS_RELEASE-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_STATIC_LINKER_FLAGS_RELWITHDEBINFO
+CMAKE_STATIC_LINKER_FLAGS_RELWITHDEBINFO-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_STRIP
+CMAKE_STRIP-ADVANCED:INTERNAL=1
+//uname command
+CMAKE_UNAME:INTERNAL=/usr/bin/uname
+//ADVANCED property for variable: CMAKE_USE_RELATIVE_PATHS
+CMAKE_USE_RELATIVE_PATHS-ADVANCED:INTERNAL=1
+//ADVANCED property for variable: CMAKE_VERBOSE_MAKEFILE
+CMAKE_VERBOSE_MAKEFILE-ADVANCED:INTERNAL=1
+
diff --git a/SAS/TMSS/src/remakemigrations.py b/SAS/TMSS/src/remakemigrations.py
index ed3475278af7c5c1fee2d8232bbe0250630e3131..6a4ee430ffd683388eb4c0ba5523dfc4d89d4c39 100755
--- a/SAS/TMSS/src/remakemigrations.py
+++ b/SAS/TMSS/src/remakemigrations.py
@@ -78,7 +78,6 @@ class Migration(migrations.Migration):
migrations.RunPython(populate_choices),
migrations.RunPython(populate_settings),
migrations.RunPython(populate_misc),
- migrations.RunPython(populate_lofar_json_schemas),
migrations.RunPython(populate_resources),
migrations.RunPython(populate_cycles),
migrations.RunPython(populate_projects) ]
diff --git a/SAS/TMSS/src/templates/josdejong_jsoneditor_widget.html b/SAS/TMSS/src/templates/josdejong_jsoneditor_widget.html
index d715008e7355ad179527119f2b5a3b43003f446b..68cfb36e9a12b1040eb87f3e8e8175582193d2ca 100644
--- a/SAS/TMSS/src/templates/josdejong_jsoneditor_widget.html
+++ b/SAS/TMSS/src/templates/josdejong_jsoneditor_widget.html
@@ -1,12 +1,8 @@
<!-- EXTERNAL RESOURCES -->
-<!--<link rel="stylesheet" id="theme" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.6/css/bootstrap.min.css">-->
-<!--<script src="https://unpkg.com/react@16/umd/react.development.js"></script>-->
-<!--<script src="https://unpkg.com/react-dom@16/umd/react-dom.development.js"></script>-->
-<!--<script src="https://unpkg.com/babel-standalone@6.15.0/babel.min.js"></script>-->
-<link rel="stylesheet" id="theme" href="https://cdnjs.cloudflare.com/ajax/libs/jsoneditor/5.24.7/jsoneditor.css">
-<!--<script src="https://cdnjs.cloudflare.com/ajax/libs/jsoneditor/5.24.7/img/jsoneditor-icons.svg"></script>-->
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jsoneditor/5.24.7/jsoneditor.js"></script>
+<link rel="stylesheet" id="theme" href="https://cdnjs.cloudflare.com/ajax/libs/jsoneditor/9.0.3/jsoneditor.css">
+<script src="https://cdnjs.cloudflare.com/ajax/libs/ajv/6.12.4/ajv.bundle.js"></script>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/jsoneditor/9.0.3/jsoneditor.js"></script>
<!-- WIDGET HTML -->
<div class="form-group {% if field.errors %}has-error{% endif %}">
diff --git a/SAS/TMSS/src/tmss/settings.py b/SAS/TMSS/src/tmss/settings.py
index 015d4ccfe18a3aedd281138016784898533bb472..31f27ec817a8a58b1444c98b9806c3d65b84ea63 100644
--- a/SAS/TMSS/src/tmss/settings.py
+++ b/SAS/TMSS/src/tmss/settings.py
@@ -91,9 +91,14 @@ INSTALLED_APPS = [
'jsoneditor',
'drf_yasg',
'django_filters',
+ 'material',
+ 'material.frontend',
+ 'viewflow',
+ 'viewflow.frontend',
]
MIDDLEWARE = [
+ 'django.middleware.gzip.GZipMiddleware',
'django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
diff --git a/SAS/TMSS/src/tmss/tmssapp/CMakeLists.txt b/SAS/TMSS/src/tmss/tmssapp/CMakeLists.txt
index 47a6fc110c6e09c09bf272f1ee0f0f04a5a65407..e24af6998d0ad9240a454cd41fdb389a38cb4208 100644
--- a/SAS/TMSS/src/tmss/tmssapp/CMakeLists.txt
+++ b/SAS/TMSS/src/tmss/tmssapp/CMakeLists.txt
@@ -7,9 +7,9 @@ set(_py_files
apps.py
views.py
populate.py
- validation.py
subtasks.py
tasks.py
+ conversions.py
)
python_install(${_py_files}
@@ -23,4 +23,5 @@ add_subdirectory(serializers)
add_subdirectory(viewsets)
add_subdirectory(adapters)
add_subdirectory(schemas)
+add_subdirectory(workflows)
diff --git a/SAS/TMSS/src/tmss/tmssapp/adapters/CMakeLists.txt b/SAS/TMSS/src/tmss/tmssapp/adapters/CMakeLists.txt
index cac38d8903a5029a44d71f97f4230a30bbaed405..457bdbabeb7c04db158abe1c7a6a6a9b0f5dd90e 100644
--- a/SAS/TMSS/src/tmss/tmssapp/adapters/CMakeLists.txt
+++ b/SAS/TMSS/src/tmss/tmssapp/adapters/CMakeLists.txt
@@ -4,6 +4,7 @@ include(PythonInstall)
set(_py_files
parset.py
sip.py
+ feedback.py
)
python_install(${_py_files}
diff --git a/SAS/TMSS/src/tmss/tmssapp/adapters/feedback.py b/SAS/TMSS/src/tmss/tmssapp/adapters/feedback.py
new file mode 100644
index 0000000000000000000000000000000000000000..55df3b79bd73c349cdac0b4681adcf92668f98f5
--- /dev/null
+++ b/SAS/TMSS/src/tmss/tmssapp/adapters/feedback.py
@@ -0,0 +1,174 @@
+#!/usr/bin/python3
+
+# Copyright (C) 2020 ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+
+from lofar.sas.tmss.tmss.tmssapp.models import *
+import logging
+logger = logging.getLogger(__name__)
+
+def parse_feedback(raw_feedback):
+ feedback_dict = {}
+ for line in raw_feedback.split('\n'):
+ line = line.strip()
+ if line and '=' in line:
+ k, v = line.split('=', 1)
+ feedback_dict[k] = v
+ return feedback_dict
+
+
+def check_feedback_is_complete(raw_feedback):
+ if raw_feedback is None:
+ return False
+ feedback_dict = parse_feedback(raw_feedback)
+ empty = True
+ for dataproduct_type in ['Correlated', 'Beamformed']:
+ nr_key = "Observation.DataProducts.nrOfOutput_%s_" % (dataproduct_type)
+ if nr_key in feedback_dict.keys():
+ empty = False
+ for i in range(int(feedback_dict[nr_key])):
+ dp_keys = ["%sOutput_%s_[%s].subband" % (prefix, dataproduct_type, i) for prefix in ['LOFAR.ObsSW.Observation.DataProducts.', 'Observation.DataProducts.']]
+ if not any([dp_key in feedback_dict.keys() for dp_key in dp_keys]):
+ raise ValueError("Feedback is missing any of %s" % dp_keys)
+ logger.debug("All expected %s %s Dataproducts are present in feedback" % (dataproduct_type, feedback_dict[nr_key]))
+ return not empty
+
+
+def process_subtask_feedback(subtask:Subtask):
+ logger.info('Now processing feedback of subtask id=%s type=%s' % (subtask.id, subtask.specifications_template.type.value))
+ feedback_dict = parse_feedback(subtask.raw_feedback)
+
+ dataproduct_feedback_docs = {}
+ if subtask.specifications_template.type.value == SubtaskType.Choices.OBSERVATION.value:
+ prefix = 'Observation.DataProducts.'
+ elif subtask.specifications_template.type.value == SubtaskType.Choices.PIPELINE.value:
+ prefix = 'LOFAR.ObsSW.Observation.DataProducts.'
+ else:
+ raise ValueError("Cannot process feedback of subtask id=%s since type=%s not in %s" %
+ (subtask.id, subtask.specifications_template.type.value,
+ [SubtaskType.Choices.OBSERVATION.value, SubtaskType.Choices.PIPELINE.value]))
+
+ for dataproduct_type in ['Correlated', 'Beamformed']:
+ # iterate over dataproducts in feedback
+ i = 0
+ while True:
+ dpkey = "%sOutput_%s_[%s]" % (prefix, dataproduct_type, i)
+ if dpkey + '.subband' not in feedback_dict.keys():
+ break
+
+ # determine corresponding TMSS dataproduct
+ dataproduct = Dataproduct.objects.get(filename=feedback_dict[dpkey+'.filename'])
+ dataproduct.feedback_template = DataproductFeedbackTemplate.objects.get(name='feedback')
+ logger.debug('Found dataproduct %s' % dataproduct.filename)
+
+ # derive values or collect for different subtask types
+ storagewriter = feedback_dict[dpkey + '.storageWriter'].lower()
+ if storagewriter == "casa":
+ storagewriter = "standard" # todo: is that correct?
+ elif storagewriter == "lofar":
+ storagewriter = "lofarstman"
+
+ if subtask.specifications_template.type.value == SubtaskType.Choices.OBSERVATION.value:
+ subbands = [int(feedback_dict[dpkey+'.stationSubband'])]
+ duration = (subtask.stop_time - subtask.start_time).total_seconds()
+ antennaset = subtask.specifications_doc['stations']['antenna_set']
+ stationlist = subtask.specifications_doc['stations']['station_list']
+ antennatype = antennaset.split('_')[0] # LBA or HBA
+ antennafields = []
+ for station in stationlist:
+ if antennaset.startswith('LBA'):
+ fields = ['LBA']
+ elif antennaset.startswith('HBA') and not station.startswith('CS'):
+ fields = ['HBA']
+ elif antennaset.startswith('HBA_DUAL'):
+ fields = ['HBA0', 'HBA1']
+ elif antennaset.startswith('HBA_ZERO'):
+ fields = ['HBA0']
+ elif antennaset.startswith('HBA_ONE'):
+ fields = ['HBA1']
+ else:
+ raise ValueError('Cannot determine antennafields for station=%s antennaset=%s' % (station, antennaset))
+ antennafields += [{"station": station, "field": field, "type": antennatype} for field in fields]
+ pointing = subtask.specifications_doc['stations']['digital_pointings'][int(feedback_dict[dpkey+'.SAP'])]['pointing']
+ else:
+ input_dataproduct = DataproductTransform.objects.get(output=dataproduct).input
+ logger.debug('Found input dataproduct %s' % input_dataproduct.filename)
+ subbands = input_dataproduct.feedback_doc["frequency"]['subbands']
+ duration = float(feedback_dict[dpkey + '.duration'])
+ antennaset = input_dataproduct.feedback_doc["antennas"]['set']
+ antennafields = input_dataproduct.feedback_doc["antennas"]['fields']
+ pointing = input_dataproduct.feedback_doc["target"]['pointing']
+
+ # add feedback doc to dataproduct
+ dataproduct.feedback_doc={
+ "percentage_written": int(feedback_dict[dpkey+'.percentageWritten']),
+ "frequency": {
+ "subbands": subbands,
+ "central_frequencies": [float(feedback_dict[dpkey+'.centralFrequency'])],
+ "channel_width": float(feedback_dict[dpkey + '.channelWidth']),
+ "channels_per_subband": int(feedback_dict[dpkey + '.channelsPerSubband'])
+ },
+ "time": {
+ "start_time": feedback_dict[dpkey+'.startTime'],
+ "duration": duration,
+ "sample_width": float(feedback_dict[dpkey+'.integrationInterval']),
+ },
+ "antennas": {
+ "set": antennaset,
+ "fields": antennafields
+ },
+ "target": {
+ "pointing": pointing
+ },
+ "samples": {
+ "polarisations": ["XX","XY","YX","YY"], # fixed
+ "type": "float", # fixed
+ "bits": 32, # fixed
+ "writer": storagewriter,
+ "writer_version": feedback_dict[dpkey + '.storageWriterVersion'],
+ "complex": True # fixed
+ }
+ }
+ i += 1
+ dataproduct.save()
+ logger.info('Saved %s %s' % (dataproduct.filename, dataproduct.feedback_doc))
+
+
+def generate_dataproduct_feedback_from_subtask_feedback_and_set_finished(subtask:Subtask):
+ """
+ Translates raw feedback from a subtask (which has been provided by Cobalt or pipelines) and translate it to
+ json documents for the individual dataproducts.
+ """
+ # check we are in finishing state and all feedback has arrived
+ if subtask.state != SubtaskState.objects.get(value='finishing'):
+ raise ValueError('Subtask id=%s state=%s is not in state %s' % (subtask.id, subtask.state, SubtaskState.Choices.FINISHING.value))
+ raw_feedback = subtask.raw_feedback
+ try:
+ check_feedback_is_complete(raw_feedback)
+ except ValueError as original_error:
+ raise ValueError("Feedback of subtask_id=%s is not complete: %s " % (subtask.id, original_error))
+
+ # convert raw feedback to dataproduct feedback docs
+ process_subtask_feedback(subtask)
+
+ # set subtask state to finished
+ subtask.state = SubtaskState.objects.get(value='finished')
+ subtask.save()
+ return subtask
+
+
+
diff --git a/SAS/TMSS/src/tmss/tmssapp/adapters/parset.py b/SAS/TMSS/src/tmss/tmssapp/adapters/parset.py
index 29ea31b133d257425ec9804adeb7ce165ea0d478..daa7a72c21d57a1a6c9ae9f7ce02f32afc4854b3 100644
--- a/SAS/TMSS/src/tmss/tmssapp/adapters/parset.py
+++ b/SAS/TMSS/src/tmss/tmssapp/adapters/parset.py
@@ -310,8 +310,8 @@ def _convert_to_parset_dict_for_pipelinecontrol_schema(subtask: models.Subtask)
# dict to store conversion methods based on subtask.specifications_template.name
-_convertors = {'observationcontrol schema': _convert_to_parset_dict_for_observationcontrol_schema,
- 'pipelinecontrol schema': _convert_to_parset_dict_for_pipelinecontrol_schema }
+_convertors = {'observation control': _convert_to_parset_dict_for_observationcontrol_schema,
+ 'pipeline control': _convert_to_parset_dict_for_pipelinecontrol_schema }
def convert_to_parset(subtask: models.Subtask) -> parameterset:
diff --git a/SAS/TMSS/src/tmss/tmssapp/adapters/sip.py b/SAS/TMSS/src/tmss/tmssapp/adapters/sip.py
index 50e2a205555195be071027cdc0894164c2726335..49e555448b91d3587c7f39db9a1c7b8021dc5f90 100644
--- a/SAS/TMSS/src/tmss/tmssapp/adapters/sip.py
+++ b/SAS/TMSS/src/tmss/tmssapp/adapters/sip.py
@@ -6,6 +6,7 @@ from lofar.lta.sip import siplib, ltasip, validator, constants
import uuid
import logging
import isodate
+import datetime
logger = logging.getLogger(__name__)
mapping_antennaset_type_TMSS_2_SIP = {
@@ -31,6 +32,7 @@ mapping_filterset_type_TMSS_2_SIP = {
"HBA_210_250": constants.FILTERSELECTIONTYPE_210_250_MHZ
}
+
def get_number_of_dataproducts_of_type(subtask, dataproduct_datatype):
"""
Retrieve the number of dataproducts of given data type in subtask:
@@ -42,7 +44,7 @@ def get_number_of_dataproducts_of_type(subtask, dataproduct_datatype):
subtask_outputs = list(SubtaskOutput.objects.filter(subtask_id=subtask.id))
for subtask_output in subtask_outputs:
dataproducts = list(Dataproduct.objects.filter(producer_id=subtask_output.id,
- dataformat=dataproduct_datatype))
+ datatype=dataproduct_datatype))
nbr_dataproduct += len(dataproducts)
return nbr_dataproduct
@@ -50,14 +52,27 @@ def get_number_of_dataproducts_of_type(subtask, dataproduct_datatype):
def get_siplib_stations_list(subtask):
"""
Retrieve a list of siplib Stations objects which is extracted from the station_list and the antennaset
- TODO Correct mapping of all different HBA/LBA antennaset flavours to antenna fieldtypes required for SIP
:param subtask:
:return:
+
+ Conversion logic see here: https://support.astron.nl/confluence/display/TMSS/Dataproduct+Provenance
"""
- siplib_station_list =[]
+ siplib_station_list = []
list_stations = subtask.specifications_doc['stations']['station_list']
- antennafieldtypes = ["HBA"] if "HBA" in subtask.specifications_doc['stations']['antenna_set'] else ["LBA"]
+ antennaset = subtask.specifications_doc['stations']['antenna_set']
for station in list_stations:
+ if antennaset.startswith('LBA'):
+ antennafieldtypes = ['LBA']
+ elif antennaset.startswith('HBA') and not station.startswith('CS'):
+ antennafieldtypes = ['HBA']
+ elif antennaset.startswith('HBA_DUAL'):
+ antennafieldtypes = ['HBA0', 'HBA1']
+ elif antennaset.startswith('HBA_ZERO'):
+ antennafieldtypes = ['HBA0']
+ elif antennaset.startswith('HBA_ONE'):
+ antennafieldtypes = ['HBA1']
+ else:
+ raise ValueError('Cannot determine antennafields for station=%s antennaset=%s' % (station, antennaset))
siplib_station_list.append(siplib.Station.preconfigured(station, antennafieldtypes))
return siplib_station_list
@@ -127,6 +142,7 @@ def create_sip_representation_for_subtask(subtask: Subtask):
# determine subtask specific properties and add subtask representation to Sip object
if subtask.specifications_template.type.value == SubtaskType.Choices.OBSERVATION.value:
+ subarraypointings=None # todo, subtask.specifications_doc, probably more complex than it looks -> RGOE yes complex type for later -> JK: assuming this is done in TMSS-308?
observation = siplib.Observation(observingmode=constants.OBSERVINGMODETYPE_BEAM_OBSERVATION, # can be hardcoded for an observation
instrumentfilter=mapping_filterset_type_TMSS_2_SIP[subtask.specifications_doc['stations']['filter']],
clock_frequency="200", # fixed,
@@ -146,7 +162,7 @@ def create_sip_representation_for_subtask(subtask: Subtask):
channelwidth_frequencyunit=constants.FREQUENCYUNIT_HZ, # fixed
observationdescription=subtask.task_blueprint.description,
channelspersubband=0, # NA any more ('BlueGene compatibility' see comment in LTA-SIP.xsd)
- subarraypointings=None, # todo, subtask.specifications_doc, probably more complex than it looks -> RGOE yes complex type for later
+ subarraypointings=subarraypointings,
transientbufferboardevents=None # fixed
)
@@ -160,11 +176,11 @@ def create_sip_representation_for_subtask(subtask: Subtask):
pipeline_map = siplib.PipelineMap(
name=subtask.task_blueprint.name,
- version='unknown', # todo subtask.specifications_doc, from feedback?
+ version='unknown', # todo from subtask.specifications_doc? from feedback (we have feedback and storagewriter versions there, not pipeline version or sth)?
sourcedata_identifiers=sourcedata_identifiers,
process_map=process_map)
- if subtask.specifications_template.name == "pipelinecontrol schema": # todo: re-evaluate this because schema name might change
+ if subtask.specifications_template.name == "pipeline control": # todo: re-evaluate this because schema name might change
pipeline = siplib.AveragingPipeline( # <-- this is what we need for UC1
pipeline_map,
numberofcorrelateddataproducts=get_number_of_dataproducts_of_type(subtask, Dataformat.Choices.MEASUREMENTSET.value),
@@ -229,6 +245,11 @@ def create_sip_representation_for_dataproduct(dataproduct: Dataproduct):
:param dataproduct:
:return: One of the siplib dataproduct object flavors.
"""
+
+ # Make sure feedback is not empty
+ if 'percentage_written' not in dataproduct.feedback_doc:
+ raise ValueError("The feedback_doc of dataproduct id=%s is incomplete. Has feedback of the producing subtask been processed?" % dataproduct.id)
+
# Note: this is for the type property present on all dataproduct flavors, dataproduct classes are
# differentiated in addition to that below
type_map = {Datatype.Choices.VISIBILITIES.value: constants.DATAPRODUCTTYPE_CORRELATOR_DATA,
@@ -245,11 +266,11 @@ def create_sip_representation_for_dataproduct(dataproduct: Dataproduct):
Dataformat.Choices.QA_PLOTS.value: constants.FILEFORMATTYPE_UNDOCUMENTED}
storage_writer_map = {"dysco": constants.STORAGEWRITERTYPE_DYSCOSTORAGEMANAGER,
- "unknown": constants.STORAGEWRITERTYPE_UNKNOWN}
+ "unknown": constants.STORAGEWRITERTYPE_UNKNOWN,
+ "standard": constants.STORAGEWRITERTYPE_LOFARSTORAGEMANAGER}
try:
- # todo: why is this not with the data but with the connector? The type of data should not depend on what it is used for? I don't get it... - what if we have several connectors?
- dataproduct_type = type_map[dataproduct.producer.subtask.task_blueprint.specifications_template.output_connector_types.first().datatype.value] # todo: does not work on the main dataproduct?
+ dataproduct_type = type_map[dataproduct.datatype.value]
except Exception as err:
dataproduct_type = constants.DATAPRODUCTTYPE_UNKNOWN
logger.warning("Could not determine the type of dataproduct id %s (%s). Falling back to %s" % (dataproduct.id, err, dataproduct_type))
@@ -265,25 +286,24 @@ def create_sip_representation_for_dataproduct(dataproduct: Dataproduct):
size=dataproduct.size if dataproduct.size else 0,
filename=dataproduct.filename,
fileformat=dataproduct_fileformat,
- storage_writer=storage_writer_map[dataproduct.producer.subtask.task_blueprint.specifications_doc.get("storagemanager", 'unknown')], # note: default required because storagemanager is not required by all schemas!
- storage_writer_version='Unknown', # todo: not modeled? needs to come from feedback eventually.
+ storage_writer=storage_writer_map[dataproduct.feedback_doc["samples"]["writer"] if 'samples' in dataproduct.feedback_doc else 'unknown'], # todo: verify we can use the feedback_doc here and remove the old method | storage_writer_map[dataproduct.producer.subtask.task_blueprint.specifications_doc.get("storagemanager", 'unknown')],
+ storage_writer_version=dataproduct.feedback_doc["samples"]["writer_version"] if 'samples' in dataproduct.feedback_doc else 'unknown',
process_identifier=create_fake_identifier_for_testing(unique_id=dataproduct.producer.subtask.id))
- # next TODOs: TMSS-300
if dataproduct.dataformat.value == Dataformat.Choices.MEASUREMENTSET.value: # <- This is the only one we currently need for UC1
sip_dataproduct = siplib.CorrelatedDataProduct(
dataproduct_map,
subarraypointing_identifier=create_fake_identifier_for_testing(), # todo, from dataproduct.specifications_doc, Jan David checks how to translate int -> Identifier object
- subband="1", # todo, from dataproduct.specifications_doc
- starttime="1980-03-23T10:20:15", # todo, from dataproduct.specifications_doc
- duration="P6Y3M10DT15H", # todo, from dataproduct.specifications_doc
- integrationinterval=10, # todo, double, from dataproduct.specifications_doc
+ subband=dataproduct.feedback_doc['frequency']['subbands'][0],
+ starttime=dataproduct.feedback_doc['time']['start_time'],
+ duration=isodate.duration_isoformat(datetime.timedelta(seconds=dataproduct.feedback_doc['time']['duration'])),
+ integrationinterval=dataproduct.feedback_doc['time']['sample_width'],
integrationintervalunit="s",
- central_frequency=160, # todo, from dataproduct.specifications_doc
+ central_frequency=dataproduct.feedback_doc['frequency']['central_frequencies'][0],
central_frequencyunit="Hz",
- channelwidth_frequency=200, # todo, from dataproduct.specifications_doc
+ channelwidth_frequency=dataproduct.feedback_doc['frequency']['channel_width'],
channelwidth_frequencyunit="Hz",
- channelspersubband=122, # todo, from dataproduct.specifications_doc
+ channelspersubband=dataproduct.feedback_doc['frequency']['channels_per_subband'],
stationsubband=0 # not correct ;) (see metadata recipe CEP/Pipeline/recipes/sip/helpers/metadata.py)
)
# todo: distinguish and create other dataproduct types. Probably most of these can be filled in over time as needed,
diff --git a/SAS/TMSS/src/tmss/tmssapp/conversions.py b/SAS/TMSS/src/tmss/tmssapp/conversions.py
new file mode 100644
index 0000000000000000000000000000000000000000..e851ecbe396955955f1ae9dc1f32890cb819b53d
--- /dev/null
+++ b/SAS/TMSS/src/tmss/tmssapp/conversions.py
@@ -0,0 +1,41 @@
+from astropy.time import Time
+import astropy.units
+from lofar.lta.sip import station_coordinates
+from datetime import datetime
+from astropy.coordinates.earth import EarthLocation
+
+
+def local_sidereal_time_for_utc_and_station(timestamp: datetime = None,
+ station: str = 'CS002',
+ field: str = 'LBA',
+ kind: str = "apparent"):
+ """
+ calculate local sidereal time for given utc time and station
+ :param timestamp: timestamp as datetime object
+ :param station: station name
+ :param field: antennafield, 'LBA' or 'HBA'
+ :param kind: 'mean' or 'apparent'
+ :return:
+ """
+ if timestamp is None:
+ timestamp = datetime.utcnow()
+ station_coords = station_coordinates.parse_station_coordinates()
+ field_coords = station_coords["%s_%s" % (station, field)]
+ location = EarthLocation.from_geocentric(x=field_coords['x'], y=field_coords['y'], z=field_coords['z'], unit=astropy.units.m)
+ return local_sidereal_time_for_utc_and_longitude(timestamp=timestamp, longitude=location.lon.to_string(decimal=True), kind=kind)
+
+
+def local_sidereal_time_for_utc_and_longitude(timestamp: datetime = None,
+ longitude: float = 6.8693028,
+ kind: str = "apparent"):
+ """
+ :param timestamp: timestamp as datetime object
+ :param longitude: decimal longitude of observer location (defaults to CS002 LBA center)
+ :param kind: 'mean' or 'apparent'
+ :return:
+ """
+ if timestamp is None:
+ timestamp = datetime.utcnow()
+ t = Time(timestamp, format='datetime', scale='utc')
+ return t.sidereal_time(kind=kind, longitude=longitude)
+
diff --git a/SAS/TMSS/src/tmss/tmssapp/migrations/0001_initial.py b/SAS/TMSS/src/tmss/tmssapp/migrations/0001_initial.py
index 2b968bf0913cf065246ceed216a2220d001b2873..e1fef009d8d810546b0392046ffb36c836e899b9 100644
--- a/SAS/TMSS/src/tmss/tmssapp/migrations/0001_initial.py
+++ b/SAS/TMSS/src/tmss/tmssapp/migrations/0001_initial.py
@@ -1,4 +1,4 @@
-# Generated by Django 3.0.9 on 2020-08-19 13:24
+# Generated by Django 3.0.9 on 2020-09-24 15:47
from django.conf import settings
import django.contrib.postgres.fields
@@ -14,6 +14,7 @@ class Migration(migrations.Migration):
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+ ('viewflow', '0008_jsonfield_and_artifact'),
]
operations = [
@@ -52,6 +53,23 @@ class Migration(migrations.Migration):
('name', models.CharField(help_text='Human-readable name of this object.', max_length=128)),
('description', models.CharField(help_text='A longer description of this object.', max_length=255)),
('location', models.CharField(help_text='Human-readable location of the cluster.', max_length=128)),
+ ('archive_site', models.BooleanField(default=False, help_text='TRUE if this cluster is an archive site, FALSE if not (f.e. a local cluster, or user-owned cluster).')),
+ ],
+ options={
+ 'abstract': False,
+ },
+ ),
+ migrations.CreateModel(
+ name='CommonSchemaTemplate',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('tags', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=128), blank=True, default=list, help_text='User-defined search keywords for object.', size=8)),
+ ('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
+ ('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
+ ('name', models.CharField(help_text='Human-readable name of this object.', max_length=128)),
+ ('description', models.CharField(help_text='A longer description of this object.', max_length=255)),
+ ('version', models.IntegerField(editable=False, help_text='Version of this template (with respect to other templates of the same name)')),
+ ('schema', django.contrib.postgres.fields.jsonb.JSONField(help_text='Schema for the configurable parameters needed to use this template.')),
],
options={
'abstract': False,
@@ -142,7 +160,7 @@ class Migration(migrations.Migration):
('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
('name', models.CharField(help_text='Human-readable name of this object.', max_length=128)),
('description', models.CharField(help_text='A longer description of this object.', max_length=255)),
- ('version', models.CharField(help_text='Version of this template (with respect to other templates of the same name).', max_length=128)),
+ ('version', models.IntegerField(editable=False, help_text='Version of this template (with respect to other templates of the same name)')),
('schema', django.contrib.postgres.fields.jsonb.JSONField(help_text='Schema for the configurable parameters needed to use this template.')),
],
options={
@@ -171,7 +189,7 @@ class Migration(migrations.Migration):
('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
('name', models.CharField(help_text='Human-readable name of this object.', max_length=128)),
('description', models.CharField(help_text='A longer description of this object.', max_length=255)),
- ('version', models.CharField(help_text='Version of this template (with respect to other templates of the same name).', max_length=128)),
+ ('version', models.IntegerField(editable=False, help_text='Version of this template (with respect to other templates of the same name)')),
('schema', django.contrib.postgres.fields.jsonb.JSONField(help_text='Schema for the configurable parameters needed to use this template.')),
],
options={
@@ -226,6 +244,19 @@ class Migration(migrations.Migration):
'abstract': False,
},
),
+ migrations.CreateModel(
+ name='DefaultSchedulingConstraintsTemplate',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('tags', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=128), blank=True, default=list, help_text='User-defined search keywords for object.', size=8)),
+ ('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
+ ('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
+ ('name', models.CharField(max_length=128, unique=True)),
+ ],
+ options={
+ 'abstract': False,
+ },
+ ),
migrations.CreateModel(
name='DefaultSchedulingUnitTemplate',
fields=[
@@ -288,6 +319,7 @@ class Migration(migrations.Migration):
('name', models.CharField(help_text='Human-readable name of this object.', max_length=128)),
('description', models.CharField(help_text='A longer description of this object.', max_length=255)),
('capacity', models.BigIntegerField(help_text='Capacity in bytes')),
+ ('directory', models.CharField(help_text='Root directory under which we are allowed to write our data.', max_length=1024)),
],
options={
'abstract': False,
@@ -311,7 +343,7 @@ class Migration(migrations.Migration):
('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
('name', models.CharField(help_text='Human-readable name of this object.', max_length=128)),
('description', models.CharField(help_text='A longer description of this object.', max_length=255)),
- ('version', models.CharField(help_text='Version of this template (with respect to other templates of the same name).', max_length=128)),
+ ('version', models.IntegerField(editable=False, help_text='Version of this template (with respect to other templates of the same name)')),
('schema', django.contrib.postgres.fields.jsonb.JSONField(help_text='Schema for the configurable parameters needed to use this template.')),
('create_function', models.CharField(help_text='Python function to call to execute the generator.', max_length=128)),
],
@@ -342,6 +374,7 @@ class Migration(migrations.Migration):
('private_data', models.BooleanField(default=True, help_text='True if data of this project is sensitive. Sensitive data is not made public.')),
('expert', models.BooleanField(default=False, help_text='Expert projects put more responsibility on the PI.')),
('filler', models.BooleanField(default=False, help_text='Use this project to fill up idle telescope time.')),
+ ('archive_subdirectory', models.CharField(help_text='Subdirectory in which this project will store its data in the LTA. The full directory is constructed by prefixing with archive_location→directory.', max_length=1024)),
],
options={
'abstract': False,
@@ -403,6 +436,22 @@ class Migration(migrations.Migration):
'abstract': False,
},
),
+ migrations.CreateModel(
+ name='SchedulingConstraintsTemplate',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('tags', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=128), blank=True, default=list, help_text='User-defined search keywords for object.', size=8)),
+ ('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
+ ('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
+ ('name', models.CharField(help_text='Human-readable name of this object.', max_length=128)),
+ ('description', models.CharField(help_text='A longer description of this object.', max_length=255)),
+ ('version', models.IntegerField(editable=False, help_text='Version of this template (with respect to other templates of the same name)')),
+ ('schema', django.contrib.postgres.fields.jsonb.JSONField(help_text='Schema for the configurable parameters needed to use this template.')),
+ ],
+ options={
+ 'abstract': False,
+ },
+ ),
migrations.CreateModel(
name='SchedulingRelationPlacement',
fields=[
@@ -443,6 +492,26 @@ class Migration(migrations.Migration):
'abstract': False,
},
),
+ migrations.CreateModel(
+ name='SchedulingUnitDemo',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('name', models.CharField(max_length=50)),
+ ('state', models.IntegerField()),
+ ],
+ ),
+ migrations.CreateModel(
+ name='SchedulingUnitDemoProcess',
+ fields=[
+ ('process_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='viewflow.Process')),
+ ('text', models.CharField(max_length=150)),
+ ('approved', models.BooleanField(default=False)),
+ ],
+ options={
+ 'abstract': False,
+ },
+ bases=('viewflow.process',),
+ ),
migrations.CreateModel(
name='SchedulingUnitDraft',
fields=[
@@ -454,6 +523,7 @@ class Migration(migrations.Migration):
('description', models.CharField(help_text='A longer description of this object.', max_length=255)),
('requirements_doc', django.contrib.postgres.fields.jsonb.JSONField(help_text='Scheduling and/or quality requirements for this run.')),
('generator_instance_doc', django.contrib.postgres.fields.jsonb.JSONField(help_text='Parameter value that generated this run draft (NULLable).', null=True)),
+ ('scheduling_constraints_doc', django.contrib.postgres.fields.jsonb.JSONField(help_text='Scheduling Constraints for this run.', null=True)),
],
options={
'abstract': False,
@@ -484,7 +554,7 @@ class Migration(migrations.Migration):
('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
('name', models.CharField(help_text='Human-readable name of this object.', max_length=128)),
('description', models.CharField(help_text='A longer description of this object.', max_length=255)),
- ('version', models.CharField(help_text='Version of this template (with respect to other templates of the same name).', max_length=128)),
+ ('version', models.IntegerField(editable=False, help_text='Version of this template (with respect to other templates of the same name)')),
('schema', django.contrib.postgres.fields.jsonb.JSONField(help_text='Schema for the configurable parameters needed to use this template.')),
],
options={
@@ -512,6 +582,7 @@ class Migration(migrations.Migration):
('specifications_doc', django.contrib.postgres.fields.jsonb.JSONField(help_text='Final specifications, as input for the controller.')),
('do_cancel', models.DateTimeField(help_text='Timestamp when the subtask has been ordered to cancel (NULLable).', null=True)),
('priority', models.IntegerField(help_text='Absolute priority of this subtask (higher value means more important).')),
+ ('raw_feedback', models.CharField(help_text='The raw feedback for this Subtask', max_length=1048576, null=True)),
],
options={
'abstract': False,
@@ -573,7 +644,7 @@ class Migration(migrations.Migration):
('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
('name', models.CharField(help_text='Human-readable name of this object.', max_length=128)),
('description', models.CharField(help_text='A longer description of this object.', max_length=255)),
- ('version', models.CharField(help_text='Version of this template (with respect to other templates of the same name).', max_length=128)),
+ ('version', models.IntegerField(editable=False, help_text='Version of this template (with respect to other templates of the same name)')),
('schema', django.contrib.postgres.fields.jsonb.JSONField(help_text='Schema for the configurable parameters needed to use this template.')),
('queue', models.BooleanField(default=False)),
('realtime', models.BooleanField(default=False)),
@@ -677,7 +748,7 @@ class Migration(migrations.Migration):
('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
('name', models.CharField(help_text='Human-readable name of this object.', max_length=128)),
('description', models.CharField(help_text='A longer description of this object.', max_length=255)),
- ('version', models.CharField(help_text='Version of this template (with respect to other templates of the same name).', max_length=128)),
+ ('version', models.IntegerField(editable=False, help_text='Version of this template (with respect to other templates of the same name)')),
('schema', django.contrib.postgres.fields.jsonb.JSONField(help_text='Schema for the configurable parameters needed to use this template.')),
],
options={
@@ -693,6 +764,19 @@ class Migration(migrations.Migration):
'abstract': False,
},
),
+ migrations.CreateModel(
+ name='HelloWorldProcess',
+ fields=[
+ ],
+ options={
+ 'verbose_name': 'World Request',
+ 'verbose_name_plural': 'World Requests',
+ 'proxy': True,
+ 'indexes': [],
+ 'constraints': [],
+ },
+ bases=('viewflow.process',),
+ ),
migrations.CreateModel(
name='Setting',
fields=[
@@ -715,9 +799,9 @@ class Migration(migrations.Migration):
('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
('name', models.CharField(help_text='Human-readable name of this object.', max_length=128)),
('description', models.CharField(help_text='A longer description of this object.', max_length=255)),
- ('version', models.CharField(help_text='Version of this template (with respect to other templates of the same name).', max_length=128)),
+ ('version', models.IntegerField(editable=False, help_text='Version of this template (with respect to other templates of the same name)')),
('schema', django.contrib.postgres.fields.jsonb.JSONField(help_text='Schema for the configurable parameters needed to use this template.')),
- ('validation_code_js', models.CharField(help_text='JavaScript code for additional (complex) validation.', max_length=128)),
+ ('validation_code_js', models.CharField(blank=True, default='', help_text='JavaScript code for additional (complex) validation.', max_length=128)),
('type', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='tmssapp.TaskType')),
],
options={
@@ -999,11 +1083,21 @@ class Migration(migrations.Migration):
name='requirements_template',
field=models.ForeignKey(help_text='Schema used for requirements_doc.', on_delete=django.db.models.deletion.CASCADE, to='tmssapp.SchedulingUnitTemplate'),
),
+ migrations.AddField(
+ model_name='schedulingunitdraft',
+ name='scheduling_constraints_template',
+ field=models.ForeignKey(help_text='Schema used for scheduling_constraints_doc.', null=True, on_delete=django.db.models.deletion.CASCADE, to='tmssapp.SchedulingConstraintsTemplate'),
+ ),
migrations.AddField(
model_name='schedulingunitdraft',
name='scheduling_set',
field=models.ForeignKey(help_text='Set to which this scheduling unit draft belongs.', on_delete=django.db.models.deletion.CASCADE, related_name='scheduling_unit_drafts', to='tmssapp.SchedulingSet'),
),
+ migrations.AddField(
+ model_name='schedulingunitdemoprocess',
+ name='su',
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='tmssapp.SchedulingUnitDemo'),
+ ),
migrations.AddField(
model_name='schedulingunitblueprint',
name='draft',
@@ -1029,6 +1123,10 @@ class Migration(migrations.Migration):
name='project',
field=models.ForeignKey(help_text='Project to which this scheduling set belongs.', on_delete=django.db.models.deletion.PROTECT, related_name='scheduling_sets', to='tmssapp.Project'),
),
+ migrations.AddConstraint(
+ model_name='schedulingconstraintstemplate',
+ constraint=models.UniqueConstraint(fields=('name', 'version'), name='schedulingconstraintstemplate_unique_name_version'),
+ ),
migrations.AddField(
model_name='resourcetype',
name='quantity',
@@ -1044,6 +1142,11 @@ class Migration(migrations.Migration):
name='resource_type',
field=models.ForeignKey(help_text='Resource type.', on_delete=django.db.models.deletion.PROTECT, to='tmssapp.ResourceType'),
),
+ migrations.AddField(
+ model_name='project',
+ name='archive_location',
+ field=models.ForeignKey(help_text='Ingest data to this LTA cluster only (NULLable). NULL means: no preference.', null=True, on_delete=django.db.models.deletion.PROTECT, to='tmssapp.Filesystem'),
+ ),
migrations.AddField(
model_name='project',
name='cycles',
@@ -1088,6 +1191,11 @@ class Migration(migrations.Migration):
name='template',
field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='tmssapp.SchedulingUnitTemplate'),
),
+ migrations.AddField(
+ model_name='defaultschedulingconstraintstemplate',
+ name='template',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='tmssapp.SchedulingConstraintsTemplate'),
+ ),
migrations.AddField(
model_name='defaultgeneratortemplate',
name='template',
@@ -1136,6 +1244,11 @@ class Migration(migrations.Migration):
name='dataformat',
field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='tmssapp.Dataformat'),
),
+ migrations.AddField(
+ model_name='dataproduct',
+ name='datatype',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='tmssapp.Datatype'),
+ ),
migrations.AddField(
model_name='dataproduct',
name='feedback_template',
@@ -1161,6 +1274,10 @@ class Migration(migrations.Migration):
name='resource_type',
field=models.ForeignKey(help_text='Resource type.', on_delete=django.db.models.deletion.PROTECT, to='tmssapp.ResourceType'),
),
+ migrations.AddConstraint(
+ model_name='commonschematemplate',
+ constraint=models.UniqueConstraint(fields=('name', 'version'), name='commonschematemplate_unique_name_version'),
+ ),
migrations.AddField(
model_name='antennaset',
name='station_type',
@@ -1230,6 +1347,10 @@ class Migration(migrations.Migration):
model_name='defaultschedulingunittemplate',
index=django.contrib.postgres.indexes.GinIndex(fields=['tags'], name='tmssapp_def_tags_3ab2d6_gin'),
),
+ migrations.AddIndex(
+ model_name='defaultschedulingconstraintstemplate',
+ index=django.contrib.postgres.indexes.GinIndex(fields=['tags'], name='tmssapp_def_tags_480bbd_gin'),
+ ),
migrations.AddIndex(
model_name='defaultgeneratortemplate',
index=django.contrib.postgres.indexes.GinIndex(fields=['tags'], name='tmssapp_def_tags_89c89d_gin'),
diff --git a/SAS/TMSS/src/tmss/tmssapp/migrations/0002_populate.py b/SAS/TMSS/src/tmss/tmssapp/migrations/0002_populate.py
index 042d87b37a76af6f3dab5b706b252873af6c1846..92baffd4c15a8c025d234eeffed61ae9f443fabf 100644
--- a/SAS/TMSS/src/tmss/tmssapp/migrations/0002_populate.py
+++ b/SAS/TMSS/src/tmss/tmssapp/migrations/0002_populate.py
@@ -19,7 +19,6 @@ class Migration(migrations.Migration):
migrations.RunPython(populate_choices),
migrations.RunPython(populate_settings),
migrations.RunPython(populate_misc),
- migrations.RunPython(populate_lofar_json_schemas),
migrations.RunPython(populate_resources),
migrations.RunPython(populate_cycles),
migrations.RunPython(populate_projects) ]
diff --git a/SAS/TMSS/src/tmss/tmssapp/models/CMakeLists.txt b/SAS/TMSS/src/tmss/tmssapp/models/CMakeLists.txt
index 7598bc12c79161c19b95275e001a28adb92d3b56..2ac64b115ecf2f4bc700c614a3ba9572f3af6aa6 100644
--- a/SAS/TMSS/src/tmss/tmssapp/models/CMakeLists.txt
+++ b/SAS/TMSS/src/tmss/tmssapp/models/CMakeLists.txt
@@ -5,6 +5,8 @@ set(_py_files
__init__.py
specification.py
scheduling.py
+ helloworldflow.py
+ schedulingunitdemoflow.py
)
python_install(${_py_files}
diff --git a/SAS/TMSS/src/tmss/tmssapp/models/__init__.py b/SAS/TMSS/src/tmss/tmssapp/models/__init__.py
index 93f3c7e6d54f95c40d6d9484aad802b13f9991ba..be7a174d740d60b255c47117cb8abfc657cc9bde 100644
--- a/SAS/TMSS/src/tmss/tmssapp/models/__init__.py
+++ b/SAS/TMSS/src/tmss/tmssapp/models/__init__.py
@@ -1,2 +1,4 @@
from .specification import *
-from .scheduling import *
\ No newline at end of file
+from .scheduling import *
+from .helloworldflow import *
+from .schedulingunitdemoflow import *
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/models/helloworldflow.py b/SAS/TMSS/src/tmss/tmssapp/models/helloworldflow.py
new file mode 100644
index 0000000000000000000000000000000000000000..d92015dba2a5865f080c8a86b0bef28bd15e53ee
--- /dev/null
+++ b/SAS/TMSS/src/tmss/tmssapp/models/helloworldflow.py
@@ -0,0 +1,13 @@
+import jsonstore
+from viewflow.models import Process
+from viewflow.compat import _
+
+
+class HelloWorldProcess(Process):
+ text = jsonstore.CharField(_('Message'), max_length=50)
+ approved = jsonstore.BooleanField(_('Approved'), default=False)
+
+ class Meta:
+ proxy = True
+ verbose_name = _("World Request")
+ verbose_name_plural = _('World Requests')
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/models/scheduling.py b/SAS/TMSS/src/tmss/tmssapp/models/scheduling.py
index 238dd1e0e9bcd39a9257a5e874cf08a1822e5010..9d8a1d48c82d0a81a12397985dab0c03097f0e8e 100644
--- a/SAS/TMSS/src/tmss/tmssapp/models/scheduling.py
+++ b/SAS/TMSS/src/tmss/tmssapp/models/scheduling.py
@@ -9,18 +9,14 @@ logger = logging.getLogger(__name__)
from datetime import datetime, timedelta
from django.db.models import ForeignKey, CharField, DateTimeField, BooleanField, IntegerField, BigIntegerField, \
- ManyToManyField, CASCADE, SET_NULL, PROTECT, UniqueConstraint, QuerySet
+ ManyToManyField, CASCADE, SET_NULL, PROTECT, QuerySet
from django.contrib.postgres.fields import ArrayField, JSONField
from django.contrib.auth.models import User
-from .specification import AbstractChoice, BasicCommon, Template, NamedCommon # , <TaskBlueprint
+from .specification import AbstractChoice, BasicCommon, Template, NamedCommon, annotate_validate_add_defaults_to_doc_using_template
from enum import Enum
-from rest_framework.serializers import HyperlinkedRelatedField
-from django.dispatch import receiver
from django.db.models.expressions import RawSQL
-from lofar.sas.tmss.tmss.tmssapp.validation import validate_json_against_schema
from lofar.sas.tmss.tmss.exceptions import SubtaskSchedulingException
-from lofar.common.datetimeutils import formatDatetime
from lofar.messaging.messagebus import ToBus, DEFAULT_BROKER, DEFAULT_BUSNAME
from lofar.messaging.messages import EventMessage
from lofar.sas.tmss.client.tmssbuslistener import DEFAULT_TMSS_SUBTASK_NOTIFICATION_PREFIX
@@ -148,6 +144,7 @@ class Subtask(BasicCommon):
cluster = ForeignKey('Cluster', null=True, on_delete=PROTECT, help_text='Where the Subtask is scheduled to run (NULLable).')
# resource_claim = ForeignKey("ResourceClaim", null=False, on_delete=PROTECT) # todo <-- how is this external reference supposed to work?
created_or_updated_by_user = ForeignKey(User, null=True, editable=False, on_delete=PROTECT, help_text='The user who created / updated the subtask.')
+ raw_feedback = CharField(null=True, max_length=1048576, help_text='The raw feedback for this Subtask')
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@@ -219,8 +216,7 @@ class Subtask(BasicCommon):
def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
creating = self._state.adding # True on create, False on update
- if self.specifications_doc and self.specifications_template_id and self.specifications_template.schema:
- validate_json_against_schema(self.specifications_doc, self.specifications_template.schema)
+ annotate_validate_add_defaults_to_doc_using_template(self, 'specifications_doc', 'specifications_template')
if self.state.value == SubtaskState.Choices.SCHEDULED.value and self.__original_state.value == SubtaskState.Choices.SCHEDULING.value:
if self.start_time is None:
@@ -278,9 +274,7 @@ class SubtaskInput(BasicCommon):
selection_template = ForeignKey('TaskRelationSelectionTemplate', on_delete=PROTECT, help_text='Schema used for selection_doc.')
def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
- if self.selection_doc and self.selection_template_id and self.selection_template.schema:
- validate_json_against_schema(self.selection_doc, self.selection_template.schema)
-
+ annotate_validate_add_defaults_to_doc_using_template(self, 'selection_doc', 'selection_template')
super().save(force_insert, force_update, using, update_fields)
@@ -298,6 +292,7 @@ class Dataproduct(BasicCommon):
filename = CharField(max_length=128, help_text='Name of the file (or top-level directory) of the dataproduct. Adheres to a naming convention, but is not meant for parsing.')
directory = CharField(max_length=1024, help_text='Directory where this dataproduct is (to be) stored.')
dataformat = ForeignKey('Dataformat', null=False, on_delete=PROTECT)
+ datatype = ForeignKey('Datatype', null=False, on_delete=PROTECT)
deleted_since = DateTimeField(null=True, help_text='When this dataproduct was removed from disk, or NULL if not deleted (NULLable).')
pinned_since = DateTimeField(null=True, help_text='When this dataproduct was pinned to disk, that is, forbidden to be removed, or NULL if not pinned (NULLable).')
specifications_doc = JSONField(help_text='Dataproduct properties (f.e. beam, subband), to distinguish them when produced by the same task, and to act as input for selections in the Task Input and Work Request Relation Blueprint objects.')
@@ -310,11 +305,8 @@ class Dataproduct(BasicCommon):
feedback_template = ForeignKey('DataproductFeedbackTemplate', on_delete=PROTECT, help_text='Schema used for feedback_doc.')
def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
- if self.specifications_doc and self.specifications_template_id and self.specifications_template.schema:
- validate_json_against_schema(self.specifications_doc, self.specifications_template.schema)
-
- if self.feedback_doc and self.feedback_template_id and self.feedback_template.schema:
- validate_json_against_schema(self.feedback_doc, self.feedback_template.schema)
+ annotate_validate_add_defaults_to_doc_using_template(self, 'specifications_doc', 'specifications_template')
+ annotate_validate_add_defaults_to_doc_using_template(self, 'feedback_doc', 'feedback_template')
super().save(force_insert, force_update, using, update_fields)
@@ -337,10 +329,18 @@ class DataproductTransform(BasicCommon):
class Filesystem(NamedCommon):
capacity = BigIntegerField(help_text='Capacity in bytes')
cluster = ForeignKey('Cluster', on_delete=PROTECT, help_text='Cluster hosting this filesystem.')
+ directory = CharField(max_length=1024, help_text='Root directory under which we are allowed to write our data.')
+
+ def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
+ if self.directory and not self.directory.endswith('/'):
+ raise ValueError('directory value must end with a trailing slash!') # todo: ...and needs to start with slash?
+
+ super().save(force_insert, force_update, using, update_fields)
class Cluster(NamedCommon):
location = CharField(max_length=128, help_text='Human-readable location of the cluster.')
+ archive_site = BooleanField(default=False, null=False, help_text='TRUE if this cluster is an archive site, FALSE if not (f.e. a local cluster, or user-owned cluster).')
class DataproductArchiveInfo(BasicCommon):
diff --git a/SAS/TMSS/src/tmss/tmssapp/models/schedulingunitdemoflow.py b/SAS/TMSS/src/tmss/tmssapp/models/schedulingunitdemoflow.py
new file mode 100644
index 0000000000000000000000000000000000000000..b9797a0b12e56ffb6f284da503f43263561522c4
--- /dev/null
+++ b/SAS/TMSS/src/tmss/tmssapp/models/schedulingunitdemoflow.py
@@ -0,0 +1,13 @@
+# Create your models here.
+
+from django.db.models import CharField, IntegerField,BooleanField, ForeignKey, CASCADE, Model
+from viewflow.models import Process
+
+class SchedulingUnitDemo(Model):
+ name = CharField(max_length=50)
+ state = IntegerField()
+
+class SchedulingUnitDemoProcess(Process):
+ text = CharField(max_length=150)
+ approved = BooleanField(default=False)
+ su = ForeignKey(SchedulingUnitDemo, blank=True, null=True, on_delete=CASCADE)
diff --git a/SAS/TMSS/src/tmss/tmssapp/models/specification.py b/SAS/TMSS/src/tmss/tmssapp/models/specification.py
index 6b21c1512c76f804399afed4ae448be2678a4297..bf76dd37904a8e7d7cb5b5a5a03e53ea50ff1bc7 100644
--- a/SAS/TMSS/src/tmss/tmssapp/models/specification.py
+++ b/SAS/TMSS/src/tmss/tmssapp/models/specification.py
@@ -4,17 +4,22 @@ This file contains the database models
import logging
logger = logging.getLogger(__name__)
+import logging
+logger = logging.getLogger(__name__)
+
from django.db.models import Model, CharField, DateTimeField, BooleanField, ForeignKey, CASCADE, IntegerField, FloatField, SET_NULL, PROTECT, ManyToManyField, UniqueConstraint, QuerySet
from django.contrib.postgres.fields import ArrayField, JSONField
from django.contrib.postgres.indexes import GinIndex
from enum import Enum
from django.db.models.expressions import RawSQL
from django.db.models.deletion import ProtectedError
-
-from lofar.sas.tmss.tmss.tmssapp.validation import validate_json_against_schema
+from lofar.common.json_utils import validate_json_against_schema, validate_json_against_its_schema, add_defaults_to_json_object_for_schema
+from lofar.sas.tmss.tmss.exceptions import SchemaValidationException
from django.core.exceptions import ValidationError
-from rest_framework import status
import datetime
+import json
+import jsonschema
+from django.urls import reverse as revese_url
from lofar.messaging.messagebus import ToBus, DEFAULT_BROKER, DEFAULT_BUSNAME
from lofar.messaging.messages import EventMessage
from lofar.common.util import single_line_with_single_spaces
@@ -149,7 +154,7 @@ class SchedulingRelationPlacement(AbstractChoice):
AFTER = "after"
BEFORE = "before"
PARALLEL = "parallel"
-
+
class Flag(AbstractChoice):
"""Defines the model and predefined list of possible Flags to be used in Setting.
The items in the Choises class below are automagically populated into the database via a data migration."""
@@ -227,15 +232,91 @@ class TaskConnectorType(BasicCommon):
# abstract models
class Template(NamedCommon):
- version = CharField(max_length=128, help_text='Version of this template (with respect to other templates of the same name).')
+ version = IntegerField(editable=False, null=False, help_text='Version of this template (with respect to other templates of the same name)')
schema = JSONField(help_text='Schema for the configurable parameters needed to use this template.')
class Meta:
abstract = True
constraints = [UniqueConstraint(fields=['name', 'version'], name='%(class)s_unique_name_version')]
+ def validate_and_annotate_schema(self):
+ '''validate this template's schema, check for the required properties '$id', '$schema', 'title', 'description',
+ and annotate this schema with the template's name, description and version.'''
+ try:
+ if isinstance(self.schema, str):
+ self.schema = json.loads(self.schema)
+ except json.JSONDecodeError as e:
+ raise SchemaValidationException(str(e))
+
+ # sync up the template properties with the schema
+ self.schema['title'] = self.name
+ self.schema['description'] = self.description
+ self.schema['version'] = self.version
+
+ # check for missing properties
+ missing_properties = [property for property in ['$id', '$schema', 'title', 'description'] if property not in self.schema]
+ if missing_properties:
+ raise SchemaValidationException("Missing required properties '%s' for %s name='%s' version=%s in schema:\n%s" % (', '.join(missing_properties),
+ self.__class__.__name__, self.name, self.version,
+ json.dumps(self.schema, indent=2)))
+
+ # check for valid url's
+ invalid_url_properties = [property for property in ['$id', '$schema'] if not self.schema[property].startswith('http')]
+ if invalid_url_properties:
+ raise SchemaValidationException("Properties '%s' should contain a valid URL's for %s name='%s' version=%s in schema:\n%s" % (', '.join(invalid_url_properties),
+ self.__class__.__name__, self.name, self.version,
+ json.dumps(self.schema, indent=2)))
+
+ try:
+ # construct full url for $id of this schema
+ path = revese_url('get_template_json_schema', kwargs={'template': self._meta.model_name,
+ 'name': self.name,
+ 'version': self.version}).rstrip('/')
+ parts = self.schema['$id'].split('/')
+ scheme_host = '%s//%s' % (parts[0], parts[2])
+ id_url = '%s%s#' % (scheme_host, path)
+ self.schema['$id'] = id_url
+ except Exception as e:
+ logger.error("Could not override schema $id with auto-generated url: %s", e)
+
+ # this template's schema has a schema of its own (usually the draft-06 meta schema). Validate it.
+ validate_json_against_its_schema(self.schema)
+
+ @property
+ def is_used(self) -> bool:
+ '''Is this template used by any of its related objects?'''
+ for rel_obj in self._meta.related_objects:
+ if rel_obj.related_model.objects.filter(**{rel_obj.field.attname: self}).count() > 0:
+ return True
+ return False
+
+ def auto_set_version_number(self):
+ '''A template cannot/shouldnot be updated if it is already being used.
+ So, update the version number if the template is already used, else keep it.'''
+ if self.pk is None:
+ # this is a new instance. auto-assign new unique version number
+ self.version = self.__class__.objects.filter(name=self.name).count() + 1
+ else:
+ # this is a known template. Check if it is being used.
+ if self.is_used:
+ # yes, this template is used by others, so "editing"/updating is forbidden,
+ # so create new instance (by setting pk=None) and assign new unique version number
+ self.pk = None
+ self.version = self.__class__.objects.filter(name=self.name).count() + 1
+
+ def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
+ self.auto_set_version_number()
+ self.validate_and_annotate_schema()
+ super().save(force_insert or self.pk is None, force_update, using, update_fields)
+
+
# concrete models
+class CommonSchemaTemplate(Template):
+ '''A Template model for common (reusable) schema's'''
+ pass
+
+
class GeneratorTemplate(Template):
create_function = CharField(max_length=128, help_text='Python function to call to execute the generator.')
@@ -271,8 +352,17 @@ class DefaultSchedulingUnitTemplate(BasicCommon):
template = ForeignKey("SchedulingUnitTemplate", on_delete=PROTECT)
+class SchedulingConstraintsTemplate(Template):
+ pass
+
+
+class DefaultSchedulingConstraintsTemplate(BasicCommon):
+ name = CharField(max_length=128, unique=True)
+ template = ForeignKey("SchedulingConstraintsTemplate", on_delete=PROTECT)
+
+
class TaskTemplate(Template):
- validation_code_js = CharField(max_length=128, help_text='JavaScript code for additional (complex) validation.')
+ validation_code_js = CharField(max_length=128, blank=True, default="", help_text='JavaScript code for additional (complex) validation.')
type = ForeignKey('TaskType', null=False, on_delete=PROTECT)
@@ -331,6 +421,16 @@ class Project(NamedCommonPK):
filler = BooleanField(default=False, help_text='Use this project to fill up idle telescope time.')
project_category = ForeignKey('ProjectCategory', null=True, on_delete=PROTECT, help_text='Project category.')
period_category = ForeignKey('PeriodCategory', null=True, on_delete=PROTECT, help_text='Period category.')
+ archive_location = ForeignKey('Filesystem', null=True, on_delete=PROTECT, help_text='Ingest data to this LTA cluster only (NULLable). NULL means: no preference.')
+ archive_subdirectory = CharField(max_length=1024, help_text='Subdirectory in which this project will store its data in the LTA. The full directory is constructed by prefixing with archive_location→directory.')
+
+ def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
+ if self.archive_subdirectory and not self.archive_subdirectory.endswith('/'):
+ raise ValueError('directory value must end with a trailing slash!')
+ if self.archive_subdirectory and self.archive_subdirectory.startswith('/'):
+ raise ValueError('directory value must be a relative path (and not start with a slash)!')
+
+ super().save(force_insert, force_update, using, update_fields)
# JK, 29/07/20 - after discussion with Sander, it turns out that the ticket TMSS-277 was a misunderstanding.
# 'default' does not refer to 'default values' that are supposed to be filled in by the backend.
@@ -366,6 +466,38 @@ class ProjectQuota(Model):
class ResourceType(NamedCommonPK):
quantity = ForeignKey('Quantity', null=False, on_delete=PROTECT, help_text='The quantity of this resource type.')
+def annotate_validate_add_defaults_to_doc_using_template(model: Model, document_attr:str, template_attr:str) -> None:
+ '''
+ annotate, validate and add defaults to the JSON document in the model instance using the schema of the given template.
+ '''
+ try:
+ # fetch the actual JSON document and template-model-instance
+ document = getattr(model, document_attr)
+ template = getattr(model, template_attr)
+
+ if document is not None and template is not None:
+ try:
+ if isinstance(document, str):
+ document = json.loads(document)
+
+ # always annotate the json data document with a $schema URI to the schema that it is based on.
+ # this enables all users using this document (inside or outside of TMSS) to do their own validation and usage of editors which use the schema as UI template
+ document['$schema'] = template.schema['$id']
+ except (KeyError, TypeError, AttributeError) as e:
+ raise SchemaValidationException("Cannot set $schema in json_doc to the schema's $id.\nError: %s \njson_doc: %s\nschema: %s" % (str(e), document, template.schema))
+
+ # add defaults for missing properies, and validate on the fly
+ document = add_defaults_to_json_object_for_schema(document, template.schema)
+
+ # update the model instance with the updated and validated document
+ setattr(model, document_attr, document)
+ except AttributeError:
+ pass
+ except json.JSONDecodeError as e:
+ raise SchemaValidationException("Invalid JSON.\nError: %s \ndata: %s" % (str(e), document))
+ except jsonschema.ValidationError as e:
+ raise SchemaValidationException(str(e))
+
class SchedulingSet(NamedCommon):
generator_doc = JSONField(null=True, help_text='Parameters for the generator (NULLable).')
@@ -374,9 +506,7 @@ class SchedulingSet(NamedCommon):
project = ForeignKey('Project', related_name="scheduling_sets", on_delete=PROTECT, help_text='Project to which this scheduling set belongs.') # protected to avoid accidents
def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
- if self.generator_doc and self.generator_template_id and self.generator_template.schema:
- validate_json_against_schema(self.generator_doc, self.generator_template.schema)
-
+ annotate_validate_add_defaults_to_doc_using_template(self, 'generator_doc', 'generator_template')
super().save(force_insert, force_update, using, update_fields)
@@ -388,17 +518,21 @@ class SchedulingUnitDraft(NamedCommon):
scheduling_set = ForeignKey('SchedulingSet', related_name='scheduling_unit_drafts', on_delete=CASCADE, help_text='Set to which this scheduling unit draft belongs.')
requirements_template = ForeignKey('SchedulingUnitTemplate', on_delete=CASCADE, help_text='Schema used for requirements_doc.')
observation_strategy_template = ForeignKey('SchedulingUnitObservingStrategyTemplate', on_delete=PROTECT, null=True, help_text='Observation Strategy Template used to create the requirements_doc.')
+ scheduling_constraints_doc = JSONField(help_text='Scheduling Constraints for this run.', null=True)
+ scheduling_constraints_template = ForeignKey('SchedulingConstraintsTemplate', on_delete=CASCADE, null=True, help_text='Schema used for scheduling_constraints_doc.')
def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
- if self.requirements_doc:
- if self.requirements_template_id and self.requirements_template.schema:
- # If this scheduling unit was created from an observation_strategy_template,
- # then make sure that the observation_strategy_template validates against this unit's requirements_template.schema
- if self.observation_strategy_template_id and self.observation_strategy_template.template:
- validate_json_against_schema(self.observation_strategy_template.template, self.requirements_template.schema)
+ if self.requirements_doc is not None and self.requirements_template_id and self.requirements_template.schema is not None:
+ # If this scheduling unit was created from an observation_strategy_template,
+ # then make sure that the observation_strategy_template validates against this unit's requirements_template.schema
+ if self.observation_strategy_template_id and self.observation_strategy_template.template:
+ validate_json_against_schema(self.observation_strategy_template.template, self.requirements_template.schema)
- validate_json_against_schema(self.requirements_doc, self.requirements_template.schema)
+ if self.scheduling_constraints_doc is not None and self.scheduling_constraints_template_id and self.scheduling_constraints_template.schema is not None:
+ validate_json_against_schema(self.scheduling_constraints_doc, self.scheduling_constraints_template.schema)
+ annotate_validate_add_defaults_to_doc_using_template(self, 'requirements_doc', 'requirements_template')
+ annotate_validate_add_defaults_to_doc_using_template(self, 'scheduling_constraints_doc', 'scheduling_constraints_template')
super().save(force_insert, force_update, using, update_fields)
@property
@@ -435,8 +569,7 @@ class SchedulingUnitBlueprint(NamedCommon):
draft = ForeignKey('SchedulingUnitDraft', related_name='scheduling_unit_blueprints', on_delete=CASCADE, help_text='Scheduling Unit Draft which this run instantiates.')
def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
- if self.requirements_doc and self.requirements_template_id and self.requirements_template.schema:
- validate_json_against_schema(self.requirements_doc, self.requirements_template.schema)
+ annotate_validate_add_defaults_to_doc_using_template(self, 'requirements_doc', 'requirements_template')
is_new_instance = self.id is None
@@ -510,9 +643,7 @@ class TaskDraft(NamedCommon):
specifications_template = ForeignKey('TaskTemplate', on_delete=CASCADE, help_text='Schema used for requirements_doc.') # todo: 'schema'?
def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
- if self.specifications_doc and self.specifications_template_id and self.specifications_template.schema:
- validate_json_against_schema(self.specifications_doc, self.specifications_template.schema)
-
+ annotate_validate_add_defaults_to_doc_using_template(self, 'specifications_doc', 'specifications_template')
super().save(force_insert, force_update, using, update_fields)
@property
@@ -626,9 +757,7 @@ class TaskBlueprint(NamedCommon):
scheduling_unit_blueprint = ForeignKey('SchedulingUnitBlueprint', related_name='task_blueprints', on_delete=CASCADE, help_text='Scheduling Unit Blueprint to which this task belongs.')
def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
- if self.specifications_doc and self.specifications_template_id and self.specifications_template.schema:
- validate_json_against_schema(self.specifications_doc, self.specifications_template.schema)
-
+ annotate_validate_add_defaults_to_doc_using_template(self, 'specifications_doc', 'specifications_template')
super().save(force_insert, force_update, using, update_fields)
@property
@@ -731,9 +860,7 @@ class TaskRelationDraft(BasicCommon):
output_role = ForeignKey('TaskConnectorType', related_name='taskrelationdraft_output_roles', on_delete=CASCADE, help_text='Output connector type (what kind of data can be created as output).')
def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
- if self.selection_doc and self.selection_template_id and self.selection_template.schema:
- validate_json_against_schema(self.selection_doc, self.selection_template.schema)
-
+ annotate_validate_add_defaults_to_doc_using_template(self, 'selection_doc', 'selection_template')
super().save(force_insert, force_update, using, update_fields)
@@ -752,9 +879,7 @@ class TaskRelationBlueprint(BasicCommon):
selection_template = ForeignKey('TaskRelationSelectionTemplate', on_delete=CASCADE, help_text='Schema used for selection_doc.') # todo: 'schema'?
def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
- if self.selection_doc and self.selection_template_id and self.selection_template.schema:
- validate_json_against_schema(self.selection_doc, self.selection_template.schema)
-
+ annotate_validate_add_defaults_to_doc_using_template(self, 'selection_doc', 'selection_template')
super().save(force_insert, force_update, using, update_fields)
diff --git a/SAS/TMSS/src/tmss/tmssapp/populate.py b/SAS/TMSS/src/tmss/tmssapp/populate.py
index d0daaa77770e6339d847f2bd6814d3521656f021..661c0e10d5ec3b04f0f40975c56c7747f33d86a8 100644
--- a/SAS/TMSS/src/tmss/tmssapp/populate.py
+++ b/SAS/TMSS/src/tmss/tmssapp/populate.py
@@ -23,7 +23,6 @@ from datetime import datetime, timezone
from lofar.sas.tmss.tmss.tmssapp import models
from lofar.sas.tmss.tmss.tmssapp.models.specification import *
from lofar.sas.tmss.tmss.tmssapp.models.scheduling import *
-from lofar.common.json_utils import *
from lofar.common import isTestEnvironment, isDevelopmentEnvironment
working_dir = os.path.dirname(os.path.abspath(__file__))
@@ -43,28 +42,6 @@ def populate_choices(apps, schema_editor):
def populate_settings(apps, schema_editor):
Setting.objects.create(name=Flag.objects.get(value='allow_scheduling_observations'), value=True)
-
-def populate_lofar_json_schemas(apps, schema_editor):
-
- _populate_scheduling_unit_schema()
- _populate_scheduling_unit_observation_strategry_schema()
-
- # populate task schema's
- _populate_preprocessing_schema()
- _populate_observation_with_stations_schema()
- _populate_calibrator_addon_schema()
-
- _populate_dataproduct_specifications_templates()
- _populate_taskrelation_selection_templates()
- _populate_dataproduct_feedback_templates()
- _populate_obscontrol_schema()
- _populate_pipelinecontrol_schema()
- _populate_connectors()
-
- _populate_qa_files_subtask_template()
- _populate_qa_plots_subtask_template()
-
-
def populate_test_data():
"""
Create a Test Schedule Set to be able to refer to when Scheduling Unit Draft is created from a
@@ -77,6 +54,7 @@ def populate_test_data():
from lofar.sas.tmss.tmss.exceptions import TMSSException
from lofar.sas.tmss.test.tmss_test_data_django_models import SchedulingSet_test_data, SchedulingUnitDraft_test_data
from lofar.sas.tmss.tmss.tmssapp.tasks import create_task_blueprints_and_subtasks_from_scheduling_unit_draft, create_task_blueprints_and_subtasks_and_schedule_subtasks_from_scheduling_unit_draft
+ from lofar.sas.tmss.tmss.tmssapp.subtasks import schedule_subtask
tmss_projects = models.Project.objects.filter(name__startswith="TMSS-Commissioning").all()
for tmss_project in tmss_projects:
@@ -167,217 +145,48 @@ def populate_resources(apps, schema_editor):
ResourceType.objects.create(name="observing_time_commissioning", description="Observing time for Commissioning/DDT (in seconds)", quantity=Quantity.objects.get(value=Quantity.Choices.TIME.value))
ResourceType.objects.create(name="support_time", description="Support time by human (in seconds)", quantity=Quantity.objects.get(value=Quantity.Choices.TIME.value))
ResourceType.objects.create(name="number_of_triggers", description="Number of trigger events (as integer)", quantity=Quantity.objects.get(value=Quantity.Choices.NUMBER.value))
+ # TODO these duplicates have names that front-end expects.
+ # TODO We should not have doubles.
+ ResourceType.objects.create(name="LTA Storage", description="Amount of storage in the LTA (in bytes)", quantity=Quantity.objects.get(value=Quantity.Choices.BYTES.value))
+ ResourceType.objects.create(name="CEP Storage", description="Amount of storage on the CEP processing cluster (in bytes)", quantity=Quantity.objects.get(value=Quantity.Choices.BYTES.value))
+ ResourceType.objects.create(name="CEP Processing Time", description="Processing time on the CEP processing cluster (in seconds)", quantity=Quantity.objects.get(value=Quantity.Choices.TIME.value))
+ ResourceType.objects.create(name="LOFAR Observing Time", description="Observing time (in seconds)", quantity=Quantity.objects.get(value=Quantity.Choices.TIME.value))
+ ResourceType.objects.create(name="LOFAR Observing Time prio A", description="Observing time with priority A (in seconds)", quantity=Quantity.objects.get(value=Quantity.Choices.TIME.value))
+ ResourceType.objects.create(name="LOFAR Observing Time prio B", description="Observing time with priority B (in seconds)", quantity=Quantity.objects.get(value=Quantity.Choices.TIME.value))
+ ResourceType.objects.create(name="LOFAR Observing Time Commissioning", description="Observing time for Commissioning/DDT (in seconds)", quantity=Quantity.objects.get(value=Quantity.Choices.TIME.value))
+ ResourceType.objects.create(name="LOFAR Support Time", description="Support time by human (in seconds)", quantity=Quantity.objects.get(value=Quantity.Choices.TIME.value))
+ ResourceType.objects.create(name="Number of triggers", description="Number of trigger events (as integer)", quantity=Quantity.objects.get(value=Quantity.Choices.NUMBER.value))
def populate_misc(apps, schema_editor):
- cluster = Cluster.objects.create(name="CEP4", location="CIT")
+ cluster = Cluster.objects.create(name="CEP4", location="CIT", archive_site=False)
fs = Filesystem.objects.create(name="LustreFS", cluster=cluster, capacity=3.6e15)
+ sara_cluster = Cluster.objects.create(name="SARA", location="SARA", archive_site=True)
+ juelich_cluster = Cluster.objects.create(name="Jülich", location="Jülich", archive_site=True)
+ poznan_cluster = Cluster.objects.create(name="Poznan", location="Poznan", archive_site=True)
+
+ sara_fs = Filesystem.objects.create(name="Lofar Storage (SARA)", cluster=sara_cluster, capacity=3.6e15,
+ directory="srm://srm.grid.sara.nl:8443/pnfs/grid.sara.nl/data/lofar/ops/projects/")
+ sara_test_fs = Filesystem.objects.create(name="Lofar Test Storage (SARA)", cluster=sara_cluster, capacity=3.6e15,
+ directory="srm://srm.grid.sara.nl:8443/pnfs/grid.sara.nl/data/lofar/ops/test/projects/")
+ sara_user_fs = Filesystem.objects.create(name="Lofar User Disk Storage (SARA)", cluster=sara_cluster, capacity=3.6e15,
+ directory="srm://srm.grid.sara.nl/pnfs/grid.sara.nl/data/lofar/user/disk/projects/")
+ juelich_fs = Filesystem.objects.create(name="Lofar Storage (Jülich)", cluster=juelich_cluster, capacity=3.6e15,
+ directory="srm://lofar-srm.fz-juelich.de:8443/pnfs/fz-juelich.de/data/lofar/ops/projects/")
+ pozname_fs = Filesystem.objects.create(name="Lofar Storage (Poznan)", cluster=poznan_cluster, capacity=3.6e15,
+ directory="srm://lta-head.lofar.psnc.pl:8443/lofar/ops/projects/")
+
-def _populate_scheduling_unit_schema():
- with open(os.path.join(working_dir, "schemas/scheduling-unit.json")) as json_file:
- json_data = json.loads(json_file.read())
- scheduling_unit_template_data = {"name": "scheduling unit schema",
- "description": 'Schema for scheduling unit',
- "version": '0.1',
- "tags": [],
- "schema": json_data}
- SchedulingUnitTemplate.objects.create(**scheduling_unit_template_data)
-
-
-def _populate_scheduling_unit_observation_strategry_schema():
- with open(os.path.join(working_dir, "schemas/UC1-scheduling-unit-observation-strategy.json")) as json_file:
- json_data = json.loads(json_file.read())
- scheduling_unit_template = models.SchedulingUnitTemplate.objects.get(name="scheduling unit schema")
-
- template_data = {"name": "UC1 observation strategy template",
- "description": 'UC1 observation strategy template',
- "scheduling_unit_template": scheduling_unit_template,
- "version": '0.1',
- "tags": ["UC1"],
- "template": json_data}
- SchedulingUnitObservingStrategyTemplate.objects.create(**template_data)
-
-
-def _populate_observation_with_stations_schema():
- with open(os.path.join(working_dir, "schemas/task-observation-with-stations.json")) as json_file:
- json_data = json.loads(json_file.read())
- task_template_data = {"name": "observation schema",
- "type": TaskType.objects.get(value='observation'),
- "description": 'schema for observations',
- "version": '0.1',
- "tags": [],
- "schema": json_data}
- TaskTemplate.objects.create(**task_template_data)
-
-
-def _populate_calibrator_addon_schema():
- with open(os.path.join(working_dir, "schemas/task-calibrator-addon.json")) as json_file:
- json_data = json.loads(json_file.read())
- task_template_data = {"name": "calibrator schema",
- "type": TaskType.objects.get(value='observation'),
- "description": 'addon schema for calibrator observations',
- "version": '0.1',
- "tags": [],
- "schema": json_data}
- TaskTemplate.objects.create(**task_template_data)
-
-
-def _populate_correlator_calibrator_schema():
- with open(os.path.join(working_dir, "schemas/task-correlator.json")) as json_file:
- json_data = json.loads(json_file.read())
- task_template_data = {"name": "correlator schema",
- "description": 'correlator schema for calibrator observations',
- "version": '0.1',
- "tags": ["obsolete?"],
- "schema": json_data}
- TaskTemplate.objects.create(**task_template_data)
-
-
-def _populate_obscontrol_schema():
- with open(os.path.join(working_dir, "schemas/subtask-observation-control.json")) as json_file:
- json_data = json.loads(json_file.read())
- subtask_template_data = {"type": SubtaskType.objects.get(value='observation'),
- "name": "observationcontrol schema",
- "description": 'observationcontrol schema for observation subtask',
- "version": '0.1',
- "realtime": True,
- "queue": False,
- "tags": [],
- "schema": json_data}
- SubtaskTemplate.objects.create(**subtask_template_data)
-
-
-def _populate_stations_schema():
- with open(os.path.join(working_dir, "schemas/task-stations.json")) as json_file:
- json_data = json.loads(json_file.read())
- task_template_data = {"name": "stations schema",
- "description": 'Generic station settings and selection',
- "version": '0.1',
- "tags": [],
- "schema": json_data}
- TaskTemplate.objects.create(**task_template_data)
-
-
-def _populate_taskrelation_selection_templates():
- # All
- with open(os.path.join(working_dir, "schemas/empty-schema.json")) as json_file:
- json_data = json.loads(json_file.read())
- data = {"name": "All",
- "description": 'Select all, apply no filtering.',
- "version": '1',
- "schema": json_data}
- TaskRelationSelectionTemplate.objects.create(**data)
-
- # SAP
- with open(os.path.join(working_dir, "schemas/task-relation-sap.json")) as json_file:
- json_data = json.loads(json_file.read())
- data = {"name": "SAP",
- "description": 'Select by SAP.',
- "version": '1',
- "tags": [],
- "schema": json_data}
- TaskRelationSelectionTemplate.objects.create(**data)
-
-
-def _populate_dataproduct_specifications_templates():
- # Note: to some extend, this reflects the TaskRelationSelectionTemplates. That is expected since they define
- # the filters that operate on the DataproductSpecificationTemplates defined here. However, filters probably
- # will only use a subset of dataproduct specs, but could allow selecting several values of which only one
- # can be met by a single dataproduct.
- with open(os.path.join(working_dir, "schemas/empty-schema.json")) as json_file:
- json_data = json.loads(json_file.read())
- template_data = {"name": "Empty",
- "description": 'Empty DataproductSpecificationsTemplate with an empty schema',
- "version": '1',
- "tags": [],
- "schema": json_data}
- DataproductSpecificationsTemplate.objects.create(**template_data)
-
- # SAP
- with open(os.path.join(working_dir, "schemas/task-relation-sap.json")) as json_file:
- json_data = json.loads(json_file.read())
- template_data = {"name": "SAP",
- "description": 'Select by SAP.',
- "version": '1',
- "schema": json_data}
- DataproductSpecificationsTemplate.objects.create(**template_data)
-
-
-def _populate_dataproduct_feedback_templates():
- with open(os.path.join(working_dir, "schemas/empty-schema.json")) as json_file:
- json_data = json.loads(json_file.read())
- template_data = {"name": "Empty",
- "description": 'Empty DataproductFeedbackTemplate with an empty schema',
- "version": '1',
- "tags": [],
- "schema": json_data}
- DataproductFeedbackTemplate.objects.create(**template_data)
-
-
-def _populate_qa_files_subtask_template():
- with open(os.path.join(working_dir, "schemas/subtask-qa-files.json")) as json_file:
- json_data = json.loads(json_file.read())
- subtask_template_data = {"type": SubtaskType.objects.get(value=SubtaskType.Choices.QA_FILES.value),
- "name": "QA file conversion",
- "description": 'QA file conversion subtask template',
- "version": '0.1',
- "realtime": False,
- "queue": True,
- "tags": [],
- "schema": json_data}
- SubtaskTemplate.objects.create(**subtask_template_data)
-
-
-def _populate_qa_plots_subtask_template():
- with open(os.path.join(working_dir, "schemas/subtask-qa-plots.json")) as json_file:
- json_data = json.loads(json_file.read())
- subtask_template_data = {"type": SubtaskType.objects.get(value=SubtaskType.Choices.QA_PLOTS.value),
- "name": "QA plots",
- "description": 'QA plots subtask template',
- "version": '0.1',
- "realtime": False,
- "queue": True,
- "tags": [],
- "schema": json_data}
- SubtaskTemplate.objects.create(**subtask_template_data)
-
-
-def _populate_preprocessing_schema():
- with open(os.path.join(working_dir, "schemas/task-preprocessing.json")) as json_file:
- json_data = json.loads(json_file.read())
- task_template_data = {"name": "preprocessing schema",
- "type": TaskType.objects.get(value='pipeline'),
- "description": 'preprocessing settings',
- "version": '0.1',
- "tags": [],
- "schema": json_data}
- TaskTemplate.objects.create(**task_template_data)
-
-
-def _populate_pipelinecontrol_schema():
- with open(os.path.join(working_dir, "schemas/subtask-pipeline-control.json")) as json_file:
- json_data = json.loads(json_file.read())
- subtask_template_data = {"type": SubtaskType.objects.get(value='pipeline'),
- "name": "pipelinecontrol schema",
- "description": 'pipelinecontrol schema for pipeline subtask',
- "version": '0.1',
- "realtime": True,
- "queue": False,
- "tags": [],
- "schema": json_data}
- SubtaskTemplate.objects.create(**subtask_template_data)
-
-
-def _populate_connectors():
+def populate_connectors():
# the TaskConnectorType's define how the Task[Draft/Blueprint] *can* be connected.
# TODO Need overview which we do actually need
TaskConnectorType.objects.create(role=Role.objects.get(value=Role.Choices.INPUT.value),
datatype=Datatype.objects.get(value=Datatype.Choices.VISIBILITIES.value),
- output_of=TaskTemplate.objects.get(name='observation schema'),
- input_of=TaskTemplate.objects.get(name='preprocessing schema'))
+ output_of=TaskTemplate.objects.get(name='calibrator observation'),
+ input_of=TaskTemplate.objects.get(name='preprocessing pipeline'))
TaskConnectorType.objects.create(role=Role.objects.get(value=Role.Choices.CORRELATOR.value),
datatype=Datatype.objects.get(value=Datatype.Choices.VISIBILITIES.value),
- output_of=TaskTemplate.objects.get(name='calibrator schema'),
- input_of=TaskTemplate.objects.get(name='preprocessing schema'))
\ No newline at end of file
+ output_of=TaskTemplate.objects.get(name='calibrator observation'),
+ input_of=TaskTemplate.objects.get(name='preprocessing pipeline'))
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/CMakeLists.txt b/SAS/TMSS/src/tmss/tmssapp/schemas/CMakeLists.txt
index f192559794af5108cca56446981e32d39eb070da..32542a45d0f26b4f8647455b7b8777fa52f5d8e3 100644
--- a/SAS/TMSS/src/tmss/tmssapp/schemas/CMakeLists.txt
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/CMakeLists.txt
@@ -1,21 +1,7 @@
include(PythonInstall)
-set(_json_schema_files
- scheduling-unit.json
- UC1-scheduling-unit-observation-strategy.json
- task-calibrator-addon.json
- task-observation-with-stations.json
- task-stations.json
- task-correlator.json
- task-preprocessing.json
- task-relation-sap.json
- subtask-observation-control.json
- subtask-pipeline-control.json
- subtask-qa-files.json
- subtask-qa-plots.json
- empty-schema.json
- )
+file(GLOB json_schema_files *.json)
+lofar_add_data_files(${json_schema_files} DESTINATION tmss/schemas)
-python_install(${_json_schema_files}
- DESTINATION lofar/sas/tmss/tmss/tmssapp/schemas)
+lofar_add_data_files(Readme.txt DESTINATION tmss/schemas)
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/Readme.txt b/SAS/TMSS/src/tmss/tmssapp/schemas/Readme.txt
new file mode 100644
index 0000000000000000000000000000000000000000..833ea65c6964c91c686612efa1ce285cb2f20367
--- /dev/null
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/Readme.txt
@@ -0,0 +1,6 @@
+For easy administration of the various templates, please use the following file naming convention: <snake_cased_template_name>-<name>-<version>.json
+
+These json files (should) contain a valid json schema, which can be uploaded to TMSS.
+
+Because there are various different types of Template models in TMSS, each with possible extra parameters,
+we've created this litte helper program tmss_populate which can upload all templates defined in the templates.json file.
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/UC1-scheduling-unit-observation-strategy.json b/SAS/TMSS/src/tmss/tmssapp/schemas/UC1-scheduling-unit-observation-strategy.json
index 10700c8a5f4f530e172ed654ceb3fee5a9e9fe73..111d0b1501934eab541b9c51b0f525f50854df38 100644
--- a/SAS/TMSS/src/tmss/tmssapp/schemas/UC1-scheduling-unit-observation-strategy.json
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/UC1-scheduling-unit-observation-strategy.json
@@ -13,7 +13,7 @@
"angle3": 0
}
},
- "specifications_template": "calibrator schema"
+ "specifications_template": "calibrator observation"
},
"Pipeline 1": {
"description": "Preprocessing Pipeline for Calibrator Observation 1",
@@ -36,7 +36,7 @@
},
"storagemanager": "dysco"
},
- "specifications_template": "preprocessing schema"
+ "specifications_template": "preprocessing pipeline"
},
"Target Observation": {
"description": "Target Observation for UC1 HBA scheduling unit",
@@ -98,7 +98,7 @@
}
]
},
- "specifications_template": "observation schema"
+ "specifications_template": "target observation"
},
"Pipeline SAP0": {
"description": "Preprocessing Pipeline for Target Observation SAP0",
@@ -121,7 +121,7 @@
},
"storagemanager": "dysco"
},
- "specifications_template": "preprocessing schema"
+ "specifications_template": "preprocessing pipeline"
},
"Pipeline SAP1": {
"description": "Preprocessing Pipeline for Target Observation SAP1",
@@ -144,7 +144,7 @@
},
"storagemanager": "dysco"
},
- "specifications_template": "preprocessing schema"
+ "specifications_template": "preprocessing pipeline"
},
"Calibrator Observation 2": {
"description": "Calibrator Observation for UC1 HBA scheduling unit",
@@ -159,7 +159,7 @@
"angle3": 0
}
},
- "specifications_template": "calibrator schema"
+ "specifications_template": "calibrator observation"
},
"Pipeline 2": {
"description": "Preprocessing Pipeline for Calibrator Observation 2",
@@ -182,7 +182,7 @@
},
"storagemanager": "dysco"
},
- "specifications_template": "preprocessing schema"
+ "specifications_template": "preprocessing pipeline"
}
},
"task_relations": [
@@ -200,7 +200,7 @@
},
"dataformat": "MeasurementSet",
"selection_doc": {},
- "selection_template": "All"
+ "selection_template": "all"
},
{
"producer": "Calibrator Observation 2",
@@ -216,7 +216,7 @@
},
"dataformat": "MeasurementSet",
"selection_doc": {},
- "selection_template": "All"
+ "selection_template": "all"
},
{
"producer": "Target Observation",
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/common_schema_template-pipeline-1.json b/SAS/TMSS/src/tmss/tmssapp/schemas/common_schema_template-pipeline-1.json
new file mode 100644
index 0000000000000000000000000000000000000000..ba6dd5cb38d09ca9bd53637cd6120c5485c78de8
--- /dev/null
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/common_schema_template-pipeline-1.json
@@ -0,0 +1,19 @@
+{
+ "$id": "http://tmss.lofar.org/api/schemas/commonschematemplate/pipeline/1#",
+ "$schema": "http://json-schema.org/draft-06/schema#",
+ "title": "pipeline",
+ "description": "This schema defines common parameters for pipelines.",
+ "version": 1,
+ "type": "object",
+ "definitions": {
+ "demix_strategy": {
+ "type": "string",
+ "default": "auto",
+ "enum": [
+ "auto",
+ "yes",
+ "no"
+ ]
+ }
+ }
+}
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/task-calibrator-addon.json b/SAS/TMSS/src/tmss/tmssapp/schemas/common_schema_template-pointing-1.json
similarity index 55%
rename from SAS/TMSS/src/tmss/tmssapp/schemas/task-calibrator-addon.json
rename to SAS/TMSS/src/tmss/tmssapp/schemas/common_schema_template-pointing-1.json
index a1642f634d20f905c7dbca91b0ad078c27c0479b..88668838c82f03c889baee2825b7f8bf9823d3a4 100644
--- a/SAS/TMSS/src/tmss/tmssapp/schemas/task-calibrator-addon.json
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/common_schema_template-pointing-1.json
@@ -1,7 +1,10 @@
{
- "$id": "http://example.com/example.json",
- "type": "object",
+ "$id": "http://tmss.lofar.org/api/schemas/commonschematemplate/pointing/1#",
"$schema": "http://json-schema.org/draft-06/schema#",
+ "title": "pointing",
+ "description": "This schema provives a definition for the pointings used in TMSS/LOFAR",
+ "version": "1",
+ "type": "object",
"definitions": {
"pointing": {
"type": "object",
@@ -31,19 +34,19 @@
"angle1": {
"type": "number",
"title": "Angle 1",
- "description": "First angle [rad] (e.g. RA)",
+ "description": "First angle (e.g. RA)",
"default": 0
},
"angle2": {
"type": "number",
"title": "Angle 2",
- "description": "Second angle [rad] (e.g. DEC)",
+ "description": "Second angle (e.g. DEC)",
"default": 0
},
"angle3": {
"type": "number",
"title": "Angle 3",
- "description": "Third angle [rad] (e.g. N in LMN)",
+ "description": "Third angle (e.g. N in LMN)",
"default": 0
}
},
@@ -52,30 +55,5 @@
"angle2"
]
}
- },
- "additionalProperties": false,
- "properties": {
- "duration": {
- "type": "number",
- "title": "Duration (seconds)",
- "description": "Duration of this observation",
- "default": 600,
- "minimum": 1
- },
- "autoselect": {
- "type": "boolean",
- "title": "Auto-select",
- "description": "Auto-select calibrator based on elevation",
- "default": true
- },
- "pointing": {
- "title": "Digital pointing",
- "description": "Manually selected calibrator",
- "$ref": "#/definitions/pointing",
- "default": {}
- }
- },
- "required": [
- "autoselect", "duration", "pointing"
- ]
+ }
}
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/common_schema_template-qa-1.json b/SAS/TMSS/src/tmss/tmssapp/schemas/common_schema_template-qa-1.json
new file mode 100644
index 0000000000000000000000000000000000000000..a023ce3c2a30ddb590e83aad0c244b49702d7dc2
--- /dev/null
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/common_schema_template-qa-1.json
@@ -0,0 +1,92 @@
+{
+ "$id":"http://tmss.lofar.org/api/schemas/commonschematemplate/QA/1#",
+ "$schema": "http://json-schema.org/draft-06/schema#",
+ "title":"QA",
+ "description":"This schema defines the parameters to setup and control the Quality Assurance (QA) tasks.",
+ "version":1,
+ "definitions":{
+ "file_conversion":{
+ "type":"object",
+ "title":"File Conversion",
+ "default":{},
+ "description":"Create a QA file for the observation",
+ "properties":{
+ "enabled":{
+ "type":"boolean",
+ "title":"enabled",
+ "default":true,
+ "description":"Do/Don't create a QA file for the observation"
+ },
+ "nr_of_subbands":{
+ "type":"integer",
+ "title":"#subbands",
+ "default":-1,
+ "description":"Keep this number of subbands from the observation in the QA file, or all if -1"
+ },
+ "nr_of_timestamps":{
+ "type":"integer",
+ "title":"#timestamps",
+ "default":256,
+ "minimum":1,
+ "description":"Extract this number of timestamps from the observation in the QA file (equidistantanly sampled, no averaging/interpolation)"
+ }
+ },
+ "additionalProperties":false,
+ "required": [
+ "enabled",
+ "nr_of_subbands",
+ "nr_of_timestamps"]
+ },
+ "plots":{
+ "type":"object",
+ "title":"Plots",
+ "default":{},
+ "description":"Create dynamic spectrum plots",
+ "properties":{
+ "enabled":{
+ "type":"boolean",
+ "title":"enabled",
+ "default":true,
+ "description":"Do/Don't create plots from the QA file from the observation"
+ },
+ "autocorrelation":{
+ "type":"boolean",
+ "title":"autocorrelation",
+ "default":true,
+ "description":"Create autocorrelation plots for all stations"
+ },
+ "crosscorrelation":{
+ "type":"boolean",
+ "title":"crosscorrelation",
+ "default":true,
+ "description":"Create crosscorrelation plots for all baselines"
+ }
+ },
+ "additionalProperties":false,
+ "required": [
+ "enabled",
+ "autocorrelation",
+ "crosscorrelation"]
+ },
+ "QA": {
+ "type":"object",
+ "title":"QA",
+ "description":"Perform all Quality Assurance (QA) tasks, including file conversion and plotting.",
+ "default":{},
+ "properties": {
+ "file_conversion" : {
+ "$ref": "http://tmss.lofar.org/api/schemas/commonschematemplate/QA/1#/definitions/file_conversion",
+ "default": {}
+ },
+ "plots" : {
+ "$ref": "http://tmss.lofar.org/api/schemas/commonschematemplate/QA/1#/definitions/plots",
+ "default": {}
+ }
+ },
+ "additionalProperties":false,
+ "required": [
+ "file_conversion",
+ "plots"]
+ }
+ }
+}
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/common_schema_template-stations-1.json b/SAS/TMSS/src/tmss/tmssapp/schemas/common_schema_template-stations-1.json
new file mode 100644
index 0000000000000000000000000000000000000000..2029524ff1cc8ba2dacd6214157d9854a9490aee
--- /dev/null
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/common_schema_template-stations-1.json
@@ -0,0 +1,172 @@
+{
+ "$id":"http://tmss.lofar.org/api/schemas/commonschematemplate/stations/1#",
+ "$schema": "http://json-schema.org/draft-06/schema#",
+ "title":"stations",
+ "description":"This schema provives a definitions for the LOFAR stations and their antenna sets and filters",
+ "version":"1",
+ "type":"object",
+ "definitions":{
+ "station":{
+ "type":"string",
+ "title":"Station",
+ "description":"",
+ "enum":[
+ "CS001",
+ "CS002",
+ "CS003",
+ "CS004",
+ "CS005",
+ "CS006",
+ "CS007",
+ "CS011",
+ "CS013",
+ "CS017",
+ "CS021",
+ "CS024",
+ "CS026",
+ "CS028",
+ "CS030",
+ "CS031",
+ "CS032",
+ "CS101",
+ "CS103",
+ "CS201",
+ "CS301",
+ "CS302",
+ "CS401",
+ "CS501",
+ "RS104",
+ "RS106",
+ "RS205",
+ "RS208",
+ "RS210",
+ "RS305",
+ "RS306",
+ "RS307",
+ "RS310",
+ "RS406",
+ "RS407",
+ "RS409",
+ "RS410",
+ "RS503",
+ "RS508",
+ "RS509",
+ "DE601",
+ "DE602",
+ "DE603",
+ "DE604",
+ "DE605",
+ "FR606",
+ "SE607",
+ "UK608",
+ "DE609",
+ "PL610",
+ "PL611",
+ "PL612",
+ "IE613",
+ "LV614"
+ ]
+ },
+ "station_list":{
+ "title":"fixed station list",
+ "default":[
+ "CS001"
+ ],
+ "type":"array",
+ "additionalItems":false,
+ "additionalProperties":false,
+ "items":{
+ "$ref": "http://tmss.lofar.org/api/schemas/commonschematemplate/stations/1#/definitions/station"
+ },
+ "minItems":1,
+ "uniqueItems":true
+ },
+ "station_set":{
+ "title":"dynamic station set",
+ "type":"object",
+ "default":{},
+ "additionalItems":false,
+ "items":{
+ "type":"object",
+ "title":"Station set",
+ "headerTemplate":"{{ self.group }}",
+ "additionalProperties":false,
+ "properties":{
+ "group":{
+ "type":"string",
+ "title":"Group/station",
+ "description":"Which (group of) station(s) to select from",
+ "default":"ALL",
+ "enum":[
+ "ALL",
+ "SUPERTERP",
+ "CORE",
+ "REMOTE",
+ "DUTCH",
+ "INTERNATIONAL"
+ ]
+ },
+ "min_stations":{
+ "type":"integer",
+ "title":"Minimum nr of stations",
+ "description":"Number of stations to use within group/station",
+ "default":1,
+ "minimum":0
+ }
+ },
+ "required":[
+ "group",
+ "min_stations"
+ ]
+ }
+ },
+ "stations": {
+ "title":"stations",
+ "description":"Use either the fixed station list, or one of the dynamic station sets.",
+ "oneOf": [ {
+ "$ref": "http://tmss.lofar.org/api/schemas/commonschematemplate/stations/1#/definitions/station_list"
+ },
+ {
+ "$ref": "http://tmss.lofar.org/api/schemas/commonschematemplate/stations/1#/definitions/station_set"
+ }
+ ],
+ "default": {
+ "group": "ALL",
+ "min_stations": 1
+ }
+ },
+ "antenna_set":{
+ "type":"string",
+ "title":"Antenna set",
+ "description":"Fields & antennas to use",
+ "default":"HBA_DUAL",
+ "enum":[
+ "HBA_DUAL",
+ "HBA_DUAL_INNER",
+ "HBA_ONE",
+ "HBA_ONE_INNER",
+ "HBA_ZERO",
+ "HBA_ZERO_INNER",
+ "LBA_INNER",
+ "LBA_OUTER",
+ "LBA_SPARSE_EVEN",
+ "LBA_SPARSE_ODD",
+ "LBA_ALL"
+ ]
+ },
+ "filter":{
+ "type":"string",
+ "title":"Band-pass filter",
+ "description":"Must match antenna type",
+ "default":"HBA_110_190",
+ "enum":[
+ "LBA_10_70",
+ "LBA_30_70",
+ "LBA_10_90",
+ "LBA_30_90",
+ "HBA_110_190",
+ "HBA_210_250"
+ ]
+ }
+ }
+}
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/common_schema_template-tasks-1.json b/SAS/TMSS/src/tmss/tmssapp/schemas/common_schema_template-tasks-1.json
new file mode 100644
index 0000000000000000000000000000000000000000..01c7c91fdb8cccbc94aae63ac1539fb006d136e3
--- /dev/null
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/common_schema_template-tasks-1.json
@@ -0,0 +1,29 @@
+{
+ "$id": "http://tmss.lofar.org/api/schemas/commonschematemplate/tasks/1#",
+ "$schema": "http://json-schema.org/draft-06/schema#",
+ "title": "tasks",
+ "description": "This schema provives a definitions for modelling task connections and relations",
+ "version": "1",
+ "type": "object",
+ "definitions": {
+ "task_connector": {
+ "type": "object",
+ "additionalProperties": false,
+ "default": {},
+ "properties": {
+ "role": {
+ "type": "string",
+ "title": "Role"
+ },
+ "datatype": {
+ "type": "string",
+ "title": "Data Type"
+ }
+ },
+ "required": [
+ "role",
+ "datatype"
+ ]
+ }
+ }
+}
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/dataproduct_feedback_template-empty-1.json b/SAS/TMSS/src/tmss/tmssapp/schemas/dataproduct_feedback_template-empty-1.json
new file mode 100644
index 0000000000000000000000000000000000000000..f7753d7b557a4230116e227d31661150f8e9d183
--- /dev/null
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/dataproduct_feedback_template-empty-1.json
@@ -0,0 +1,9 @@
+{
+ "$id":"http://tmss.lofar.org/api/schemas/dataproductfeedbacktemplate/empty/1#",
+ "$schema": "http://json-schema.org/draft-06/schema#",
+ "title":"empty",
+ "description":"empty",
+ "version":1,
+ "type": "object",
+ "properties": {}
+}
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/dataproduct_feedback_template-feedback-1.json b/SAS/TMSS/src/tmss/tmssapp/schemas/dataproduct_feedback_template-feedback-1.json
new file mode 100644
index 0000000000000000000000000000000000000000..ec18c35a2e37b575f746f06d40c2de992e6c2fd2
--- /dev/null
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/dataproduct_feedback_template-feedback-1.json
@@ -0,0 +1,211 @@
+{
+ "$id":"http://tmss.lofar.org/api/schemas/dataproductfeedbacktemplate/feedback/1#",
+ "$schema": "http://json-schema.org/draft-06/schema#",
+ "title": "feedback",
+ "type": "object",
+ "properties": {
+ "percentage_written": {
+ "title": "Percentage written",
+ "type": "integer",
+ "default": 0
+ },
+ "frequency": {
+ "title": "Frequency",
+ "type": "object",
+ "properties": {
+ "subbands": {
+ "title": "Subbands",
+ "type": "array",
+ "default": [],
+ "items": {
+ "title": "Subband",
+ "type": "integer",
+ "minimum": 0,
+ "maximum": 511
+ }
+ },
+ "central_frequencies": {
+ "title": "Central frequencies",
+ "type": "array",
+ "default": [],
+ "items": {
+ "title": "frequency",
+ "type": "number",
+ "default": 0.0,
+ "minimum": 0.0
+ }
+ },
+ "channel_width": {
+ "title": "Channel width",
+ "type": "number",
+ "default": 3051.8,
+ "minimum": 0.0
+ },
+ "channels_per_subband": {
+ "title": "Channels per subband",
+ "type": "integer",
+ "default": 64,
+ "minimum": 1
+ }
+ },
+ "required": [ "subbands", "central_frequencies", "channel_width", "channels_per_subband" ]
+ },
+ "time": {
+ "title": "Time",
+ "type": "object",
+ "properties": {
+ "start_time": {
+ "title": "Start time",
+ "type": "string",
+ "default": ""
+ },
+ "duration": {
+ "title": "Duration",
+ "type": "number",
+ "default": 0.0
+ },
+ "sample_width": {
+ "title": "Sample width",
+ "type": "number",
+ "default": 0.0
+ }
+ },
+ "required": [ "start_time", "duration", "sample_width" ]
+ },
+ "antennas": {
+ "title": "Antennas",
+ "type": "object",
+ "properties": {
+ "set": {
+ "$ref": "http://tmss.lofar.org/api/schemas/commonschematemplate/stations/1/#/definitions/antenna_set",
+ "default": "HBA_DUAL"
+ },
+ "fields": {
+ "title": "Fields",
+ "type": "array",
+ "default": [],
+ "items": {
+ "title": "Field",
+ "type": "object",
+ "properties": {
+ "station": {
+ "$ref": "http://tmss.lofar.org/api/schemas/commonschematemplate/stations/1/#/definitions/station"
+ },
+ "field": {
+ "title": "Field",
+ "type": "string",
+ "default": "HBA",
+ "enum": [
+ "LBA",
+ "HBA",
+ "HBA0",
+ "HBA1"
+ ]
+ },
+ "type": {
+ "title": "Type",
+ "type": "string",
+ "default": "HBA",
+ "enum": [
+ "LBA",
+ "HBA"
+ ]
+ }
+ },
+ "required": [ "station", "field", "type" ]
+ }
+ }
+ },
+ "required": [ "fields" ]
+ },
+ "target": {
+ "title": "Target",
+ "type": "object",
+ "properties": {
+ "pointing": {
+ "title": "Pointing",
+ "$ref": "http://tmss.lofar.org/api/schemas/commonschematemplate/pointing/1/#/definitions/pointing"
+ }
+ },
+ "required": [ "pointing" ]
+ },
+ "samples": {
+ "title": "Samples",
+ "type": "object",
+ "properties": {
+ "polarisations": {
+ "title": "Polarisations",
+ "type": "array",
+ "default": [
+ "XX",
+ "XY",
+ "YX",
+ "YY"
+ ],
+ "items": {
+ "title": "Polarisation",
+ "type": "string",
+ "default": "I",
+ "enum": [
+ "XX",
+ "XY",
+ "YX",
+ "YY",
+ "I",
+ "Q",
+ "U",
+ "V",
+ "Xr",
+ "Xi",
+ "Yr",
+ "Yi"
+ ]
+ }
+ },
+ "type": {
+ "title": "Type",
+ "type": "string",
+ "default": "float",
+ "enum": [
+ "float",
+ "integer"
+ ]
+ },
+ "complex": {
+ "title": "Complex values",
+ "type": "boolean",
+ "default": true
+ },
+ "bits": {
+ "title": "Bits per sample",
+ "type": "integer",
+ "default": 32,
+ "enum": [
+ 4,
+ 8,
+ 16,
+ 32,
+ 64
+ ]
+ },
+ "writer": {
+ "title": "Writer",
+ "type": "string",
+ "default": "standard",
+ "enum": [
+ "lofarstman",
+ "standard",
+ "dysco"
+ ]
+ },
+ "writer_version": {
+ "title": "Writer version",
+ "type": "string",
+ "default": "UNKNOWN"
+ }
+ },
+ "required": [ "polarisations", "type", "complex", "bits", "writer" ]
+ }
+ },
+ "required": [ "percentage_written", "frequency", "time", "antennas", "target", "samples" ]
+}
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/task-relation-sap.json b/SAS/TMSS/src/tmss/tmssapp/schemas/dataproduct_specifications_template-SAP-1.json
similarity index 65%
rename from SAS/TMSS/src/tmss/tmssapp/schemas/task-relation-sap.json
rename to SAS/TMSS/src/tmss/tmssapp/schemas/dataproduct_specifications_template-SAP-1.json
index 5a3fcd971304b297f152dc42d98780ff586b875e..16265b5fcc2f0080cef17c7c927ea7ea369bfe85 100644
--- a/SAS/TMSS/src/tmss/tmssapp/schemas/task-relation-sap.json
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/dataproduct_specifications_template-SAP-1.json
@@ -1,8 +1,10 @@
{
- "$id": "http://example.com/example.json",
+ "$id":"http://tmss.lofar.org/api/schemas/dataproductspecificationstemplate/SAP/1#",
"$schema": "http://json-schema.org/draft-06/schema#",
- "additionalProperties": false,
- "definitions": {},
+ "title":"SAP",
+ "description":"SAP",
+ "version":1,
+ "type": "object",
"properties": {
"sap": {
"type": "array",
@@ -16,6 +18,5 @@
"maximum": 1
}
}
- },
- "type": "object"
+ }
}
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/dataproduct_specifications_template-empty-1.json b/SAS/TMSS/src/tmss/tmssapp/schemas/dataproduct_specifications_template-empty-1.json
new file mode 100644
index 0000000000000000000000000000000000000000..abad10b57f882eed0f3588c6714bf888a4b00d3a
--- /dev/null
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/dataproduct_specifications_template-empty-1.json
@@ -0,0 +1,9 @@
+{
+ "$id":"http://tmss.lofar.org/api/schemas/dataproductspecificationstemplate/empty/1#",
+ "$schema": "http://json-schema.org/draft-06/schema#",
+ "title":"empty",
+ "description":"empty",
+ "version":1,
+ "type": "object",
+ "properties": {}
+}
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/empty-schema.json b/SAS/TMSS/src/tmss/tmssapp/schemas/empty-schema.json
deleted file mode 100644
index 68b2c85fdb23fc17e374619884793e9290d39321..0000000000000000000000000000000000000000
--- a/SAS/TMSS/src/tmss/tmssapp/schemas/empty-schema.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{
- "$id": "http://example.com/example.json",
- "type": "object",
- "$schema": "http://json-schema.org/draft-06/schema#",
- "definitions": {},
- "additionalProperties": false,
- "properties": {}
-}
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/scheduling_constraints_template-constraints-1.json b/SAS/TMSS/src/tmss/tmssapp/schemas/scheduling_constraints_template-constraints-1.json
new file mode 100644
index 0000000000000000000000000000000000000000..04bb208f0b4deff2d4a7d0491ef4108afe335922
--- /dev/null
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/scheduling_constraints_template-constraints-1.json
@@ -0,0 +1,164 @@
+{
+ "$schema": "http://json-schema.org/draft-06/schema#",
+ "title": "Constraints",
+ "description": "This schema defines the constraints for a scheduling unit",
+ "version": 1,
+ "definitions": {
+ "timestamp": {
+ "type": "string",
+ "pattern": "\\d{4}-[01]\\d-[0-3]\\dT[0-2]\\d:[0-5]\\d:[0-5]\\d\\.\\d+Z",
+ "format": "datetime"
+ },
+ "distance_on_sky": {
+ "type": "number",
+ "minimum": 0,
+ "maximum": 3.142
+ },
+ "elevation": {
+ "type": "number",
+ "minimum": 0,
+ "maximum": 1.571
+ }
+ },
+ "type": "object",
+ "properties": {
+ "scheduler": {
+ "name": "Scheduler",
+ "description": "Which scheduling system wiil schedule this",
+ "type": "string",
+ "enum": [
+ "manual",
+ "online"
+ ],
+ "default": "online"
+ },
+ "time": {
+ "type": "object",
+ "properties": {
+ "at": {
+ "description": "Start at this moment",
+ "$ref": "#/definitions/timestamp"
+ },
+ "after": {
+ "description": "Start after this moment",
+ "$ref": "#/definitions/timestamp"
+ },
+ "before": {
+ "description": "End before this moment",
+ "$ref": "#/definitions/timestamp"
+ },
+ "between": {
+ "description": "Run within one of these time windows",
+ "type": "array",
+ "items": {
+ "from": {
+ "$ref": "#/definitions/timestamp"
+ },
+ "to": {
+ "$ref": "#/definitions/timestamp"
+ },
+ "required": [
+ "from",
+ "to"
+ ]
+ },
+ "additionalItems": false
+ },
+ "not_between": {
+ "description": "NOT run within one of these time windows",
+ "type": "array",
+ "items": {
+ "from": {
+ "$ref": "#/definitions/timestamp"
+ },
+ "to": {
+ "$ref": "#/definitions/timestamp"
+ },
+ "required": [
+ "from",
+ "to"
+ ]
+ },
+ "additionalItems": false
+ }
+ },
+ "additionalProperties": false
+ },
+ "daily": {
+ "type": "object",
+ "properties": {
+ "require_night": {
+ "description": "Must run at night",
+ "type": "boolean",
+ "default": false
+ },
+ "require_day": {
+ "description": "Must run in daylight",
+ "type": "boolean",
+ "default": false
+ },
+ "avoid_twilight": {
+ "description": "Do not run during sunrise or sunset",
+ "type": "boolean",
+ "default": false
+ }
+ },
+ "additionalProperties": false
+ },
+ "sky": {
+ "type": "object",
+ "properties": {
+ "min_calibrator_elevation": {
+ "description": "Minimum elevation for all calibrator sources",
+ "$ref": "#/definitions/elevation",
+ "default": 0.5
+ },
+ "min_target_elevation": {
+ "description": "Minimum elevation for all target sources",
+ "$ref": "#/definitions/elevation",
+ "default": 0.5
+ },
+ "transit_offset": {
+ "description": "Offset window to LST centering",
+ "type": "object",
+ "properties": {
+ "from": {
+ "type": "integer",
+ "minimum": -43200,
+ "maximum": 43200
+ },
+ "to": {
+ "type": "integer",
+ "minimum": -43200,
+ "maximum": 43200
+ }
+ },
+ "additionalProperties": false
+ },
+ "min_distance": {
+ "type": "object",
+ "properties": {
+ "sun": {
+ "$ref": "#/definitions/distance_on_sky",
+ "default": 0.5
+ },
+ "moon": {
+ "$ref": "#/definitions/distance_on_sky",
+ "default": 0.5
+ },
+ "jupiter": {
+ "$ref": "#/definitions/distance_on_sky",
+ "default": 0.5
+ }
+ },
+ "additionalProperties": false
+ }
+ },
+ "additionalProperties": false
+ }
+ },
+ "additionalProperties": false,
+ "required": [
+ "scheduler"
+ ]
+}
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/scheduling-unit.json b/SAS/TMSS/src/tmss/tmssapp/schemas/scheduling_unit_template-scheduling_unit-1.json
similarity index 89%
rename from SAS/TMSS/src/tmss/tmssapp/schemas/scheduling-unit.json
rename to SAS/TMSS/src/tmss/tmssapp/schemas/scheduling_unit_template-scheduling_unit-1.json
index d792ba7893922198058d75ff403561fe684e4a5c..00af272aa1318b9628e974edd49baed3be4ec25a 100644
--- a/SAS/TMSS/src/tmss/tmssapp/schemas/scheduling-unit.json
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/scheduling_unit_template-scheduling_unit-1.json
@@ -1,29 +1,10 @@
{
- "$id": "http://example.com/example.json",
- "type": "object",
+ "$id": "http://tmss.lofar.org/api/schemas/schedulingunittemplate/scheduling unit/1#",
"$schema": "http://json-schema.org/draft-06/schema#",
- "additionalProperties": false,
- "definitions": {
- "task_connector": {
- "type": "object",
- "additionalProperties": false,
- "default": {},
- "properties": {
- "role": {
- "type": "string",
- "title": "Role"
- },
- "datatype": {
- "type": "string",
- "title": "Data Type"
- }
- },
- "required": [
- "role",
- "datatype"
- ]
- }
- },
+ "title": "scheduling unit",
+ "description": "This schema defines the structure of all tasks in a scheduling unit",
+ "version": 1,
+ "type": "object",
"properties": {
"tasks": {
"title": "Tasks",
@@ -101,12 +82,12 @@
},
"input": {
"title": "Input I/O Connector",
- "$ref": "#/definitions/task_connector",
+ "$ref": "http://tmss.lofar.org/api/schemas/commonschematemplate/tasks/1/#/definitions/task_connector",
"default": {}
},
"output": {
"title": "Output I/O Connector",
- "$ref": "#/definitions/task_connector",
+ "$ref": "http://tmss.lofar.org/api/schemas/commonschematemplate/tasks/1/#/definitions/task_connector",
"default": {}
},
"dataformat": {
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/subtask-observation-control.json b/SAS/TMSS/src/tmss/tmssapp/schemas/subtask-observation-control.json
deleted file mode 100644
index b225aab1d3104443231952f98859e2b557add3b2..0000000000000000000000000000000000000000
--- a/SAS/TMSS/src/tmss/tmssapp/schemas/subtask-observation-control.json
+++ /dev/null
@@ -1,316 +0,0 @@
-{
- "$id": "http://example.com/example.json",
- "type": "object",
- "$schema": "http://json-schema.org/draft-06/schema#",
- "definitions": {
- "pointing": {
- "type": "object",
- "additionalProperties": false,
- "properties": {
- "direction_type": {
- "type": "string",
- "title": "Reference frame",
- "description": "",
- "default": "J2000",
- "enum": [
- "J2000",
- "SUN",
- "MOON",
- "MERCURY",
- "VENUS",
- "MARS",
- "JUPITER",
- "SATURN",
- "URANUS",
- "NEPTUNE",
- "PLUTO"
- ]
- },
- "angle1": {
- "type": "number",
- "title": "Angle 1",
- "description": "First angle (f.e. RA)",
- "default": 0
- },
- "angle2": {
- "type": "number",
- "title": "Angle 2",
- "description": "Second angle (f.e. DEC)",
- "default": 0
- }
- }
- }
- },
- "additionalProperties": false,
- "properties": {
- "stations": {
- "type": "object",
- "default": {},
- "properties": {
- "station_list": {
- "title": "Station list",
- "type": "array",
- "additionalItems": false,
- "additionalProperties": false,
- "default": ["CS001"],
- "items": {
- "type": "string",
- "enum": [
- "CS001",
- "CS002",
- "CS003",
- "CS004",
- "CS005",
- "CS006",
- "CS007",
- "CS011",
- "CS013",
- "CS017",
- "CS021",
- "CS024",
- "CS026",
- "CS028",
- "CS030",
- "CS031",
- "CS032",
- "CS101",
- "CS103",
- "CS201",
- "CS301",
- "CS302",
- "CS401",
- "CS501",
- "RS106",
- "RS205",
- "RS208",
- "RS210",
- "RS305",
- "RS306",
- "RS307",
- "RS310",
- "RS406",
- "RS407",
- "RS409",
- "RS503",
- "RS508",
- "RS509",
- "DE601",
- "DE602",
- "DE603",
- "DE604",
- "DE605",
- "FR606",
- "SE607",
- "UK608",
- "DE609",
- "PL610",
- "PL611",
- "PL612",
- "IE613",
- "LV614"
- ],
- "title": "Station",
- "description": ""
- },
- "minItems": 1,
- "uniqueItems": true
- },
- "antenna_set": {
- "type": "string",
- "title": "Antenna set",
- "description": "Which antennas & fields to use on each station",
- "default": "HBA_DUAL",
- "enum": [
- "HBA_DUAL",
- "HBA_DUAL_INNER",
- "HBA_ONE",
- "HBA_ONE_INNER",
- "HBA_ZERO",
- "HBA_ZERO_INNER",
- "LBA_INNER",
- "LBA_OUTER",
- "LBA_SPARSE_EVEN",
- "LBA_SPARSE_ODD",
- "LBA_ALL"
- ]
- },
- "filter": {
- "type": "string",
- "title": "Band-pass filter",
- "description": "Must match antenna type",
- "default": "HBA_110_190",
- "enum": [
- "LBA_10_90",
- "LBA_30_90",
- "HBA_110_190",
- "HBA_210_250"
- ]
- },
- "analog_pointing": {
- "title": "Analog pointing",
- "description": "HBA only",
- "$ref": "#/definitions/pointing",
- "default": {}
- },
- "digital_pointings": {
- "type": "array",
- "title": "Beams",
- "additionalItems": false,
- "default": [{}],
- "items": {
- "title": "Beam",
- "headerTemplate": "{{ i0 }} - {{ self.name }}",
- "type": "object",
- "additionalProperties": false,
- "properties": {
- "name": {
- "type": "string",
- "title": "Name/target",
- "description": "Custom identifier for this beam. Same name is same beam.",
- "default": ""
- },
- "pointing": {
- "title": "Digital pointing",
- "$ref": "#/definitions/pointing",
- "default": {}
- },
- "subbands": {
- "type": "array",
- "title": "Subband list",
- "additionalItems": false,
- "default": [],
- "items": {
- "type": "integer",
- "title": "Subband",
- "minimum": 0,
- "maximum": 511
- }
- }
- }
- }
- }
- }
- },
- "COBALT": {
- "type": "object",
- "title": "COBALT correlator/beamformer",
- "additionalProperties": false,
- "default": {},
- "properties": {
- "blocksize": {
- "type": "integer",
- "title": "Block size (samples)",
- "description": "Size of blocks COBALT works on, must be a multiple of all processing requirements",
- "default": 196608,
- "minimum": 97656,
- "maximum": 292968
- },
- "delay_compensation": {
- "type": "boolean",
- "title": "Apply delay compensation",
- "description": "Compensate for geometric and clock differences",
- "default": true
- },
- "bandpass_correction": {
- "type": "boolean",
- "title": "Apply band-pass correction",
- "description": "Compensate for differences in station sensitivity within a subband",
- "default": true
- },
- "correlator": {
- "title": "Correlator",
- "type": "object",
- "default": {},
- "oneOf": [
- {
- "type": "object",
- "title": "Enabled",
- "additionalProperties": false,
- "default": {},
- "properties": {
- "enabled": {
- "type": "boolean",
- "title": "Enabled",
- "description": "",
- "default": true,
- "options": {
- "hidden": true
- },
- "enum": [
- true
- ]
- },
- "channels_per_subband": {
- "type": "integer",
- "title": "Channels/subband",
- "description": "Number of frequency bands per subband",
- "default": 64,
- "minimum": 1,
- "enum": [
- 1,
- 8,
- 16,
- 32,
- 64,
- 128,
- 256,
- 512,
- 1024
- ]
- },
- "blocks_per_integration": {
- "type": "integer",
- "title": "Blocks per integration",
- "description": "Number of blocks to integrate",
- "default": 1,
- "minimum": 1
- },
- "integrations_per_block": {
- "type": "integer",
- "title": "Integrations per block",
- "description": "Number of integrations to fit within each block",
- "default": 1,
- "minimum": 1
- },
- "phase_centers": {
- "type": "array",
- "title": "Custom phase centers",
- "additionalItems": false,
- "default": [{}],
- "items": {
- "title": "Beam",
- "headerTemplate": "Beam {{ self.index }}",
- "type": "object",
- "additionalProperties": false,
- "default": {},
- "properties": {
- "index": {
- "type": "integer",
- "title": "Station beam index",
- "description": "Apply to this station beam",
- "minimum": 0,
- "default": 0
- },
- "pointing": {
- "title": "Correlator pointing",
- "$ref": "#/definitions/pointing",
- "default": {}
- }
- }
- }
- }
- }
- },
- {
- "type": "object",
- "title": "Disabled",
- "additionalProperties": false,
- "default": {},
- "properties": {}
- }
- ]
- }
- }
- }
- }
-}
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/subtask-qa-files.json b/SAS/TMSS/src/tmss/tmssapp/schemas/subtask-qa-files.json
deleted file mode 100644
index 905f0a9285b65d721c8a8f243ee1b0a479b3475b..0000000000000000000000000000000000000000
--- a/SAS/TMSS/src/tmss/tmssapp/schemas/subtask-qa-files.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
- "$id": "http://example.com/example.json",
- "type": "object",
- "$schema": "http://json-schema.org/draft-06/schema#",
- "definitions": {},
- "additionalProperties": false,
- "properties": {
- "nr_of_subbands": {
- "type": "integer",
- "title": "#subbands",
- "default": -1,
- "description": "Keep this number of subbands from the observation in the QA file, or all if -1"
- },
- "nr_of_timestamps": {
- "type": "integer",
- "title": "#timestamps",
- "default": 256,
- "minimum": 1,
- "description": "Extract this number of timestamps from the observation in the QA file (equidistantanly sampled, no averaging/interpolation)"
- }
- }
-}
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/subtask-qa-plots.json b/SAS/TMSS/src/tmss/tmssapp/schemas/subtask-qa-plots.json
deleted file mode 100644
index 461305537ec18d3c123c94a2573a0dedfbeb47a8..0000000000000000000000000000000000000000
--- a/SAS/TMSS/src/tmss/tmssapp/schemas/subtask-qa-plots.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
- "$id": "http://example.com/example.json",
- "type": "object",
- "$schema": "http://json-schema.org/draft-06/schema#",
- "definitions": {},
- "additionalProperties": false,
- "properties": {
- "autocorrelation": {
- "type": "boolean",
- "title": "autocorrelation",
- "default": true,
- "description": "Create autocorrelation plots for all stations"
- },
- "crosscorrelation": {
- "type": "boolean",
- "title": "crosscorrelation",
- "default": true,
- "description": "Create crosscorrelation plots for all baselines"
- }
- }
-}
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/subtask_template-observation-1.json b/SAS/TMSS/src/tmss/tmssapp/schemas/subtask_template-observation-1.json
new file mode 100644
index 0000000000000000000000000000000000000000..c016a1412a74225fa66047319f0d2bba2f75a89e
--- /dev/null
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/subtask_template-observation-1.json
@@ -0,0 +1,216 @@
+{
+ "$id":"http://tmss.lofar.org/api/schemas/subtasktemplate/observation control/1#",
+ "$schema": "http://json-schema.org/draft-06/schema#",
+ "title":"observation control",
+ "description":"This schema defines the parameters to setup and control the observation subtask.",
+ "version":1,
+ "type":"object",
+ "default":{},
+ "properties":{
+ "stations":{
+ "type":"object",
+ "default":{},
+ "properties": {
+ "station_list": {
+ "$ref": "http://tmss.lofar.org/api/schemas/commonschematemplate/stations/1/#/definitions/station_list",
+ "default": [
+ "CS001"
+ ]
+ },
+ "antenna_set": {
+ "$ref": "http://tmss.lofar.org/api/schemas/commonschematemplate/stations/1/#/definitions/antenna_set",
+ "default": "HBA_DUAL"
+ },
+ "filter": {
+ "$ref": "http://tmss.lofar.org/api/schemas/commonschematemplate/stations/1/#/definitions/filter",
+ "default": "HBA_110_190"
+ },
+ "analog_pointing": {
+ "title": "Analog pointing",
+ "description": "HBA only",
+ "$ref": "http://tmss.lofar.org/api/schemas/commonschematemplate/pointing/1/#/definitions/pointing",
+ "default": {}
+ },
+ "digital_pointings": {
+ "type": "array",
+ "title": "Beams",
+ "additionalItems": false,
+ "default": [
+ {}
+ ],
+ "items": {
+ "title": "Beam",
+ "headerTemplate": "{{ i0 }} - {{ self.name }}",
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "name": {
+ "type": "string",
+ "title": "Name/target",
+ "description": "Custom identifier for this beam. Same name is same beam.",
+ "default": ""
+ },
+ "pointing": {
+ "title": "Digital pointing",
+ "$ref": "http://tmss.lofar.org/api/schemas/commonschematemplate/pointing/1/#/definitions/pointing",
+ "default": {}
+ },
+ "subbands": {
+ "type": "array",
+ "title": "Subband list",
+ "additionalItems": false,
+ "default": [],
+ "items": {
+ "type": "integer",
+ "title": "Subband",
+ "minimum": 0,
+ "maximum": 511
+ }
+ }
+ }
+ }
+ }
+ },
+ "additionalProperties": false
+ },
+ "COBALT":{
+ "type":"object",
+ "title":"COBALT correlator/beamformer",
+ "additionalProperties":false,
+ "default":{
+
+ },
+ "properties":{
+ "blocksize":{
+ "type":"integer",
+ "title":"Block size (samples)",
+ "description":"Size of blocks COBALT works on, must be a multiple of all processing requirements",
+ "default":196608,
+ "minimum":97656,
+ "maximum":292968
+ },
+ "delay_compensation":{
+ "type":"boolean",
+ "title":"Apply delay compensation",
+ "description":"Compensate for geometric and clock differences",
+ "default":true
+ },
+ "bandpass_correction":{
+ "type":"boolean",
+ "title":"Apply band-pass correction",
+ "description":"Compensate for differences in station sensitivity within a subband",
+ "default":true
+ },
+ "correlator":{
+ "title":"Correlator",
+ "type":"object",
+ "default":{
+
+ },
+ "oneOf":[
+ {
+ "type":"object",
+ "title":"Enabled",
+ "additionalProperties":false,
+ "default":{
+
+ },
+ "properties":{
+ "enabled":{
+ "type":"boolean",
+ "title":"Enabled",
+ "description":"",
+ "default":true,
+ "options":{
+ "hidden":true
+ },
+ "enum":[
+ true
+ ]
+ },
+ "channels_per_subband":{
+ "type":"integer",
+ "title":"Channels/subband",
+ "description":"Number of frequency bands per subband",
+ "default":64,
+ "minimum":1,
+ "enum":[
+ 1,
+ 8,
+ 16,
+ 32,
+ 64,
+ 128,
+ 256,
+ 512,
+ 1024
+ ]
+ },
+ "blocks_per_integration":{
+ "type":"integer",
+ "title":"Blocks per integration",
+ "description":"Number of blocks to integrate",
+ "default":1,
+ "minimum":1
+ },
+ "integrations_per_block":{
+ "type":"integer",
+ "title":"Integrations per block",
+ "description":"Number of integrations to fit within each block",
+ "default":1,
+ "minimum":1
+ },
+ "phase_centers":{
+ "type":"array",
+ "title":"Custom phase centers",
+ "additionalItems":false,
+ "default":[
+ {
+
+ }
+ ],
+ "items":{
+ "title":"Beam",
+ "headerTemplate":"Beam {{ self.index }}",
+ "type":"object",
+ "additionalProperties":false,
+ "default":{
+
+ },
+ "properties":{
+ "index":{
+ "type":"integer",
+ "title":"Station beam index",
+ "description":"Apply to this station beam",
+ "minimum":0,
+ "default":0
+ },
+ "pointing":{
+ "title":"Correlator pointing",
+ "$ref":"http://tmss.lofar.org/api/schemas/commonschematemplate/pointing/1/#/definitions/pointing",
+ "default":{
+
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ {
+ "type":"object",
+ "title":"Disabled",
+ "additionalProperties":false,
+ "default":{
+
+ },
+ "properties":{
+
+ }
+ }
+ ]
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/subtask-pipeline-control.json b/SAS/TMSS/src/tmss/tmssapp/schemas/subtask_template-pipeline-1.json
similarity index 92%
rename from SAS/TMSS/src/tmss/tmssapp/schemas/subtask-pipeline-control.json
rename to SAS/TMSS/src/tmss/tmssapp/schemas/subtask_template-pipeline-1.json
index f49c523fe5b217029da32dddde280dca384b31ca..cc6b1e86bdb5d0b1145042a323672c1228d9767f 100644
--- a/SAS/TMSS/src/tmss/tmssapp/schemas/subtask-pipeline-control.json
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/subtask_template-pipeline-1.json
@@ -1,8 +1,11 @@
{
- "$id": "http://example.com/example.json",
+ "$id":"http://tmss.lofar.org/api/schemas/subtasktemplate/pipeline control/1#",
+ "$schema": "http://json-schema.org/draft-06/schema#",
+ "title":"pipeline control",
+ "description":"This schema defines the parameters to setup and control a (preprocessing) pipeline subtask.",
+ "version":1,
"type": "object",
"$schema": "http://json-schema.org/draft-06/schema#",
- "additionalProperties": false,
"properties": {
"preflagger0": {
"title": "Preflagger0",
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/subtask_template-qa_file-1.json b/SAS/TMSS/src/tmss/tmssapp/schemas/subtask_template-qa_file-1.json
new file mode 100644
index 0000000000000000000000000000000000000000..1328385133006b38adce5a0be98ea22f094d756b
--- /dev/null
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/subtask_template-qa_file-1.json
@@ -0,0 +1,14 @@
+{
+ "$id": "http://tmss.lofar.org/api/schemas/subtasktemplate/QA file conversion/1#",
+ "$schema": "http://json-schema.org/draft-06/schema#",
+ "title": "QA file conversion",
+ "description": "This schema defines the parameters to setup and control the QA file creation subtask.",
+ "version": 1,
+ "type": "object",
+ "properties": {
+ "file_conversion": {
+ "$ref": "http://tmss.lofar.org/api/schemas/commonschematemplate/QA/1/#/definitions/file_conversion",
+ "default": {}
+ }
+ }
+}
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/subtask_template-qa_plots-1.json b/SAS/TMSS/src/tmss/tmssapp/schemas/subtask_template-qa_plots-1.json
new file mode 100644
index 0000000000000000000000000000000000000000..f823b284b4c33f12d59ad395cdabadee31e665ed
--- /dev/null
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/subtask_template-qa_plots-1.json
@@ -0,0 +1,14 @@
+{
+ "$id":"http://tmss.lofar.org/api/schemas/subtasktemplate/QA plots/1#",
+ "$schema": "http://json-schema.org/draft-06/schema#",
+ "title":"QA plots",
+ "description":"This schema defines the parameters to setup and control the QA plotting subtask.",
+ "version":1,
+ "type": "object",
+ "properties": {
+ "plots": {
+ "$ref": "http://tmss.lofar.org/api/schemas/commonschematemplate/QA/1/#/definitions/plots",
+ "default": {}
+ }
+ }
+}
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/task-correlator.json b/SAS/TMSS/src/tmss/tmssapp/schemas/task-correlator.json
deleted file mode 100644
index 55b73899eb0e499455bb37d14df1207eca65a43a..0000000000000000000000000000000000000000
--- a/SAS/TMSS/src/tmss/tmssapp/schemas/task-correlator.json
+++ /dev/null
@@ -1,176 +0,0 @@
-{
- "$id": "http://example.com/example.json",
- "type": "object",
- "$schema": "http://json-schema.org/draft-06/schema#",
- "definitions": {
- "pointing": {
- "type": "object",
- "additionalProperties": false,
- "properties": {
- "direction_type": {
- "type": "string",
- "title": "Reference frame",
- "description": "",
- "default": "J2000",
- "enum": [
- "J2000",
- "SUN",
- "MOON",
- "MERCURY",
- "VENUS",
- "MARS",
- "JUPITER",
- "SATURN",
- "URANUS",
- "NEPTUNE",
- "PLUTO"
- ]
- },
- "angle1": {
- "type": "number",
- "title": "Angle 1",
- "description": "First angle (f.e. RA)",
- "default": 0
- },
- "angle2": {
- "type": "number",
- "title": "Angle 2",
- "description": "Second angle (f.e. DEC)",
- "default": 0
- }
- }
- }
- },
- "additionalProperties": false,
- "properties": {
- "duration": {
- "type": "number",
- "title": "Duration (seconds)",
- "description": "Duration of this observation",
- "default": 60,
- "minimum": 1
- },
- "calibrator": {
- "type": "object",
- "additionalProperties": false,
- "default": {},
- "properties": {
- "enabled": {
- "type": "boolean",
- "title": "Calibrator",
- "description": "Replace targets by calibrators",
- "default": false
- },
- "autoselect": {
- "type": "boolean",
- "title": "Auto-select",
- "description": "Auto-select calibrator based on elevation",
- "default": false
- },
- "pointing": {
- "title": "Digital pointing",
- "$ref": "#/definitions/pointing",
- "default": {}
- }
- }
- },
- "channels_per_subband": {
- "type": "integer",
- "title": "Channels/subband",
- "description": "Number of frequency bands per subband",
- "default": 64,
- "minimum": 8,
- "enum": [
- 8,
- 16,
- 32,
- 64,
- 128,
- 256,
- 512,
- 1024
- ]
- },
- "integration_time": {
- "type": "number",
- "title": "Integration time (seconds)",
- "description": "Desired integration period",
- "default": 1,
- "minimum": 0.1
- },
- "storage_cluster": {
- "type": "string",
- "title": "Storage cluster",
- "description": "Cluster to write output to",
- "default": "CEP4",
- "enum": [
- "CEP4",
- "DragNet"
- ]
- },
- "QA": {
- "type": "object",
- "title": "Quality Assurance",
- "default": {},
- "description": "Specify Quality Assurance steps for this observation",
- "properties": {
- "file_conversion": {
- "type": "object",
- "title": "File Conversion",
- "default": {},
- "description": "Create a QA file for the observation",
- "properties": {
- "enabled": {
- "type": "boolean",
- "title": "enabled",
- "default": true,
- "description": "Do/Don't create a QA file for the observation"
- },
- "nr_of_subbands": {
- "type": "integer",
- "title": "#subbands",
- "default": -1,
- "description": "Keep this number of subbands from the observation in the QA file, or all if -1"
- },
- "nr_of_timestamps": {
- "type": "integer",
- "title": "#timestamps",
- "default": 256,
- "minimum": 1,
- "description": "Extract this number of timestamps from the observation in the QA file (equidistantanly sampled, no averaging/interpolation)"
- }
- },
- "additionalProperties": false
- },
- "plots": {
- "type": "object",
- "title": "Plots",
- "default": {},
- "description": "Create dynamic spectrum plots",
- "properties": {
- "enabled": {
- "type": "boolean",
- "title": "enabled",
- "default": true,
- "description": "Do/Don't create plots from the QA file from the observation"
- },
- "autocorrelation": {
- "type": "boolean",
- "title": "autocorrelation",
- "default": true,
- "description": "Create autocorrelation plots for all stations"
- },
- "crosscorrelation": {
- "type": "boolean",
- "title": "crosscorrelation",
- "default": true,
- "description": "Create crosscorrelation plots for all baselines"
- }
- },
- "additionalProperties": false
- }
- },
- "additionalProperties": false
- }
- }
-}
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/task-observation-with-stations.json b/SAS/TMSS/src/tmss/tmssapp/schemas/task-observation-with-stations.json
deleted file mode 100644
index 9249751e45316a485a76766a8e4835b2e4110f92..0000000000000000000000000000000000000000
--- a/SAS/TMSS/src/tmss/tmssapp/schemas/task-observation-with-stations.json
+++ /dev/null
@@ -1,382 +0,0 @@
-{
- "$id": "http://example.com/example.json",
- "type": "object",
- "$schema": "http://json-schema.org/draft-06/schema#",
- "definitions": {
- "pointing": {
- "type": "object",
- "additionalProperties": false,
- "properties": {
- "direction_type": {
- "type": "string",
- "title": "Reference frame",
- "description": "",
- "default": "J2000",
- "enum": [
- "J2000",
- "AZELGEO",
- "LMN",
- "SUN",
- "MOON",
- "MERCURY",
- "VENUS",
- "MARS",
- "JUPITER",
- "SATURN",
- "URANUS",
- "NEPTUNE",
- "PLUTO"
- ]
- },
- "angle1": {
- "type": "number",
- "title": "Angle 1",
- "description": "First angle (e.g. RA)",
- "default": 0
- },
- "angle2": {
- "type": "number",
- "title": "Angle 2",
- "description": "Second angle (e.g. DEC)",
- "default": 0
- },
- "angle3": {
- "type": "number",
- "title": "Angle 3",
- "description": "Third angle (e.g. N in LMN)",
- "default": 0
- }
- },
- "required": [
- "angle1",
- "angle2"
- ]
- }
- },
- "additionalProperties": false,
- "properties": {
- "stations": {
- "title": "Station list",
- "default": ["CS001"],
- "oneOf": [
- {
- "type": "array",
- "title": "Fixed list",
- "additionalItems": false,
- "additionalProperties": false,
- "default": ["CS001"],
- "items": {
- "type": "string",
- "enum": [
- "CS001",
- "CS002",
- "CS003",
- "CS004",
- "CS005",
- "CS006",
- "CS007",
- "CS011",
- "CS013",
- "CS017",
- "CS021",
- "CS024",
- "CS026",
- "CS028",
- "CS030",
- "CS031",
- "CS032",
- "CS101",
- "CS103",
- "CS201",
- "CS301",
- "CS302",
- "CS401",
- "CS501",
- "RS104",
- "RS106",
- "RS205",
- "RS208",
- "RS210",
- "RS305",
- "RS306",
- "RS307",
- "RS310",
- "RS406",
- "RS407",
- "RS409",
- "RS410",
- "RS503",
- "RS508",
- "RS509",
- "DE601",
- "DE602",
- "DE603",
- "DE604",
- "DE605",
- "FR606",
- "SE607",
- "UK608",
- "DE609",
- "PL610",
- "PL611",
- "PL612",
- "IE613",
- "LV614"
- ],
- "title": "Station",
- "description": ""
- },
- "minItems": 1,
- "uniqueItems": true
- },
- {
- "title": "Dynamic list",
- "type": "array",
- "default": [{}],
- "additionalItems": false,
- "items": {
- "type": "object",
- "title": "Station set",
- "headerTemplate": "{{ self.group }}",
- "additionalProperties": false,
- "properties": {
- "group": {
- "type": "string",
- "title": "Group/station",
- "description": "Which (group of) station(s) to select from",
- "default": "ALL",
- "enum": [
- "ALL",
- "SUPERTERP",
- "CORE",
- "REMOTE",
- "DUTCH",
- "INTERNATIONAL"
- ]
- },
- "min_stations": {
- "type": "integer",
- "title": "Minimum nr of stations",
- "description": "Number of stations to use within group/station",
- "default": 1,
- "minimum": 0
- }
- },
- "required": [
- "group",
- "min_stations"
- ]
- }
- }
- ]
- },
- "antenna_set": {
- "type": "string",
- "title": "Antenna set",
- "description": "Fields & antennas to use",
- "default": "HBA_DUAL",
- "enum": [
- "HBA_DUAL",
- "HBA_DUAL_INNER",
- "HBA_ONE",
- "HBA_ONE_INNER",
- "HBA_ZERO",
- "HBA_ZERO_INNER",
- "LBA_INNER",
- "LBA_OUTER",
- "LBA_SPARSE_EVEN",
- "LBA_SPARSE_ODD",
- "LBA_ALL"
- ]
- },
- "filter": {
- "type": "string",
- "title": "Band-pass filter",
- "description": "Must match antenna type",
- "default": "HBA_110_190",
- "enum": [
- "LBA_10_70",
- "LBA_30_70",
- "LBA_10_90",
- "LBA_30_90",
- "HBA_110_190",
- "HBA_210_250"
- ]
- },
- "tile_beam": {
- "title": "Tile beam",
- "description": "HBA only",
- "$ref": "#/definitions/pointing"
- },
- "SAPs": {
- "type": "array",
- "title": "SAPs",
- "description": "Station beams",
- "additionalItems": false,
- "default": [{}],
- "items": {
- "title": "SAP",
- "headerTemplate": "{{ i0 }} - {{ self.name }}",
- "type": "object",
- "additionalProperties": false,
- "default": {},
- "properties": {
- "name": {
- "type": "string",
- "title": "Name/target",
- "description": "Identifier for this beam",
- "default": ""
- },
- "digital_pointing": {
- "title": "Digital pointing",
- "default": {},
- "$ref": "#/definitions/pointing"
- },
- "subbands": {
- "type": "array",
- "title": "Subband list",
- "additionalItems": false,
- "minItems": 1,
- "default": [0],
- "items": {
- "type": "integer",
- "title": "Subband",
- "minimum": 0,
- "maximum": 511
- }
- }
- },
- "required": [
- "digital_pointing",
- "subbands"
- ]
- }
- },
- "duration": {
- "type": "number",
- "title": "Duration (seconds)",
- "description": "Duration of this observation",
- "default": 300,
- "minimum": 1
- },
- "correlator": {
- "title": "Correlator Settings",
- "type": "object",
- "additionalProperties": false,
- "default": {},
- "properties": {
- "channels_per_subband": {
- "type": "integer",
- "title": "Channels/subband",
- "description": "Number of frequency bands per subband",
- "default": 64,
- "minimum": 8,
- "enum": [
- 8,
- 16,
- 32,
- 64,
- 128,
- 256,
- 512,
- 1024
- ]
- },
- "integration_time": {
- "type": "number",
- "title": "Integration time (seconds)",
- "description": "Desired integration period",
- "default": 1,
- "minimum": 0.1
- },
- "storage_cluster": {
- "type": "string",
- "title": "Storage cluster",
- "description": "Cluster to write output to",
- "default": "CEP4",
- "enum": [
- "CEP4",
- "DragNet"
- ]
- }
- },
- "required": [
- "channels_per_subband",
- "integration_time",
- "storage_cluster"
- ]
- },
- "QA": {
- "type": "object",
- "title": "Quality Assurance",
- "default": {},
- "description": "Specify Quality Assurance steps for this observation",
- "properties": {
- "file_conversion": {
- "type": "object",
- "title": "File Conversion",
- "default": {},
- "description": "Create a QA file for the observation",
- "properties": {
- "enabled": {
- "type": "boolean",
- "title": "enabled",
- "default": true,
- "description": "Do/Don't create a QA file for the observation"
- },
- "nr_of_subbands": {
- "type": "integer",
- "title": "#subbands",
- "default": -1,
- "description": "Keep this number of subbands from the observation in the QA file, or all if -1"
- },
- "nr_of_timestamps": {
- "type": "integer",
- "title": "#timestamps",
- "default": 256,
- "minimum": 1,
- "description": "Extract this number of timestamps from the observation in the QA file (equidistantanly sampled, no averaging/interpolation)"
- }
- },
- "additionalProperties": false
- },
- "plots": {
- "type": "object",
- "title": "Plots",
- "default": {},
- "description": "Create dynamic spectrum plots",
- "properties": {
- "enabled": {
- "type": "boolean",
- "title": "enabled",
- "default": true,
- "description": "Do/Don't create plots from the QA file from the observation"
- },
- "autocorrelation": {
- "type": "boolean",
- "title": "autocorrelation",
- "default": true,
- "description": "Create autocorrelation plots for all stations"
- },
- "crosscorrelation": {
- "type": "boolean",
- "title": "crosscorrelation",
- "default": true,
- "description": "Create crosscorrelation plots for all baselines"
- }
- },
- "additionalProperties": false
- }
- },
- "additionalProperties": false
- }
- },
- "required": [
- "stations",
- "antenna_set",
- "filter",
- "SAPs",
- "duration",
- "correlator"
- ]
-}
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/task-stations.json b/SAS/TMSS/src/tmss/tmssapp/schemas/task-stations.json
deleted file mode 100644
index 26f6690bcfe44c0d03f415ef2dcf8900e9be5c02..0000000000000000000000000000000000000000
--- a/SAS/TMSS/src/tmss/tmssapp/schemas/task-stations.json
+++ /dev/null
@@ -1,240 +0,0 @@
-{
- "$id": "http://example.com/example.json",
- "type": "object",
- "$schema": "http://json-schema.org/draft-06/schema#",
- "definitions": {
- "pointing": {
- "type": "object",
- "additionalProperties": false,
- "properties": {
- "direction_type": {
- "type": "string",
- "title": "Reference frame",
- "description": "",
- "default": "J2000",
- "enum": [
- "J2000",
- "SUN",
- "MOON",
- "MERCURY",
- "VENUS",
- "MARS",
- "JUPITER",
- "SATURN",
- "URANUS",
- "NEPTUNE",
- "PLUTO"
- ]
- },
- "angle1": {
- "type": "number",
- "title": "Angle 1",
- "description": "First angle (f.e. RA)",
- "default": 0
- },
- "angle2": {
- "type": "number",
- "title": "Angle 2",
- "description": "Second angle (f.e. DEC)",
- "default": 0
- }
- }
- }
- },
- "additionalProperties": false,
- "properties": {
- "stations": {
- "title": "Station list",
- "type": "array",
- "default": [{}],
- "oneOf": [{
- "type": "array",
- "title": "Fixed list",
- "additionalItems": false,
- "additionalProperties": false,
- "default": ["CS001"],
- "items": {
- "type": "string",
- "enum": [
- "CS001",
- "CS002",
- "CS003",
- "CS004",
- "CS005",
- "CS006",
- "CS007",
- "CS011",
- "CS013",
- "CS017",
- "CS021",
- "CS024",
- "CS026",
- "CS028",
- "CS030",
- "CS031",
- "CS032",
- "CS101",
- "CS103",
- "CS201",
- "CS301",
- "CS302",
- "CS401",
- "CS501",
- "RS104",
- "RS106",
- "RS205",
- "RS208",
- "RS210",
- "RS305",
- "RS306",
- "RS307",
- "RS310",
- "RS406",
- "RS407",
- "RS409",
- "RS410",
- "RS503",
- "RS508",
- "RS509",
- "DE601",
- "DE602",
- "DE603",
- "DE604",
- "DE605",
- "FR606",
- "SE607",
- "UK608",
- "DE609",
- "PL610",
- "PL611",
- "PL612",
- "IE613",
- "LV614"
- ],
- "title": "Station",
- "description": ""
- },
- "minItems": 1,
- "uniqueItems": true
- },
- {
- "title": "Dynamic list",
- "type": "array",
- "additionalItems": false,
- "default": [{}],
- "minItems": 1,
- "uniqueItems": true,
- "items": {
- "type": "object",
- "title": "Station set",
- "headerTemplate": "{{ self.group }}",
- "additionalProperties": false,
- "default": {},
- "properties": {
- "group": {
- "type": "string",
- "title": "Group/station",
- "description": "Which (group of) station(s) to select from",
- "default": "ALL",
- "enum": [
- "ALL",
- "SUPERTERP",
- "CORE",
- "REMOTE",
- "DUTCH",
- "INTERNATIONAL"
- ]
- },
- "min_stations": {
- "type": "integer",
- "title": "Minimum nr of stations",
- "description": "Number of stations to use within group/station",
- "default": 1,
- "minimum": 0
- }
- }
- }
- }
- ]
- },
- "antenna_set": {
- "type": "string",
- "title": "Antenna set",
- "description": "Fields & antennas to use",
- "default": "HBA_DUAL",
- "enum": [
- "HBA_DUAL",
- "HBA_DUAL_INNER",
- "HBA_JOINED",
- "HBA_JOINED_INNER",
- "HBA_ONE",
- "HBA_ONE_INNER",
- "HBA_ZERO",
- "HBA_ZERO_INNER",
- "LBA_INNER",
- "LBA_OUTER",
- "LBA_SPARSE_EVEN",
- "LBA_SPARSE_ODD",
- "LBA_ALL"
- ]
- },
- "filter": {
- "type": "string",
- "title": "Band-pass filter",
- "description": "Must match antenna type",
- "default": "HBA_110_190",
- "enum": [
- "LBA_10_70",
- "LBA_30_70",
- "LBA_10_90",
- "LBA_30_90",
- "HBA_110_190",
- "HBA_210_250"
- ]
- },
- "analog_pointing": {
- "title": "Analog pointing",
- "description": "HBA only",
- "default": {},
- "$ref": "#/definitions/pointing"
- },
- "beams": {
- "type": "array",
- "title": "Beams",
- "additionalItems": false,
- "default": [{}],
- "items": {
- "title": "Beam",
- "headerTemplate": "{{ i0 }} - {{ self.name }}",
- "type": "object",
- "additionalProperties": false,
- "default": {},
- "properties": {
- "name": {
- "type": "string",
- "title": "Name/target",
- "description": "Identifier for this beam",
- "default": ""
- },
- "digital_pointing": {
- "title": "Digital pointing",
- "default": {},
- "$ref": "#/definitions/pointing"
- },
- "subbands": {
- "type": "array",
- "title": "Subband list",
- "additionalItems": false,
- "default": [],
- "items": {
- "type": "integer",
- "title": "Subband",
- "minimum": 0,
- "maximum": 511
- }
- }
- }
- }
- }
- }
-}
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/task_relation_selection_template-SAP-1.json b/SAS/TMSS/src/tmss/tmssapp/schemas/task_relation_selection_template-SAP-1.json
new file mode 100644
index 0000000000000000000000000000000000000000..bc62ea8d7b4493cf3ff11bea012a9f962229fbd9
--- /dev/null
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/task_relation_selection_template-SAP-1.json
@@ -0,0 +1,22 @@
+{
+ "$id":"http://tmss.lofar.org/api/schemas/taskrelationselection/SAP/1#",
+ "$schema": "http://json-schema.org/draft-06/schema#",
+ "title":"SAP",
+ "description":"This task relation selection schema defines the select by SAP parameter.",
+ "version":1,
+ "type": "object",
+ "properties": {
+ "sap": {
+ "type": "array",
+ "title": "sap list",
+ "additionalItems": false,
+ "default": [],
+ "items": {
+ "type": "integer",
+ "title": "sap",
+ "minimum": 0,
+ "maximum": 1
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/task_relation_selection_template-all-1.json b/SAS/TMSS/src/tmss/tmssapp/schemas/task_relation_selection_template-all-1.json
new file mode 100644
index 0000000000000000000000000000000000000000..0d0cac9b06b00b60fff3c2a0732d1151bdfc01a6
--- /dev/null
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/task_relation_selection_template-all-1.json
@@ -0,0 +1,9 @@
+{
+ "$id":"http://tmss.lofar.org/api/schemas/taskrelationselection/all/1#",
+ "$schema": "http://json-schema.org/draft-06/schema#",
+ "title":"all",
+ "description":"This task relation selection schema defines no restrictions, and hence selects 'all'.",
+ "version":1,
+ "type": "object",
+ "properties": {}
+}
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/task_template-calibrator_observation-1.json b/SAS/TMSS/src/tmss/tmssapp/schemas/task_template-calibrator_observation-1.json
new file mode 100644
index 0000000000000000000000000000000000000000..0e32bb1da081fbee61a559f8a07364787282bdb7
--- /dev/null
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/task_template-calibrator_observation-1.json
@@ -0,0 +1,32 @@
+{
+ "$id": "http://tmss.lofar.org/api/schemas/tasktemplate/calibrator observation/1#",
+ "$schema": "http://json-schema.org/draft-06/schema#",
+ "title": "calibrator observation",
+ "description": "This schema defines the (extra) parameters to setup a calibrator observation task, which uses all paramters from the target observation task which it is linked to, plus these calibrator overrides.",
+ "version": 1,
+ "type": "object",
+ "properties": {
+ "duration": {
+ "type": "number",
+ "title": "Duration (seconds)",
+ "description": "Duration of this observation",
+ "default": 600,
+ "minimum": 1
+ },
+ "autoselect": {
+ "type": "boolean",
+ "title": "Auto-select",
+ "description": "Auto-select calibrator based on elevation",
+ "default": true
+ },
+ "pointing": {
+ "title": "Digital pointing",
+ "$ref": "http://tmss.lofar.org/api/schemas/commonschematemplate/pointing/1/#/definitions/pointing",
+ "description": "Manually selected calibrator",
+ "default": {}
+ }
+ },
+ "required": [
+ "autoselect", "duration", "pointing"
+ ]
+}
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/task-preprocessing.json b/SAS/TMSS/src/tmss/tmssapp/schemas/task_template-preprocessing_pipeline-1.json
similarity index 77%
rename from SAS/TMSS/src/tmss/tmssapp/schemas/task-preprocessing.json
rename to SAS/TMSS/src/tmss/tmssapp/schemas/task_template-preprocessing_pipeline-1.json
index e23f297b3abde73a2cde2291b084bcc9d5129224..74278f49310705212c20f65d8afe9aa61fb6ed97 100644
--- a/SAS/TMSS/src/tmss/tmssapp/schemas/task-preprocessing.json
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/task_template-preprocessing_pipeline-1.json
@@ -1,19 +1,10 @@
{
- "$id": "http://example.com/example.json",
- "type": "object",
+ "$id": "http://tmss.lofar.org/api/schemas/tasktemplate/preprocessing pipeline/1#",
"$schema": "http://json-schema.org/draft-06/schema#",
- "additionalProperties": false,
- "definitions": {
- "demix_strategy": {
- "type": "string",
- "default": "auto",
- "enum": [
- "auto",
- "yes",
- "no"
- ]
- }
- },
+ "title": "preprocessing pipeline",
+ "description": "This schema defines the parameters to setup a preprocessing pipeline task.",
+ "version": 1,
+ "type": "object",
"properties": {
"flag": {
"title": "Flagging",
@@ -104,27 +95,27 @@
"properties": {
"CasA": {
"title": "CasA",
- "$ref": "#/definitions/demix_strategy"
+ "$ref": "http://tmss.lofar.org/api/schemas/commonschematemplate/pipeline/1#/definitions/demix_strategy"
},
"CygA": {
"title": "CygA",
- "$ref": "#/definitions/demix_strategy"
+ "$ref": "http://tmss.lofar.org/api/schemas/commonschematemplate/pipeline/1#/definitions/demix_strategy"
},
"HerA": {
"title": "HerA",
- "$ref": "#/definitions/demix_strategy"
+ "$ref": "http://tmss.lofar.org/api/schemas/commonschematemplate/pipeline/1#/definitions/demix_strategy"
},
"HydraA": {
"title": "HyrdraA",
- "$ref": "#/definitions/demix_strategy"
+ "$ref": "http://tmss.lofar.org/api/schemas/commonschematemplate/pipeline/1#/definitions/demix_strategy"
},
"TauA": {
"title": "TauA",
- "$ref": "#/definitions/demix_strategy"
+ "$ref": "http://tmss.lofar.org/api/schemas/commonschematemplate/pipeline/1#/definitions/demix_strategy"
},
"VirA": {
"title": "VirA",
- "$ref": "#/definitions/demix_strategy"
+ "$ref": "http://tmss.lofar.org/api/schemas/commonschematemplate/pipeline/1#/definitions/demix_strategy"
}
},
"default": {}
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/task_template-target_observation-1.json b/SAS/TMSS/src/tmss/tmssapp/schemas/task_template-target_observation-1.json
new file mode 100644
index 0000000000000000000000000000000000000000..e987d0d2fc1d0628be24ef009833f712601cf05f
--- /dev/null
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/task_template-target_observation-1.json
@@ -0,0 +1,139 @@
+{
+ "$id": "http://tmss.lofar.org/api/schemas/tasktemplate/target observation/1#",
+ "$schema": "http://json-schema.org/draft-06/schema#",
+ "title": "target observation",
+ "description": "This schema defines the parameters to setup a target observation task.",
+ "version": 1,
+ "type": "object",
+ "properties": {
+ "stations": {
+ "$ref": "http://tmss.lofar.org/api/schemas/commonschematemplate/stations/1#/definitions/stations",
+ "default": ["CS001"]
+ },
+ "antenna_set": {
+ "$ref": "http://tmss.lofar.org/api/schemas/commonschematemplate/stations/1/#/definitions/antenna_set",
+ "default": "HBA_DUAL"
+ },
+ "filter": {
+ "$ref": "http://tmss.lofar.org/api/schemas/commonschematemplate/stations/1/#/definitions/filter",
+ "default": "HBA_110_190"
+ },
+ "tile_beam": {
+ "title": "Tile beam",
+ "description": "HBA only",
+ "default": {},
+ "$ref": "http://tmss.lofar.org/api/schemas/commonschematemplate/pointing/1/#/definitions/pointing"
+ },
+ "SAPs": {
+ "type": "array",
+ "title": "SAPs",
+ "description": "Station beams",
+ "additionalItems": false,
+ "default": [
+ {}
+ ],
+ "items": {
+ "title": "SAP",
+ "headerTemplate": "{{ i0 }} - {{ self.name }}",
+ "type": "object",
+ "additionalProperties": false,
+ "default": {},
+ "properties": {
+ "name": {
+ "type": "string",
+ "title": "Name/target",
+ "description": "Identifier for this beam",
+ "default": ""
+ },
+ "digital_pointing": {
+ "title": "Digital pointing",
+ "default": {},
+ "$ref": "http://tmss.lofar.org/api/schemas/commonschematemplate/pointing/1/#/definitions/pointing"
+ },
+ "subbands": {
+ "type": "array",
+ "title": "Subband list",
+ "additionalItems": false,
+ "default": [],
+ "items": {
+ "type": "integer",
+ "title": "Subband",
+ "minimum": 0,
+ "maximum": 511
+ }
+ }
+ },
+ "required": [
+ "digital_pointing",
+ "subbands"
+ ]
+ }
+ },
+ "duration": {
+ "type": "number",
+ "title": "Duration (seconds)",
+ "description": "Duration of this observation",
+ "default": 300,
+ "minimum": 1
+ },
+ "correlator": {
+ "title": "Correlator Settings",
+ "type": "object",
+ "additionalProperties": false,
+ "default": {},
+ "properties": {
+ "channels_per_subband": {
+ "type": "integer",
+ "title": "Channels/subband",
+ "description": "Number of frequency bands per subband",
+ "default": 64,
+ "minimum": 8,
+ "enum": [
+ 8,
+ 16,
+ 32,
+ 64,
+ 128,
+ 256,
+ 512,
+ 1024
+ ]
+ },
+ "integration_time": {
+ "type": "number",
+ "title": "Integration time (seconds)",
+ "description": "Desired integration period",
+ "default": 1,
+ "minimum": 0.1
+ },
+ "storage_cluster": {
+ "type": "string",
+ "title": "Storage cluster",
+ "description": "Cluster to write output to",
+ "default": "CEP4",
+ "enum": [
+ "CEP4",
+ "DragNet"
+ ]
+ }
+ },
+ "required": [
+ "channels_per_subband",
+ "integration_time",
+ "storage_cluster"
+ ]
+ },
+ "QA": {
+ "default":{},
+ "$ref": "http://tmss.lofar.org/api/schemas/commonschematemplate/QA/1#/definitions/QA"
+ }
+ },
+ "required": [
+ "stations",
+ "antenna_set",
+ "filter",
+ "SAPs",
+ "duration",
+ "correlator"
+ ]
+}
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/templates.json b/SAS/TMSS/src/tmss/tmssapp/schemas/templates.json
new file mode 100644
index 0000000000000000000000000000000000000000..09698f2cd23c880f8f6638b35640d5fc9b6c3917
--- /dev/null
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/templates.json
@@ -0,0 +1,112 @@
+[
+ {
+ "file_name": "common_schema_template-pointing-1.json",
+ "template": "common_schema_template"
+ },
+ {
+ "file_name": "common_schema_template-stations-1.json",
+ "template": "common_schema_template"
+ },
+ {
+ "file_name": "common_schema_template-qa-1.json",
+ "template": "common_schema_template"
+ },
+ {
+ "file_name": "common_schema_template-tasks-1.json",
+ "template": "common_schema_template"
+ },
+ {
+ "file_name": "common_schema_template-pipeline-1.json",
+ "template": "common_schema_template"
+ },
+ {
+ "file_name": "dataproduct_specifications_template-SAP-1.json",
+ "template": "dataproduct_specifications_template"
+ },
+ {
+ "file_name": "dataproduct_specifications_template-empty-1.json",
+ "template": "dataproduct_specifications_template"
+ },
+ {
+ "file_name": "dataproduct_feedback_template-empty-1.json",
+ "template": "dataproduct_feedback_template"
+ },
+ {
+ "file_name": "dataproduct_feedback_template-feedback-1.json",
+ "template": "dataproduct_feedback_template"
+ },
+ {
+ "file_name": "scheduling_unit_template-scheduling_unit-1.json",
+ "template": "scheduling_unit_template"
+ },
+ {
+ "file_name": "task_relation_selection_template-SAP-1.json",
+ "template": "task_relation_selection_template"
+ },
+ {
+ "file_name": "task_relation_selection_template-all-1.json",
+ "template": "task_relation_selection_template"
+ },
+ {
+ "file_name": "task_template-calibrator_observation-1.json",
+ "template": "task_template",
+ "type": "observation",
+ "validation_code_js": ""
+ },
+ {
+ "file_name": "task_template-target_observation-1.json",
+ "template": "task_template",
+ "type": "observation",
+ "validation_code_js": ""
+ },
+ {
+ "file_name": "task_template-preprocessing_pipeline-1.json",
+ "template": "task_template",
+ "name": "preprocessing pipeline",
+ "type": "pipeline",
+ "version": 1,
+ "validation_code_js": "",
+ "description": "This schema defines the parameters for a preprocessing pipeline."
+ },
+ {
+ "file_name": "subtask_template-observation-1.json",
+ "template": "subtask_template",
+ "type": "observation",
+ "realtime": true,
+ "queue": false
+ },
+ {
+ "file_name": "subtask_template-pipeline-1.json",
+ "template": "subtask_template",
+ "type": "pipeline",
+ "realtime": true,
+ "queue": false
+ },
+ {
+ "file_name": "subtask_template-qa_file-1.json",
+ "template": "subtask_template",
+ "type": "qa_files",
+ "realtime": true,
+ "queue": false
+ },
+ {
+ "file_name": "subtask_template-qa_plots-1.json",
+ "template": "subtask_template",
+ "type": "qa_plots",
+ "realtime": true,
+ "queue": false
+ },
+ {
+ "file_name": "UC1-scheduling-unit-observation-strategy.json",
+ "template": "scheduling_unit_observing_strategy_template",
+ "scheduling_unit_template_name": "scheduling unit",
+ "scheduling_unit_template_version": "1",
+ "name": "UC1 CTC+pipelines",
+ "description": "This observation strategy template defines a Calibrator-Target-Calibrator observation chain, plus a preprocessing pipeline for each.",
+ "version": 1
+ },
+ {
+ "file_name": "scheduling_constraints_template-constraints-1.json",
+ "template": "scheduling_constraints_template"
+ }
+]
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/serializers/scheduling.py b/SAS/TMSS/src/tmss/tmssapp/serializers/scheduling.py
index e5d0a521a9eb121b8ea254199de128188a9f2d10..889ecdefb101cd9e175a125f065e22f74f36cda7 100644
--- a/SAS/TMSS/src/tmss/tmssapp/serializers/scheduling.py
+++ b/SAS/TMSS/src/tmss/tmssapp/serializers/scheduling.py
@@ -7,7 +7,7 @@ logger = logging.getLogger(__name__)
from rest_framework import serializers
from .. import models
-from .specification import RelationalHyperlinkedModelSerializer
+from .specification import RelationalHyperlinkedModelSerializer, AbstractTemplateSerializer
from .widgets import JSONEditorField
class SubtaskStateSerializer(RelationalHyperlinkedModelSerializer):
@@ -46,7 +46,7 @@ class ScheduleMethodSerializer(RelationalHyperlinkedModelSerializer):
fields = '__all__'
-class SubtaskTemplateSerializer(RelationalHyperlinkedModelSerializer):
+class SubtaskTemplateSerializer(AbstractTemplateSerializer):
class Meta:
model = models.SubtaskTemplate
fields = '__all__'
@@ -58,7 +58,7 @@ class DefaultSubtaskTemplateSerializer(RelationalHyperlinkedModelSerializer):
fields = '__all__'
-class DataproductSpecificationsTemplateSerializer(RelationalHyperlinkedModelSerializer):
+class DataproductSpecificationsTemplateSerializer(AbstractTemplateSerializer):
class Meta:
model = models.DataproductSpecificationsTemplate
fields = '__all__'
@@ -71,7 +71,7 @@ class DefaultDataproductSpecificationsTemplateSerializer(RelationalHyperlinkedMo
-class DataproductFeedbackTemplateSerializer(RelationalHyperlinkedModelSerializer):
+class DataproductFeedbackTemplateSerializer(AbstractTemplateSerializer):
class Meta:
model = models.DataproductFeedbackTemplate
fields = '__all__'
@@ -80,34 +80,16 @@ class DataproductFeedbackTemplateSerializer(RelationalHyperlinkedModelSerializer
class SubtaskSerializer(RelationalHyperlinkedModelSerializer):
# If this is OK then we can extend API with NO url ('flat' values) on more places if required
cluster_value = serializers.StringRelatedField(source='cluster', label='cluster_value', read_only=True)
+ specifications_doc = JSONEditorField(schema_source='specifications_template.schema')
class Meta:
model = models.Subtask
fields = '__all__'
extra_fields = ['cluster_value']
- # Create a JSON editor form to replace the simple text field based on the schema in the template that this
- # draft refers to. If that fails, the JSONField remains a standard text input.
- def __init__(self, *args, **kwargs):
- super().__init__(*args, **kwargs)
- try:
- self.fields['specifications_doc'] = JSONEditorField(self.instance.specifications_template.schema)
- except Exception as e:
- print('Could not initialize JSONEditorField (%s), hence no fancy JSON form. This is expected for e.g. the list view.' % e)
- # todo: Shall we use the schema for one of the default templates in this case instead?
-
class SubtaskInputSerializer(RelationalHyperlinkedModelSerializer):
-
- # Create a JSON editor form to replace the simple text field based on the schema in the template that this
- # draft refers to. If that fails, the JSONField remains a standard text input.
- def __init__(self, *args, **kwargs):
- super().__init__(*args, **kwargs)
- try:
- self.fields['selection_doc'] = JSONEditorField(self.instance.selection_template.schema)
- except Exception as e:
- print('Could not initialize JSONEditorField (%s), hence no fancy JSON form. This is expected for e.g. the list view.' % e)
- # todo: Shall we use the schema for one of the default templates in this case instead?
+ selection_doc = JSONEditorField(schema_source='selection_template.schema')
class Meta:
model = models.SubtaskInput
@@ -122,17 +104,8 @@ class SubtaskOutputSerializer(RelationalHyperlinkedModelSerializer):
class DataproductSerializer(RelationalHyperlinkedModelSerializer):
-
- # Create a JSON editor form to replace the simple text field based on the schema in the template that this
- # draft refers to. If that fails, the JSONField remains a standard text input.
- def __init__(self, *args, **kwargs):
- super().__init__(*args, **kwargs)
- try:
- self.fields['specifications_doc'] = JSONEditorField(self.instance.specifications_template.schema)
- self.fields['feedback_doc'] = JSONEditorField(self.instance.feedback_template.schema)
- except Exception as e:
- print('Could not initialize JSONEditorField (%s), hence no fancy JSON form. This is expected for e.g. the list view.' % e)
- # todo: Shall we use the schema for one of the default templates in this case instead?
+ specifications_doc = JSONEditorField(schema_source='specifications_template.schema')
+ feedback_doc = JSONEditorField(schema_source='feedback_template.schema')
class Meta:
model = models.Dataproduct
diff --git a/SAS/TMSS/src/tmss/tmssapp/serializers/specification.py b/SAS/TMSS/src/tmss/tmssapp/serializers/specification.py
index fd6a1284babfff6bda958d69a11e62d5a1909ce6..69d62cea5acae788b2c381b6db088c25b5aefdb1 100644
--- a/SAS/TMSS/src/tmss/tmssapp/serializers/specification.py
+++ b/SAS/TMSS/src/tmss/tmssapp/serializers/specification.py
@@ -16,7 +16,10 @@ class RelationalHyperlinkedModelSerializer(serializers.HyperlinkedModelSerialize
def get_field_names(self, declared_fields, info):
field_names = super().get_field_names(declared_fields, info)
- field_names.remove(self.url_field_name) # is added later, see retun statement
+ try:
+ field_names.remove(self.url_field_name) # is added later, see retun statement
+ except ValueError:
+ pass
if getattr(self.Meta, 'extra_fields', None):
field_names += self.Meta.extra_fields
@@ -88,7 +91,23 @@ class TagsSerializer(RelationalHyperlinkedModelSerializer):
fields = '__all__'
-class GeneratorTemplateSerializer(RelationalHyperlinkedModelSerializer):
+
+
+class AbstractTemplateSerializer(RelationalHyperlinkedModelSerializer):
+ schema = JSONEditorField(schema_source=None)
+
+
+ class Meta:
+ abstract = True
+
+
+class CommonSchemaTemplateSerializer(AbstractTemplateSerializer):
+ class Meta:
+ model = models.CommonSchemaTemplate
+ fields = '__all__'
+
+
+class GeneratorTemplateSerializer(AbstractTemplateSerializer):
class Meta:
model = models.GeneratorTemplate
fields = '__all__'
@@ -101,12 +120,14 @@ class DefaultGeneratorTemplateSerializer(RelationalHyperlinkedModelSerializer):
class SchedulingUnitObservingStrategyTemplateSerializer(RelationalHyperlinkedModelSerializer):
+ template = JSONEditorField(schema_source="scheduling_unit_template.schema")
+
class Meta:
model = models.SchedulingUnitObservingStrategyTemplate
fields = '__all__'
-class SchedulingUnitTemplateSerializer(RelationalHyperlinkedModelSerializer):
+class SchedulingUnitTemplateSerializer(AbstractTemplateSerializer):
class Meta:
model = models.SchedulingUnitTemplate
fields = '__all__'
@@ -118,7 +139,19 @@ class DefaultSchedulingUnitTemplateSerializer(RelationalHyperlinkedModelSerializ
fields = '__all__'
-class TaskTemplateSerializer(RelationalHyperlinkedModelSerializer):
+class SchedulingConstraintsTemplateSerializer(AbstractTemplateSerializer):
+ class Meta:
+ model = models.SchedulingConstraintsTemplate
+ fields = '__all__'
+
+
+class DefaultSchedulingConstraintsTemplateSerializer(RelationalHyperlinkedModelSerializer):
+ class Meta:
+ model = models.DefaultSchedulingConstraintsTemplate
+ fields = '__all__'
+
+
+class TaskTemplateSerializer(AbstractTemplateSerializer):
class Meta:
model = models.TaskTemplate
fields = '__all__'
@@ -130,7 +163,7 @@ class DefaultTaskTemplateSerializer(RelationalHyperlinkedModelSerializer):
fields = '__all__'
-class TaskRelationSelectionTemplateSerializer(RelationalHyperlinkedModelSerializer):
+class TaskRelationSelectionTemplateSerializer(AbstractTemplateSerializer):
class Meta:
model = models.TaskRelationSelectionTemplate
fields = '__all__'
@@ -245,15 +278,7 @@ class PeriodCategorySerializer(RelationalHyperlinkedModelSerializer):
class SchedulingSetSerializer(RelationalHyperlinkedModelSerializer):
-
- # Create a JSON editor form to replace the simple text field based on the schema in the template that this
- # draft refers to. If that fails, the JSONField remains a standard text input.
- def __init__(self, *args, **kwargs):
- super().__init__(*args, **kwargs)
- try:
- self.fields['generator_doc'] = JSONEditorField(self.instance.generator_template.schema)
- except Exception as e:
- pass
+ generator_doc = JSONEditorField(schema_source="generator_template.schema")
class Meta:
model = models.SchedulingSet
@@ -262,57 +287,48 @@ class SchedulingSetSerializer(RelationalHyperlinkedModelSerializer):
class SchedulingUnitDraftSerializer(RelationalHyperlinkedModelSerializer):
-
+ requirements_doc = JSONEditorField(schema_source="requirements_template.schema")
duration = FloatDurationField(required=False)
- # Create a JSON editor form to replace the simple text field based on the schema in the template that this
- # draft refers to. If that fails, the JSONField remains a standard text input.
- def __init__(self, *args, **kwargs):
- super().__init__(*args, **kwargs)
- try:
- self.fields['requirements_doc'] = JSONEditorField(self.instance.requirements_template.schema)
- except Exception as e:
- pass
-
class Meta:
model = models.SchedulingUnitDraft
fields = '__all__'
extra_fields = ['scheduling_unit_blueprints', 'task_drafts', 'duration']
+class SchedulingUnitDraftCopySerializer(SchedulingUnitDraftSerializer):
+ class Meta(SchedulingUnitDraftSerializer.Meta):
+ fields = ['copy_reason']
+ extra_fields =['scheduling_set_id']
+ read_only_fields = ['scheduling_unit_blueprints','task_drafts']
-class SchedulingUnitBlueprintSerializer(RelationalHyperlinkedModelSerializer):
+class SchedulingUnitDraftCopyFromSchedulingSetSerializer(SchedulingUnitDraftSerializer):
+ class Meta(SchedulingUnitDraftSerializer.Meta):
+ fields = ['copy_reason']
+ read_only_fields = ['scheduling_unit_blueprints','task_drafts']
+class SchedulingUnitBlueprintSerializer(RelationalHyperlinkedModelSerializer):
+ requirements_doc = JSONEditorField(schema_source="requirements_template.schema")
duration = FloatDurationField(required=False)
- # Create a JSON editor form to replace the simple text field based on the schema in the template that this
- # draft refers to. If that fails, the JSONField remains a standard text input.
- def __init__(self, *args, **kwargs):
- super().__init__(*args, **kwargs)
- try:
- self.fields['requirements_doc'] = JSONEditorField(self.instance.requirements_template.schema)
- except Exception as e:
- pass
-
class Meta:
model = models.SchedulingUnitBlueprint
fields = '__all__'
extra_fields = ['task_blueprints', 'duration', 'start_time', 'stop_time']
+class SchedulingUnitBlueprintCopyToSchedulingUnitDraftSerializer(SchedulingUnitBlueprintSerializer):
+ class Meta(SchedulingUnitDraftSerializer.Meta):
+ fields = ['copy_reason']
+ extra_fields =['scheduling_set_id']
+ read_only_fields = ['scheduling_unit_blueprints','task_drafts']
+
+
class TaskDraftSerializer(RelationalHyperlinkedModelSerializer):
duration = FloatDurationField(required=False)
relative_start_time = FloatDurationField(required=False)
relative_stop_time = FloatDurationField(required=False)
-
- # Create a JSON editor form to replace the simple text field based on the schema in the template that this
- # draft refers to. If that fails, the JSONField remains a standard text input.
- def __init__(self, *args, **kwargs):
- super().__init__(*args, **kwargs)
- try:
- self.fields['specifications_doc'] = JSONEditorField(self.instance.specifications_template.schema)
- except Exception as e:
- pass
+ specifications_doc = JSONEditorField(schema_source='specifications_template.schema')
class Meta:
model = models.TaskDraft
@@ -325,15 +341,7 @@ class TaskBlueprintSerializer(RelationalHyperlinkedModelSerializer):
duration = FloatDurationField(required=False)
relative_start_time = FloatDurationField(required=False)
relative_stop_time = FloatDurationField(required=False)
-
- # Create a JSON editor form to replace the simple text field based on the schema in the template that this
- # draft refers to. If that fails, the JSONField remains a standard text input.
- def __init__(self, *args, **kwargs):
- super().__init__(*args, **kwargs)
- try:
- self.fields['specifications_doc'] = JSONEditorField(self.instance.specifications_template.schema)
- except Exception as e:
- pass
+ specifications_doc = JSONEditorField(schema_source='specifications_template.schema')
class Meta:
model = models.TaskBlueprint
@@ -342,15 +350,7 @@ class TaskBlueprintSerializer(RelationalHyperlinkedModelSerializer):
class TaskRelationDraftSerializer(RelationalHyperlinkedModelSerializer):
-
- # Create a JSON editor form to replace the simple text field based on the schema in the template that this
- # draft refers to. If that fails, the JSONField remains a standard text input.
- def __init__(self, *args, **kwargs):
- super().__init__(*args, **kwargs)
- try:
- self.fields['selection_doc'] = JSONEditorField(self.instance.selection_template.schema)
- except Exception as e:
- pass
+ selection_doc = JSONEditorField(schema_source='selection_template.schema')
class Meta:
model = models.TaskRelationDraft
@@ -359,15 +359,7 @@ class TaskRelationDraftSerializer(RelationalHyperlinkedModelSerializer):
class TaskRelationBlueprintSerializer(RelationalHyperlinkedModelSerializer):
-
- # Create a JSON editor form to replace the simple text field based on the schema in the template that this
- # draft refers to. If that fails, the JSONField remains a standard text input.
- def __init__(self, *args, **kwargs):
- super().__init__(*args, **kwargs)
- try:
- self.fields['selection_doc'] = JSONEditorField(self.instance.selection_template.schema)
- except Exception as e:
- pass
+ selection_doc = JSONEditorField(schema_source='selection_template.schema')
class Meta:
model = models.TaskRelationBlueprint
diff --git a/SAS/TMSS/src/tmss/tmssapp/serializers/widgets.py b/SAS/TMSS/src/tmss/tmssapp/serializers/widgets.py
index 3f621dc899332041702a3a0e1320f8c61f033e45..f19ee6f1913f297bed9f97fcfc920ebb150954b7 100644
--- a/SAS/TMSS/src/tmss/tmssapp/serializers/widgets.py
+++ b/SAS/TMSS/src/tmss/tmssapp/serializers/widgets.py
@@ -1,14 +1,64 @@
"""
This file contains customized UI elements for use in the viewsets (based on the elsewhere defined data models and serializers)
"""
-from rest_framework import serializers
+from rest_framework import serializers, fields
+import requests
+import re
import json
+from lofar.sas.tmss.tmss.tmssapp import models
+from lofar.common import json_utils
+from lofar.common.json_utils import get_default_json_object_for_schema
class JSONEditorField(serializers.JSONField):
"""
- An enhanced JSONfield that provides a nice editor widget with validation against the provided schema.
+ An enhanced JSONField that provides a nice editor widget with validation against the $schema in the json field value.
"""
- def __init__(self, schema, *args, **kwargs):
- kwargs['style'] = {'template': 'josdejong_jsoneditor_widget.html', 'schema': json.dumps(schema)}
+ def __init__(self, schema_source: str=None, *args, **kwargs):
+ '''schema_source should be a string 'pointing to' the used template and it's schema property.
+ For example in the SubtaskSerializer, we point to the specifications_template's schema like so:
+ specifications_doc = JSONEditorField(schema_source='specifications_template.schema')
+ '''
+ self.schema_source = schema_source
super().__init__(*args, **kwargs)
+
+ def get_schema(self, json_data=None):
+ '''get the schema that this JSONEditorField is using via the schema_source'''
+ try:
+ if isinstance(self.parent.instance, list):
+ # the serializer is serializing a list of model instances, so we cannot get a single schema from a single instance
+ return None
+ return fields.get_attribute(self.parent.instance, self.schema_source.split('.'))
+ except (AttributeError, TypeError):
+ try:
+ if json_data and '$schema' in json_data:
+ schema_url = json_data['$schema']
+ response = requests.get(schema_url)
+ if response.status_code == 200:
+ schema = response.text
+ return json.loads(schema)
+ except (KeyError, TypeError, json.JSONDecodeError):
+ pass
+ return None
+
+ def to_representation(self, value):
+ '''create representation of the json-schema-value,
+ with all common json schema $ref's pointing to the correct host,
+ and inject the josdejong_jsoneditor_widget.html in the render style based on the requests accepted_media_type'''
+ self.style = {}
+
+ if self.parent.context['request'].accepted_media_type == 'text/html':
+ # get the used schema...
+ schema = self.get_schema(value)
+
+ if schema:
+ # ...and 'massage' the served schema such that our rendered html json-editor understands it.
+ # the josdejong_jsoneditor_widget cannot resolve absolute URL's in the schema
+ # although this should be possible according to the JSON schema standard.
+ # so, let's do the resolving here and feed the resolved schema to the josdejong_jsoneditor_widget
+ schema = json_utils.resolved_refs(schema)
+
+ self.style = {'template': 'josdejong_jsoneditor_widget.html',
+ 'schema': json.dumps(schema)}
+
+ return super().to_representation(value)
diff --git a/SAS/TMSS/src/tmss/tmssapp/subtasks.py b/SAS/TMSS/src/tmss/tmssapp/subtasks.py
index ebce1cc48e442a336559f69f323e9b687c81031b..5b59f2fdc1a5082bc25408ec2ca0595af942f4c5 100644
--- a/SAS/TMSS/src/tmss/tmssapp/subtasks.py
+++ b/SAS/TMSS/src/tmss/tmssapp/subtasks.py
@@ -44,11 +44,11 @@ def create_subtasks_from_task_blueprint(task_blueprint: TaskBlueprint) -> [Subta
return subtasks
# fixed mapping from template name to generator functions which create the list of subtask(s) for this task_blueprint
- generators_mapping = {'observation schema': [create_observation_control_subtask_from_task_blueprint,
+ generators_mapping = {'target observation': [create_observation_control_subtask_from_task_blueprint,
create_qafile_subtask_from_task_blueprint,
create_qaplots_subtask_from_task_blueprint],
- 'preprocessing schema': [create_preprocessing_subtask_from_task_blueprint]}
- generators_mapping['calibrator schema'] = generators_mapping['observation schema']
+ 'preprocessing pipeline': [create_preprocessing_subtask_from_task_blueprint]}
+ generators_mapping['calibrator observation'] = generators_mapping['target observation']
template_name = task_blueprint.specifications_template.name
if template_name in generators_mapping:
@@ -69,16 +69,16 @@ def create_subtasks_from_task_blueprint(task_blueprint: TaskBlueprint) -> [Subta
def create_observation_subtask_specifications_from_observation_task_blueprint(task_blueprint: TaskBlueprint) -> (dict, SubtaskTemplate):
"""
- Create a valid observation subtask specification ('observationcontrol schema' SubtaskTemplate schema) based on the task_blueprint's settings
+ Create a valid observation subtask specification ('observation control' SubtaskTemplate schema) based on the task_blueprint's settings
"""
# check if task_blueprint has an observation-like specification
- if task_blueprint.specifications_template.name.lower() not in ['observation schema', 'calibrator schema']:
+ if task_blueprint.specifications_template.name.lower() not in ['target observation', 'calibrator observation']:
raise SubtaskCreationException("Cannot create observation subtask specifications from task_blueprint id=%s with template name='%s'" % (
task_blueprint.id, task_blueprint.specifications_template.name))
# start with an observation subtask specification with all the defaults and the right structure according to the schema
- subtask_template = SubtaskTemplate.objects.get(name='observationcontrol schema')
+ subtask_template = SubtaskTemplate.objects.get(name='observation control')
subtask_spec = get_default_json_object_for_schema(subtask_template.schema)
# wipe the default pointings, these should come from the task_spec
@@ -130,10 +130,10 @@ def create_observation_subtask_specifications_from_observation_task_blueprint(ta
subtask_spec['stations']["filter"] = task_spec["filter"]
if "stations" in task_spec:
- if "group" in task_spec["stations"][0]:
+ if "group" in task_spec["stations"]:
try:
# retrieve stations in group from RADB virtual instrument
- station_group_name = task_spec["stations"][0]["group"]
+ station_group_name = task_spec["stations"]["group"]
subtask_spec['stations']['station_list'] = get_stations_in_group(station_group_name)
except Exception as e:
raise SubtaskCreationException("Could not determine stations in group '%s' for task_blueprint id=%s. Error: %s" % (
@@ -199,11 +199,11 @@ def get_related_target_observation_task_blueprint(calibrator_task_blueprint: Tas
try:
return next(relation.second for relation in TaskSchedulingRelationBlueprint.objects.filter(first=calibrator_task_blueprint).all()
- if relation.second is not None and relation.second.specifications_template.name.lower() == 'observation schema')
+ if relation.second is not None and relation.second.specifications_template.name.lower() == 'target observation')
except StopIteration:
try:
return next(relation.first for relation in TaskSchedulingRelationBlueprint.objects.filter(second=calibrator_task_blueprint).all()
- if relation.first is not None and relation.first.specifications_template.name.lower() == 'observation schema')
+ if relation.first is not None and relation.first.specifications_template.name.lower() == 'target observation')
except StopIteration:
logger.info("No related target observation task_blueprint found for calibrator observation task_blueprint id=%d", calibrator_task_blueprint.id)
@@ -298,7 +298,7 @@ def create_qafile_subtask_from_observation_subtask(observation_subtask: Subtask)
qafile_subtask = Subtask.objects.create(**qafile_subtask_data)
# step 2: create and link subtask input/output
- selection_template = TaskRelationSelectionTemplate.objects.get(name="All")
+ selection_template = TaskRelationSelectionTemplate.objects.get(name="all")
selection_doc = get_default_json_object_for_schema(selection_template.schema)
qafile_subtask_input = SubtaskInput.objects.create(subtask=qafile_subtask,
producer=observation_subtask.outputs.first(), # TODO: determine proper producer based on spec in task_relation_blueprint
@@ -362,7 +362,7 @@ def create_qaplots_subtask_from_qafile_subtask(qafile_subtask: Subtask) -> Subta
qaplots_subtask = Subtask.objects.create(**qaplots_subtask_data)
# step 2: create and link subtask input/output
- selection_template = TaskRelationSelectionTemplate.objects.get(name="All")
+ selection_template = TaskRelationSelectionTemplate.objects.get(name="all")
selection_doc = get_default_json_object_for_schema(selection_template.schema)
qaplots_subtask_input = SubtaskInput.objects.create(subtask=qaplots_subtask,
producer=qafile_subtask.outputs.first(),
@@ -393,7 +393,7 @@ def create_preprocessing_subtask_from_task_blueprint(task_blueprint: TaskBluepri
"to an observation predecessor (sub)task." % task_blueprint.pk)
# step 1: create subtask in defining state, with filled-in subtask_template
- subtask_template = SubtaskTemplate.objects.get(name='pipelinecontrol schema')
+ subtask_template = SubtaskTemplate.objects.get(name='pipeline control')
default_subtask_specs = get_default_json_object_for_schema(subtask_template.schema)
subtask_specs = _generate_subtask_specs_from_preprocessing_task_specs(task_blueprint.specifications_doc, default_subtask_specs)
cluster_name = task_blueprint.specifications_doc.get("storage_cluster", "CEP4")
@@ -606,11 +606,12 @@ def schedule_qafile_subtask(qafile_subtask: Subtask):
qafile_subtask_dataproduct = Dataproduct.objects.create(filename="L%s_QA.h5" % (qa_input.producer.subtask_id, ),
directory="/data/qa/qa_files",
dataformat=Dataformat.objects.get(value=Dataformat.Choices.QA_HDF5.value),
+ datatype=Datatype.objects.get(value=Datatype.Choices.QUALITY.value), # todo: is this correct?
producer=qafile_subtask.outputs.first(),
- specifications_doc="",
- specifications_template=DataproductSpecificationsTemplate.objects.first(), # ?????
- feedback_doc="",
- feedback_template=DataproductFeedbackTemplate.objects.first() # ?????
+ specifications_doc=get_default_json_object_for_schema(DataproductSpecificationsTemplate.objects.get(name="empty").schema),
+ specifications_template=DataproductSpecificationsTemplate.objects.get(name="empty"),
+ feedback_doc=get_default_json_object_for_schema(DataproductFeedbackTemplate.objects.get(name="empty").schema),
+ feedback_template=DataproductFeedbackTemplate.objects.get(name="empty")
)
# step 5: set state to SCHEDULED (resulting in the qaservice to pick this subtask up and run it)
@@ -656,12 +657,13 @@ def schedule_qaplots_subtask(qaplots_subtask: Subtask):
obs_subtask = qafile_subtask.predecessors.first()
qaplots_subtask_dataproduct = Dataproduct.objects.create(directory="/data/qa/plots/L%s" % (obs_subtask.id, ),
dataformat=Dataformat.objects.get(value=Dataformat.Choices.QA_PLOTS.value),
+ datatype=Datatype.objects.get(value=Datatype.Choices.QUALITY.value), # todo: is this correct?
producer=qaplots_subtask.outputs.first(),
- specifications_doc="",
- specifications_template=DataproductSpecificationsTemplate.objects.first(), # ?????
- feedback_doc="",
- feedback_template=DataproductFeedbackTemplate.objects.first() # ?????
- )
+ specifications_doc=get_default_json_object_for_schema(DataproductSpecificationsTemplate.objects.get(name="empty").schema),
+ specifications_template=DataproductSpecificationsTemplate.objects.get(name="empty"),
+ feedback_doc=get_default_json_object_for_schema(DataproductFeedbackTemplate.objects.get(name="empty").schema),
+ feedback_template=DataproductFeedbackTemplate.objects.get(name="empty")
+ )
# step 5: set state to SCHEDULED (resulting in the qaservice to pick this subtask up and run it)
qaplots_subtask.state = SubtaskState.objects.get(value=SubtaskState.Choices.SCHEDULED.value)
@@ -761,7 +763,7 @@ def schedule_observation_subtask(observation_subtask: Subtask):
# step 3: create output dataproducts, and link these to the output
specifications_doc = observation_subtask.specifications_doc
dataproduct_specifications_template = DataproductSpecificationsTemplate.objects.get(name="SAP") # todo: should this be derived from the task relation specification template?
- dataproduct_feedback_template = DataproductFeedbackTemplate.objects.get(name="Empty")
+ dataproduct_feedback_template = DataproductFeedbackTemplate.objects.get(name="empty")
subtask_output = observation_subtask.outputs.first() # TODO: make proper selection, not default first()
directory = "/data/%s/%s/L%s/uv" % ("projects" if isProductionEnvironment() else "test-projects",
observation_subtask.task_blueprint.scheduling_unit_blueprint.draft.scheduling_set.project.name,
@@ -770,11 +772,14 @@ def schedule_observation_subtask(observation_subtask: Subtask):
Dataproduct.objects.bulk_create([Dataproduct(filename="L%d_SAP%03d_SB%03d_uv.MS" % (observation_subtask.id, sap_nr, sb_nr),
directory=directory,
dataformat=Dataformat.objects.get(value="MeasurementSet"),
+ datatype=Datatype.objects.get(value="visibilities"),
producer=subtask_output,
specifications_doc={"sap": [sap_nr]}, # todo: set correct value. This will be provided by the RA somehow
specifications_template=dataproduct_specifications_template,
- feedback_doc="",
- feedback_template=dataproduct_feedback_template) for sb_nr in pointing['subbands']])
+ feedback_doc=get_default_json_object_for_schema(dataproduct_feedback_template.schema),
+ feedback_template=dataproduct_feedback_template,
+ size=0 if sb_nr%10==0 else 1024*1024*1024*sb_nr,
+ expected_size=1024*1024*1024*sb_nr) for sb_nr in pointing['subbands']])
# step 4: resource assigner (if possible)
_assign_or_unassign_resources(observation_subtask)
@@ -819,9 +824,9 @@ def schedule_pipeline_subtask(pipeline_subtask: Subtask):
raise SubtaskSchedulingException("Cannot schedule subtask id=%d type=%s because it has no input(s)" % (pipeline_subtask.pk,
pipeline_subtask.specifications_template.type))
- # TODO: use existing and reasonable selection and specification templates for output when we have those, for now, use "Empty"
- dataproduct_specifications_template = DataproductSpecificationsTemplate.objects.get(name="Empty")
- dataproduct_feedback_template = DataproductFeedbackTemplate.objects.get(name="Empty")
+ # TODO: use existing and reasonable selection and specification templates for output when we have those, for now, use "empty"
+ dataproduct_specifications_template = DataproductSpecificationsTemplate.objects.get(name="empty")
+ dataproduct_feedback_template = DataproductFeedbackTemplate.objects.get(name="empty")
# iterate over all inputs
for pipeline_subtask_input in pipeline_subtask.inputs.all():
@@ -848,10 +853,11 @@ def schedule_pipeline_subtask(pipeline_subtask: Subtask):
output_dp = Dataproduct(filename=filename,
directory=input_dp.directory.replace(str(pipeline_subtask_input.producer.subtask.pk), str(pipeline_subtask.pk)),
dataformat=dataformat,
+ datatype=Datatype.objects.get(value="visibilities"), # todo: is this correct?
producer=pipeline_subtask_output,
- specifications_doc={},
+ specifications_doc=get_default_json_object_for_schema(dataproduct_specifications_template.schema),
specifications_template=dataproduct_specifications_template,
- feedback_doc="",
+ feedback_doc=get_default_json_object_for_schema(dataproduct_feedback_template.schema),
feedback_template=dataproduct_feedback_template)
output_dp_objects.append(output_dp)
diff --git a/SAS/TMSS/src/tmss/tmssapp/tasks.py b/SAS/TMSS/src/tmss/tmssapp/tasks.py
index 9f6b129b42a7b6ece4084e65252263e6b61f5b8e..dd3957360a0d51d9047fa2a3eb186be573d64e22 100644
--- a/SAS/TMSS/src/tmss/tmssapp/tasks.py
+++ b/SAS/TMSS/src/tmss/tmssapp/tasks.py
@@ -1,7 +1,6 @@
from lofar.sas.tmss.tmss.exceptions import *
from lofar.sas.tmss.tmss.tmssapp import models
-from lofar.sas.tmss.tmss.tmssapp.models.specification import TaskBlueprint, SchedulingUnitBlueprint, TaskDraft, SchedulingRelationPlacement
-from lofar.sas.tmss.tmss.tmssapp.subtasks import create_and_schedule_subtasks_from_task_blueprint, unschedule_subtasks_in_task_blueprint
+from lofar.sas.tmss.tmss.tmssapp.subtasks import unschedule_subtasks_in_task_blueprint
from lofar.sas.tmss.tmss.tmssapp.models.specification import TaskBlueprint, SchedulingUnitBlueprint
from lofar.sas.tmss.tmss.tmssapp.subtasks import create_and_schedule_subtasks_from_task_blueprint, create_subtasks_from_task_blueprint, schedule_independent_subtasks_in_task_blueprint
from lofar.messaging.messagebus import ToBus, DEFAULT_BROKER, DEFAULT_BUSNAME
@@ -10,12 +9,12 @@ from lofar.common.util import single_line_with_single_spaces
from lofar.sas.tmss.client.tmssbuslistener import DEFAULT_TMSS_SCHEDULINGUNITBLUEPRINT_NOTIFICATION_PREFIX
from functools import cmp_to_key
import os
+from copy import deepcopy
from lofar.common.json_utils import add_defaults_to_json_object_for_schema
-
import logging
from datetime import datetime
-logger = logging.getLogger(__name__)
+logger = logging.getLogger(__name__)
def create_scheduling_unit_blueprint_from_scheduling_unit_draft(scheduling_unit_draft: models.SchedulingUnitDraft) -> models.SchedulingUnitBlueprint:
"""
@@ -37,6 +36,108 @@ def create_scheduling_unit_blueprint_from_scheduling_unit_draft(scheduling_unit_
return scheduling_unit_blueprint
+def copy_scheduling_unit_draft(scheduling_unit_draft: models.SchedulingUnitDraft, scheduling_set_dst: models.SchedulingSet, copy_reason: str) -> models.SchedulingUnitDraft:
+ """
+ Copy a SchedulingUnitDraft
+ :raises Exception if instantiate fails.
+ """
+ task_drafts = list(scheduling_unit_draft.task_drafts.all())
+ scheduling_unit_draft_copy = deepcopy(scheduling_unit_draft)
+ scheduling_unit_draft_copy.id = None
+ scheduling_unit_draft_copy.copies=scheduling_unit_draft
+ if copy_reason is not None:
+ scheduling_unit_draft_copy.copy_reason = models.CopyReason.objects.get(value=copy_reason)
+
+ scheduling_unit_draft_copy.name="%s (Copy)" % (scheduling_unit_draft.name,)
+ scheduling_unit_draft_copy.description="%s (Copy from %s)" % (scheduling_unit_draft.description or "<no description>",scheduling_unit_draft.name,)
+
+ scheduling_unit_draft_copy.scheduling_set=scheduling_set_dst
+
+ task_drafts_copy = []
+ scheduling_unit_draft_copy.save()
+ for td in task_drafts:
+ task_drafts_copy.append(copy_task_draft(td))
+ scheduling_unit_draft_copy.task_drafts.set(task_drafts_copy)
+ scheduling_unit_draft_copy.save()
+
+ logger.info("copy_scheduling_unit_draft(scheduling_unit_draft.id=%s) created copy_scheduling_unit_draft id=%s", scheduling_unit_draft.pk, scheduling_unit_draft_copy.pk)
+ return scheduling_unit_draft_copy
+
+
+def create_scheduling_unit_draft_from_scheduling_unit_blueprint(scheduling_unit_blueprint: models.SchedulingUnitBlueprint, copy_reason: str) -> models.SchedulingUnitDraft:
+ """
+ Create a SchedulingUnitDraft from the SchedulingUnitBlueprint
+ :raises Exception if instantiate fails.
+ """
+ logger.debug("create_scheduling_unit_draft_from_scheduling_unit_blueprint(scheduling_unit_blueprint.id=%s)", scheduling_unit_blueprint.pk)
+
+ original_scheduling_unit_draft = scheduling_unit_blueprint.draft
+
+ # TODO: fix the copy reason
+ scheduling_unit_draft = SchedulingUnitDraft.objects.create(
+ name="%s (Copied from SchedulingUnitBlueprint)" % (scheduling_unit_blueprint.name,),
+ description="%s (Copied from %s SchedulingUnitBlueprint)" % (scheduling_unit_blueprint.description or "<no description>",scheduling_unit_blueprint.name,),
+ requirements_doc=scheduling_unit_blueprint.requirements_doc,
+ copy_reason=copy_reason,
+ generator_instance_doc=original_scheduling_unit_draft.generator_instance_doc,
+ scheduling_set=original_scheduling_unit_draft.scheduling_set,
+ requirements_template=scheduling_unit_blueprint.requirements_template)
+
+ task_drafts_copy = []
+ task_blueprints = list(scheduling_unit_blueprint.task_blueprints.all())
+
+ for tb in task_blueprints:
+ task_drafts_copy.append(copy_task_blueprint_to_task_draft(tb))
+ scheduling_unit_draft.task_drafts.set(task_drafts_copy)
+ scheduling_unit_draft.save()
+
+ logger.info("create_scheduling_unit_draft_from_scheduling_unit_blueprint(scheduling_unit_blueprint.id=%s) created scheduling_unit_draft id=%s", scheduling_unit_blueprint.pk, scheduling_unit_draft.pk)
+ return scheduling_unit_draft
+
+
+def copy_task_draft(task_draft: models.TaskDraft, copy_reason: str) -> models.TaskDraft:
+
+ task_template_name = task_draft.specifications_template
+ task_template = models.TaskTemplate.objects.get(name=task_template_name)
+
+ task_draft_copy = models.TaskDraft.objects.create(
+ name=task_draft.name,
+ description=task_draft.description,
+ tags=task_draft.tags,
+ specifications_doc=task_draft.specifications_doc,
+ copy_reason=copy_reason,
+ copies=task_draft,
+ scheduling_unit_draft=task_draft.scheduling_unit_draft,
+ specifications_template=task_template)
+ logger.info("task draft with id %s created successfully" % task_draft.id)
+ logger.info("copy_task_draft(task_draft.id=%s) created copy_task_draft id=%s", task_draft.pk, task_draft_copy.pk)
+ return task_draft_copy
+
+
+def copy_task_blueprint_to_task_draft(task_blueprint:models.TaskBlueprint ) -> models.TaskDraft:
+ """
+ Create a copy of a task_blueprint to a task_draft
+ :raises Exception if instantiate fails.
+ """
+ logger.debug("Create Task Draft from Task Blueprint (id=%s)", task_blueprint.pk)
+
+ original_task_draft = task_blueprint.draft
+ task_template_name = original_task_draft.specifications_template
+ task_template = models.TaskTemplate.objects.get(name=task_template_name)
+
+ task_draft_copy = models.TaskDraft.objects.create(
+ name="%s (Copied from Task Blueprint)" % (task_blueprint.name,),
+ description="%s (Copied from %s Task Blueprint)" % (task_blueprint.description or "<no description>",task_blueprint.name,),
+ tags=original_task_draft.tags,
+ specifications_doc=original_task_draft.specifications_doc,
+ copy_reason=original_task_draft.copy_reason,
+ copies=original_task_draft,
+ scheduling_unit_draft=original_task_draft.scheduling_unit_draft,
+ specifications_template=task_template)
+ logger.info("copy_task_blueprint_to_task_draft(task_blueprint.id=%s) created task_draft id=%s", task_blueprint.pk, task_draft_copy.pk)
+ return task_draft_copy
+
+
def create_task_drafts_from_scheduling_unit_draft(scheduling_unit_draft: models.SchedulingUnitDraft) -> models.SchedulingUnitDraft:
"""
Create all defined task_drafts in the scheduling_unit_draft's requirements_doc, connect them, and return the updated scheduling_unit_draft.
diff --git a/SAS/TMSS/src/tmss/tmssapp/validation.py b/SAS/TMSS/src/tmss/tmssapp/validation.py
deleted file mode 100644
index 2908a80cad68da0c71aea006a2aa9b6787768033..0000000000000000000000000000000000000000
--- a/SAS/TMSS/src/tmss/tmssapp/validation.py
+++ /dev/null
@@ -1,32 +0,0 @@
-import json
-import jsonschema
-from lofar.sas.tmss.tmss.exceptions import *
-
-def validate_json_against_schema(json_string: str, schema: str):
- '''validate the given json_string against the given schema.
- If no exception if thrown, then the given json_string validates against the given schema.
- :raises SchemaValidationException if the json_string does not validate against the schema
- '''
-
- # ensure the given arguments are strings
- if type(json_string) != str:
- json_string = json.dumps(json_string)
- if type(schema) != str:
- schema = json.dumps(schema)
-
- # ensure the specification and schema are both valid json in the first place
- try:
- json_object = json.loads(json_string)
- except json.decoder.JSONDecodeError as e:
- raise SchemaValidationException("Invalid JSON: %s\n%s" % (str(e), json_string))
-
- try:
- schema_object = json.loads(schema)
- except json.decoder.JSONDecodeError as e:
- raise SchemaValidationException("Invalid JSON: %s\n%s" % (str(e), schema))
-
- # now do the actual validation
- try:
- jsonschema.validate(json_object, schema_object)
- except jsonschema.ValidationError as e:
- raise SchemaValidationException(str(e))
diff --git a/SAS/TMSS/src/tmss/tmssapp/views.py b/SAS/TMSS/src/tmss/tmssapp/views.py
index 4614c940953d2a277b00cf1eb0589ef6efb1edd5..3b163882b38e08b032d7700efe3145b8d70c02f2 100644
--- a/SAS/TMSS/src/tmss/tmssapp/views.py
+++ b/SAS/TMSS/src/tmss/tmssapp/views.py
@@ -5,7 +5,14 @@ from django.shortcuts import get_object_or_404, render
from lofar.sas.tmss.tmss.tmssapp import models
from lofar.common.json_utils import get_default_json_object_for_schema
from lofar.sas.tmss.tmss.tmssapp.adapters.parset import convert_to_parset
+from drf_yasg.utils import swagger_auto_schema
+from rest_framework.permissions import AllowAny
+from rest_framework.decorators import authentication_classes, permission_classes
+from django.apps import apps
+from datetime import datetime
+import dateutil.parser
+from lofar.sas.tmss.tmss.tmssapp.conversions import local_sidereal_time_for_utc_and_station, local_sidereal_time_for_utc_and_longitude
def subtask_template_default_specification(request, subtask_template_pk:int):
subtask_template = get_object_or_404(models.SubtaskTemplate, pk=subtask_template_pk)
@@ -23,11 +30,59 @@ def subtask_parset(request, subtask_pk:int):
subtask = get_object_or_404(models.Subtask, pk=subtask_pk)
parset = convert_to_parset(subtask)
return HttpResponse(str(parset), content_type='text/plain')
-
+
+
def index(request):
return render(request, os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), '../../frontend','tmss_webapp/build/index.html'))
#return render(request, "../../../frontend/frontend_poc/build/index.html")
+
def task_specify_observation(request, pk=None):
task = get_object_or_404(models.TaskDraft, pk=pk)
return HttpResponse("response", content_type='text/plain')
+
+# Allow everybody to GET our publicly available template-json-schema's
+@permission_classes([AllowAny])
+@authentication_classes([AllowAny])
+@swagger_auto_schema(responses={200: 'Get the JSON schema from the template with the requested <template>, <name> and <version>',
+ 404: 'the schema with requested <template>, <name> and <version> is not available'},
+ operation_description="Get the JSON schema for the given <template> with the given <name> and <version> as application/json content response.")
+def get_template_json_schema(request, template:str, name:str, version:str):
+ template_model = apps.get_model("tmssapp", template)
+ template_instance = get_object_or_404(template_model, name=name, version=version)
+ schema = template_instance.schema
+ response = JsonResponse(schema, json_dumps_params={"indent":2})
+
+ # config Access-Control. Our schemas use $ref url's to other schemas, mainly pointing to our own common schemas with base definitions.
+ # We instruct the client to allow fetching those.
+ response["Access-Control-Allow-Origin"] = "*"
+ response["Access-Control-Allow-Methods"] = "GET, OPTIONS"
+ return response
+
+
+def utc(request):
+ return HttpResponse(datetime.utcnow().isoformat(), content_type='text/plain')
+
+
+def lst(request):
+ # Handling optional parameters via django paths in urls.py is a pain, we access them on the request directly instead.
+ timestamp = request.GET.get('timestamp', None)
+ station = request.GET.get('station', None)
+ longitude = request.GET.get('longitude', None)
+
+ # conversions
+ if timestamp:
+ timestamp = dateutil.parser.parse(timestamp) # isot to datetime
+ if longitude:
+ longitude = float(longitude)
+
+ if station:
+ lst_lon = local_sidereal_time_for_utc_and_station(timestamp, station)
+ elif longitude:
+ lst_lon = local_sidereal_time_for_utc_and_longitude(timestamp, longitude)
+ else:
+ # fall back to defaults
+ lst_lon = local_sidereal_time_for_utc_and_station(timestamp)
+
+ # todo: do we want to return a dict, so users can make sure their parameters were parsed correctly instead?
+ return HttpResponse(str(lst_lon), content_type='text/plain')
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/viewsets/CMakeLists.txt b/SAS/TMSS/src/tmss/tmssapp/viewsets/CMakeLists.txt
index fc0325a523508e371b2456d96b3467274dae748d..445e0bbe4672e5cdad3a5a41be8575dbf2169ff0 100644
--- a/SAS/TMSS/src/tmss/tmssapp/viewsets/CMakeLists.txt
+++ b/SAS/TMSS/src/tmss/tmssapp/viewsets/CMakeLists.txt
@@ -6,6 +6,8 @@ set(_py_files
lofar_viewset.py
specification.py
scheduling.py
+ helloworldflow.py
+ schedulingunitdemoflow.py
)
python_install(${_py_files}
diff --git a/SAS/TMSS/src/tmss/tmssapp/viewsets/__init__.py b/SAS/TMSS/src/tmss/tmssapp/viewsets/__init__.py
index 93f3c7e6d54f95c40d6d9484aad802b13f9991ba..882458975ee4be50507620471ed1026433ddf589 100644
--- a/SAS/TMSS/src/tmss/tmssapp/viewsets/__init__.py
+++ b/SAS/TMSS/src/tmss/tmssapp/viewsets/__init__.py
@@ -1,2 +1,3 @@
from .specification import *
-from .scheduling import *
\ No newline at end of file
+from .scheduling import *
+from .schedulingunitdemoflow import *
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/viewsets/helloworldflow.py b/SAS/TMSS/src/tmss/tmssapp/viewsets/helloworldflow.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/SAS/TMSS/src/tmss/tmssapp/viewsets/lofar_viewset.py b/SAS/TMSS/src/tmss/tmssapp/viewsets/lofar_viewset.py
index 017d8e13c90b3b6f7e4dce84d8136e1553a38393..b30fe6463f3e2d652f1b53db5c818c999aa3bfee 100644
--- a/SAS/TMSS/src/tmss/tmssapp/viewsets/lofar_viewset.py
+++ b/SAS/TMSS/src/tmss/tmssapp/viewsets/lofar_viewset.py
@@ -5,9 +5,18 @@ Adds the following functionality:
- Swagger API annotation
"""
+import logging
+logger = logging.getLogger(__name__)
+
from rest_framework import viewsets
from drf_yasg.utils import swagger_auto_schema
from rest_framework import mixins
+import json
+from django.shortcuts import get_object_or_404
+from django.http import JsonResponse
+from django.urls import reverse as revese_url
+from rest_framework.decorators import action
+from lofar.common import json_utils
class LOFARViewSet(viewsets.ModelViewSet):
"""
@@ -50,4 +59,56 @@ class LOFARNestedViewSet(mixins.CreateModelMixin,
@swagger_auto_schema(responses={400: 'invalid specification', 403: 'forbidden'})
def create(self, request, **kwargs):
- return super(LOFARNestedViewSet, self).create(request, **kwargs)
\ No newline at end of file
+ return super(LOFARNestedViewSet, self).create(request, **kwargs)
+
+
+
+class LOFARCopyViewSet(mixins.CreateModelMixin, viewsets.GenericViewSet):
+
+ """
+ @swagger_auto_schema(responses={403: 'forbidden'})
+ def list(self, request, **kwargs):
+ return super(LOFARCopyViewSet, self).list(request, **kwargs)
+ """
+ @swagger_auto_schema(responses={400: 'invalid specification', 403: 'forbidden'})
+ def create(self, request, **kwargs):
+ return super(LOFARCopyViewSet, self).create(request, **kwargs)
+
+
+class AbstractTemplateViewSet(LOFARViewSet):
+ def retrieve(self, request, pk=None, **kwargs):
+ response = super().retrieve(request, pk, **kwargs)
+
+ # config Access-Control. Our schemas use $ref url's to other schemas, mainly pointing to our own common schemas with base definitions.
+ # We instruct the client to allow fetching those.
+ response["Access-Control-Allow-Origin"] = "*"
+ response["Access-Control-Allow-Methods"] = "GET, OPTIONS"
+ return response
+
+ @swagger_auto_schema(responses={200: 'The schema as a JSON object',
+ 403: 'forbidden'},
+ operation_description="Get the schema as a JSON object.")
+ @action(methods=['get'], detail=True)
+ def schema(self, request, pk=None):
+ template = get_object_or_404(self.queryset.model, pk=pk)
+ return JsonResponse(template.schema, json_dumps_params={'indent': 2})
+
+ @swagger_auto_schema(responses={200: 'The schema as a JSON object',
+ 403: 'forbidden'},
+ operation_description="Get the schema as a JSON object with all $ref URL's resolved, resulting in a complete and self describing schema.")
+ @action(methods=['get'], detail=True)
+ def ref_resolved_schema(self, request, pk=None):
+ template = get_object_or_404(self.queryset.model, pk=pk)
+ schema = json_utils.resolved_refs(template.schema)
+ return JsonResponse(schema, json_dumps_params={'indent': 2})
+
+ @swagger_auto_schema(responses={200: 'JSON object with all the defaults from the schema filled in',
+ 403: 'forbidden'},
+ operation_description="Get a JSON object with all the defaults from the schema filled in.")
+ @action(methods=['get'], detail=True)
+ def default(self, request, pk=None):
+ template = get_object_or_404(self.queryset.model, pk=pk)
+ spec = json_utils.get_default_json_object_for_schema(template.schema)
+ return JsonResponse(spec, json_dumps_params={'indent': 2})
+
+
diff --git a/SAS/TMSS/src/tmss/tmssapp/viewsets/scheduling.py b/SAS/TMSS/src/tmss/tmssapp/viewsets/scheduling.py
index 040ff55be026a4cb8217ffab272a2eff80300117..af49948d1615bac654f06ea03f55f8b09f679d6a 100644
--- a/SAS/TMSS/src/tmss/tmssapp/viewsets/scheduling.py
+++ b/SAS/TMSS/src/tmss/tmssapp/viewsets/scheduling.py
@@ -4,7 +4,7 @@ This file contains the viewsets (based on the elsewhere defined data models and
from django.shortcuts import get_object_or_404
from rest_framework import viewsets
-from .lofar_viewset import LOFARViewSet, LOFARNestedViewSet
+from .lofar_viewset import LOFARViewSet, LOFARNestedViewSet, AbstractTemplateViewSet
from .. import models
from .. import serializers
from django_filters import rest_framework as filters
@@ -90,7 +90,7 @@ class SubtaskTemplateFilter(filters.FilterSet):
'version': ['lt', 'gt', 'exact']
}
-class SubtaskTemplateViewSet(LOFARViewSet):
+class SubtaskTemplateViewSet(AbstractTemplateViewSet):
queryset = models.SubtaskTemplate.objects.all()
serializer_class = serializers.SubtaskTemplateSerializer
filter_class = SubtaskTemplateFilter
@@ -105,30 +105,13 @@ class SubtaskTemplateViewSet(LOFARViewSet):
return queryset
- @swagger_auto_schema(responses={200: 'The schema as a JSON object',
- 403: 'forbidden'},
- operation_description="Get the schema as a JSON object.")
- @action(methods=['get'], detail=True)
- def schema(self, request, pk=None):
- subtask_template = get_object_or_404(models.SubtaskTemplate, pk=pk)
- return JsonResponse(subtask_template.schema)
-
- @swagger_auto_schema(responses={200: 'JSON object with all the defaults from the schema filled in',
- 403: 'forbidden'},
- operation_description="Get a JSON object with all the defaults from the schema filled in.")
- @action(methods=['get'], detail=True)
- def default_specification(self, request, pk=None):
- subtask_template = get_object_or_404(models.SubtaskTemplate, pk=pk)
- spec = get_default_json_object_for_schema(subtask_template.schema)
- return JsonResponse(spec)
-
class DefaultSubtaskTemplateViewSet(LOFARViewSet):
queryset = models.DefaultSubtaskTemplate.objects.all()
serializer_class = serializers.DefaultSubtaskTemplateSerializer
-class DataproductSpecificationsTemplateViewSet(LOFARViewSet):
+class DataproductSpecificationsTemplateViewSet(AbstractTemplateViewSet):
queryset = models.DataproductSpecificationsTemplate.objects.all()
serializer_class = serializers.DataproductSpecificationsTemplateSerializer
@@ -137,7 +120,8 @@ class DefaultDataproductSpecificationsTemplateViewSet(LOFARViewSet):
queryset = models.DefaultDataproductSpecificationsTemplate.objects.all()
serializer_class = serializers.DefaultDataproductSpecificationsTemplateSerializer
-class DataproductFeedbackTemplateViewSet(LOFARViewSet):
+
+class DataproductFeedbackTemplateViewSet(AbstractTemplateViewSet):
queryset = models.DataproductFeedbackTemplate.objects.all()
serializer_class = serializers.DataproductFeedbackTemplateSerializer
@@ -242,6 +226,18 @@ class SubtaskViewSet(LOFARViewSet):
serializer = serializers.DataproductSerializer(dataproducts, many=True, context={'request': request})
return RestResponse(serializer.data)
+ @swagger_auto_schema(responses={200: 'The finished version of this subtask.',
+ 403: 'forbidden',
+ 500: 'The feedback of this subtask could not be processed'},
+ operation_description="Generate feedback_doc of subtask output dataproducts from the subtask raw_feedback and set subtask state to finished.")
+ @action(methods=['post'], detail=True, url_name='process_feedback_and_set_finished')
+ def process_feedback(self, request, pk=None):
+ from lofar.sas.tmss.tmss.tmssapp.adapters.feedback import generate_dataproduct_feedback_from_subtask_feedback_and_set_finished
+ subtask = get_object_or_404(models.Subtask, pk=pk)
+ finished_subtask = generate_dataproduct_feedback_from_subtask_feedback_and_set_finished(subtask)
+ serializer = self.get_serializer(finished_subtask)
+ return RestResponse(serializer.data)
+
class SubtaskNestedViewSet(LOFARNestedViewSet):
queryset = models.Subtask.objects.all()
diff --git a/SAS/TMSS/src/tmss/tmssapp/viewsets/schedulingunitdemoflow.py b/SAS/TMSS/src/tmss/tmssapp/viewsets/schedulingunitdemoflow.py
new file mode 100644
index 0000000000000000000000000000000000000000..ea117c0f9c27a4324fe76c77fe1256e1b1eca446
--- /dev/null
+++ b/SAS/TMSS/src/tmss/tmssapp/viewsets/schedulingunitdemoflow.py
@@ -0,0 +1,22 @@
+from django.shortcuts import render
+from rest_framework import viewsets
+from rest_framework.response import Response
+from rest_framework.decorators import action
+from rest_framework.serializers import ModelSerializer
+from lofar.sas.tmss.tmss.tmssapp import models
+
+# Create your views here.
+
+class SchedulingUnitDemoSerializer(ModelSerializer):
+ class Meta:
+ model = models.SchedulingUnitDemo
+ fields = '__all__'
+
+class SchedulingUnitFlowViewSet(viewsets.ModelViewSet):
+ queryset = models.SchedulingUnitDemo.objects.all()
+ serializer_class = SchedulingUnitDemoSerializer
+
+ @action(methods=['get'], detail=True)
+ def trigger(self, request, pk=None):
+ SchedulingUnitDemoFlow
+ return Response("ok")
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/viewsets/specification.py b/SAS/TMSS/src/tmss/tmssapp/viewsets/specification.py
index f7b2aeeafdd2d57ac65b7da5aa5d1df3e9b3fc2b..ce3fa163142398bbaf6ae6bf7e197b33b6311cd2 100644
--- a/SAS/TMSS/src/tmss/tmssapp/viewsets/specification.py
+++ b/SAS/TMSS/src/tmss/tmssapp/viewsets/specification.py
@@ -2,7 +2,8 @@
This file contains the viewsets (based on the elsewhere defined data models and serializers)
"""
-from django.shortcuts import get_object_or_404
+from django.shortcuts import get_object_or_404, render
+
from django.http import JsonResponse
from django.contrib.auth.models import User
from django_filters import rest_framework as filters
@@ -17,7 +18,7 @@ from rest_framework.decorators import action
from drf_yasg.utils import swagger_auto_schema
from drf_yasg.openapi import Parameter
-from lofar.sas.tmss.tmss.tmssapp.viewsets.lofar_viewset import LOFARViewSet, LOFARNestedViewSet
+from lofar.sas.tmss.tmss.tmssapp.viewsets.lofar_viewset import LOFARViewSet, LOFARNestedViewSet, AbstractTemplateViewSet, LOFARCopyViewSet
from lofar.sas.tmss.tmss.tmssapp import models
from lofar.sas.tmss.tmss.tmssapp import serializers
@@ -27,8 +28,16 @@ from lofar.common.datetimeutils import formatDatetime
from lofar.sas.tmss.tmss.tmssapp.tasks import *
from lofar.sas.tmss.tmss.tmssapp.subtasks import *
+from django.urls import resolve, get_script_prefix,Resolver404
+
+import json
+import logging
+
+from django.core.exceptions import ObjectDoesNotExist
+logger = logging.getLogger(__name__)
+
# This is required for keeping a user reference as ForeignKey in other models
# (I think so that the HyperlinkedModelSerializer can generate a URI)
@@ -42,7 +51,12 @@ class TagsViewSet(LOFARViewSet):
serializer_class = serializers.TagsSerializer
-class GeneratorTemplateViewSet(LOFARViewSet):
+class CommonSchemaTemplateViewSet(AbstractTemplateViewSet):
+ queryset = models.CommonSchemaTemplate.objects.all()
+ serializer_class = serializers.CommonSchemaTemplateSerializer
+
+
+class GeneratorTemplateViewSet(AbstractTemplateViewSet):
queryset = models.GeneratorTemplate.objects.all()
serializer_class = serializers.GeneratorTemplateSerializer
@@ -98,25 +112,37 @@ class SchedulingUnitTemplateFilter(filters.FilterSet):
'version': ['lt', 'gt', 'exact']
}
-class SchedulingUnitTemplateViewSet(LOFARViewSet):
+class SchedulingUnitTemplateViewSet(AbstractTemplateViewSet):
queryset = models.SchedulingUnitTemplate.objects.all()
serializer_class = serializers.SchedulingUnitTemplateSerializer
filter_class = SchedulingUnitTemplateFilter
- @swagger_auto_schema(responses={200: 'JSON object with all the defaults from the schema filled in',
- 403: 'forbidden'},
- operation_description="Get a JSON object with all the defaults from the schema filled in.")
- @action(methods=['get'], detail=True)
- def default_specification(self, request, pk=None):
- schedulingunit_template = get_object_or_404(models.SchedulingUnitTemplate, pk=pk)
- spec = get_default_json_object_for_schema(schedulingunit_template.schema)
- return JsonResponse(spec)
-
class DefaultSchedulingUnitTemplateViewSet(LOFARViewSet):
queryset = models.DefaultSchedulingUnitTemplate.objects.all()
serializer_class = serializers.DefaultSchedulingUnitTemplateSerializer
+
+class SchedulingConstraintsTemplateFilter(filters.FilterSet):
+ class Meta:
+ model = models.SchedulingConstraintsTemplate
+ fields = {
+ 'name': ['exact'],
+ 'version': ['lt', 'gt', 'exact']
+ }
+
+
+class SchedulingConstraintsTemplateViewSet(AbstractTemplateViewSet):
+ queryset = models.SchedulingConstraintsTemplate.objects.all()
+ serializer_class = serializers.SchedulingConstraintsTemplateSerializer
+ filter_class = SchedulingConstraintsTemplateFilter
+
+
+class DefaultSchedulingConstraintsTemplateViewSet(LOFARViewSet):
+ queryset = models.DefaultSchedulingConstraintsTemplate.objects.all()
+ serializer_class = serializers.DefaultSchedulingConstraintsTemplateSerializer
+
+
class TaskTemplateFilter(filters.FilterSet):
class Meta:
model = models.TaskTemplate
@@ -125,35 +151,19 @@ class TaskTemplateFilter(filters.FilterSet):
'version': ['lt', 'gt', 'exact']
}
-class TaskTemplateViewSet(LOFARViewSet):
+
+class TaskTemplateViewSet(AbstractTemplateViewSet):
queryset = models.TaskTemplate.objects.all()
serializer_class = serializers.TaskTemplateSerializer
filter_class = TaskTemplateFilter
- @swagger_auto_schema(responses={200: 'The schema as a JSON object',
- 403: 'forbidden'},
- operation_description="Get the schema as a JSON object.")
- @action(methods=['get'], detail=True)
- def schema(self, request, pk=None):
- template = get_object_or_404(models.TaskTemplate, pk=pk)
- return JsonResponse(template.schema)
-
- @swagger_auto_schema(responses={200: 'JSON object with all the defaults from the schema filled in',
- 403: 'forbidden'},
- operation_description="Get a JSON object with all the defaults from the schema filled in.")
- @action(methods=['get'], detail=True)
- def default_specification(self, request, pk=None):
- template = get_object_or_404(models.TaskTemplate, pk=pk)
- spec = get_default_json_object_for_schema(template.schema)
- return JsonResponse(spec)
-
class DefaultTaskTemplateViewSet(LOFARViewSet):
queryset = models.DefaultTaskTemplate.objects.all()
serializer_class = serializers.DefaultTaskTemplateSerializer
-class TaskRelationSelectionTemplateViewSet(LOFARViewSet):
+class TaskRelationSelectionTemplateViewSet(AbstractTemplateViewSet):
queryset = models.TaskRelationSelectionTemplate.objects.all()
serializer_class = serializers.TaskRelationSelectionTemplateSerializer
@@ -376,6 +386,195 @@ class SchedulingUnitDraftNestedViewSet(LOFARNestedViewSet):
else:
return models.SchedulingUnitDraft.objects.all()
+class TaskDraftCopyViewSet(LOFARCopyViewSet):
+ queryset = models.TaskDraft.objects.all()
+ serializer_class = serializers.TaskDraftSerializer
+
+ @swagger_auto_schema(responses={201: 'The new Task Draft',
+ 403: 'forbidden'},
+ operation_description="Copy a Task Draft to a new Task Draft")
+ def create(self, request, *args, **kwargs):
+ if 'task_draft_id' in kwargs:
+ task_draft = get_object_or_404(models.TaskDraft, pk=kwargs["task_draft_id"])
+
+ body_unicode = request.body.decode('utf-8')
+ body_data = json.loads(body_unicode)
+
+ copy_reason = body_data.get('copy_reason', None)
+
+ try:
+ copy_reason_obj = models.CopyReason.objects.get(value=copy_reason)
+ except ObjectDoesNotExist:
+ logger.info("CopyReason matching query does not exist.")
+ #if a non valid copy_reason is specified, set copy_reason to None
+ copy_reason = None
+
+ task_draft_copy = copy_task_draft(task_draft,copy_reason)
+
+
+ # url path magic to construct the new task_draft_path url
+ task_draft_path = request._request.path
+ base_path = task_draft_path[:task_draft_path.find('/task_draft')]
+ task_draft_copy_path = '%s/task_draft/%s/' % (base_path, task_draft_copy.id,)
+
+
+ # return a response with the new serialized SchedulingUnitBlueprintSerializer, and a Location to the new instance in the header
+ return Response(serializers.TaskDraftSerializer(task_draft_copy, context={'request':request}).data,
+ status=status.HTTP_201_CREATED,
+ headers={'Location': task_draft_copy_path})
+ else:
+ content = {'Error': 'scheduling_unit_draft_id is missing'}
+ return Response(content, status=status.HTTP_404_NOT_FOUND)
+
+
+class SchedulingUnitDraftCopyViewSet(LOFARCopyViewSet):
+ queryset = models.SchedulingUnitDraft.objects.all()
+ serializer_class = serializers.SchedulingUnitDraftCopySerializer
+
+ @swagger_auto_schema(responses={201: 'The new scheduling_unit_draft',
+ 403: 'forbidden'},
+ operation_description="Copy a Scheduling Unit Draft to a new Scheduling Unit Draft")
+ def create(self, request, *args, **kwargs):
+ if 'scheduling_unit_draft_id' in kwargs:
+ scheduling_unit_draft = get_object_or_404(models.SchedulingUnitDraft, pk=kwargs['scheduling_unit_draft_id'])
+ scheduling_set = scheduling_unit_draft.scheduling_set
+
+ body_unicode = request.body.decode('utf-8')
+ body_data = json.loads(body_unicode)
+
+ copy_reason = body_data.get('copy_reason', None)
+
+ try:
+ copy_reason_obj = models.CopyReason.objects.get(value=copy_reason)
+ except ObjectDoesNotExist:
+ logger.info("CopyReason matching query does not exist.")
+ #if a non valid copy_reason is specified, set copy_reason to None
+ copy_reason = None
+
+ scheduling_set_id = body_data.get('scheduling_set_id', None)
+ logger.info(scheduling_set_id)
+ if scheduling_set_id is not None:
+ try:
+ scheduling_set = models.SchedulingSet.objects.get(id=scheduling_set_id)
+ except ObjectDoesNotExist:
+ logger.info("scheduling Set does not exist.")
+
+ scheduling_unit_draft_copy = copy_scheduling_unit_draft(scheduling_unit_draft,scheduling_set,copy_reason)
+ # url path magic to construct the new scheduling_unit_draft_path url
+ scheduling_unit_draft_path = request._request.path
+ base_path = scheduling_unit_draft_path[:scheduling_unit_draft_path.find('/scheduling_unit_draft')]
+ scheduling_unit_draft_copy_path = '%s/scheduling_unit_draft/%s/' % (base_path, scheduling_unit_draft_copy.id,)
+
+ # return a response with the new serialized SchedulingUnitBlueprintSerializer, and a Location to the new instance in the header
+ return Response(serializers.SchedulingUnitDraftSerializer(scheduling_unit_draft_copy, context={'request':request}).data,
+ status=status.HTTP_201_CREATED,
+ headers={'Location': scheduling_unit_draft_copy_path})
+ else:
+ content = {'Error': 'scheduling_unit_draft_id is missing'}
+ return Response(content, status=status.HTTP_404_NOT_FOUND)
+
+
+class SchedulingUnitDraftCopyFromSchedulingSetViewSet(LOFARCopyViewSet):
+ queryset = models.SchedulingUnitDraft.objects.all()
+ serializer_class = serializers.SchedulingUnitDraftCopyFromSchedulingSetSerializer
+
+ def get_queryset(self):
+ if 'scheduling_set_id' in self.kwargs:
+ scheduling_set = get_object_or_404(models.SchedulingSet, pk=self.kwargs['scheduling_set_id'])
+ return scheduling_set.scheduling_unit_drafts.all()
+ else:
+ return models.SchedulingUnitDraft.objects.all()
+
+ @swagger_auto_schema(responses={201: "The TaskDrafts copied from the TaskDrafts in this Scheduling Unit Set",
+ 403: 'forbidden'},
+ operation_description="Create a copy of all the TaskDrafts in this Scheduling Unit Set.")
+ def create(self, request, *args, **kwargs):
+ if 'scheduling_set_id' in kwargs:
+ scheduling_set = get_object_or_404(models.SchedulingSet, pk=kwargs['scheduling_set_id'])
+ scheduling_unit_drafts = scheduling_set.scheduling_unit_drafts.all()
+
+ body_unicode = request.body.decode('utf-8')
+ body_data = json.loads(body_unicode)
+
+
+ copy_reason = body_data.get('copy_reason', None)
+
+ try:
+ copy_reason_obj = models.CopyReason.objects.get(value=copy_reason)
+ except ObjectDoesNotExist:
+ logger.info("CopyReason matching query does not exist.")
+ #if a non valid copy_reason is specified, set copy_reason to None
+ copy_reason = None
+
+ scheduling_unit_draft_copy_path=[]
+ for scheduling_unit_draft in scheduling_unit_drafts:
+ scheduling_unit_draft_copy = copy_scheduling_unit_draft(scheduling_unit_draft,scheduling_set,copy_reason)
+ # url path magic to construct the new scheduling_unit_draft url
+ copy_scheduling_unit_draft_path = request._request.path
+ base_path = copy_scheduling_unit_draft_path[:copy_scheduling_unit_draft_path.find('/copy_scheduling_unit_drafts')]
+ scheduling_unit_draft_copy_path += ['%s/copy_scheduling_unit_drafts/%s/' % (base_path, scheduling_unit_draft_copy.id,)]
+
+ # just return as a response the serialized scheduling_set (with references to the created copy_scheduling_unit_draft(s)
+ return Response(serializers.SchedulingSetSerializer(scheduling_set, context={'request':request}).data,status=status.HTTP_201_CREATED)
+ else:
+ content = {'Error': 'scheduling_set_id is missing'}
+ return Response(content, status=status.HTTP_404_NOT_FOUND)
+
+class SchedulingUnitBlueprintCopyToSchedulingUnitDraftViewSet(LOFARCopyViewSet):
+ queryset = models.SchedulingUnitBlueprint.objects.all()
+ serializer_class = serializers.SchedulingUnitBlueprintCopyToSchedulingUnitDraftSerializer
+
+ @swagger_auto_schema(responses={201: "The copy of the SchedulingUnitDraft",
+ 403: 'forbidden'},
+ operation_description="Create a SchedulingUnitDraft from the SchedulingUnitBlueprint")
+ def create(self, request, *args, **kwargs):
+
+ if 'scheduling_unit_blueprint_id' in kwargs:
+ scheduling_unit_blueprint = get_object_or_404(models.SchedulingUnitBlueprint, pk=kwargs['scheduling_unit_blueprint_id'])
+
+ body_unicode = request.body.decode('utf-8')
+ body_data = json.loads(body_unicode)
+
+
+ copy_reason = body_data.get('copy_reason', None)
+
+ try:
+ copy_reason_obj = models.CopyReason.objects.get(value=copy_reason)
+ except ObjectDoesNotExist:
+ logger.info("CopyReason matching query does not exist.")
+ #if a non valid copy_reason is specified, set copy_reason to None
+ copy_reason = None
+
+ scheduling_unit_draft = create_scheduling_unit_draft_from_scheduling_unit_blueprint(scheduling_unit_blueprint,copy_reason)
+
+ # return a response with the new serialized scheduling_unit_blueprint (with references to the created task_blueprint(s) and (scheduled) subtasks)
+ return Response(serializers.SchedulingUnitDraftSerializer(scheduling_unit_draft, context={'request':request}).data,
+ status=status.HTTP_201_CREATED)
+ else:
+ content = {'Error': 'scheduling_unit_draft_id is missing'}
+ return Response(content, status=status.HTTP_404_NOT_FOUND)
+
+
+class TaskBlueprintCopyToTaskDraftViewSet(LOFARCopyViewSet):
+ queryset = models.SchedulingUnitBlueprint.objects.all()
+ serializer_class = serializers.SchedulingUnitBlueprintCopyToSchedulingUnitDraftSerializer
+
+ @swagger_auto_schema(responses={201: "The TaskDraft created from this TaskBlueprint",
+ 403: 'forbidden'},
+ operation_description="Copy this TaskBlueprint to a new TaskDraft.")
+ def create(self, request, *args, **kwargs):
+ if 'task_blueprint_id' in kwargs:
+ task_blueprint = get_object_or_404(models.TaskBlueprint, pk=kwargs['task_blueprint_id'])
+ task_draft = copy_task_blueprint_to_task_draft(task_blueprint)
+
+ # return a response with the new serialized scheduling_unit_blueprint (with references to the created task_blueprint(s) and (scheduled) subtasks)
+ return Response(serializers.TaskDraftSerializer(task_draft, context={'request':request}).data,
+ status=status.HTTP_201_CREATED)
+
+ else:
+ content = {'Error': 'task_blueprint_id is missing'}
+ return Response(content, status=status.HTTP_404_NOT_FOUND)
+
class SchedulingUnitBlueprintViewSet(LOFARViewSet):
queryset = models.SchedulingUnitBlueprint.objects.all()
diff --git a/SAS/TMSS/src/tmss/tmssapp/workflows/CMakeLists.txt b/SAS/TMSS/src/tmss/tmssapp/workflows/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..474aada33041160e598ac2b1a126d68971d75afd
--- /dev/null
+++ b/SAS/TMSS/src/tmss/tmssapp/workflows/CMakeLists.txt
@@ -0,0 +1,11 @@
+
+include(PythonInstall)
+
+set(_py_files
+ __init__.py
+ helloworldflow.py
+ schedulingunitdemoflow.py
+ )
+
+python_install(${_py_files}
+ DESTINATION lofar/sas/tmss/tmss/tmssapp/workflows)
diff --git a/SAS/TMSS/src/tmss/tmssapp/workflows/__init__.py b/SAS/TMSS/src/tmss/tmssapp/workflows/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..45516795a25730483ebfa40c1fbdb5f533df8ebe
--- /dev/null
+++ b/SAS/TMSS/src/tmss/tmssapp/workflows/__init__.py
@@ -0,0 +1,2 @@
+from .helloworldflow import *
+from .schedulingunitdemoflow import *
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/workflows/helloworldflow.py b/SAS/TMSS/src/tmss/tmssapp/workflows/helloworldflow.py
new file mode 100644
index 0000000000000000000000000000000000000000..d3307efe5f773359de58c89bea4a8728fa809c05
--- /dev/null
+++ b/SAS/TMSS/src/tmss/tmssapp/workflows/helloworldflow.py
@@ -0,0 +1,69 @@
+import os
+
+from viewflow import flow, frontend, lock
+from viewflow.base import this, Flow
+from viewflow.compat import _
+from viewflow.flow import views as flow_views
+
+
+from lofar.sas.tmss.tmss.tmssapp import models
+
+
+
+@frontend.register
+class HelloWorldFlow(Flow):
+ """
+ Hello world
+ This process demonstrates hello world approval request flow.
+ """
+ process_class = models.HelloWorldProcess
+ process_title = _('Hello world')
+ process_description = _('This process demonstrates hello world approval request flow.')
+
+ lock_impl = lock.select_for_update_lock
+
+ summary_template = _("'{{ process.text }}' message to the world")
+
+ start = (
+ flow.Start(
+ flow_views.CreateProcessView,
+ fields=['text'],
+ task_title=_('New message'))
+ .Permission(auto_create=True)
+ .Next(this.approve)
+ )
+
+ approve = (
+ flow.View(
+ flow_views.UpdateProcessView, fields=['approved'],
+ task_title=_('Approve'),
+ task_description=_("{{ process.text }} approvement required"),
+ task_result_summary=_("Messsage was {{ process.approved|yesno:'Approved,Rejected' }}"))
+ .Permission(auto_create=True)
+ .Next(this.check_approve)
+ )
+
+ check_approve = (
+ flow.If(
+ cond=lambda act: act.process.approved,
+ task_title=_('Approvement check'),
+ )
+ .Then(this.send)
+ .Else(this.end)
+ )
+
+ send = (
+ flow.Handler(
+ this.send_hello_world_request,
+ task_title=_('Send message'),
+ )
+ .Next(this.end)
+ )
+
+ end = flow.End(
+ task_title=_('End'),
+ )
+
+ def send_hello_world_request(self, activation):
+ with open(os.devnull, "w") as world:
+ world.write(activation.process.text)
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/workflows/schedulingunitdemoflow.py b/SAS/TMSS/src/tmss/tmssapp/workflows/schedulingunitdemoflow.py
new file mode 100644
index 0000000000000000000000000000000000000000..a35c72db8e9430b929e9ada4f424bbf6a58527c9
--- /dev/null
+++ b/SAS/TMSS/src/tmss/tmssapp/workflows/schedulingunitdemoflow.py
@@ -0,0 +1,187 @@
+from django.utils.decorators import method_decorator
+from django.db.models.signals import post_save
+from viewflow import flow
+from viewflow.models import Task
+from viewflow.base import this, Flow
+from viewflow.flow.views import CreateProcessView, UpdateProcessView, AssignTaskView
+from viewflow.activation import FuncActivation, ViewActivation
+from viewflow.flow.nodes import Signal
+from viewflow import mixins
+
+from lofar.sas.tmss.tmss.tmssapp import models
+
+from viewflow import frontend, ThisObject
+from viewflow.activation import STATUS
+
+class ConditionActivation(FuncActivation):
+ @classmethod
+ def activate(cls, flow_task, prev_activation, token):
+ activation = super(ConditionActivation, cls).activate(flow_task, prev_activation, token)
+
+ if flow_task.condition_check(activation, None):
+ # condition holds on activation
+ activation.prepare()
+ activation.done()
+
+ return activation
+
+class Condition(Signal):
+ #task_type = "HUMAN" # makes it show up in the unassigned task lists
+ activation_class = ConditionActivation
+
+ def __init__(self, condition_check, signal, sender=None, task_loader=None, **kwargs):
+ """
+ Instantiate a Signal task.
+
+ :param signal: A django signal to connect
+ :param receiver: Callable[activation, **kwargs]
+ :param sender: Optional signal sender
+ :param task_loader: Callable[**kwargs] -> Task
+ :param allow_skip: If True task_loader can return None if
+ signal could be skipped.
+
+ You can skip a `task_loader` if the signal going to be
+ sent with Task instance.
+ """
+ self.condition_check = condition_check
+
+ super(Condition, self).__init__(signal, self.signal_handler, sender, task_loader, **kwargs)
+
+ @method_decorator(flow.flow_signal)
+ def signal_handler(self, activation, sender, instance, **signal_kwargs):
+ if activation.get_status() == STATUS.DONE:
+ # race condition -- condition was true on activation but we also receive the signal now
+ return
+
+ activation.prepare()
+ if activation.flow_task.condition_check(activation, instance):
+ activation.done()
+
+ def ready(self):
+ """Resolve internal `this`-references. and subscribe to the signal."""
+ if isinstance(self.condition_check, ThisObject):
+ self.condition_check = getattr(self.flow_class.instance, self.condition_check.name)
+
+ super(Condition, self).ready()
+
+@frontend.register
+class SchedulingUnitDemoFlow(Flow):
+ process_class = models.SchedulingUnitDemoProcess
+
+ # 0. Start on SU instantiation
+ # 1. To be Manually scheduled? -> Go to 1a
+ # 1a. Present view to manually schedule.
+ # 2. Wait on signal SU got finished/error/cancelled (might have already!!) ->
+ # - Wait for assignment to RO user
+ # View: - Present any quality plots
+ # - Present any error info
+ # - Present fixing options
+ # - Present choice to fix & redo, discard, or continue.
+ # Continue:
+ # View: - Present any quality plots
+ # - Present any error info
+ # - Submit quality report/score
+ # - Submit recommendation
+ # 3. - Assign ticket to Contact Author
+ # - Present quality plots to user
+ # - Present quality report/score, and recommendation
+ # - Submit acceptance & report
+ # 4. - Assign ticket to owner in step 2.
+ # - Present quality report/score, and recommendation
+ # - Present acceptance & report
+ # - Present choice to ingest or discard.
+ # Ingest:
+ # Set ingestable flag on SU.
+ # Discard: - Cancel SU (triggering garbage collection
+ #
+ # Fix & Redo:
+ # - Wait for user to confirm SU is fixed
+ # - Go to 2
+ #
+
+ # Consider adding to any/all views:
+ # - Present any opened JIRA tickets
+ # - Present opportunity to open JIRA ticket
+ # Note that previously submitted info can be found by clicking through the task. So
+ # we only need to show whats nominally needed.
+ # Note that orthogonally to the above flow:
+ # - Users need to be informed tasks are assigned to them (e-mail?)
+ # - Users already have an overview in viewflow of tickets assigned to them
+ # - We likely want to control what e-mails are sent.
+
+ start = (
+ flow.StartSignal(
+ post_save,
+ this.on_save_can_start,
+ sender=models.SchedulingUnitDemo
+ ).Next(this.wait_schedulable)
+ )
+
+ wait_schedulable = (
+ Condition(
+ this.check_condition,
+ post_save,
+ sender=models.SchedulingUnitDemo,
+ task_loader=this.get_scheduling_unit_task
+ )
+ .Next(this.form)
+ )
+
+ form = (
+ flow.View(
+ UpdateProcessView,
+ fields=["text"]
+ ).Permission(
+ auto_create=True
+ ).Next(this.approve)
+ )
+
+ approve = (
+ flow.View(
+ UpdateProcessView,
+ fields=["approved"]
+ ).Permission(
+ auto_create=True
+ ).Next(this.check_approve)
+ )
+
+ check_approve = (
+ flow.If(lambda activation: activation.process.approved)
+ .Then(this.send)
+ .Else(this.end)
+ )
+
+ send = (
+ flow.Handler(
+ this.send_hello_world_request
+ ).Next(this.end)
+ )
+
+ end = flow.End()
+
+ @method_decorator(flow.flow_start_signal)
+ def on_save_can_start(self, activation, sender, instance, created, **signal_kwargs):
+ if created:
+ activation.prepare()
+ activation.process.su = instance
+ activation.done()
+ print("workflow started")
+ else:
+ print("no workflow started")
+ return activation
+
+ def send_hello_world_request(self, activation):
+ print(activation.process.text)
+
+ def check_condition(self, activation, instance):
+ if instance is None:
+ instance = activation.process.su
+
+ condition = instance.state == 5
+ print("condition is ",condition)
+ return condition
+
+ def get_scheduling_unit_task(self, flow_task, sender, instance, **kwargs):
+ print(kwargs)
+ process = models.SchedulingUnitDemoProcess.objects.get(su=instance)
+ return Task.objects.get(process=process,flow_task=flow_task)
diff --git a/SAS/TMSS/src/tmss/urls.py b/SAS/TMSS/src/tmss/urls.py
index 53146045e08986f1cb8930e993b04129df909610..37d9d081a3a3ecbee61e876ea3ee365b7c111ace 100644
--- a/SAS/TMSS/src/tmss/urls.py
+++ b/SAS/TMSS/src/tmss/urls.py
@@ -23,11 +23,16 @@ from django.views.generic.base import TemplateView, RedirectView
from collections import OrderedDict
from rest_framework import routers, permissions
-from .tmssapp import viewsets, models, serializers, views
+from .tmssapp import viewsets, models, serializers, views, workflows
from rest_framework.documentation import include_docs_urls
from drf_yasg.views import get_schema_view
from drf_yasg import openapi
+from datetime import datetime
+
+from material.frontend import urls as frontend_urls
+from viewflow.flow.viewset import FlowViewSet
+
#
# Django style patterns
#
@@ -54,7 +59,11 @@ urlpatterns = [
path('docs/', include_docs_urls(title='TMSS API')),
re_path(r'^swagger(?P<format>\.json|\.yaml)$', swagger_schema_view.without_ui(cache_timeout=0), name='schema-json'),
path('swagger/', swagger_schema_view.with_ui('swagger', cache_timeout=0), name='schema-swagger-ui'),
- path('redoc/', swagger_schema_view.with_ui('redoc', cache_timeout=0), name='schema-redoc')
+ path('redoc/', swagger_schema_view.with_ui('redoc', cache_timeout=0), name='schema-redoc'),
+ path('schemas/<str:template>/<str:name>/<str:version>', views.get_template_json_schema, name='get_template_json_schema'), #TODO: how to make trailing slash optional?
+ path('schemas/<str:template>/<str:name>/<str:version>/', views.get_template_json_schema, name='get_template_json_schema'),
+ path(r'util/utc', views.utc, name="system-utc"),
+ path(r'util/lst', views.lst, name="conversion-lst")
]
@@ -100,14 +109,17 @@ router.register(r'quantity', viewsets.QuantityViewSet)
router.register(r'task_type', viewsets.TaskTypeViewSet)
# templates
+router.register(r'common_schema_template', viewsets.CommonSchemaTemplateViewSet)
router.register(r'generator_template', viewsets.GeneratorTemplateViewSet)
router.register(r'scheduling_unit_observing_strategy_template', viewsets.SchedulingUnitObservingStrategyTemplateViewSet)
router.register(r'scheduling_unit_template', viewsets.SchedulingUnitTemplateViewSet)
+router.register(r'scheduling_constraints_template', viewsets.SchedulingConstraintsTemplateViewSet)
router.register(r'task_template', viewsets.TaskTemplateViewSet)
router.register(r'task_relation_selection_template', viewsets.TaskRelationSelectionTemplateViewSet)
router.register(r'task_connector_type', viewsets.TaskConnectorTypeViewSet)
router.register(r'default_generator_template', viewsets.DefaultGeneratorTemplateViewSet)
router.register(r'default_scheduling_unit_template', viewsets.DefaultSchedulingUnitTemplateViewSet)
+router.register(r'default_scheduling_constraints_template', viewsets.DefaultSchedulingConstraintsTemplateViewSet)
router.register(r'default_task_template', viewsets.DefaultTaskTemplateViewSet)
router.register(r'default_task_relation_selection_template', viewsets.DefaultTaskRelationSelectionTemplateViewSet)
@@ -140,6 +152,14 @@ router.register(r'task_relation_draft/(?P<task_relation_draft_id>\d+)/task_relat
router.register(r'task_blueprint/(?P<task_blueprint_id>\d+)/task_relation_blueprint', viewsets.TaskRelationBlueprintNestedViewSet)
router.register(r'task_blueprint/(?P<task_blueprint_id>\d+)/subtask', viewsets.SubtaskNestedViewSet)
+# copy
+router.register(r'task_draft/(?P<task_draft_id>\d+)/copy', viewsets.TaskDraftCopyViewSet)
+router.register(r'task_blueprint/(?P<task_blueprint_id>\d+)/copy_to_task_draft', viewsets.TaskBlueprintCopyToTaskDraftViewSet)
+router.register(r'scheduling_set/(?P<scheduling_set_id>\d+)/copy_scheduling_unit_drafts', viewsets.SchedulingUnitDraftCopyFromSchedulingSetViewSet)
+router.register(r'scheduling_unit_draft/(?P<scheduling_unit_draft_id>\d+)/copy', viewsets.SchedulingUnitDraftCopyViewSet)
+router.register(r'scheduling_unit_blueprint/(?P<scheduling_unit_blueprint_id>\d+)/copy_to_scheduling_unit_draft', viewsets.SchedulingUnitBlueprintCopyToSchedulingUnitDraftViewSet)
+
+
# SCHEDULING
# choices
@@ -175,12 +195,16 @@ router.register(r'user', viewsets.UserViewSet)
urlpatterns.extend(router.urls)
-
frontend_urlpatterns = [
path("", views.index, name="index")
]
-# prefix everything for proxy
-#urlpatterns = [url(r'^api/', include(urlpatterns)), url(r'^oidc/', include('mozilla_django_oidc.urls')),]
-urlpatterns = [url(r'^api$', RedirectView.as_view(url='/api/')), url(r'^api/', include(urlpatterns)), url(r'^oidc$', RedirectView.as_view(url='/oidc/')), url(r'^oidc/', include('mozilla_django_oidc.urls')), url(r'^.*', include(frontend_urlpatterns)),]
+urlpatterns = [url(r'^api$', RedirectView.as_view(url='/api/')),
+ url(r'^api/', include(urlpatterns)), url(r'^oidc$',
+ RedirectView.as_view(url='/oidc/')),
+ url(r'^oidc/', include('mozilla_django_oidc.urls')),
+ url(r'^workflow$', RedirectView.as_view(url='/workflow/', permanent=False)),
+ url(r'', include(frontend_urls)),
+ url(r'^.*', include(frontend_urlpatterns)),
+]
diff --git a/SAS/TMSS/test/CMakeLists.txt b/SAS/TMSS/test/CMakeLists.txt
index b19ddcd546e283f0e176ecaf57711bb3b8b8f03c..769fce231ac3bc18470ae3c974456d4ec089ff68 100644
--- a/SAS/TMSS/test/CMakeLists.txt
+++ b/SAS/TMSS/test/CMakeLists.txt
@@ -32,6 +32,7 @@ if(BUILD_TESTING)
lofar_add_test(t_adapter)
lofar_add_test(t_tasks)
lofar_add_test(t_scheduling)
+ lofar_add_test(t_conversions)
# To get ctest running
file(COPY testdata DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
diff --git a/SAS/TMSS/test/t_adapter.py b/SAS/TMSS/test/t_adapter.py
index 43c8b010e4a3ef0f17003b3c0e0665a016e60b4e..379f3a37a4b53165882be3579af3eef08a8b40fe 100755
--- a/SAS/TMSS/test/t_adapter.py
+++ b/SAS/TMSS/test/t_adapter.py
@@ -31,6 +31,7 @@ logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=loggin
# use setup/teardown magic for tmss test database, ldap server and django server
# (ignore pycharm unused import statement, python unittests does use at RunTime the tmss_test_environment_unittest_setup module)
from lofar.sas.tmss.test.tmss_test_environment_unittest_setup import *
+tmss_test_env.populate_schemas()
from lofar.sas.tmss.test.tmss_test_data_django_models import *
@@ -42,11 +43,12 @@ from lofar.sas.tmss.tmss.tmssapp import models
from lofar.sas.tmss.tmss.tmssapp.adapters.parset import convert_to_parset
from lofar.common.json_utils import get_default_json_object_for_schema
from lofar.sas.tmss.tmss.tmssapp.adapters.sip import generate_sip_for_dataproduct
-
+from lofar.sas.tmss.tmss.tmssapp.adapters.feedback import generate_dataproduct_feedback_from_subtask_feedback_and_set_finished
+from lofar.lta.sip import constants
class ParsetAdapterTest(unittest.TestCase):
def test_01(self):
- subtask_template = models.SubtaskTemplate.objects.get(name='observationcontrol schema')
+ subtask_template = models.SubtaskTemplate.objects.get(name='observation control')
specifications_doc = get_default_json_object_for_schema(subtask_template.schema)
for dp in specifications_doc['stations']['digital_pointings']:
dp['subbands'] = list(range(8))
@@ -60,18 +62,202 @@ class ParsetAdapterTest(unittest.TestCase):
class SIPdapterTest(unittest.TestCase):
def test_simple_sip_generate_from_dataproduct(self):
- subtask_template = models.SubtaskTemplate.objects.get(name='observationcontrol schema')
+ subtask_template = models.SubtaskTemplate.objects.get(name='observation control')
specifications_doc = get_default_json_object_for_schema(subtask_template.schema)
+ specifications_doc['stations']['filter'] = "HBA_210_250"
+ feedback_template = models.DataproductFeedbackTemplate.objects.get(name='feedback')
+ # feedback_doc = get_default_json_object_for_schema(feedback_template.schema) # todo <- fix the default generator, for some reason it does not produce valid json here...
+ feedback_doc = {'percentage_written': 100, 'frequency': {'subbands': [156], 'central_frequencies': [33593750.0], 'channel_width': 6103.515625, 'channels_per_subband': 32}, 'time': {'start_time': '2013-02-16T17:00:00', 'duration': 5.02732992172, 'sample_width': 2.00278016}, 'antennas': {'set': 'HBA_DUAL', 'fields': [{'type': 'HBA', 'field': 'HBA0', 'station': 'CS001'}, {'type': 'HBA', 'field': 'HBA1', 'station': 'CS001'}]}, 'target': {'pointing': {'angle1': 0, 'angle2': 0, 'angle3': 0, 'direction_type': 'J2000'}}, 'samples': {'polarisations': ['XX', 'XY', 'YX', 'YY'], 'type': 'float', 'bits': 32, 'writer': 'standard', 'writer_version': '2.2.0', 'complex': True}, '$schema': 'http://127.0.0.1:8001/api/schemas/dataproductfeedbacktemplate/feedback/1#'}
for dp in specifications_doc['stations']['digital_pointings']:
dp['subbands'] = list(range(8))
subtask_data = Subtask_test_data(subtask_template=subtask_template, specifications_doc=specifications_doc)
subtask:models.Subtask = models.Subtask.objects.create(**subtask_data)
subtask_output = models.SubtaskOutput.objects.create(**SubtaskOutput_test_data(subtask=subtask))
- dataproduct:models.Dataproduct = models.Dataproduct.objects.create(**Dataproduct_test_data(producer=subtask_output))
+ dataproduct:models.Dataproduct = models.Dataproduct.objects.create(**Dataproduct_test_data(feedback_doc=feedback_doc, producer=subtask_output))
+ # make sure we can create a valid SIP
sip = generate_sip_for_dataproduct(dataproduct)
- # TODO: Although validate succeed at this step, would be interesting to check some xml values
- # print(sip.get_prettyxml())
+
+ # double-check that SIP contains values from feedback and specifications docs
+ self.assertIn(str(feedback_doc['frequency']['channel_width']), sip.get_prettyxml())
+ self.assertIn(str(feedback_doc['time']['start_time']), sip.get_prettyxml())
+ self.assertIn(constants.FILTERSELECTIONTYPE_210_250_MHZ, sip.get_prettyxml()) # specifications_doc: "HBA_210_250"
+
+
+class FeedbackAdapterTest(unittest.TestCase):
+
+ feedback_pipe_complete = """
+feedback_version=03.01.00
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[0].centralFrequency=33593750.0
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[0].channelsPerSubband=32
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[0].channelWidth=6103.515625
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[0].duration=5.02732992172
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[0].fileFormat=AIPS++/CASA
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[0].filename=L99307_SB000_uv.dppp.MS
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[0].integrationInterval=2.00278016
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[0].location=locus001:/data/L99307
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[0].percentageWritten=100
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[0].returncode=0
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[0].size=15606123742
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[0].startTime=2013-02-16T17:00:00.000
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[0].stationSubband=0
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[0].storageWriter=CASA
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[0].storageWriterVersion=2.2.0
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[0].subband=0
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[1].centralFrequency=33789062.5
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[1].channelsPerSubband=32
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[1].channelWidth=6103.515625
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[1].duration=5.02513194084
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[1].fileFormat=AIPS++/CASA
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[1].filename=L99307_SB001_uv.dppp.MS
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[1].integrationInterval=2.00278016
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[1].location=locus003:/data/L99307
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[1].percentageWritten=100
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[1].returncode=0
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[1].size=15606156518
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[1].startTime=2013-02-16T17:00:00.000
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[1].stationSubband=0
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[1].storageWriter=CASA
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[1].storageWriterVersion=2.2.0
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[1].subband=1
+Observation.DataProducts.nrOfOutput_Beamformed_=0
+Observation.DataProducts.nrOfOutput_Correlated_=2
+_isCobalt=T
+feedback_version=03.01.00
+"""
+
+ feedback_pipe_incomplete = """
+feedback_version=03.01.00
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[0].centralFrequency=33593750.0
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[0].channelsPerSubband=32
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[0].channelWidth=6103.515625
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[0].duration=5.02732992172
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[0].fileFormat=AIPS++/CASA
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[0].filename=L99307_SB000_uv.dppp.MS
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[0].integrationInterval=2.00278016
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[0].location=locus001:/data/L99307
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[0].percentageWritten=100
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[0].returncode=0
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[0].size=15606123742
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[0].startTime=2013-02-16T17:00:00.000
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[0].stationSubband=0
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[0].storageWriter=CASA
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[0].storageWriterVersion=2.2.0
+LOFAR.ObsSW.Observation.DataProducts.Output_Correlated_[0].subband=0
+Observation.DataProducts.nrOfOutput_Beamformed_=0
+Observation.DataProducts.nrOfOutput_Correlated_=2
+_isCobalt=T
+feedback_version=03.01.00
+"""
+
+ feedback_obs_complete = """
+ Observation.Correlator.channelWidth=3051.7578125
+Observation.Correlator.channelsPerSubband=64
+Observation.Correlator.integrationInterval=1.00663296
+Observation.DataProducts.Output_Correlated_[0].SAP=0
+Observation.DataProducts.Output_Correlated_[0].centralFrequency=30468750.000000
+Observation.DataProducts.Output_Correlated_[0].channelWidth=3051.757812
+Observation.DataProducts.Output_Correlated_[0].channelsPerSubband=64
+Observation.DataProducts.Output_Correlated_[0].duration=0
+Observation.DataProducts.Output_Correlated_[0].fileFormat=AIPS++/CASA
+Observation.DataProducts.Output_Correlated_[0].filename=L220133_SAP000_SB000_uv.MS
+Observation.DataProducts.Output_Correlated_[0].integrationInterval=1.006633
+Observation.DataProducts.Output_Correlated_[0].location=CEP2:/data/L220133/
+Observation.DataProducts.Output_Correlated_[0].percentageWritten=0
+Observation.DataProducts.Output_Correlated_[0].size=0
+Observation.DataProducts.Output_Correlated_[0].startTime=2014-04-18 15:02:00
+Observation.DataProducts.Output_Correlated_[0].stationSubband=156
+Observation.DataProducts.Output_Correlated_[0].storageWriter=LOFAR
+Observation.DataProducts.Output_Correlated_[0].storageWriterVersion=3
+Observation.DataProducts.Output_Correlated_[0].subband=0
+Observation.DataProducts.Output_Correlated_[1].SAP=0
+Observation.DataProducts.Output_Correlated_[1].centralFrequency=30664062.500000
+Observation.DataProducts.Output_Correlated_[1].channelWidth=3051.757812
+Observation.DataProducts.Output_Correlated_[1].channelsPerSubband=64
+Observation.DataProducts.Output_Correlated_[1].duration=0
+Observation.DataProducts.Output_Correlated_[1].fileFormat=AIPS++/CASA
+Observation.DataProducts.Output_Correlated_[1].filename=L220133_SAP000_SB001_uv.MS
+Observation.DataProducts.Output_Correlated_[1].integrationInterval=1.006633
+Observation.DataProducts.Output_Correlated_[1].location=CEP2:/data/L220133/
+Observation.DataProducts.Output_Correlated_[1].percentageWritten=0
+Observation.DataProducts.Output_Correlated_[1].size=0
+Observation.DataProducts.Output_Correlated_[1].startTime=2014-04-18 15:02:00
+Observation.DataProducts.Output_Correlated_[1].stationSubband=157
+Observation.DataProducts.Output_Correlated_[1].storageWriter=LOFAR
+Observation.DataProducts.Output_Correlated_[1].storageWriterVersion=3
+Observation.DataProducts.Output_Correlated_[1].subband=1
+Observation.DataProducts.nrOfOutput_Beamformed_=0
+Observation.DataProducts.nrOfOutput_Correlated_=2
+_isCobalt=T
+feedback_version=03.01.00
+"""
+
+ def test_generate_dataproduct_feedback_from_subtask_feedback_and_set_finished_fails_on_wrong_subtask_state(self):
+ subtask_data = Subtask_test_data()
+ subtask:models.Subtask = models.Subtask.objects.create(**subtask_data)
+
+ with self.assertRaises(ValueError) as cm:
+ generate_dataproduct_feedback_from_subtask_feedback_and_set_finished(subtask)
+
+ self.assertIn("not in state finishing", str(cm.exception))
+
+
+ def test_generate_dataproduct_feedback_from_subtask_feedback_and_set_finished_fails_on_incomplete_feedback(self):
+ subtask_data = Subtask_test_data(raw_feedback=self.feedback_pipe_incomplete,
+ state=models.SubtaskState.objects.get(value='finishing'))
+ subtask:models.Subtask = models.Subtask.objects.create(**subtask_data)
+
+ with self.assertRaises(ValueError) as cm:
+ generate_dataproduct_feedback_from_subtask_feedback_and_set_finished(subtask)
+
+ self.assertIn("is not complete", str(cm.exception))
+
+
+ def test_generate_dataproduct_feedback_from_subtask_feedback_and_set_finished(self):
+ subtask_data = Subtask_test_data(raw_feedback=self.feedback_obs_complete,
+ state=models.SubtaskState.objects.get(value='finishing'),
+ subtask_template=models.SubtaskTemplate.objects.get(name='observation control'))
+ subtask_obs:models.Subtask = models.Subtask.objects.create(**subtask_data)
+
+ subtask_data = Subtask_test_data(raw_feedback=self.feedback_pipe_complete,
+ state=models.SubtaskState.objects.get(value='finishing'),
+ subtask_template=models.SubtaskTemplate.objects.get(name='pipeline control'))
+ subtask_pipe: models.Subtask = models.Subtask.objects.create(**subtask_data)
+
+ dataproduct_obs_out1:models.Dataproduct = models.Dataproduct.objects.create(**Dataproduct_test_data(filename='L220133_SAP000_SB000_uv.MS'))
+ dataproduct_obs_out2: models.Dataproduct = models.Dataproduct.objects.create(**Dataproduct_test_data(filename='L220133_SAP000_SB001_uv.MS'))
+ dataproduct_pipe_out1: models.Dataproduct = models.Dataproduct.objects.create(**Dataproduct_test_data(filename='L99307_SB000_uv.dppp.MS'))
+ dataproduct_pipe_out2: models.Dataproduct = models.Dataproduct.objects.create(**Dataproduct_test_data(filename='L99307_SB001_uv.dppp.MS'))
+ models.DataproductTransform.objects.create(input=dataproduct_obs_out1, output=dataproduct_pipe_out1, identity=True)
+ models.DataproductTransform.objects.create(input=dataproduct_obs_out2, output=dataproduct_pipe_out2, identity=True)
+
+ # assert dataproducts have no feedback docs before conversion
+ for dataproduct in [dataproduct_obs_out1, dataproduct_obs_out2, dataproduct_pipe_out1, dataproduct_pipe_out2]:
+ self.assertNotIn('percentage_written', dataproduct.feedback_doc)
+
+ generate_dataproduct_feedback_from_subtask_feedback_and_set_finished(subtask_obs)
+ generate_dataproduct_feedback_from_subtask_feedback_and_set_finished(subtask_pipe)
+
+ # reload dataproducts and assert dataproduct feedback docs have feedback after conversion
+ for dataproduct in [dataproduct_obs_out1, dataproduct_obs_out2, dataproduct_pipe_out1, dataproduct_pipe_out2]:
+ dataproduct.refresh_from_db()
+ self.assertIsNotNone(dataproduct.feedback_doc)
+ self.assertIn('percentage_written', dataproduct.feedback_doc)
+
+ # assert correct relations of feedback docs
+ self.assertNotEqual(dataproduct_obs_out1.feedback_doc['frequency']['subbands'],
+ dataproduct_obs_out2.feedback_doc['frequency']['subbands'])
+ self.assertNotEqual(dataproduct_obs_out1.feedback_doc['frequency']['subbands'],
+ dataproduct_pipe_out2.feedback_doc['frequency']['subbands'])
+ self.assertEqual(dataproduct_obs_out1.feedback_doc['frequency']['subbands'],
+ dataproduct_pipe_out1.feedback_doc['frequency']['subbands'])
+ self.assertEqual(dataproduct_obs_out1.feedback_doc['frequency']['subbands'],
+ dataproduct_pipe_out1.feedback_doc['frequency']['subbands'])
+
+ # assert FINISHED states
+ for subtask in [subtask_obs, subtask_pipe]:
+ self.assertEqual(models.SubtaskState.objects.get(value='finished'), subtask.state)
+
if __name__ == "__main__":
diff --git a/SAS/TMSS/test/t_conversions.py b/SAS/TMSS/test/t_conversions.py
new file mode 100755
index 0000000000000000000000000000000000000000..ccd4025f6c4c21a43d63f5ccb6a55c3b764f0963
--- /dev/null
+++ b/SAS/TMSS/test/t_conversions.py
@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+
+# Copyright (C) 2018 ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+
+# $Id: $
+
+import os
+import unittest
+import datetime
+import logging
+import requests
+import dateutil.parser
+import astropy.coordinates
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO)
+from lofar.sas.tmss.tmss.tmssapp.conversions import local_sidereal_time_for_utc_and_station, local_sidereal_time_for_utc_and_longitude
+
+# Do Mandatory setup step:
+# use setup/teardown magic for tmss test database, ldap server and django server
+# (ignore pycharm unused import statement, python unittests does use at RunTime the tmss_test_environment_unittest_setup module)
+from lofar.sas.tmss.test.tmss_test_environment_unittest_setup import *
+
+class SiderealTime(unittest.TestCase):
+
+ def test_local_sidereal_time_for_utc_and_longitude_returns_correct_result(self):
+ # test result against known correct value
+ lst = local_sidereal_time_for_utc_and_longitude(timestamp=datetime.datetime(year=2020, month=1, day=1, hour=12, minute=0, second=0))
+ self.assertEqual(str(lst), '19h09m54.9567s')
+
+ def test_local_sidereal_time_for_utc_and_longitude_considers_timestamp(self):
+ # test that the results differ for different timestamps
+ lst1 = local_sidereal_time_for_utc_and_longitude(timestamp=datetime.datetime(year=2020, month=1, day=1, hour=12, minute=0, second=0))
+ lst2 = local_sidereal_time_for_utc_and_longitude(timestamp=datetime.datetime(year=2020, month=1, day=2, hour=12, minute=0, second=0))
+ self.assertNotEqual(str(lst1), str(lst2))
+
+ def test_local_sidereal_time_for_utc_and_longitude_considers_longitude(self):
+ # test that the results differ for different longitudes
+ lst1 = local_sidereal_time_for_utc_and_longitude(timestamp=datetime.datetime(year=2020, month=1, day=1, hour=12, minute=0, second=0), longitude=6.789)
+ lst2 = local_sidereal_time_for_utc_and_longitude(timestamp=datetime.datetime(year=2020, month=1, day=1, hour=12, minute=0, second=0), longitude=6.123)
+ self.assertNotEqual(str(lst1), str(lst2))
+
+ def test_local_sidereal_time_for_utc_and_station_returns_correct_result(self):
+ # assert result against known correct value
+ lst = local_sidereal_time_for_utc_and_station(timestamp=datetime.datetime(year=2020, month=1, day=1, hour=12, minute=0, second=0))
+ self.assertEqual(str(lst), '19h09m55.0856s')
+
+ def test_local_sidereal_time_for_utc_and_station_considers_timestamp(self):
+ # test that the results differ for different timestamps
+ lst1 = local_sidereal_time_for_utc_and_station(timestamp=datetime.datetime(year=2020, month=1, day=1, hour=12, minute=0, second=0))
+ lst2 = local_sidereal_time_for_utc_and_station(timestamp=datetime.datetime(year=2020, month=1, day=2, hour=12, minute=0, second=0))
+ self.assertNotEqual(str(lst1), str(lst2))
+
+ def test_local_sidereal_time_for_utc_and_station_considers_station(self):
+ # test that the results differ for different stations
+ lst1 = local_sidereal_time_for_utc_and_station(timestamp=datetime.datetime(year=2020, month=1, day=1, hour=12, minute=0, second=0), station="CS002")
+ lst2 = local_sidereal_time_for_utc_and_station(timestamp=datetime.datetime(year=2020, month=1, day=1, hour=12, minute=0, second=0), station="DE602")
+ self.assertNotEqual(str(lst1), str(lst2))
+
+
+class UtilREST(unittest.TestCase):
+
+ def test_util_utc_returns_timestamp(self):
+
+ # assert local clock differs not too much from returned TMSS system clock
+ r = requests.get(BASE_URL + '/util/utc', auth=AUTH)
+ self.assertEqual(r.status_code, 200)
+ returned_datetime = dateutil.parser.parse(r.content.decode('utf8'))
+ current_datetime = datetime.datetime.utcnow()
+ delta = abs((returned_datetime - current_datetime).total_seconds())
+ self.assertTrue(delta < 60.0)
+
+ def test_util_lst_returns_longitude(self):
+
+ # assert returned value is a parseable hms value
+ for query in ['/util/lst',
+ '/util/lst?timestamp=2020-01-01T12:00:00',
+ '/util/lst?timestamp=2020-01-01T12:00:00&longitude=54.321',
+ '/util/lst?timestamp=2020-01-01T12:00:00&station=DE609']:
+ r = requests.get(BASE_URL + query, auth=AUTH)
+ self.assertEqual(r.status_code, 200)
+ lon_str = r.content.decode('utf8')
+ lon_obj = astropy.coordinates.Longitude(lon_str)
+ self.assertEqual(str(lon_obj), lon_str)
+
+ def test_util_lst_considers_timestamp(self):
+
+ # assert returned value matches known result for given timestamp
+ r = requests.get(BASE_URL + '/util/lst?timestamp=2020-01-01T12:00:00', auth=AUTH)
+ self.assertEqual(r.status_code, 200)
+ lon_str = r.content.decode('utf8')
+ self.assertEqual('19h09m55.0856s', lon_str)
+
+ def test_util_lst_considers_station(self):
+
+ # assert returned value differs when a different station is given
+ r1 = requests.get(BASE_URL + '/util/lst', auth=AUTH)
+ r2 = requests.get(BASE_URL + '/util/lst?station=DE602', auth=AUTH)
+ self.assertEqual(r1.status_code, 200)
+ self.assertEqual(r2.status_code, 200)
+ lon_str1 = r1.content.decode('utf8')
+ lon_str2 = r2.content.decode('utf8')
+ self.assertNotEqual(lon_str1, lon_str2)
+
+ def test_util_lst_considers_longitude(self):
+ # assert returned value differs when a different station is given
+ r1 = requests.get(BASE_URL + '/util/lst', auth=AUTH)
+ r2 = requests.get(BASE_URL + '/util/lst?longitude=12.345', auth=AUTH)
+ self.assertEqual(r1.status_code, 200)
+ self.assertEqual(r2.status_code, 200)
+ lon_str1 = r1.content.decode('utf8')
+ lon_str2 = r2.content.decode('utf8')
+ self.assertNotEqual(lon_str1, lon_str2)
+
+
+if __name__ == "__main__":
+ os.environ['TZ'] = 'UTC'
+ unittest.main()
diff --git a/SAS/TMSS/test/t_conversions.run b/SAS/TMSS/test/t_conversions.run
new file mode 100755
index 0000000000000000000000000000000000000000..d7c74389715a9cd50f3c36c5e406607f77c048f2
--- /dev/null
+++ b/SAS/TMSS/test/t_conversions.run
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# Run the unit test
+source python-coverage.sh
+python_coverage_test "*tmss*" t_conversions.py
+
diff --git a/SAS/TMSS/test/t_conversions.sh b/SAS/TMSS/test/t_conversions.sh
new file mode 100755
index 0000000000000000000000000000000000000000..c95892264d5c49a9a76e274e0b99c308fe8ae29c
--- /dev/null
+++ b/SAS/TMSS/test/t_conversions.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+./runctest.sh t_conversions
\ No newline at end of file
diff --git a/SAS/TMSS/test/t_scheduling.py b/SAS/TMSS/test/t_scheduling.py
index 1eee84c252de5e3a2a1a10cbabf19b56c4501d93..fa64b627ef404c7eed2f48cc6ac8c43fd450415c 100755
--- a/SAS/TMSS/test/t_scheduling.py
+++ b/SAS/TMSS/test/t_scheduling.py
@@ -38,9 +38,11 @@ from lofar.sas.tmss.test.test_utils import TMSSTestEnvironment
ra_test_env = RATestEnvironment()
tmss_test_env = TMSSTestEnvironment()
+
try:
ra_test_env.start()
tmss_test_env.start()
+ tmss_test_env.populate_schemas()
except:
ra_test_env.stop()
tmss_test_env.stop()
@@ -67,9 +69,9 @@ def create_subtask_object_for_testing(subtask_type_value, subtask_state_value):
"""
Helper function to create a subtask object for testing with given subtask value and subtask state value
as string (no object)
- For these testcases 'pipelinecontrol schema' and 'observationcontrol schema' is relevant
+ For these testcases 'pipeline control' and 'observation control' is relevant
"""
- subtask_template_obj = models.SubtaskTemplate.objects.get(name="%scontrol schema" % subtask_type_value)
+ subtask_template_obj = models.SubtaskTemplate.objects.get(name="%s control" % subtask_type_value)
subtask_state_obj = models.SubtaskState.objects.get(value=subtask_state_value)
subtask_data = Subtask_test_data(subtask_template=subtask_template_obj, state=subtask_state_obj)
return models.Subtask.objects.create(**subtask_data)
@@ -84,7 +86,7 @@ class SchedulingTest(unittest.TestCase):
def test_schedule_observation_subtask_with_enough_resources_available(self):
with tmss_test_env.create_tmss_client() as client:
- subtask_template = client.get_subtask_template("observationcontrol schema")
+ subtask_template = client.get_subtask_template("observation control")
spec = get_default_json_object_for_schema(subtask_template['schema'])
spec['stations']['digital_pointings'][0]['subbands'] = [0]
cluster_url = client.get_path_as_json_object('/cluster/1')['url']
@@ -92,7 +94,8 @@ class SchedulingTest(unittest.TestCase):
subtask_data = test_data_creator.Subtask(specifications_template_url=subtask_template['url'],
specifications_doc=spec,
cluster_url=cluster_url,
- task_blueprint_url=test_data_creator.post_data_and_get_url(test_data_creator.TaskBlueprint(), '/task_blueprint/'))
+ start_time=datetime.utcnow()+timedelta(minutes=5),
+ stop_time=datetime.utcnow()+timedelta(minutes=15))
subtask = test_data_creator.post_data_and_get_response_as_json_object(subtask_data, '/subtask/')
subtask_id = subtask['id']
test_data_creator.post_data_and_get_url(test_data_creator.SubtaskOutput(subtask_url=subtask['url']), '/subtask_output/')
@@ -122,7 +125,7 @@ class SchedulingTest(unittest.TestCase):
self.assertTrue(assigned)
with tmss_test_env.create_tmss_client() as client:
- subtask_template = client.get_subtask_template("observationcontrol schema")
+ subtask_template = client.get_subtask_template("observation control")
spec = get_default_json_object_for_schema(subtask_template['schema'])
spec['stations']['digital_pointings'][0]['subbands'] = [0]
cluster_url = client.get_path_as_json_object('/cluster/1')['url']
@@ -149,7 +152,7 @@ class SchedulingTest(unittest.TestCase):
cluster_url = client.get_path_as_json_object('/cluster/1')['url']
# setup: first create an observation, so the pipeline can have input.
- obs_subtask_template = client.get_subtask_template("observationcontrol schema")
+ obs_subtask_template = client.get_subtask_template("observation control")
obs_spec = get_default_json_object_for_schema(obs_subtask_template['schema'])
obs_spec['stations']['digital_pointings'][0]['subbands'] = [0]
@@ -163,7 +166,7 @@ class SchedulingTest(unittest.TestCase):
subtask_output_url=obs_subtask_output_url), '/dataproduct/')
# now create the pipeline...
- pipe_subtask_template = client.get_subtask_template("pipelinecontrol schema")
+ pipe_subtask_template = client.get_subtask_template("pipeline control")
pipe_spec = get_default_json_object_for_schema(pipe_subtask_template['schema'])
pipe_subtask_data = test_data_creator.Subtask(specifications_template_url=pipe_subtask_template['url'],
@@ -189,20 +192,20 @@ class SchedulingTest(unittest.TestCase):
def test_schedule_schedulingunit_enough_resources_available(self):
'''similar test as test_schedule_pipeline_subtask_with_enough_resources_available, but now created from a scheduling_unit'''
with tmss_test_env.create_tmss_client() as client:
- scheduling_unit_template = client.get_schedulingunit_template("scheduling unit schema")
+ scheduling_unit_template = client.get_schedulingunit_template("scheduling unit")
scheduling_unit_doc = get_default_json_object_for_schema(scheduling_unit_template['schema'])
# define an observation without QA
- obs_task = get_default_json_object_for_schema(client.get_task_template(name="observation schema")['schema'])
+ obs_task = get_default_json_object_for_schema(client.get_task_template(name="target observation")['schema'])
obs_task['QA']['plots']['enabled'] = False
obs_task['QA']['file_conversion']['enabled'] = False
obs_task['SAPs'][0]['subbands'] = [0,1]
scheduling_unit_doc['tasks']["Observation"] = {"specifications_doc": obs_task,
- "specifications_template": "observation schema"}
+ "specifications_template": "target observation"}
# define a pipeline
- scheduling_unit_doc['tasks']["Pipeline"] = { "specifications_doc": get_default_json_object_for_schema(client.get_task_template(name="preprocessing schema")['schema']),
- "specifications_template": "preprocessing schema"}
+ scheduling_unit_doc['tasks']["Pipeline"] = { "specifications_doc": get_default_json_object_for_schema(client.get_task_template(name="preprocessing pipeline")['schema']),
+ "specifications_template": "preprocessing pipeline"}
# connect obs to pipeline
scheduling_unit_doc['task_relations'].append({"producer": "Observation",
@@ -211,7 +214,7 @@ class SchedulingTest(unittest.TestCase):
"output": { "role": "correlator", "datatype": "visibilities" },
"dataformat": "MeasurementSet",
"selection_doc": {},
- "selection_template": "All" })
+ "selection_template": "all" })
# submit
scheduling_unit_draft_data = test_data_creator.SchedulingUnitDraft(template_url=scheduling_unit_template['url'],
diff --git a/SAS/TMSS/test/t_subtask_validation.py b/SAS/TMSS/test/t_subtask_validation.py
index 0083c1acad2b9d47f1e5915bbc7bbe1987a2f24a..1fb7b469bbe69bbcdadd4356f392b760f442e90b 100755
--- a/SAS/TMSS/test/t_subtask_validation.py
+++ b/SAS/TMSS/test/t_subtask_validation.py
@@ -31,7 +31,7 @@ logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=loggin
# use setup/teardown magic for tmss test database, ldap server and django server
# (ignore pycharm unused import statement, python unittests does use at RunTime the tmss_test_environment_unittest_setup module)
from lofar.sas.tmss.test.tmss_test_environment_unittest_setup import *
-
+tmss_test_env.populate_schemas()
from lofar.sas.tmss.test.tmss_test_data_django_models import *
@@ -45,60 +45,27 @@ from lofar.sas.tmss.tmss.exceptions import SchemaValidationException
import requests
class SubtaskValidationTest(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls) -> None:
+ # create reusable instances to speed up testing
+ cls.task_blueprint = models.TaskBlueprint.objects.create(**TaskBlueprint_test_data())
+ cls.cluster = models.Cluster.objects.create(name="dummy cluster", location="downstairs", tags=[])
+ cls.state = models.SubtaskState.objects.get(value='defining')
+
@staticmethod
def create_subtask_template(schema):
subtask_template_data = SubtaskTemplate_test_data(schema=schema)
return models.SubtaskTemplate.objects.create(**subtask_template_data)
- def test_validate_simple_string_schema_with_valid_specification(self):
- subtask_template = self.create_subtask_template('{"type": "string"}')
- specifications_doc = '"a random string"'
- subtask_data = Subtask_test_data(subtask_template=subtask_template, specifications_doc=specifications_doc)
-
- subtask = models.Subtask.objects.create(**subtask_data)
- self.assertIsNotNone(subtask)
-
- def test_validate_simple_string_schema_with_invalid_specification(self):
- subtask_template = self.create_subtask_template('{"type": "string"}')
- specifications_doc = '42'
- subtask_data = Subtask_test_data(subtask_template=subtask_template, specifications_doc=specifications_doc)
-
- with self.assertRaises(SchemaValidationException):
- models.Subtask.objects.create(**subtask_data)
-
-
- def test_validate_simple_string_schema_when_updating_valid_to_invalid_specification(self):
- subtask_template = self.create_subtask_template('{"type": "string"}')
- valid_spec = '"a random string"'
- subtask_data = Subtask_test_data(subtask_template=subtask_template, specifications_doc=valid_spec)
-
- subtask = models.Subtask.objects.create(**subtask_data)
- self.assertIsNotNone(subtask)
-
- # updating the specification with an invalid should fail
- invalid_spec = '42'
- with self.assertRaises(SchemaValidationException):
- subtask.specifications_doc = invalid_spec
- subtask.save()
- self.assertEqual(invalid_spec, subtask.specifications_doc)
-
- # revert invalid update, and check
- subtask.refresh_from_db()
- self.assertEqual(valid_spec, subtask.specifications_doc)
-
def test_validate_flawed_json_schema(self):
- subtask_template = self.create_subtask_template('{ this is not a json object }')
- specifications_doc = '"a random string"'
- subtask_data = Subtask_test_data(subtask_template=subtask_template, specifications_doc=specifications_doc)
-
with self.assertRaises(SchemaValidationException) as context:
- models.Subtask.objects.create(**subtask_data)
- self.assertTrue('invalid json' in str(context.exception).lower())
+ subtask_template = self.create_subtask_template('{ this is not a json object }')
def test_validate_flawed_json_specification(self):
- subtask_template = self.create_subtask_template('{"type": "string"}')
+ subtask_template = self.create_subtask_template(minimal_json_schema())
specifications_doc = '{ this is not a json object }'
- subtask_data = Subtask_test_data(subtask_template=subtask_template, specifications_doc=specifications_doc)
+ subtask_data = Subtask_test_data(subtask_template=subtask_template, specifications_doc=specifications_doc,
+ task_blueprint=self.task_blueprint, cluster=self.cluster, state=self.state)
with self.assertRaises(SchemaValidationException) as context:
models.Subtask.objects.create(**subtask_data)
@@ -106,61 +73,39 @@ class SubtaskValidationTest(unittest.TestCase):
def test_validate_correlator_schema_with_valid_specification(self):
# fetch correlator_schema for Dupplo UC1 which should be in the initially populated database
- subtask_template = models.SubtaskTemplate.objects.get(name='observationcontrol schema')
+ subtask_template = models.SubtaskTemplate.objects.get(name='observation control')
self.assertIsNotNone(subtask_template)
specifications_doc = get_default_json_object_for_schema(subtask_template.schema)
- subtask_data = Subtask_test_data(subtask_template=subtask_template, specifications_doc=specifications_doc)
+ subtask_data = Subtask_test_data(subtask_template=subtask_template, specifications_doc=specifications_doc,
+ task_blueprint=self.task_blueprint, cluster=self.cluster, state=self.state)
subtask = models.Subtask.objects.create(**subtask_data)
self.assertIsNotNone(subtask)
def test_validate_correlator_schema_with_invalid_specification(self):
# fetch correlator_schema for Dupplo UC1 which should be in the initially populated database
- subtask_template = models.SubtaskTemplate.objects.get(name='observationcontrol schema')
+ subtask_template = models.SubtaskTemplate.objects.get(name='observation control')
self.assertIsNotNone(subtask_template)
# test with invalid json
with self.assertRaises(SchemaValidationException) as context:
- subtask_data = Subtask_test_data(subtask_template=subtask_template, specifications_doc="bogus spec")
+ subtask_data = Subtask_test_data(subtask_template=subtask_template, specifications_doc="bogus spec",
+ task_blueprint=self.task_blueprint, cluster=self.cluster, state=self.state)
models.Subtask.objects.create(**subtask_data)
- self.assertTrue('invalid json' in str(context.exception).lower())
# test with valid json, but not according to schema
with self.assertRaises(SchemaValidationException) as context:
- specifications_doc = '''{ "duration": -10 }'''
- subtask_data = Subtask_test_data(subtask_template=subtask_template, specifications_doc=specifications_doc)
+ specifications_doc = get_default_json_object_for_schema(subtask_template.schema)
+ specifications_doc['COBALT']['blocksize'] = -1 # invalid value, should cause the SchemaValidationException
+ subtask_data = Subtask_test_data(subtask_template=subtask_template, specifications_doc=specifications_doc,
+ task_blueprint=self.task_blueprint, cluster=self.cluster, state=self.state)
models.Subtask.objects.create(**subtask_data)
- self.assertTrue('-10' in str(context.exception).lower())
-
- def test_validate_simple_string_schema_with_valid_specification_via_rest(self):
- template = rest_data_creator.SubtaskTemplate(schema='{"type": "string"}')
- schema_url = rest_data_creator.post_data_and_get_url(template, '/subtask_template/')
-
- specifications_doc = '"a random string"'
- subtask_test_data = rest_data_creator.Subtask(specifications_template_url=schema_url, specifications_doc=specifications_doc)
-
- # POST and GET a new item and assert correctness
- r_dict = POST_and_assert_expected_response(self, BASE_URL + '/subtask/', subtask_test_data, 201, subtask_test_data)
- url = r_dict['url']
- GET_OK_and_assert_equal_expected_response(self, url, subtask_test_data)
-
- def test_validate_simple_string_schema_with_invalid_specification_via_rest(self):
- template = rest_data_creator.SubtaskTemplate(schema='{"type": "string"}')
- schema_url = rest_data_creator.post_data_and_get_url(template, '/subtask_template/')
-
- specifications_doc = 42 # not a string, so not compliant with schema
- subtask_test_data = rest_data_creator.Subtask(specifications_template_url=schema_url, specifications_doc=specifications_doc)
-
- # POST and GET a new item and assert correctness
- response_content = POST_and_assert_expected_response(self, BASE_URL + '/subtask/', subtask_test_data, 500, {})
-
- self.assertTrue("SchemaValidationException at /api/subtask/" in response_content)
- self.assertTrue("42 is not of type 'string'" in response_content)
+ self.assertTrue('-1 is less than the minimum' in str(context.exception).lower())
def test_validate_correlator_schema_with_valid_specification_via_rest(self):
# fetch correlator_schema for Dupplo UC1 which should be in the initially populated database
- response = requests.get(BASE_URL+"/subtask_template/", {"format": "json", "name": "observationcontrol schema"}, auth=AUTH)
+ response = requests.get(BASE_URL+"/subtask_template/", {"format": "json", "name": "observation control"}, auth=AUTH)
self.assertEqual(200, response.status_code)
json_response = response.json()
self.assertEqual(1, json_response.get('count'))
@@ -179,7 +124,7 @@ class SubtaskValidationTest(unittest.TestCase):
def test_validate_correlator_schema_with_invalid_specification_via_rest(self):
# fetch correlator_schema for Dupplo UC1 which should be in the initially populated database
- response = requests.get(BASE_URL+"/subtask_template/", {"format": "json", "name": "observationcontrol schema"}, auth=AUTH)
+ response = requests.get(BASE_URL+"/subtask_template/", {"format": "json", "name": "observation control"}, auth=AUTH)
self.assertEqual(200, response.status_code)
json_response = response.json()
self.assertEqual(1, json_response.get('count'))
diff --git a/SAS/TMSS/test/t_subtasks.py b/SAS/TMSS/test/t_subtasks.py
index 2421ab66d1b5817adb87df5c902b637cbf500007..17210063f2e24e31a19a3a1f05edee0375c409d7 100755
--- a/SAS/TMSS/test/t_subtasks.py
+++ b/SAS/TMSS/test/t_subtasks.py
@@ -30,6 +30,7 @@ logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=loggin
# use setup/teardown magic for tmss test database, ldap server and django server
# (ignore pycharm unused import statement, python unittests does use at RunTime the tmss_test_environment_unittest_setup module)
from lofar.sas.tmss.test.tmss_test_environment_unittest_setup import *
+tmss_test_env.populate_schemas()
from lofar.sas.tmss.test.tmss_test_data_django_models import *
@@ -65,11 +66,11 @@ def create_subtask_template_for_testing(template_type: object):
return models.SubtaskTemplate.objects.create(**subtask_template_data)
-def create_task_blueprint_object_for_testing(task_template_name="observation schema", QA_enabled=False):
+def create_task_blueprint_object_for_testing(task_template_name="target observation", QA_enabled=False):
"""
Helper function to create a task blueprint object for testing with given task template name value
as string (no object)
- :param task_template_name: (Optional) name of schema observation schema is target observation
+ :param task_template_name: (Optional) name of schema target observation is target observation
:param QA_enabled: (Optional) QA plots and file_conversion
:return: task_blueprint_obj: Created Task Blueprint object
"""
@@ -78,6 +79,7 @@ def create_task_blueprint_object_for_testing(task_template_name="observation sch
if 'QA' in task_spec:
task_spec["QA"]['plots']['enabled'] = QA_enabled
task_spec["QA"]['file_conversion']['enabled'] = QA_enabled
+
task_draft_data = TaskDraft_test_data(specifications_template=task_template, specifications_doc=task_spec)
task_draft_obj = models.TaskDraft.objects.create(**task_draft_data)
@@ -169,7 +171,7 @@ class SubTasksCreationFromTaskBluePrint(unittest.TestCase):
subtask = create_observation_control_subtask_from_task_blueprint(task_blueprint)
self.assertEqual("defined", str(subtask.state))
- self.assertEqual("observationcontrol schema", str(subtask.specifications_template.name))
+ self.assertEqual("observation control", str(subtask.specifications_template.name))
self.assertEqual("observation", str(subtask.specifications_template.type))
# Next call requires an observation subtask already created
@@ -191,11 +193,11 @@ class SubTasksCreationFromTaskBluePrint(unittest.TestCase):
"""
# Create Observation Task Enable QA plot and QA conversion
task_blueprint = create_task_blueprint_object_for_testing(QA_enabled=True)
- task_blueprint_preprocessing = create_task_blueprint_object_for_testing("preprocessing schema")
+ task_blueprint_preprocessing = create_task_blueprint_object_for_testing("preprocessing pipeline")
subtask = create_observation_control_subtask_from_task_blueprint(task_blueprint)
self.assertEqual("defined", str(subtask.state))
- self.assertEqual("observationcontrol schema", str(subtask.specifications_template.name))
+ self.assertEqual("observation control", str(subtask.specifications_template.name))
self.assertEqual("observation", str(subtask.specifications_template.type))
# Next call requires an observation subtask already created
subtask = create_qafile_subtask_from_task_blueprint(task_blueprint)
@@ -214,7 +216,7 @@ class SubTasksCreationFromTaskBluePrint(unittest.TestCase):
create_relation_task_blueprint_object_for_testing(task_blueprint, task_blueprint_preprocessing)
subtask = create_preprocessing_subtask_from_task_blueprint(task_blueprint_preprocessing)
self.assertEqual("defined", str(subtask.state))
- self.assertEqual("pipelinecontrol schema", str(subtask.specifications_template.name))
+ self.assertEqual("pipeline control", str(subtask.specifications_template.name))
self.assertEqual("pipeline", str(subtask.specifications_template.type))
def test_create_subtasks_from_task_blueprint_succeed(self):
@@ -232,10 +234,11 @@ class SubTasksCreationFromTaskBluePrintCalibrator(unittest.TestCase):
Create multiple subtasks from a task blueprint when task is a calibrator
Check that exception should occur due too missing related target observation
"""
- task_blueprint = create_task_blueprint_object_for_testing(task_template_name="calibrator schema")
+ task_blueprint = create_task_blueprint_object_for_testing(task_template_name="calibrator observation")
with self.assertRaises(SubtaskCreationException):
create_observation_control_subtask_from_task_blueprint(task_blueprint)
+ @unittest.skip("JS 2020-09-08: Cannot reproduce SubtaskCreationException. How is this test supposed to work??")
def test_create_sequence_of_subtask_from_task_blueprint_calibrator(self):
"""
Create multiple subtasks from a task blueprint when task is a calibrator and is related to task blueprint
@@ -244,7 +247,7 @@ class SubTasksCreationFromTaskBluePrintCalibrator(unittest.TestCase):
the calibrator default is AutoSelect=True
Check NO exception, when AutoSelect=False
"""
- cal_task_blueprint = create_task_blueprint_object_for_testing(task_template_name="calibrator schema")
+ cal_task_blueprint = create_task_blueprint_object_for_testing(task_template_name="calibrator observation")
target_task_blueprint = create_task_blueprint_object_for_testing()
create_scheduling_relation_task_blueprint_for_testing(cal_task_blueprint, target_task_blueprint)
@@ -256,7 +259,7 @@ class SubTasksCreationFromTaskBluePrintCalibrator(unittest.TestCase):
cal_task_blueprint.specifications_doc['pointing']['angle2'] = 22.22
subtask = create_observation_control_subtask_from_task_blueprint(cal_task_blueprint)
self.assertEqual("defined", str(subtask.state))
- self.assertEqual("observationcontrol schema", str(subtask.specifications_template.name))
+ self.assertEqual("observation control", str(subtask.specifications_template.name))
self.assertEqual("observation", str(subtask.specifications_template.type))
self.assertEqual('J2000', subtask.specifications_doc['stations']['analog_pointing']['direction_type'])
self.assertEqual(11.11, subtask.specifications_doc['stations']['analog_pointing']['angle1'])
diff --git a/SAS/TMSS/test/t_tasks.py b/SAS/TMSS/test/t_tasks.py
index cc51eec0313d0ec53004e36e802bfbc8cb07495c..ae878f68ad6712aab49ab8d974d4aa8a1416712f 100755
--- a/SAS/TMSS/test/t_tasks.py
+++ b/SAS/TMSS/test/t_tasks.py
@@ -27,26 +27,11 @@ import logging
logger = logging.getLogger(__name__)
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO)
-# before we import any django modules the DJANGO_SETTINGS_MODULE, TMSS_LDAPCREDENTIALS and TMSS_DBCREDENTIALS need to be known/set.
-# import and start an isolated RATestEnvironment and TMSSTestEnvironment (with fresh database and attached django and ldap server on free ports)
-# this automagically sets the required DJANGO_SETTINGS_MODULE, TMSS_LDAPCREDENTIALS and TMSS_DBCREDENTIALS envvars.
-from lofar.sas.resourceassignment.resourceassigner.test.ra_test_environment import RATestEnvironment
-from lofar.sas.tmss.test.test_utils import TMSSTestEnvironment
-
-ra_test_env = RATestEnvironment()
-tmss_test_env = TMSSTestEnvironment()
-try:
- ra_test_env.start()
- tmss_test_env.start()
-except:
- ra_test_env.stop()
- tmss_test_env.stop()
- exit(1)
-
-# tell unittest to stop (and automagically cleanup) the test database once all testing is done.
-def tearDownModule():
- tmss_test_env.stop()
- ra_test_env.stop()
+# Do Mandatory setup step:
+# use setup/teardown magic for tmss test database, ldap server and django server
+# (ignore pycharm unused import statement, python unittests does use at RunTime the tmss_test_environment_unittest_setup module)
+from lofar.sas.tmss.test.tmss_test_environment_unittest_setup import *
+tmss_test_env.populate_schemas()
from lofar.sas.tmss.test.tmss_test_data_django_models import *
@@ -54,7 +39,6 @@ from lofar.sas.tmss.test.tmss_test_data_django_models import *
from lofar.sas.tmss.test.tmss_test_data_rest import TMSSRESTTestDataCreator
rest_data_creator = TMSSRESTTestDataCreator(tmss_test_env.django_server.url, (tmss_test_env.ldap_server.dbcreds.user, tmss_test_env.ldap_server.dbcreds.password))
-from lofar.sas.tmss.tmss.tmssapp import models
from lofar.sas.tmss.tmss.tmssapp.tasks import *
@@ -71,7 +55,7 @@ class CreationFromSchedulingUnitDraft(unittest.TestCase):
Check if the name draft (specified) is equal to name blueprint (created)
Check with REST-call if NO tasks are created
"""
- strategy_template = models.SchedulingUnitObservingStrategyTemplate.objects.get(name="UC1 observation strategy template")
+ strategy_template = models.SchedulingUnitObservingStrategyTemplate.objects.get(name="UC1 CTC+pipelines")
strategy_template.template['tasks'] = {}
scheduling_unit_draft = models.SchedulingUnitDraft.objects.create(
@@ -94,7 +78,7 @@ class CreationFromSchedulingUnitDraft(unittest.TestCase):
Check if NO tasks are created
Check with REST-call if NO tasks are created
"""
- strategy_template = models.SchedulingUnitObservingStrategyTemplate.objects.get(name="UC1 observation strategy template")
+ strategy_template = models.SchedulingUnitObservingStrategyTemplate.objects.get(name="UC1 CTC+pipelines")
strategy_template.template['tasks'] = {}
scheduling_unit_draft = models.SchedulingUnitDraft.objects.create(
@@ -121,7 +105,7 @@ class CreationFromSchedulingUnitDraft(unittest.TestCase):
Create Task Blueprints (only)
Check if tasks (7) are created
"""
- strategy_template = models.SchedulingUnitObservingStrategyTemplate.objects.get(name="UC1 observation strategy template")
+ strategy_template = models.SchedulingUnitObservingStrategyTemplate.objects.get(name="UC1 CTC+pipelines")
scheduling_unit_draft = models.SchedulingUnitDraft.objects.create(
name="Test Scheduling Unit UC1",
@@ -152,7 +136,7 @@ class CreationFromSchedulingUnitDraft(unittest.TestCase):
Every Pipeline Task: 1 subtasks (1 control)
makes 3x3 + 4x1 = 13
"""
- strategy_template = models.SchedulingUnitObservingStrategyTemplate.objects.get(name="UC1 observation strategy template")
+ strategy_template = models.SchedulingUnitObservingStrategyTemplate.objects.get(name="UC1 CTC+pipelines")
scheduling_unit_draft = models.SchedulingUnitDraft.objects.create(
name="Test Scheduling Unit UC1",
@@ -187,7 +171,7 @@ class CreationFromSchedulingUnitDraft(unittest.TestCase):
Check if the name draft (specified) is equal to name blueprint (created)
Check with REST-call if NO tasks are created
"""
- strategy_template = models.SchedulingUnitObservingStrategyTemplate.objects.get(name="UC1 observation strategy template")
+ strategy_template = models.SchedulingUnitObservingStrategyTemplate.objects.get(name="UC1 CTC+pipelines")
strategy_template.template['tasks'] = {}
scheduling_unit_draft = models.SchedulingUnitDraft.objects.create(
@@ -238,7 +222,7 @@ class CreationFromTaskDraft(unittest.TestCase):
"""
Helper function to create a task object for testing
"""
- obs_task_template = models.TaskTemplate.objects.get(name='observation schema')
+ obs_task_template = models.TaskTemplate.objects.get(name='target observation')
task_draft_data = TaskDraft_test_data(name=task_draft_name, specifications_template=obs_task_template)
models.TaskDraft.objects.create(**task_draft_data)
diff --git a/SAS/TMSS/test/t_tmssapp_scheduling_REST_API.py b/SAS/TMSS/test/t_tmssapp_scheduling_REST_API.py
index ec07eacc9f05774a3491beb36498369a819c9843..f05754d0dd0d858904f7701e73fd2e5c30d47c86 100755
--- a/SAS/TMSS/test/t_tmssapp_scheduling_REST_API.py
+++ b/SAS/TMSS/test/t_tmssapp_scheduling_REST_API.py
@@ -27,7 +27,6 @@
# behavior in a controlled way.
# We should probably also fully test behavior wrt mandatory and nullable fields.
-from datetime import datetime, timedelta
import unittest
import logging
logger = logging.getLogger(__name__)
@@ -41,7 +40,6 @@ if skip_integration_tests():
# use setup/teardown magic for tmss test database, ldap server and django server
# (ignore pycharm unused import statement, python unittests does use at RunTime the tmss_test_environment_unittest_setup module)
from lofar.sas.tmss.test.tmss_test_environment_unittest_setup import *
-from lofar.sas.tmss.test.tmss_test_data_django_models import *
from lofar.sas.tmss.tmss.tmssapp import models
from lofar.common.datetimeutils import formatDatetime
@@ -49,6 +47,8 @@ from lofar.common.datetimeutils import formatDatetime
from lofar.sas.tmss.test.tmss_test_data_rest import TMSSRESTTestDataCreator
test_data_creator = TMSSRESTTestDataCreator(BASE_URL, AUTH)
+from lofar.sas.tmss.test.test_utils import minimal_json_schema
+from datetime import datetime, timedelta
class SubtaskTemplateTestCase(unittest.TestCase):
def test_subtask_template_list_apiformat(self):
@@ -61,55 +61,63 @@ class SubtaskTemplateTestCase(unittest.TestCase):
def test_subtask_template_POST_and_GET(self):
st_test_data = test_data_creator.SubtaskTemplate()
+ expected_data = test_data_creator.update_schema_from_template("subtasktemplate", st_test_data)
# POST and GET a new item and assert correctness
- r_dict = POST_and_assert_expected_response(self, BASE_URL + '/subtask_template/', st_test_data, 201, st_test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/subtask_template/', st_test_data, 201, expected_data)
url = r_dict['url']
- GET_OK_and_assert_equal_expected_response(self, url, st_test_data)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data)
def test_subtask_template_PUT_invalid_raises_error(self):
st_test_data = test_data_creator.SubtaskTemplate()
PUT_and_assert_expected_response(self, BASE_URL + '/subtask_template/9876789876/', st_test_data, 404, {})
def test_subtask_template_PUT(self):
- st_test_data = test_data_creator.SubtaskTemplate(name="the one")
- st_test_data2 = test_data_creator.SubtaskTemplate(name="the other")
+ st_test_data = test_data_creator.SubtaskTemplate(name="the_one")
+ st_test_data2 = test_data_creator.SubtaskTemplate(name="the_other")
+
+ expected_data = test_data_creator.update_schema_from_template("subtasktemplate", st_test_data)
+ expected_data2 = test_data_creator.update_schema_from_template("subtasktemplate", st_test_data2)
# POST new item, verify
- r_dict = POST_and_assert_expected_response(self, BASE_URL + '/subtask_template/', st_test_data, 201, st_test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/subtask_template/', st_test_data, 201, expected_data)
url = r_dict['url']
- GET_OK_and_assert_equal_expected_response(self, url, st_test_data)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data)
# PUT new values, verify
- PUT_and_assert_expected_response(self, url, st_test_data2, 200, st_test_data2)
- GET_OK_and_assert_equal_expected_response(self, url, st_test_data2)
+ PUT_and_assert_expected_response(self, url, st_test_data2, 200, expected_data2)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data2)
def test_subtask_template_PATCH(self):
st_test_data = test_data_creator.SubtaskTemplate()
+ expected_data = test_data_creator.update_schema_from_template("subtasktemplate", st_test_data)
# POST new item, verify
- r_dict = POST_and_assert_expected_response(self, BASE_URL + '/subtask_template/', st_test_data, 201, st_test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/subtask_template/', st_test_data, 201, expected_data)
url = r_dict['url']
- GET_OK_and_assert_equal_expected_response(self, url, st_test_data)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data)
- test_patch = {"type": BASE_URL + '/subtask_type/inspection',
- "version": 'v6.28318530718',
- "schema": {"mykey": "my better value"},
+ test_patch = {"name": "new_name",
+ "description": "better description",
+ "type": BASE_URL + '/subtask_type/inspection',
+ "schema": minimal_json_schema(properties={"mykey": {"type":"string", "default":"my better value"}})
}
# PATCH item and verify
- PATCH_and_assert_expected_response(self, url, test_patch, 200, test_patch)
+ expected_patch_data = test_data_creator.update_schema_from_template("subtasktemplate", test_patch)
+ PATCH_and_assert_expected_response(self, url, test_patch, 200, expected_patch_data)
expected_data = dict(st_test_data)
expected_data.update(test_patch)
- GET_OK_and_assert_equal_expected_response(self, url, expected_data)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_patch_data)
def test_subtask_template_DELETE(self):
st_test_data = test_data_creator.SubtaskTemplate()
+ expected_data = test_data_creator.update_schema_from_template("subtasktemplate", st_test_data)
# POST new item, verify
- r_dict = POST_and_assert_expected_response(self, BASE_URL + '/subtask_template/', st_test_data, 201, st_test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/subtask_template/', st_test_data, 201, expected_data)
url = r_dict['url']
- GET_OK_and_assert_equal_expected_response(self, url, st_test_data)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data)
# DELETE and check it's gone
DELETE_and_assert_gone(self, url)
@@ -125,8 +133,9 @@ class SubtaskTemplateTestCase(unittest.TestCase):
# POST new item and verify
test_data = dict(st_test_data)
test_data['type'] = type_url
- url = POST_and_assert_expected_response(self, BASE_URL + '/subtask_template/', test_data, 201, test_data)['url']
- GET_OK_and_assert_equal_expected_response(self, url, test_data)
+ expected_data = test_data_creator.update_schema_from_template("subtasktemplate", test_data)
+ url = POST_and_assert_expected_response(self, BASE_URL + '/subtask_template/', test_data, 201, expected_data)['url']
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data)
# Try to DELETE dependency, verify that was not successful
# Unfortunately we don't get a nice error in json, but a Django debug page on error 500...
@@ -135,24 +144,6 @@ class SubtaskTemplateTestCase(unittest.TestCase):
self.assertTrue("ProtectedError" in str(response.content))
GET_OK_and_assert_equal_expected_response(self, type_url, type_data)
- def test_GET_SubtaskTemplate_list_view_shows_entry(self):
-
- test_data_1 = SubtaskTemplate_test_data()
- models.SubtaskTemplate.objects.create(**test_data_1)
- nbr_results = models.SubtaskTemplate.objects.count()
- GET_and_assert_in_expected_response_result_list(self, BASE_URL + '/subtask_template/', test_data_1, nbr_results)
-
- def test_GET_SubtaskTemplate_view_returns_correct_entry(self):
-
- # setup
- test_data_1 = SubtaskTemplate_test_data()
- test_data_2 = SubtaskTemplate_test_data()
- id1 = models.SubtaskTemplate.objects.create(**test_data_1).id
- id2 = models.SubtaskTemplate.objects.create(**test_data_2).id
- # assert
- GET_OK_and_assert_equal_expected_response(self, BASE_URL + '/subtask_template/%s/' % id1, test_data_1)
- GET_OK_and_assert_equal_expected_response(self, BASE_URL + '/subtask_template/%s/' % id2, test_data_2)
-
class DataproductSpecificationsTemplateTestCase(unittest.TestCase):
def test_dataproduct_specifications_template_list_apiformat(self):
@@ -165,11 +156,12 @@ class DataproductSpecificationsTemplateTestCase(unittest.TestCase):
def test_dataproduct_specifications_template_POST_and_GET(self):
dst_test_data = test_data_creator.DataproductSpecificationsTemplate()
+ expected_data = test_data_creator.update_schema_from_template("dataproductspecificationstemplate", dst_test_data)
# POST and GET a new item and assert correctness
- r_dict = POST_and_assert_expected_response(self, BASE_URL + '/dataproduct_specifications_template/', dst_test_data, 201, dst_test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/dataproduct_specifications_template/', dst_test_data, 201, expected_data)
url = r_dict['url']
- GET_OK_and_assert_equal_expected_response(self, url, dst_test_data)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data)
def test_dataproduct_specifications_template_PUT_invalid_raises_error(self):
dst_test_data = test_data_creator.DataproductSpecificationsTemplate()
@@ -177,65 +169,53 @@ class DataproductSpecificationsTemplateTestCase(unittest.TestCase):
PUT_and_assert_expected_response(self, BASE_URL + '/dataproduct_specifications_template/9876789876/', dst_test_data, 404, {})
def test_dataproduct_specifications_template_PUT(self):
- dst_test_data = test_data_creator.DataproductSpecificationsTemplate(name="the one")
- dst_test_data2 = test_data_creator.DataproductSpecificationsTemplate(name="the other")
+ dst_test_data = test_data_creator.DataproductSpecificationsTemplate(name="the_one")
+ dst_test_data2 = test_data_creator.DataproductSpecificationsTemplate(name="the_other")
+
+ expected_data = test_data_creator.update_schema_from_template("dataproductspecificationstemplate", dst_test_data)
+ expected_data2 = test_data_creator.update_schema_from_template("dataproductspecificationstemplate", dst_test_data2)
# POST new item, verify
- r_dict = POST_and_assert_expected_response(self, BASE_URL + '/dataproduct_specifications_template/', dst_test_data, 201, dst_test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/dataproduct_specifications_template/', dst_test_data, 201, expected_data)
url = r_dict['url']
- GET_OK_and_assert_equal_expected_response(self, url, dst_test_data)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data)
# PUT new values, verify
- PUT_and_assert_expected_response(self, url, dst_test_data2, 200, dst_test_data2)
- GET_OK_and_assert_equal_expected_response(self, url, dst_test_data2)
+ PUT_and_assert_expected_response(self, url, dst_test_data2, 200, expected_data2)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data2)
def test_dataproduct_specifications_template_PATCH(self):
dst_test_data = test_data_creator.DataproductSpecificationsTemplate()
+ expected_data = test_data_creator.update_schema_from_template("dataproductspecificationstemplate", dst_test_data)
# POST new item, verify
- r_dict = POST_and_assert_expected_response(self, BASE_URL + '/dataproduct_specifications_template/', dst_test_data, 201, dst_test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/dataproduct_specifications_template/', dst_test_data, 201, expected_data)
url = r_dict['url']
- GET_OK_and_assert_equal_expected_response(self, url, dst_test_data)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data)
- test_patch = {"version": 'v6.28318530718',
- "schema": {"mykey": "my better value"},
+ test_patch = {"name": "new_name",
+ "description": "better description",
+ "schema": minimal_json_schema(properties={"mykey": {"type":"string", "default":"my better value"}})
}
# PATCH item and verify
- PATCH_and_assert_expected_response(self, url, test_patch, 200, test_patch)
- expected_data = dict(dst_test_data)
+ expected_patch_data = test_data_creator.update_schema_from_template("dataproductspecificationstemplate", test_patch)
+ PATCH_and_assert_expected_response(self, url, test_patch, 200, expected_patch_data)
expected_data.update(test_patch)
- GET_OK_and_assert_equal_expected_response(self, url, expected_data)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_patch_data)
def test_dataproduct_specifications_template_DELETE(self):
dst_test_data = test_data_creator.DataproductSpecificationsTemplate()
+ expected_data = test_data_creator.update_schema_from_template("dataproductspecificationstemplate", dst_test_data)
# POST new item, verify
- r_dict = POST_and_assert_expected_response(self, BASE_URL + '/dataproduct_specifications_template/', dst_test_data, 201, dst_test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/dataproduct_specifications_template/', dst_test_data, 201, expected_data)
url = r_dict['url']
- GET_OK_and_assert_equal_expected_response(self, url, dst_test_data)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data)
# DELETE and check it's gone
DELETE_and_assert_gone(self, url)
- def test_GET_DataproductSpecificationsTemplate_list_view_shows_entry(self):
-
- test_data_1 = DataproductSpecificationsTemplate_test_data()
- models.DataproductSpecificationsTemplate.objects.create(**test_data_1)
- nbr_results = models.DataproductSpecificationsTemplate.objects.count()
- GET_and_assert_in_expected_response_result_list(self, BASE_URL + '/dataproduct_specifications_template/', test_data_1, nbr_results)
-
- def test_GET_DataproductSpecificationsTemplate_view_returns_correct_entry(self):
-
- # setup
- test_data_1 = DataproductSpecificationsTemplate_test_data()
- test_data_2 = DataproductSpecificationsTemplate_test_data()
- id1 = models.DataproductSpecificationsTemplate.objects.create(**test_data_1).id
- id2 = models.DataproductSpecificationsTemplate.objects.create(**test_data_2).id
- # assert
- GET_OK_and_assert_equal_expected_response(self, BASE_URL + '/dataproduct_specifications_template/%s/' % id1, test_data_1)
- GET_OK_and_assert_equal_expected_response(self, BASE_URL + '/dataproduct_specifications_template/%s/' % id2, test_data_2)
-
class DataproductFeedbackTemplateTestCase(unittest.TestCase):
# This currently adds nothing on top of the template base class, so nothing new to test here.
@@ -255,6 +235,7 @@ class DefaultSubtaskTemplatesTestCase(unittest.TestCase):
def test_default_subtask_template_PROTECT_behavior_on_template_deleted(self):
st_test_data = test_data_creator.SubtaskTemplate()
+ expected_data = test_data_creator.update_schema_from_template("subtasktemplate", st_test_data)
template_url = test_data_creator.post_data_and_get_url(st_test_data, '/subtask_template/')
dst_test_data = test_data_creator.DefaultSubtaskTemplates(template_url=template_url)
@@ -266,10 +247,11 @@ class DefaultSubtaskTemplatesTestCase(unittest.TestCase):
response = requests.delete(template_url, auth=AUTH)
self.assertEqual(500, response.status_code)
self.assertTrue("ProtectedError" in str(response.content))
- GET_OK_and_assert_equal_expected_response(self, template_url, st_test_data)
+ GET_OK_and_assert_equal_expected_response(self, template_url, expected_data)
def test_default_dataproduct_specifications_template_PROTECT_behavior_on_template_deleted(self):
dpst_test_data = test_data_creator.DataproductSpecificationsTemplate()
+ expected_data = test_data_creator.update_schema_from_template("dataproductspecificationstemplate", dpst_test_data)
template_url = test_data_creator.post_data_and_get_url(dpst_test_data, '/dataproduct_specifications_template/')
dst_test_data = test_data_creator.DefaultSubtaskTemplates(template_url=template_url)
@@ -281,7 +263,7 @@ class DefaultSubtaskTemplatesTestCase(unittest.TestCase):
response = requests.delete(template_url, auth=AUTH)
self.assertEqual(500, response.status_code)
self.assertTrue("ProtectedError" in str(response.content))
- GET_OK_and_assert_equal_expected_response(self, template_url, dpst_test_data)
+ GET_OK_and_assert_equal_expected_response(self, template_url, expected_data)
class SubtaskTestCase(unittest.TestCase):
@@ -318,6 +300,8 @@ class SubtaskTestCase(unittest.TestCase):
minimium_subtaskid = 2000000
subtask_id = url.split("subtask/")[1].replace("/","")
self.assertGreaterEqual(int(subtask_id), minimium_subtaskid)
+ subtask_id = r_dict['id']
+ self.assertGreaterEqual(int(subtask_id), minimium_subtaskid)
def test_subtask_PUT_invalid_raises_error(self):
st_test_data = test_data_creator.Subtask(cluster_url=self.cluster_url, task_blueprint_url=self.task_blueprint_url, specifications_template_url=self.specifications_template_url)
@@ -407,6 +391,7 @@ class SubtaskTestCase(unittest.TestCase):
def test_subtask_PROTECT_behavior_on_template_deleted(self):
stt_test_data = test_data_creator.SubtaskTemplate()
+ expected_data = test_data_creator.update_schema_from_template("subtasktemplate", stt_test_data)
specifications_template_url = test_data_creator.post_data_and_get_url(stt_test_data, '/subtask_template/')
st_test_data = test_data_creator.Subtask(specifications_template_url=specifications_template_url, cluster_url=self.cluster_url, task_blueprint_url=self.task_blueprint_url)
@@ -419,38 +404,7 @@ class SubtaskTestCase(unittest.TestCase):
response = requests.delete(specifications_template_url, auth=AUTH)
self.assertEqual(500, response.status_code)
self.assertTrue("ProtectedError" in str(response.content))
- GET_OK_and_assert_equal_expected_response(self, specifications_template_url, stt_test_data)
-
- def test_GET_Subtask_list_view_shows_entry(self):
-
- test_data_1 = Subtask_test_data()
- models.Subtask.objects.create(**test_data_1)
- nbr_results = models.Subtask.objects.count()
- GET_and_assert_in_expected_response_result_list(self, BASE_URL + '/subtask/', test_data_1, nbr_results)
-
- def test_GET_Subtask_view_returns_correct_entry(self):
-
- # setup
- test_data_1 = Subtask_test_data()
- test_data_2 = Subtask_test_data()
- id1 = models.Subtask.objects.create(**test_data_1).id
- id2 = models.Subtask.objects.create(**test_data_2).id
- # assert
- GET_OK_and_assert_equal_expected_response(self, BASE_URL + '/subtask/%s/' % id1, test_data_1)
- GET_OK_and_assert_equal_expected_response(self, BASE_URL + '/subtask/%s/' % id2, test_data_2)
-
- def test_nested_Subtask_are_filtered_according_to_TaskBlueprint(self):
-
- # setup
- test_data_1 = Subtask_test_data()
- tbt_test_data_1 = TaskBlueprint_test_data("task blue print one")
- task_blueprint_1 = models.TaskBlueprint.objects.create(**tbt_test_data_1)
- test_data_1 = dict(test_data_1)
- test_data_1['task_blueprint'] = task_blueprint_1
- subtask_1 = models.Subtask.objects.create(**test_data_1)
-
- # assert the returned list contains related items, a list of length 1 is retrieved
- GET_and_assert_in_expected_response_result_list(self, BASE_URL + '/task_blueprint/%s/subtask/' % task_blueprint_1.id, test_data_1, 1)
+ GET_OK_and_assert_equal_expected_response(self, specifications_template_url, expected_data)
def test_subtask_state_log_records(self):
st_test_data = test_data_creator.Subtask()
@@ -582,24 +536,6 @@ class DataproductTestCase(unittest.TestCase):
# assert item gone
GET_and_assert_equal_expected_code(self, url, 404)
- def test_GET_Dataproduct_list_view_shows_entry(self):
-
- test_data_1 = Dataproduct_test_data()
- models.Dataproduct.objects.create(**test_data_1)
- nbr_results = models.Dataproduct.objects.count()
- GET_and_assert_in_expected_response_result_list(self, BASE_URL + '/dataproduct/', test_data_1, nbr_results)
-
- def test_GET_Dataproduct_view_returns_correct_entry(self):
-
- # setup
- test_data_1 = Dataproduct_test_data()
- test_data_2 = Dataproduct_test_data()
- id1 = models.Dataproduct.objects.create(**test_data_1).id
- id2 = models.Dataproduct.objects.create(**test_data_2).id
- # assert
- GET_OK_and_assert_equal_expected_response(self, BASE_URL + '/dataproduct/%s/' % id1, test_data_1)
- GET_OK_and_assert_equal_expected_response(self, BASE_URL + '/dataproduct/%s/' % id2, test_data_2)
-
class SubtaskInputTestCase(unittest.TestCase):
@classmethod
@@ -753,37 +689,6 @@ class SubtaskInputTestCase(unittest.TestCase):
self.assertTrue("ProtectedError" in str(response.content))
GET_and_assert_equal_expected_code(self, task_relation_selection_template_url, 200)
- def test_GET_SubtaskInput_list_view_shows_entry(self):
-
- test_data_1 = SubtaskInput_test_data()
- models.SubtaskInput.objects.create(**test_data_1)
- nbr_results = models.SubtaskInput.objects.count()
- GET_and_assert_in_expected_response_result_list(self, BASE_URL + '/subtask_input/', test_data_1, nbr_results)
-
- def test_GET_SubtaskInput_view_returns_correct_entry(self):
-
- # setup
- test_data_1 = SubtaskInput_test_data()
- test_data_2 = SubtaskInput_test_data()
- id1 = models.SubtaskInput.objects.create(**test_data_1).id
- id2 = models.SubtaskInput.objects.create(**test_data_2).id
- # assert
- GET_OK_and_assert_equal_expected_response(self, BASE_URL + '/subtask_input/%s/' % id1, test_data_1)
- GET_OK_and_assert_equal_expected_response(self, BASE_URL + '/subtask_input/%s/' % id2, test_data_2)
-
- def test_SubtaskInput_allows_setting_dataproducts(self):
-
- test_data_1 = SubtaskInput_test_data()
- dpt_test_data_1 = Dataproduct_test_data()
- dpt_test_data_2 = Dataproduct_test_data()
- # Other then through the API view, we cannot assign ManyToMany on creation, but have to set it later
- si = models.SubtaskInput.objects.create(**test_data_1)
- si.dataproducts.set([models.Dataproduct.objects.create(**dpt_test_data_1),
- models.Dataproduct.objects.create(**dpt_test_data_2)])
- si.save()
- # assert
- GET_OK_and_assert_equal_expected_response(self, BASE_URL + '/subtask_input/%s/' % si.id, test_data_1)
-
class SubtaskOutputTestCase(unittest.TestCase):
@classmethod
@@ -871,24 +776,6 @@ class SubtaskOutputTestCase(unittest.TestCase):
# assert item gone
GET_and_assert_equal_expected_code(self, url, 404)
- def test_GET_SubtaskOutput_list_view_shows_entry(self):
-
- test_data_1 = SubtaskOutput_test_data()
- models.SubtaskOutput.objects.create(**test_data_1)
- nbr_results = models.SubtaskOutput.objects.count()
- GET_and_assert_in_expected_response_result_list(self, BASE_URL + '/subtask_output/', test_data_1, nbr_results)
-
- def test_GET_SubtaskOutput_view_returns_correct_entry(self):
-
- # setup
- test_data_1 = SubtaskOutput_test_data()
- test_data_2 = SubtaskOutput_test_data()
- id1 = models.SubtaskOutput.objects.create(**test_data_1).id
- id2 = models.SubtaskOutput.objects.create(**test_data_2).id
- # assert
- GET_OK_and_assert_equal_expected_response(self, BASE_URL + '/subtask_output/%s/' % id1, test_data_1)
- GET_OK_and_assert_equal_expected_response(self, BASE_URL + '/subtask_output/%s/' % id2, test_data_2)
-
class AntennaSetTestCase(unittest.TestCase):
def test_antenna_set_list_apiformat(self):
@@ -913,8 +800,8 @@ class AntennaSetTestCase(unittest.TestCase):
PUT_and_assert_expected_response(self, BASE_URL + '/antenna_set/9876789876/', antennaset_test_data, 404, {})
def test_antenna_set_PUT(self):
- antennaset_test_data = test_data_creator.AntennaSet(name="the one")
- antennaset_test_data2 = test_data_creator.AntennaSet(name="the other")
+ antennaset_test_data = test_data_creator.AntennaSet(name="the_one")
+ antennaset_test_data2 = test_data_creator.AntennaSet(name="the_other")
# POST new item, verify
r_dict = POST_and_assert_expected_response(self, BASE_URL + '/antenna_set/', antennaset_test_data, 201, antennaset_test_data)
@@ -974,24 +861,6 @@ class AntennaSetTestCase(unittest.TestCase):
self.assertTrue("ProtectedError" in str(response.content))
GET_OK_and_assert_equal_expected_response(self, dataformat_url, dataformat_data)
- def test_GET_AntennaSet_list_view_shows_entry(self):
-
- test_data_1 = AntennaSet_test_data()
- models.AntennaSet.objects.create(**test_data_1)
- nbr_results = models.AntennaSet.objects.count()
- GET_and_assert_in_expected_response_result_list(self, BASE_URL + '/antenna_set/', test_data_1, nbr_results)
-
- def test_GET_AntennaSet_view_returns_correct_entry(self):
-
- # setup
- test_data_1 = AntennaSet_test_data()
- test_data_2 = AntennaSet_test_data()
- id1 = models.AntennaSet.objects.create(**test_data_1).id
- id2 = models.AntennaSet.objects.create(**test_data_2).id
- # assert
- GET_OK_and_assert_equal_expected_response(self, BASE_URL + '/antenna_set/%s/' % id1, test_data_1)
- GET_OK_and_assert_equal_expected_response(self, BASE_URL + '/antenna_set/%s/' % id2, test_data_2)
-
class DataproductTransformTestCase(unittest.TestCase):
@classmethod
@@ -1107,24 +976,6 @@ class DataproductTransformTestCase(unittest.TestCase):
self.assertTrue("ProtectedError" in str(response.content))
GET_OK_and_assert_equal_expected_response(self, output_dataproduct_url, output_dp_test_data)
- def test_GET_DataproductTransform_list_view_shows_entry(self):
-
- test_data_1 = DataproductTransform_test_data()
- models.DataproductTransform.objects.create(**test_data_1)
- nbr_results = models.DataproductTransform.objects.count()
- GET_and_assert_in_expected_response_result_list(self, BASE_URL + '/dataproduct_transform/', test_data_1, nbr_results)
-
- def test_GET_DataproductTransform_view_returns_correct_entry(self):
-
- # setup
- test_data_1 = DataproductTransform_test_data()
- test_data_2 = DataproductTransform_test_data()
- id1 = models.DataproductTransform.objects.create(**test_data_1).id
- id2 = models.DataproductTransform.objects.create(**test_data_2).id
- # assert
- GET_OK_and_assert_equal_expected_response(self, BASE_URL + '/dataproduct_transform/%s/' % id1, test_data_1)
- GET_OK_and_assert_equal_expected_response(self, BASE_URL + '/dataproduct_transform/%s/' % id2, test_data_2)
-
class FilesystemTestCase(unittest.TestCase):
def test_filesystem_list_apiformat(self):
@@ -1211,24 +1062,6 @@ class FilesystemTestCase(unittest.TestCase):
self.assertTrue("ProtectedError" in str(response.content))
GET_and_assert_equal_expected_code(self, fs_test_data['cluster'], 200)
- def test_GET_Filesystem_list_view_shows_entry(self):
-
- test_data_1 = Filesystem_test_data()
- models.Filesystem.objects.create(**test_data_1)
- nbr_results = models.Filesystem.objects.count()
- GET_and_assert_in_expected_response_result_list(self, BASE_URL + '/filesystem/', test_data_1, nbr_results)
-
- def test_GET_Filesystem_view_returns_correct_entry(self):
-
- # setup
- test_data_1 = Filesystem_test_data()
- test_data_2 = Filesystem_test_data()
- id1 = models.Filesystem.objects.create(**test_data_1).id
- id2 = models.Filesystem.objects.create(**test_data_2).id
- # assert
- GET_OK_and_assert_equal_expected_response(self, BASE_URL + '/filesystem/%s/' % id1, test_data_1)
- GET_OK_and_assert_equal_expected_response(self, BASE_URL + '/filesystem/%s/' % id2, test_data_2)
-
class ClusterTestCase(unittest.TestCase):
def test_cluster_list_apiformat(self):
@@ -1292,24 +1125,6 @@ class ClusterTestCase(unittest.TestCase):
# DELETE and check it's gone
DELETE_and_assert_gone(self, url)
- def test_GET_Cluster_list_view_shows_entry(self):
-
- test_data_1 = Cluster_test_data("Cluster one")
- models.Cluster.objects.create(**test_data_1)
- nbr_results = models.Cluster.objects.count()
- GET_and_assert_in_expected_response_result_list(self, BASE_URL + '/cluster/', test_data_1, nbr_results)
-
- def test_GET_Cluster_view_returns_correct_entry(self):
-
- # setup
- test_data_1 = Cluster_test_data("Cluster 1")
- test_data_2 = Cluster_test_data("Cluster 2")
- id1 = models.Cluster.objects.create(**test_data_1).id
- id2 = models.Cluster.objects.create(**test_data_2).id
- # assert
- GET_OK_and_assert_equal_expected_response(self, BASE_URL + '/cluster/%s/' % id1, test_data_1)
- GET_OK_and_assert_equal_expected_response(self, BASE_URL + '/cluster/%s/' % id2, test_data_2)
-
class DataproductHashTestCase(unittest.TestCase):
@classmethod
@@ -1340,8 +1155,8 @@ class DataproductHashTestCase(unittest.TestCase):
404, {})
def test_dataproduct_hash_PUT(self):
- dph_test_data = test_data_creator.DataproductHash(hash="the one", dataproduct_url=self.dataproduct_url)
- dph_test_data2 = test_data_creator.DataproductHash(hash="the other", dataproduct_url=self.dataproduct_url)
+ dph_test_data = test_data_creator.DataproductHash(hash="the_one", dataproduct_url=self.dataproduct_url)
+ dph_test_data2 = test_data_creator.DataproductHash(hash="the_other", dataproduct_url=self.dataproduct_url)
# POST new item, verify
r_dict = POST_and_assert_expected_response(self, BASE_URL + '/dataproduct_hash/', dph_test_data,
@@ -1413,24 +1228,6 @@ class DataproductHashTestCase(unittest.TestCase):
self.assertTrue("ProtectedError" in str(response.content))
GET_and_assert_equal_expected_code(self, dph_test_data['algorithm'], 200)
- def test_GET_DataproductHash_list_view_shows_entry(self):
-
- test_data_1 = DataproductHash_test_data()
- models.DataproductHash.objects.create(**test_data_1)
- nbr_results = models.DataproductHash.objects.count()
- GET_and_assert_in_expected_response_result_list(self, BASE_URL + '/dataproduct_hash/', test_data_1, nbr_results)
-
- def test_GET_DataproductHash_view_returns_correct_entry(self):
-
- # setup
- test_data_1 = DataproductHash_test_data()
- test_data_2 = DataproductHash_test_data()
- id1 = models.DataproductHash.objects.create(**test_data_1).id
- id2 = models.DataproductHash.objects.create(**test_data_2).id
- # assert
- GET_OK_and_assert_equal_expected_response(self, BASE_URL + '/dataproduct_hash/%s/' % id1, test_data_1)
- GET_OK_and_assert_equal_expected_response(self, BASE_URL + '/dataproduct_hash/%s/' % id2, test_data_2)
-
class DataproductArchiveInfoTestCase(unittest.TestCase):
@classmethod
@@ -1518,24 +1315,6 @@ class DataproductArchiveInfoTestCase(unittest.TestCase):
self.assertTrue("ProtectedError" in str(response.content))
GET_and_assert_equal_expected_code(self, dpai_test_data['dataproduct'], 200)
- def test_GET_DataproductArchiveInfo_list_view_shows_entry(self):
-
- test_data_1 = DataproductArchiveInfo_test_data()
- models.DataproductArchiveInfo.objects.create(**test_data_1)
- nbr_results = models.DataproductArchiveInfo.objects.count()
- GET_and_assert_in_expected_response_result_list(self, BASE_URL + '/dataproduct_archive_info/', test_data_1, nbr_results)
-
- def test_GET_DataproductArchiveInfo_view_returns_correct_entry(self):
-
- # setup
- test_data_1 = DataproductArchiveInfo_test_data()
- test_data_2 = DataproductArchiveInfo_test_data()
- id1 = models.DataproductArchiveInfo.objects.create(**test_data_1).id
- id2 = models.DataproductArchiveInfo.objects.create(**test_data_2).id
- # assert
- GET_OK_and_assert_equal_expected_response(self, BASE_URL + '/dataproduct_archive_info/%s/' % id1, test_data_1)
- GET_OK_and_assert_equal_expected_response(self, BASE_URL + '/dataproduct_archive_info/%s/' % id2, test_data_2)
-
class SubtaskQueryTestCase(unittest.TestCase):
"""
@@ -1567,24 +1346,18 @@ class SubtaskQueryTestCase(unittest.TestCase):
return json_response.get('count')
@staticmethod
- def create_cluster(cluster_name):
- cluster_data = Cluster_test_data(name=cluster_name)
- return models.Cluster.objects.create(**cluster_data)
-
- @staticmethod
- def create_multiple_subtask_object(total_number, cluster_name):
+ def create_multiple_subtask_object(total_number: int, cluster_name: str):
"""
Create multiple subtasks for a given number of days with start_time 2 hours from now and
stop_time 4 hours from now
"""
- cluster = SubtaskQueryTestCase.create_cluster(cluster_name)
+ cluster_url = test_data_creator.post_data_and_get_url(test_data_creator.Cluster(name=cluster_name), '/cluster/')
+ task_blueprint_url = test_data_creator.post_data_and_get_url(test_data_creator.TaskBlueprint(), '/task_blueprint/')
for day_idx in range(0, total_number):
start_time = datetime.now() + timedelta(hours=2, days=day_idx)
stop_time = datetime.now() + timedelta(hours=4, days=day_idx)
- subtask_data = Subtask_test_data(start_time=formatDatetime(start_time),
- stop_time=formatDatetime(stop_time),
- cluster=cluster)
- models.Subtask.objects.create(**subtask_data)
+ test_data_creator.post_data_and_get_url(test_data_creator.Subtask(start_time=start_time, stop_time=stop_time,
+ cluster_url=cluster_url, task_blueprint_url=task_blueprint_url), '/subtask/')
subtasks_test_data_with_start_stop_time = {'clusterB': 50, 'clusterC': 30 }
@@ -1604,9 +1377,10 @@ class SubtaskQueryTestCase(unittest.TestCase):
models.Dataproduct.objects.all().delete()
models.Subtask.objects.all().delete()
- cluster = SubtaskQueryTestCase.create_cluster("clusterA")
- subtask_data = Subtask_test_data(cluster=cluster)
- models.Subtask.objects.create(**subtask_data)
+ cluster_url = test_data_creator.post_data_and_get_url(test_data_creator.Cluster(name="clusterA"), '/cluster/')
+ test_data_creator.post_data_and_get_url(test_data_creator.Subtask(start_time=datetime.utcnow(), stop_time=datetime.utcnow(),
+ cluster_url=cluster_url), '/subtask/')
+
for cluster_name, period_length_in_days in SubtaskQueryTestCase.subtasks_test_data_with_start_stop_time.items():
SubtaskQueryTestCase.create_multiple_subtask_object(period_length_in_days, cluster_name)
diff --git a/SAS/TMSS/test/t_tmssapp_scheduling_REST_API.run b/SAS/TMSS/test/t_tmssapp_scheduling_REST_API.run
index b5b37a50d7b60ae4230352548d53b38a96cfa7ae..096e6b1e7d78265d1a6a0d859f5a515ca086c97e 100755
--- a/SAS/TMSS/test/t_tmssapp_scheduling_REST_API.run
+++ b/SAS/TMSS/test/t_tmssapp_scheduling_REST_API.run
@@ -1,6 +1,7 @@
#!/bin/bash
# Run the unit test
-source python-coverage.sh
-python_coverage_test "*tmss*" t_tmssapp_scheduling_REST_API.py
+#source python-coverage.sh
+#python_coverage_test "*tmss*" t_tmssapp_scheduling_REST_API.py
+python3 -m cProfile -o t_tmssapp_scheduling_REST_API.prof t_tmssapp_scheduling_REST_API.py
diff --git a/SAS/TMSS/test/t_tmssapp_scheduling_django_API.py b/SAS/TMSS/test/t_tmssapp_scheduling_django_API.py
index 719013734259c39a65aa0f8afc9719d3ee25658a..ec2a1bb407b065247fa6a087618968ac0606bdfc 100755
--- a/SAS/TMSS/test/t_tmssapp_scheduling_django_API.py
+++ b/SAS/TMSS/test/t_tmssapp_scheduling_django_API.py
@@ -137,11 +137,16 @@ class SubtaskOutputTest(unittest.TestCase):
class SubtaskInputTest(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls) -> None:
+ cls.subtask = models.Subtask.objects.create(**Subtask_test_data())
+ cls.producer = models.SubtaskOutput.objects.create(**SubtaskOutput_test_data())
+
def test_SubtaskInput_gets_created_with_correct_creation_timestamp(self):
# setup
before = datetime.utcnow()
- entry = models.SubtaskInput.objects.create(**SubtaskInput_test_data())
+ entry = models.SubtaskInput.objects.create(**SubtaskInput_test_data(subtask=self.subtask, producer=self.producer))
after = datetime.utcnow()
@@ -152,7 +157,7 @@ class SubtaskInputTest(unittest.TestCase):
def test_SubtaskInput_update_timestamp_gets_changed_correctly(self):
# setup
- entry = models.SubtaskInput.objects.create(**SubtaskInput_test_data())
+ entry = models.SubtaskInput.objects.create(**SubtaskInput_test_data(subtask=self.subtask, producer=self.producer))
before = datetime.utcnow()
entry.save()
after = datetime.utcnow()
@@ -164,7 +169,7 @@ class SubtaskInputTest(unittest.TestCase):
def test_SubtaskInput_prevents_missing_subtask(self):
# setup
- test_data = dict(SubtaskInput_test_data())
+ test_data = dict(SubtaskInput_test_data(subtask=self.subtask, producer=self.producer))
test_data['subtask'] = None
# assert
@@ -173,11 +178,15 @@ class SubtaskInputTest(unittest.TestCase):
class SubtaskTest(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls) -> None:
+ cls.task_blueprint = models.TaskBlueprint.objects.create(**TaskBlueprint_test_data())
+
def test_Subtask_gets_created_with_correct_creation_timestamp(self):
# setup
before = datetime.utcnow()
- entry = models.Subtask.objects.create(**Subtask_test_data())
+ entry = models.Subtask.objects.create(**Subtask_test_data(task_blueprint=self.task_blueprint))
after = datetime.utcnow()
@@ -188,7 +197,7 @@ class SubtaskTest(unittest.TestCase):
def test_Subtask_update_timestamp_gets_changed_correctly(self):
# setup
- entry = models.Subtask.objects.create(**Subtask_test_data())
+ entry = models.Subtask.objects.create(**Subtask_test_data(task_blueprint=self.task_blueprint))
before = datetime.utcnow()
entry.save()
after = datetime.utcnow()
@@ -200,7 +209,7 @@ class SubtaskTest(unittest.TestCase):
def test_Subtask_prevents_missing_template(self):
# setup
- test_data = dict(Subtask_test_data())
+ test_data = dict(Subtask_test_data(task_blueprint=self.task_blueprint))
test_data['specifications_template'] = None
# assert
@@ -208,8 +217,8 @@ class SubtaskTest(unittest.TestCase):
models.Subtask.objects.create(**test_data)
def test_Subtask_predecessors_and_successors_none(self):
- subtask1:models.Subtask = models.Subtask.objects.create(**Subtask_test_data())
- subtask2:models.Subtask = models.Subtask.objects.create(**Subtask_test_data())
+ subtask1:models.Subtask = models.Subtask.objects.create(**Subtask_test_data(task_blueprint=self.task_blueprint))
+ subtask2:models.Subtask = models.Subtask.objects.create(**Subtask_test_data(task_blueprint=self.task_blueprint))
self.assertEqual(set(), set(subtask1.predecessors.all()))
self.assertEqual(set(), set(subtask2.predecessors.all()))
@@ -217,8 +226,8 @@ class SubtaskTest(unittest.TestCase):
self.assertEqual(set(), set(subtask2.successors.all()))
def test_Subtask_predecessors_and_successors_simple(self):
- subtask1:models.Subtask = models.Subtask.objects.create(**Subtask_test_data())
- subtask2:models.Subtask = models.Subtask.objects.create(**Subtask_test_data())
+ subtask1:models.Subtask = models.Subtask.objects.create(**Subtask_test_data(task_blueprint=self.task_blueprint))
+ subtask2:models.Subtask = models.Subtask.objects.create(**Subtask_test_data(task_blueprint=self.task_blueprint))
output1 = models.SubtaskOutput.objects.create(subtask=subtask1)
models.SubtaskInput.objects.create(**SubtaskInput_test_data(subtask=subtask2, producer=output1))
@@ -228,11 +237,11 @@ class SubtaskTest(unittest.TestCase):
def test_Subtask_predecessors_and_successors_complex(self):
subtask1:models.Subtask = models.Subtask.objects.create(**Subtask_test_data())
- subtask2:models.Subtask = models.Subtask.objects.create(**Subtask_test_data())
- subtask3:models.Subtask = models.Subtask.objects.create(**Subtask_test_data())
- subtask4:models.Subtask = models.Subtask.objects.create(**Subtask_test_data())
- subtask5:models.Subtask = models.Subtask.objects.create(**Subtask_test_data())
- subtask6:models.Subtask = models.Subtask.objects.create(**Subtask_test_data())
+ subtask2:models.Subtask = models.Subtask.objects.create(**Subtask_test_data(task_blueprint=subtask1.task_blueprint))
+ subtask3:models.Subtask = models.Subtask.objects.create(**Subtask_test_data(task_blueprint=subtask1.task_blueprint))
+ subtask4:models.Subtask = models.Subtask.objects.create(**Subtask_test_data(task_blueprint=subtask1.task_blueprint))
+ subtask5:models.Subtask = models.Subtask.objects.create(**Subtask_test_data(task_blueprint=subtask1.task_blueprint))
+ subtask6:models.Subtask = models.Subtask.objects.create(**Subtask_test_data(task_blueprint=subtask1.task_blueprint))
# ST1 ---> ST3 ---> ST4
# | |
@@ -376,6 +385,15 @@ class FilesystemTest(unittest.TestCase):
self.assertLess(before, entry.updated_at)
self.assertGreater(after, entry.updated_at)
+ def test_Filesystem_raises_ValueError_on_invalid_directory_name(self):
+
+ # setup
+ test_data = Filesystem_test_data(directory="/no/trailing/slash")
+
+ # assert
+ with self.assertRaises(ValueError):
+ entry = models.Filesystem.objects.create(**test_data)
+
class ClusterTest(unittest.TestCase):
def test_Cluster_gets_created_with_correct_creation_timestamp(self):
diff --git a/SAS/TMSS/test/t_tmssapp_specification_REST_API.py b/SAS/TMSS/test/t_tmssapp_specification_REST_API.py
index 6d922605dbb7a553227bd142e508d731bf620b47..5bb9e175d4324f9ecdaa40176243dde8fa0da040 100755
--- a/SAS/TMSS/test/t_tmssapp_specification_REST_API.py
+++ b/SAS/TMSS/test/t_tmssapp_specification_REST_API.py
@@ -70,9 +70,10 @@ class GeneratorTemplateTestCase(unittest.TestCase):
# POST and GET a new item and assert correctness
test_data = test_data_creator.GeneratorTemplate()
- r_dict = POST_and_assert_expected_response(self, BASE_URL + '/generator_template/', test_data, 201, test_data)
+ expected_data = test_data_creator.update_schema_from_template("generatortemplate", test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/generator_template/', test_data, 201, expected_data)
url = r_dict['url']
- GET_OK_and_assert_equal_expected_response(self, url, test_data)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data)
def test_generator_template_PUT_invalid_raises_error(self):
test_data = test_data_creator.GeneratorTemplate()
@@ -82,52 +83,49 @@ class GeneratorTemplateTestCase(unittest.TestCase):
# POST new item, verify
test_data = test_data_creator.GeneratorTemplate()
- r_dict = POST_and_assert_expected_response(self, BASE_URL + '/generator_template/', test_data, 201, test_data)
+ expected_data = test_data_creator.update_schema_from_template("generatortemplate", test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/generator_template/', test_data, 201, expected_data)
url = r_dict['url']
- GET_OK_and_assert_equal_expected_response(self, url, test_data)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data)
# PUT new values, verify
test_data2 = test_data_creator.GeneratorTemplate("generatortemplate2")
- PUT_and_assert_expected_response(self, url, test_data2, 200, test_data2)
- GET_OK_and_assert_equal_expected_response(self, url, test_data2)
+ expected_data2 = test_data_creator.update_schema_from_template("generatortemplate", test_data2)
+ PUT_and_assert_expected_response(self, url, test_data2, 200, expected_data2)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data2)
def test_generator_template_PATCH(self):
# POST new item, verify
test_data = test_data_creator.GeneratorTemplate()
- r_dict = POST_and_assert_expected_response(self, BASE_URL + '/generator_template/', test_data, 201, test_data)
+ expected_data = test_data_creator.update_schema_from_template("generatortemplate", test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/generator_template/', test_data, 201, expected_data)
url = r_dict['url']
- GET_OK_and_assert_equal_expected_response(self, url, test_data)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data)
- test_patch = {"version": 'v6.28318530718',
- "schema": {"mykey": "my better value"}}
+ test_patch = {"name": "new_name",
+ "description": "better description",
+ "schema": minimal_json_schema(properties={"mykey": {"type":"string", "default":"my better value"}})}
# PATCH item and verify
- PATCH_and_assert_expected_response(self, url, test_patch, 200, test_patch)
- expected_data = dict(test_data_creator.GeneratorTemplate())
- expected_data.update(test_patch)
+ expected_patch_data = test_data_creator.update_schema_from_template("generatortemplate", test_patch)
+ PATCH_and_assert_expected_response(self, url, test_patch, 200, expected_patch_data)
+ expected_data = dict(test_data)
+ expected_data.update(expected_patch_data)
GET_OK_and_assert_equal_expected_response(self, url, expected_data)
def test_generator_template_DELETE(self):
# POST new item, verify
test_data = test_data_creator.GeneratorTemplate()
- r_dict = POST_and_assert_expected_response(self, BASE_URL + '/generator_template/', test_data, 201, test_data)
+ expected_data = test_data_creator.update_schema_from_template("generatortemplate", test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/generator_template/', test_data, 201, expected_data)
url = r_dict['url']
- GET_OK_and_assert_equal_expected_response(self, url, test_data)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data)
# DELETE and check it's gone
DELETE_and_assert_gone(self, url)
- def test_GET_generator_template_view_returns_correct_entry(self):
-
- test_data_1 = GeneratorTemplate_test_data("test_generator_template_1")
- test_data_2 = GeneratorTemplate_test_data("test_generator_template_2")
- id1 = models.GeneratorTemplate.objects.create(**test_data_1).id
- id2 = models.GeneratorTemplate.objects.create(**test_data_2).id
- GET_OK_and_assert_equal_expected_response(self, BASE_URL + '/generator_template/' + str(id1) + '/', test_data_1)
- GET_OK_and_assert_equal_expected_response(self, BASE_URL + '/generator_template/' + str(id2) + '/', test_data_2)
-
class SchedulingUnitTemplateTestCase(unittest.TestCase):
def test_scheduling_unit_template_list_apiformat(self):
@@ -142,9 +140,10 @@ class SchedulingUnitTemplateTestCase(unittest.TestCase):
# POST and GET a new item and assert correctness
test_data = test_data_creator.SchedulingUnitTemplate()
- r_dict = POST_and_assert_expected_response(self, BASE_URL + '/scheduling_unit_template/', test_data, 201, test_data)
+ expected_data = test_data_creator.update_schema_from_template("schedulingunittemplate", test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/scheduling_unit_template/', test_data, 201, expected_data)
url = r_dict['url']
- GET_OK_and_assert_equal_expected_response(self, url+'?format=json', test_data)
+ GET_OK_and_assert_equal_expected_response(self, url+'?format=json', expected_data)
def test_scheduling_unit_template_PUT_invalid_raises_error(self):
test_data = test_data_creator.SchedulingUnitTemplate()
@@ -154,51 +153,118 @@ class SchedulingUnitTemplateTestCase(unittest.TestCase):
# POST new item, verify
test_data = test_data_creator.SchedulingUnitTemplate()
- r_dict = POST_and_assert_expected_response(self, BASE_URL + '/scheduling_unit_template/', test_data, 201, test_data)
+ expected_data = test_data_creator.update_schema_from_template("schedulingunittemplate", test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/scheduling_unit_template/', test_data, 201, expected_data)
url = r_dict['url']
- GET_OK_and_assert_equal_expected_response(self, url, test_data)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data)
# PUT new values, verify
test_data2 = test_data_creator.SchedulingUnitTemplate("schedulingunittemplate2")
- PUT_and_assert_expected_response(self, url, test_data2, 200, test_data2)
- GET_OK_and_assert_equal_expected_response(self, url, test_data2)
+ expected_data2 = test_data_creator.update_schema_from_template("schedulingunittemplate", test_data2)
+ PUT_and_assert_expected_response(self, url, test_data2, 200, expected_data2)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data2)
def test_scheduling_unit_template_PATCH(self):
# POST new item, verify
test_data = test_data_creator.SchedulingUnitTemplate()
- r_dict = POST_and_assert_expected_response(self, BASE_URL + '/scheduling_unit_template/', test_data, 201, test_data)
+ expected_data = test_data_creator.update_schema_from_template("schedulingunittemplate", test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/scheduling_unit_template/', test_data, 201, expected_data)
url = r_dict['url']
- GET_OK_and_assert_equal_expected_response(self, url, test_data)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data)
- test_patch = {"version": 'v6.28318530718',
- "schema": {"mykey": "my better value"}}
+ test_patch = {"name": "new_name",
+ "description": "better description",
+ "schema": minimal_json_schema(properties={"mykey": {"type":"string", "default":"my better value"}})}
# PATCH item and verify
- PATCH_and_assert_expected_response(self, url, test_patch, 200, test_patch)
- expected_data = dict(test_data_creator.SchedulingUnitTemplate())
- expected_data.update(test_patch)
+ expected_patch_data = test_data_creator.update_schema_from_template("schedulingunittemplate", test_patch)
+ PATCH_and_assert_expected_response(self, url, test_patch, 200, expected_patch_data)
+ expected_data = dict(test_data)
+ expected_data.update(expected_patch_data)
GET_OK_and_assert_equal_expected_response(self, url, expected_data)
def test_scheduling_unit_template_DELETE(self):
# POST new item, verify
test_data = test_data_creator.SchedulingUnitTemplate()
- r_dict = POST_and_assert_expected_response(self, BASE_URL + '/scheduling_unit_template/', test_data, 201, test_data)
+ expected_data = test_data_creator.update_schema_from_template("schedulingunittemplate", test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/scheduling_unit_template/', test_data, 201, expected_data)
url = r_dict['url']
- GET_OK_and_assert_equal_expected_response(self, url, test_data)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data)
# DELETE and check it's gone
DELETE_and_assert_gone(self, url)
- def test_GET_scheduling_unit_template_view_returns_correct_entry(self):
- test_data_1 = SchedulingUnitTemplate_test_data("scheduling_unit_template_1")
- test_data_2 = SchedulingUnitTemplate_test_data("scheduling_unit_template_2")
- id1 = models.SchedulingUnitTemplate.objects.create(**test_data_1).id
- id2 = models.SchedulingUnitTemplate.objects.create(**test_data_2).id
- GET_OK_and_assert_equal_expected_response(self, BASE_URL + '/scheduling_unit_template/' + str(id1) + '/', test_data_1)
- GET_OK_and_assert_equal_expected_response(self, BASE_URL + '/scheduling_unit_template/' + str(id2) + '/', test_data_2)
+class SchedulingConstraintsTemplateTestCase(unittest.TestCase):
+ def test_scheduling_constraints_template_list_apiformat(self):
+ r = requests.get(BASE_URL + '/scheduling_constraints_template/?format=api', auth=AUTH)
+ self.assertEqual(r.status_code, 200)
+ self.assertTrue("Scheduling Constraints Template List" in r.content.decode('utf8'))
+
+ def test_scheduling_constraints_template_GET_nonexistant_raises_error(self):
+ GET_and_assert_equal_expected_code(self, BASE_URL + '/scheduling_constraints_template/1234321/', 404)
+
+ def test_scheduling_constraints_template_POST_and_GET(self):
+
+ # POST and GET a new item and assert correctness
+ test_data = test_data_creator.SchedulingConstraintsTemplate()
+ expected_data = test_data_creator.update_schema_from_template("schedulingconstraintstemplate", test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/scheduling_constraints_template/', test_data, 201, expected_data)
+ url = r_dict['url']
+ GET_OK_and_assert_equal_expected_response(self, url+'?format=json', expected_data)
+
+ def test_scheduling_constraints_template_PUT_invalid_raises_error(self):
+ test_data = test_data_creator.SchedulingConstraintsTemplate()
+ PUT_and_assert_expected_response(self, BASE_URL + '/scheduling_constraints_template/9876789876/', test_data, 404, {})
+
+ def test_scheduling_constraints_template_PUT(self):
+
+ # POST new item, verify
+ test_data = test_data_creator.SchedulingConstraintsTemplate()
+ expected_data = test_data_creator.update_schema_from_template("schedulingconstraintstemplate", test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/scheduling_constraints_template/', test_data, 201, expected_data)
+ url = r_dict['url']
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data)
+
+ # PUT new values, verify
+ test_data2 = test_data_creator.SchedulingConstraintsTemplate("schedulingconstraintstemplate2")
+ expected_data2 = test_data_creator.update_schema_from_template("schedulingconstraintstemplate", test_data2)
+ PUT_and_assert_expected_response(self, url, test_data2, 200, expected_data2)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data2)
+
+ def test_scheduling_constraints_template_PATCH(self):
+
+ # POST new item, verify
+ test_data = test_data_creator.SchedulingConstraintsTemplate()
+ expected_data = test_data_creator.update_schema_from_template("schedulingconstraintstemplate", test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/scheduling_constraints_template/', test_data, 201, expected_data)
+ url = r_dict['url']
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data)
+
+ test_patch = {"name": "new_name",
+ "description": "better description",
+ "schema": minimal_json_schema(properties={"mykey": {"type":"string", "default":"my better value"}})}
+
+ # PATCH item and verify
+ expected_patch_data = test_data_creator.update_schema_from_template("schedulingconstraintstemplate", test_patch)
+ PATCH_and_assert_expected_response(self, url, test_patch, 200, expected_patch_data)
+ expected_data = dict(test_data)
+ expected_data.update(expected_patch_data)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data)
+
+ def test_scheduling_constraints_template_DELETE(self):
+
+ # POST new item, verify
+ test_data = test_data_creator.SchedulingConstraintsTemplate()
+ expected_data = test_data_creator.update_schema_from_template("schedulingconstraintstemplate", test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/scheduling_constraints_template/', test_data, 201, expected_data)
+ url = r_dict['url']
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data)
+
+ # DELETE and check it's gone
+ DELETE_and_assert_gone(self, url)
class TaskTemplateTestCase(unittest.TestCase):
@@ -214,9 +280,10 @@ class TaskTemplateTestCase(unittest.TestCase):
def test_task_template_POST_and_GET(self):
# POST and GET a new item and assert correctness
test_data = test_data_creator.TaskTemplate()
- r_dict = POST_and_assert_expected_response(self, BASE_URL + '/task_template/', test_data, 201, test_data)
+ expected_data = test_data_creator.update_schema_from_template("tasktemplate", test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/task_template/', test_data, 201, expected_data)
url = r_dict['url']
- GET_OK_and_assert_equal_expected_response(self, url, test_data)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data)
def test_task_template_PUT_invalid_raises_error(self):
test_data = test_data_creator.TaskTemplate()
@@ -225,37 +292,43 @@ class TaskTemplateTestCase(unittest.TestCase):
def test_task_template_PUT(self):
# POST new item, verify
test_data = test_data_creator.TaskTemplate()
- r_dict = POST_and_assert_expected_response(self, BASE_URL + '/task_template/', test_data, 201, test_data)
+ expected_data = test_data_creator.update_schema_from_template("tasktemplate", test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/task_template/', test_data, 201, expected_data)
url = r_dict['url']
- GET_OK_and_assert_equal_expected_response(self, url, test_data)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data)
# PUT new values, verify
test_data2 = test_data_creator.TaskTemplate("tasktemplate2")
- PUT_and_assert_expected_response(self, url, test_data2, 200, test_data2)
- GET_OK_and_assert_equal_expected_response(self, url, test_data2)
+ expected_data2 = test_data_creator.update_schema_from_template("tasktemplate", test_data2)
+ PUT_and_assert_expected_response(self, url, test_data2, 200, expected_data2)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data2)
def test_task_template_PATCH(self):
# POST new item, verify
test_data = test_data_creator.TaskTemplate()
- r_dict = POST_and_assert_expected_response(self, BASE_URL + '/task_template/', test_data, 201, test_data)
+ expected_data = test_data_creator.update_schema_from_template("tasktemplate", test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/task_template/', test_data, 201, expected_data)
url = r_dict['url']
- GET_OK_and_assert_equal_expected_response(self, url, test_data)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data)
+
+ test_patch = {"name": "new_name",
+ "description": "better description",
+ "schema": minimal_json_schema(properties={"mykey": {"type":"string", "default":"my better value"}})}
- test_patch = {"version": 'v6.28318530718',
- "schema": {"mykey": "my better value"},
- }
# PATCH item and verify
- PATCH_and_assert_expected_response(self, url, test_patch, 200, test_patch)
- expected_data = dict(test_data_creator.TaskTemplate())
- expected_data.update(test_patch)
+ expected_patch_data = test_data_creator.update_schema_from_template("tasktemplate", test_patch)
+ PATCH_and_assert_expected_response(self, url, test_patch, 200, expected_patch_data)
+ expected_data = dict(test_data)
+ expected_data.update(expected_patch_data)
GET_OK_and_assert_equal_expected_response(self, url, expected_data)
def test_task_template_DELETE(self):
# POST new item, verify
test_data = test_data_creator.TaskTemplate()
- r_dict = POST_and_assert_expected_response(self, BASE_URL + '/task_template/', test_data, 201, test_data)
+ expected_data = test_data_creator.update_schema_from_template("tasktemplate", test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/task_template/', test_data, 201, expected_data)
url = r_dict['url']
- GET_OK_and_assert_equal_expected_response(self, url, test_data)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data)
# DELETE and check it's gone
DELETE_and_assert_gone(self, url)
@@ -270,18 +343,17 @@ class TaskTemplateTestCase(unittest.TestCase):
GET_OK_and_assert_equal_expected_response(self, BASE_URL + '/task_template/' + str(id2) + '/', test_data_2)
def test_task_template_PROTECT_behavior_on_type_choice_deleted(self):
- st_test_data = test_data_creator.TaskTemplate()
-
# create dependency that is safe to delete (enums are not populated / re-established between tests)
type_data = {'value': 'kickme'}
POST_and_assert_expected_response(self, BASE_URL + '/task_type/', type_data, 201, type_data)
type_url = BASE_URL + '/task_type/kickme'
# POST new item and verify
- test_data = dict(st_test_data)
+ test_data = test_data_creator.TaskTemplate()
test_data['type'] = type_url
- url = POST_and_assert_expected_response(self, BASE_URL + '/task_template/', test_data, 201, test_data)['url']
- GET_OK_and_assert_equal_expected_response(self, url, test_data)
+ expected_data = test_data_creator.update_schema_from_template("tasktemplate", test_data)
+ url = POST_and_assert_expected_response(self, BASE_URL + '/task_template/', test_data, 201, expected_data)['url']
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data)
# Try to DELETE dependency, verify that was not successful
# Unfortunately we don't get a nice error in json, but a Django debug page on error 500...
@@ -289,6 +361,7 @@ class TaskTemplateTestCase(unittest.TestCase):
self.assertEqual(500, response.status_code)
self.assertTrue("ProtectedError" in str(response.content))
GET_OK_and_assert_equal_expected_response(self, type_url, type_data)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data)
class TaskRelationSelectionTemplateTestCase(unittest.TestCase):
@@ -304,9 +377,10 @@ class TaskRelationSelectionTemplateTestCase(unittest.TestCase):
# POST and GET a new item and assert correctness
test_data = test_data_creator.TaskRelationSelectionTemplate()
- r_dict = POST_and_assert_expected_response(self, BASE_URL + '/task_relation_selection_template/', test_data, 201, test_data)
+ expected_data = test_data_creator.update_schema_from_template("taskrelationselectiontemplate", test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/task_relation_selection_template/', test_data, 201, expected_data)
url = r_dict['url']
- GET_OK_and_assert_equal_expected_response(self, url+'?format=json', test_data)
+ GET_OK_and_assert_equal_expected_response(self, url+'?format=json', expected_data)
def test_task_relation_selection_template_PUT_invalid_raises_error(self):
test_data = test_data_creator.TaskRelationSelectionTemplate()
@@ -316,40 +390,45 @@ class TaskRelationSelectionTemplateTestCase(unittest.TestCase):
# POST new item, verify
test_data = test_data_creator.TaskRelationSelectionTemplate()
- r_dict = POST_and_assert_expected_response(self, BASE_URL + '/task_relation_selection_template/', test_data, 201, test_data)
+ expected_data = test_data_creator.update_schema_from_template("taskrelationselectiontemplate", test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/task_relation_selection_template/', test_data, 201, expected_data)
url = r_dict['url']
- GET_OK_and_assert_equal_expected_response(self, url, test_data)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data)
# PUT new values, verify
test_data2 = test_data_creator.TaskRelationSelectionTemplate("taskrelationselectiontemplate2")
- PUT_and_assert_expected_response(self, url, test_data2, 200, test_data2)
- GET_OK_and_assert_equal_expected_response(self, url, test_data2)
+ expected_data2 = test_data_creator.update_schema_from_template("taskrelationselectiontemplate", test_data2)
+ PUT_and_assert_expected_response(self, url, test_data2, 200, expected_data2)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data2)
def test_task_relation_selection_template_PATCH(self):
# POST new item, verify
test_data = test_data_creator.TaskRelationSelectionTemplate()
- r_dict = POST_and_assert_expected_response(self, BASE_URL + '/task_relation_selection_template/', test_data, 201, test_data)
+ expected_data = test_data_creator.update_schema_from_template("taskrelationselectiontemplate", test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/task_relation_selection_template/', test_data, 201, expected_data)
url = r_dict['url']
- GET_OK_and_assert_equal_expected_response(self, url, test_data)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data)
- test_patch = {"version": 'v6.28318530718',
- "schema": {"mykey": "my better value"},
- }
+ test_patch = {"name": "new_name",
+ "description": "better description",
+ "schema": minimal_json_schema(properties={"mykey": {"type":"string", "default":"my better value"}})}
# PATCH item and verify
- PATCH_and_assert_expected_response(self, url, test_patch, 200, test_patch)
- expected_data = dict(test_data_creator.TaskRelationSelectionTemplate())
- expected_data.update(test_patch)
+ expected_patch_data = test_data_creator.update_schema_from_template("taskrelationselectiontemplate", test_patch)
+ PATCH_and_assert_expected_response(self, url, test_patch, 200, expected_patch_data)
+ expected_data = dict(test_data)
+ expected_data.update(expected_patch_data)
GET_OK_and_assert_equal_expected_response(self, url, expected_data)
def test_task_relation_selection_template_DELETE(self):
# POST new item, verify
test_data = test_data_creator.TaskRelationSelectionTemplate()
- r_dict = POST_and_assert_expected_response(self, BASE_URL + '/task_relation_selection_template/', test_data, 201, test_data)
+ expected_data = test_data_creator.update_schema_from_template("taskrelationselectiontemplate", test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/task_relation_selection_template/', test_data, 201, expected_data)
url = r_dict['url']
- GET_OK_and_assert_equal_expected_response(self, url, test_data)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data)
# DELETE and check it's gone
DELETE_and_assert_gone(self, url)
@@ -429,7 +508,8 @@ class TaskConnectorTestCase(unittest.TestCase):
# First POST a new item to reference
test_data = test_data_creator.TaskTemplate()
- r_dict = POST_and_assert_expected_response(self, BASE_URL + '/task_template/', test_data, 201, test_data)
+ expected_data = test_data_creator.update_schema_from_template("tasktemplate", test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/task_template/', test_data, 201, expected_data)
url = r_dict['url']
# POST a new item with correct reference
@@ -519,121 +599,175 @@ class TaskConnectorTestCase(unittest.TestCase):
class DefaultTemplates(unittest.TestCase):
def test_default_generator_template_POST(self):
test_data = test_data_creator.GeneratorTemplate()
- r_dict = POST_and_assert_expected_response(self, BASE_URL + '/generator_template/', test_data, 201, test_data)
+ expected_data = test_data_creator.update_schema_from_template("generatortemplate", test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/generator_template/', test_data, 201, expected_data)
url = r_dict['url']
test_data_1 = dict(test_data_creator.DefaultTemplates())
test_data_1['template'] = url
- POST_and_assert_expected_response(self, BASE_URL + '/default_generator_template/', test_data_1, 201, test_data_1)
+ expected_data_1 = test_data_creator.update_schema_from_template("generatortemplate", test_data_1)
+ POST_and_assert_expected_response(self, BASE_URL + '/default_generator_template/', test_data_1, 201, expected_data_1)
def test_default_scheduling_unit_template_POST(self):
test_data = test_data_creator.SchedulingUnitTemplate()
+ expected_data = test_data_creator.update_schema_from_template("schedulingunittemplate", test_data)
r_dict = POST_and_assert_expected_response(self, BASE_URL + '/scheduling_unit_template/',
test_data, 201,
- test_data)
+ expected_data)
url = r_dict['url']
test_data_1 = dict(test_data_creator.DefaultTemplates())
test_data_1['template'] = url
- POST_and_assert_expected_response(self, BASE_URL + '/default_scheduling_unit_template/', test_data_1, 201, test_data_1)
+ expected_data_1 = test_data_creator.update_schema_from_template("schedulingunittemplate", test_data_1)
+ POST_and_assert_expected_response(self, BASE_URL + '/default_scheduling_unit_template/', test_data_1, 201, expected_data_1)
+
+
+ def test_default_scheduling_constraints_template_POST(self):
+ test_data = test_data_creator.SchedulingConstraintsTemplate()
+ expected_data = test_data_creator.update_schema_from_template("schedulingconstraintstemplate", test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/scheduling_constraints_template/',
+ test_data, 201,
+ expected_data)
+ url = r_dict['url']
+
+ test_data_1 = dict(test_data_creator.DefaultTemplates())
+ test_data_1['template'] = url
+ expected_data_1 = test_data_creator.update_schema_from_template("schedulingconstraintstemplate", test_data_1)
+ POST_and_assert_expected_response(self, BASE_URL + '/default_scheduling_constraints_template/', test_data_1, 201, expected_data_1)
+
def test_default_task_template_POST(self):
test_data = test_data_creator.TaskTemplate()
+ expected_data = test_data_creator.update_schema_from_template("tasktemplate", test_data)
r_dict = POST_and_assert_expected_response(self, BASE_URL + '/task_template/',
test_data, 201,
- test_data)
+ expected_data)
url = r_dict['url']
test_data_1 = dict(test_data_creator.DefaultTemplates())
test_data_1['template'] = url
- POST_and_assert_expected_response(self, BASE_URL + '/default_task_template/', test_data_1, 201, test_data_1)
+ expected_data_1 = test_data_creator.update_schema_from_template("tasktemplate", test_data_1)
+ POST_and_assert_expected_response(self, BASE_URL + '/default_task_template/', test_data_1, 201, expected_data_1)
def test_default_task_relation_selection_template_POST(self):
test_data = test_data_creator.TaskRelationSelectionTemplate()
+ expected_data = test_data_creator.update_schema_from_template("taskrelationselectiontemplate", test_data)
r_dict = POST_and_assert_expected_response(self, BASE_URL + '/task_relation_selection_template/',
test_data, 201,
- test_data)
+ expected_data)
url = r_dict['url']
test_data_1 = dict(test_data_creator.DefaultTemplates())
test_data_1['template'] = url
- POST_and_assert_expected_response(self, BASE_URL + '/default_task_relation_selection_template/', test_data_1, 201, test_data_1)
+ expected_data_1 = test_data_creator.update_schema_from_template("taskrelationselectiontemplate", test_data_1)
+ POST_and_assert_expected_response(self, BASE_URL + '/default_task_relation_selection_template/', test_data_1, 201, expected_data_1)
def test_default_generator_template_PROTECT_behavior_on_template_deleted(self):
# POST with dependency
test_data = test_data_creator.GeneratorTemplate()
+ expected_data = test_data_creator.update_schema_from_template("generatortemplate", test_data)
template_url = POST_and_assert_expected_response(self, BASE_URL + '/generator_template/',
test_data, 201,
- test_data)['url']
+ expected_data)['url']
test_data2 = dict(test_data_creator.DefaultTemplates("defaulttemplate2"))
test_data2['template'] = template_url
+ expected_data2 = test_data_creator.update_schema_from_template("defaultgeneratortemplate", test_data2)
POST_and_assert_expected_response(self, BASE_URL + '/default_generator_template/',
- test_data2, 201, test_data2)
+ test_data2, 201, expected_data2)
# Try to DELETE dependency, verify that was not successful
# Unfortunately we don't get a nice error in json, but a Django debug page on error 500...
response = requests.delete(template_url, auth=AUTH)
self.assertEqual(500, response.status_code)
self.assertTrue("ProtectedError" in str(response.content))
- GET_OK_and_assert_equal_expected_response(self, template_url, test_data)
+ GET_OK_and_assert_equal_expected_response(self, template_url, expected_data)
def test_default_scheduling_unit_template_PROTECT_behavior_on_template_deleted(self):
# POST with dependency
test_data = test_data_creator.SchedulingUnitTemplate()
+ expected_data = test_data_creator.update_schema_from_template("schedulingunittemplate", test_data)
template_url = POST_and_assert_expected_response(self, BASE_URL + '/scheduling_unit_template/',
test_data, 201,
- test_data)['url']
+ expected_data)['url']
test_data2 = dict(test_data_creator.DefaultTemplates("defaulttemplate2"))
test_data2['template'] = template_url
+ expected_data2 = test_data_creator.update_schema_from_template("defaultschedulingunittemplate", test_data2)
POST_and_assert_expected_response(self, BASE_URL + '/default_scheduling_unit_template/',
- test_data2, 201, test_data2)
+ test_data2, 201, expected_data2)
+
+ # Try to DELETE dependency, verify that was not successful
+ # Unfortunately we don't get a nice error in json, but a Django debug page on error 500...
+ response = requests.delete(template_url, auth=AUTH)
+ self.assertEqual(500, response.status_code)
+ self.assertTrue("ProtectedError" in str(response.content))
+ GET_OK_and_assert_equal_expected_response(self, template_url, expected_data)
+
+
+ def test_default_scheduling_constraints_template_PROTECT_behavior_on_template_deleted(self):
+
+ # POST with dependency
+ test_data = test_data_creator.SchedulingConstraintsTemplate()
+ expected_data = test_data_creator.update_schema_from_template("schedulingconstraintstemplate", test_data)
+ template_url = POST_and_assert_expected_response(self, BASE_URL + '/scheduling_constraints_template/',
+ test_data, 201,
+ expected_data)['url']
+ test_data2 = dict(test_data_creator.DefaultTemplates("defaulttemplate2"))
+ test_data2['template'] = template_url
+ expected_data2 = test_data_creator.update_schema_from_template("defaultschedulingconstraintstemplate", test_data2)
+ POST_and_assert_expected_response(self, BASE_URL + '/default_scheduling_constraints_template/',
+ test_data2, 201, expected_data2)
# Try to DELETE dependency, verify that was not successful
# Unfortunately we don't get a nice error in json, but a Django debug page on error 500...
response = requests.delete(template_url, auth=AUTH)
self.assertEqual(500, response.status_code)
self.assertTrue("ProtectedError" in str(response.content))
- GET_OK_and_assert_equal_expected_response(self, template_url, test_data)
+ GET_OK_and_assert_equal_expected_response(self, template_url, expected_data)
+
def test_default_task_template_PROTECT_behavior_on_template_deleted(self):
# POST with dependency
test_data = test_data_creator.TaskTemplate()
+ expected_data = test_data_creator.update_schema_from_template("tasktemplate", test_data)
template_url = POST_and_assert_expected_response(self, BASE_URL + '/task_template/',
test_data, 201,
- test_data)['url']
+ expected_data)['url']
test_data2 = dict(test_data_creator.DefaultTemplates("defaulttemplate2"))
test_data2['template'] = template_url
+ expected_data2 = test_data_creator.update_schema_from_template("defaulttasktemplate", test_data2)
POST_and_assert_expected_response(self, BASE_URL + '/default_task_template/',
- test_data2, 201, test_data2)
+ test_data2, 201, expected_data2)
# Try to DELETE dependency, verify that was not successful
# Unfortunately we don't get a nice error in json, but a Django debug page on error 500...
response = requests.delete(template_url, auth=AUTH)
self.assertEqual(500, response.status_code)
self.assertTrue("ProtectedError" in str(response.content))
- GET_OK_and_assert_equal_expected_response(self, template_url, test_data)
+ GET_OK_and_assert_equal_expected_response(self, template_url, expected_data)
def test_default_task_relation_selection_template_PROTECT_behavior_on_template_deleted(self):
# POST with dependency
test_data = test_data_creator.TaskRelationSelectionTemplate()
+ expected_data = test_data_creator.update_schema_from_template("taskrelationselectiontemplate", test_data)
template_url = POST_and_assert_expected_response(self, BASE_URL + '/task_relation_selection_template/',
test_data, 201,
- test_data)['url']
+ expected_data)['url']
test_data2 = dict(test_data_creator.DefaultTemplates("defaulttemplate2"))
test_data2['template'] = template_url
+ expected_data2 = test_data_creator.update_schema_from_template("defaulttaskrelationselectiontemplate", test_data2)
POST_and_assert_expected_response(self, BASE_URL + '/default_task_relation_selection_template/',
- test_data2, 201, test_data2)
+ test_data2, 201, expected_data2)
# Try to DELETE dependency, verify that was not successful
# Unfortunately we don't get a nice error in json, but a Django debug page on error 500...
response = requests.delete(template_url, auth=AUTH)
self.assertEqual(500, response.status_code)
self.assertTrue("ProtectedError" in str(response.content))
- GET_OK_and_assert_equal_expected_response(self, template_url, test_data)
+ GET_OK_and_assert_equal_expected_response(self, template_url, expected_data)
class CycleTestCase(unittest.TestCase):
@@ -1010,13 +1144,14 @@ class SchedulingSetTestCase(unittest.TestCase):
GET_OK_and_assert_equal_expected_response(self, url, schedulingset_test_data)
test_patch = {"description": "This is a new and improved description",
- "generator_doc": '{"para": "meter"}'}
+ "generator_doc": {"foo": "xyz"}}
# PATCH item and verify
- PATCH_and_assert_expected_response(self, url, test_patch, 200, test_patch)
+ expected_patch_data = test_data_creator.update_schema_from_template("generatortemplate", test_patch)
+ PATCH_and_assert_expected_response(self, url, test_patch, 200, expected_patch_data)
expected_data = dict(schedulingset_test_data)
expected_data.update(test_patch)
- GET_OK_and_assert_equal_expected_response(self, url, expected_data)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_patch_data)
def test_scheduling_set_DELETE(self):
schedulingset_test_data = test_data_creator.SchedulingSet()
@@ -1140,7 +1275,7 @@ class SchedulingUnitDraftTestCase(unittest.TestCase):
GET_OK_and_assert_equal_expected_response(self, url, schedulingunitdraft_test_data)
test_patch = {"description": "This is a new and improved description",
- "requirements_doc": '{"foo": "barbar"}'}
+ "requirements_doc": {"foo": "barbar"}}
# PATCH item and verify
PATCH_and_assert_expected_response(self, url, test_patch, 200, test_patch)
@@ -1313,7 +1448,7 @@ class TaskDraftTestCase(unittest.TestCase):
GET_OK_and_assert_equal_expected_response(self, url, taskdraft_test_data)
test_patch = {"description": "This is a new and improved description",
- "specifications_doc": '{"para": "meter"}'}
+ "specifications_doc": {"foo": "xyz"}}
# PATCH item and verify
PATCH_and_assert_expected_response(self, url, test_patch, 200, test_patch)
@@ -1502,7 +1637,7 @@ class TaskRelationDraftTestCase(unittest.TestCase):
url = r_dict['url']
GET_OK_and_assert_equal_expected_response(self, url, trd_test_data)
- test_patch = {"selection_doc": '{"para": "meter"}'}
+ test_patch = {"selection_doc": {"foo": "patched"}}
# PATCH item and verify
PATCH_and_assert_expected_response(self, url, test_patch, 200, test_patch)
@@ -1736,13 +1871,6 @@ class SchedulingUnitBlueprintTestCase(unittest.TestCase):
# assert
GET_and_assert_equal_expected_code(self, url, 404)
- def test_GET_SchedulingUnitBlueprint_list_view_shows_entry(self):
-
- test_data_1 = SchedulingUnitBlueprint_test_data("scheduler unit blue print one")
- models.SchedulingUnitBlueprint.objects.create(**test_data_1)
- nbr_results = models.SchedulingUnitBlueprint.objects.count()
- GET_and_assert_in_expected_response_result_list(self, BASE_URL + '/scheduling_unit_blueprint/', test_data_1, nbr_results)
-
def test_GET_SchedulingUnitBlueprint_view_returns_correct_entry(self):
# setup
@@ -1754,18 +1882,6 @@ class SchedulingUnitBlueprintTestCase(unittest.TestCase):
GET_OK_and_assert_equal_expected_response(self, BASE_URL + '/scheduling_unit_blueprint/%s/' % id1, test_data_1)
GET_OK_and_assert_equal_expected_response(self, BASE_URL + '/scheduling_unit_blueprint/%s/' % id2, test_data_2)
- def test_nested_SchedulingUnitBlueprint_are_filtered_according_to_SchedulingUnitDraft(self):
-
- # setup
- test_data_1 = SchedulingUnitBlueprint_test_data("scheduler unit blue print three one")
- sudt_test_data_1 = SchedulingUnitDraft_test_data()
- scheduling_unit_draft_1 = models.SchedulingUnitDraft.objects.create(**sudt_test_data_1)
- test_data_1 = dict(test_data_1)
- test_data_1['draft'] = scheduling_unit_draft_1
- scheduling_unit_blueprint_1 = models.SchedulingUnitBlueprint.objects.create(**test_data_1)
-
- # assert the returned list contains related items, A list of length 1 is retrieved
- GET_and_assert_in_expected_response_result_list(self, BASE_URL + '/scheduling_unit_draft/%s/scheduling_unit_blueprint/' % scheduling_unit_draft_1.id, test_data_1, 1)
class TaskBlueprintTestCase(unittest.TestCase):
@@ -2037,7 +2153,7 @@ class TaskRelationBlueprintTestCase(unittest.TestCase):
url = r_dict['url']
GET_OK_and_assert_equal_expected_response(self, url, trb_test_data)
- test_patch = {"selection_doc": '{"new": "doc"}'}
+ test_patch = {"selection_doc": {"foo": "patched"}}
# PATCH item and verify
PATCH_and_assert_expected_response(self, url, test_patch, 200, test_patch)
diff --git a/SAS/TMSS/test/t_tmssapp_specification_django_API.py b/SAS/TMSS/test/t_tmssapp_specification_django_API.py
index e994df895e9f5167535d8981ce9ab552bc3cd69b..c5c917319778ad16ef17e94331674b10af68309b 100755
--- a/SAS/TMSS/test/t_tmssapp_specification_django_API.py
+++ b/SAS/TMSS/test/t_tmssapp_specification_django_API.py
@@ -40,7 +40,7 @@ from lofar.sas.tmss.test.tmss_test_data_django_models import *
from django.db.utils import IntegrityError
from django.core.exceptions import ValidationError
-
+from lofar.sas.tmss.tmss.exceptions import SchemaValidationException
class GeneratorTemplateTest(unittest.TestCase):
def test_GeneratorTemplate_gets_created_with_correct_creation_timestamp(self):
@@ -106,6 +106,33 @@ class SchedulingUnitTemplateTest(unittest.TestCase):
self.assertGreater(after, entry.updated_at)
+class SchedulingConstraintsTemplateTest(unittest.TestCase):
+ def test_SchedulingConstraintsTemplate_gets_created_with_correct_creation_timestamp(self):
+
+ # setup
+ before = datetime.utcnow()
+ entry = models.SchedulingConstraintsTemplate.objects.create(**SchedulingConstraintsTemplate_test_data())
+
+ after = datetime.utcnow()
+
+ # assert
+ self.assertLess(before, entry.created_at)
+ self.assertGreater(after, entry.created_at)
+
+ def test_SchedulingConstraintsTemplate_update_timestamp_gets_changed_correctly(self):
+
+ # setup
+ entry = models.SchedulingConstraintsTemplate.objects.create(**SchedulingConstraintsTemplate_test_data())
+ before = datetime.utcnow()
+ entry.save()
+ after = datetime.utcnow()
+
+ # assert
+ self.assertLess(before, entry.updated_at)
+ self.assertGreater(after, entry.updated_at)
+
+
+
class TaskTemplateTest(unittest.TestCase):
def test_TaskTemplate_gets_created_with_correct_creation_timestamp(self):
@@ -131,20 +158,82 @@ class TaskTemplateTest(unittest.TestCase):
self.assertLess(before, entry.updated_at)
self.assertGreater(after, entry.updated_at)
+ def test_TaskTemplate_incorrect_schema_raises(self):
+ with self.assertRaises(SchemaValidationException):
+ models.TaskTemplate.objects.create(**TaskTemplate_test_data(schema=""))
+
+ with self.assertRaises(SchemaValidationException) as context:
+ models.TaskTemplate.objects.create(**TaskTemplate_test_data(schema={}))
+ self.assertTrue(True)
+
+ with self.assertRaises(SchemaValidationException) as context:
+ schema = minimal_json_schema()
+ del schema['$schema']
+ models.TaskTemplate.objects.create(**TaskTemplate_test_data(schema=schema))
+ self.assertTrue("Missing required properties" in str(context.exception))
+
+ with self.assertRaises(SchemaValidationException) as context:
+ models.TaskTemplate.objects.create(**TaskTemplate_test_data(schema= minimal_json_schema(id="my id with no url")))
+ self.assertTrue("should contain a valid URL" in str(context.exception))
+
+ def test_TaskTemplate_annotated_schema(self):
+ schema = minimal_json_schema()
+ data = TaskTemplate_test_data(schema=schema, name="foo", description="bar")
+ template = models.TaskTemplate.objects.create(**data)
+ self.assertEqual("foo", template.name)
+ self.assertEqual("foo", template.schema['title'])
+ self.assertEqual("bar", template.description)
+ self.assertEqual("bar", template.schema['description'])
+
+
def test_TaskTemplate_name_version_unique(self):
- test_data = TaskTemplate_test_data(name="my_name", version="1")
+ name = str(uuid.uuid4())
+ self.assertEqual(0, models.TaskTemplate.objects.filter(name=name).count())
+ test_data = TaskTemplate_test_data(name=name)
+ # save data twice
entry1 = models.TaskTemplate.objects.create(**test_data)
+ entry2 = models.TaskTemplate.objects.create(**test_data)
+ self.assertEqual(2, models.TaskTemplate.objects.filter(name=name).count())
- with self.assertRaises(IntegrityError):
- entry2 = models.TaskTemplate.objects.create(**test_data)
-
- test_data2 = dict(**test_data)
- test_data2['version'] = "2"
- entry2 = models.TaskTemplate.objects.create(**test_data2)
+ self.assertEqual(1, entry1.version)
+ self.assertEqual(2, entry2.version) #version is autoincremented
+ # try to modify version... should be allowed, cause the template is not used, but should raise IntegrityError (unique constraint)
+ self.assertFalse(entry2.is_used)
with self.assertRaises(IntegrityError):
- entry2.version = '1'
+ entry2.version = 1
entry2.save()
+ entry2.refresh_from_db()
+
+ # versions still the same?
+ self.assertEqual(1, entry1.version)
+ self.assertEqual(2, entry2.version)
+
+ # let's use the template in a task
+ models.TaskDraft.objects.create(**TaskDraft_test_data(specifications_template=entry2))
+ self.assertTrue(entry2.is_used)
+
+ # there should still be only 2 templates with this name
+ self.assertEqual(2, models.TaskTemplate.objects.filter(name=name).count())
+
+ # now (try to) modify the template
+ org_pk = entry2.pk
+ org_schema = dict(entry2.schema)
+ new_schema = minimal_json_schema(properties={"new_prop":{"type":"string"}})
+ entry2.schema = new_schema
+ entry2.save()
+ #this should now be a NEW instance
+ self.assertNotEqual(org_pk, entry2.pk)
+ self.assertEqual(3, models.TaskTemplate.objects.filter(name=name).count())
+
+ # lets request the "old" entry2 via name and version, so we can check if it is unchanged
+ entry2 = models.TaskTemplate.objects.get(name=name, version=2)
+ self.assertEqual(org_schema, entry2.schema)
+
+ # instead there should be a new version of the template with the new schema
+ entry3 = models.TaskTemplate.objects.get(name=name, version=3)
+ self.assertEqual(3, entry3.version)
+ self.assertEqual(new_schema, entry3.schema)
class TaskRelationSelectionTemplateTest(unittest.TestCase):
@@ -249,6 +338,19 @@ class ProjectTest(unittest.TestCase):
self.assertLess(before, entry.updated_at)
self.assertGreater(after, entry.updated_at)
+ def test_Project_raises_ValueError_on_invalid_archive_subdirectory_name(self):
+
+ # setup
+ test_data_1 = Project_test_data(archive_subdirectory="no/trailing/slash")
+ test_data_2 = Project_test_data(archive_subdirectory="/with/leading/slash/")
+
+ # assert
+ with self.assertRaises(ValueError):
+ entry = models.Project.objects.create(**test_data_1)
+
+ with self.assertRaises(ValueError):
+ entry = models.Project.objects.create(**test_data_2)
+
class ProjectQuotaTest(unittest.TestCase):
def test_ProjectQuota_prevents_missing_project(self):
@@ -551,12 +653,16 @@ class SchedulingUnitBlueprintTest(unittest.TestCase):
class TaskBlueprintTest(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls) -> None:
+ cls.task_draft = models.TaskDraft.objects.create(**TaskDraft_test_data())
+ cls.scheduling_unit_blueprint = models.SchedulingUnitBlueprint.objects.create(**SchedulingUnitBlueprint_test_data())
def test_TaskBlueprint_gets_created_with_correct_creation_timestamp(self):
# setup
before = datetime.utcnow()
- entry = models.TaskBlueprint.objects.create(**TaskBlueprint_test_data())
+ entry = models.TaskBlueprint.objects.create(**TaskBlueprint_test_data(task_draft=self.task_draft, scheduling_unit_blueprint=self.scheduling_unit_blueprint))
after = datetime.utcnow()
@@ -567,7 +673,7 @@ class TaskBlueprintTest(unittest.TestCase):
def test_TaskBlueprint_update_timestamp_gets_changed_correctly(self):
# setup
- entry = models.TaskBlueprint.objects.create(**TaskBlueprint_test_data())
+ entry = models.TaskBlueprint.objects.create(**TaskBlueprint_test_data(task_draft=self.task_draft, scheduling_unit_blueprint=self.scheduling_unit_blueprint))
before = datetime.utcnow()
entry.save()
after = datetime.utcnow()
@@ -579,7 +685,7 @@ class TaskBlueprintTest(unittest.TestCase):
def test_TaskBlueprint_prevents_missing_template(self):
# setup
- test_data = dict(TaskBlueprint_test_data())
+ test_data = dict(TaskBlueprint_test_data(task_draft=self.task_draft, scheduling_unit_blueprint=self.scheduling_unit_blueprint))
test_data['specifications_template'] = None
# assert
@@ -589,7 +695,7 @@ class TaskBlueprintTest(unittest.TestCase):
def test_TaskBlueprint_prevents_missing_draft(self):
# setup
- test_data = dict(TaskBlueprint_test_data())
+ test_data = dict(TaskBlueprint_test_data(task_draft=self.task_draft, scheduling_unit_blueprint=self.scheduling_unit_blueprint))
test_data['draft'] = None
# assert
@@ -599,7 +705,7 @@ class TaskBlueprintTest(unittest.TestCase):
def test_TaskBlueprint_prevents_missing_scheduling_unit_blueprint(self):
# setup
- test_data = dict(TaskBlueprint_test_data())
+ test_data = dict(TaskBlueprint_test_data(task_draft=self.task_draft, scheduling_unit_blueprint=self.scheduling_unit_blueprint))
test_data['scheduling_unit_blueprint'] = None
# assert
@@ -607,8 +713,8 @@ class TaskBlueprintTest(unittest.TestCase):
models.TaskBlueprint.objects.create(**test_data)
def test_TaskBlueprint_predecessors_and_successors_none(self):
- task_blueprint_1: models.TaskBlueprint = models.TaskBlueprint.objects.create(**TaskBlueprint_test_data())
- task_blueprint_2: models.TaskBlueprint = models.TaskBlueprint.objects.create(**TaskBlueprint_test_data())
+ task_blueprint_1: models.TaskBlueprint = models.TaskBlueprint.objects.create(**TaskBlueprint_test_data(task_draft=self.task_draft, scheduling_unit_blueprint=self.scheduling_unit_blueprint))
+ task_blueprint_2: models.TaskBlueprint = models.TaskBlueprint.objects.create(**TaskBlueprint_test_data(task_draft=self.task_draft, scheduling_unit_blueprint=self.scheduling_unit_blueprint))
self.assertEqual(set(), set(task_blueprint_1.predecessors.all()))
self.assertEqual(set(), set(task_blueprint_2.predecessors.all()))
@@ -616,8 +722,8 @@ class TaskBlueprintTest(unittest.TestCase):
self.assertEqual(set(), set(task_blueprint_2.successors.all()))
def test_TaskBlueprint_predecessors_and_successors_simple(self):
- task_blueprint_1: models.TaskBlueprint = models.TaskBlueprint.objects.create(**TaskBlueprint_test_data())
- task_blueprint_2: models.TaskBlueprint = models.TaskBlueprint.objects.create(**TaskBlueprint_test_data())
+ task_blueprint_1: models.TaskBlueprint = models.TaskBlueprint.objects.create(**TaskBlueprint_test_data(task_draft=self.task_draft, scheduling_unit_blueprint=self.scheduling_unit_blueprint))
+ task_blueprint_2: models.TaskBlueprint = models.TaskBlueprint.objects.create(**TaskBlueprint_test_data(task_draft=self.task_draft, scheduling_unit_blueprint=self.scheduling_unit_blueprint))
models.TaskRelationBlueprint.objects.create(**TaskRelationBlueprint_test_data(producer=task_blueprint_1,
consumer=task_blueprint_2))
@@ -627,11 +733,11 @@ class TaskBlueprintTest(unittest.TestCase):
def test_TaskBlueprint_predecessors_and_successors_complex(self):
task_blueprint_1: models.TaskBlueprint = models.TaskBlueprint.objects.create(**TaskBlueprint_test_data())
- task_blueprint_2: models.TaskBlueprint = models.TaskBlueprint.objects.create(**TaskBlueprint_test_data())
- task_blueprint_3: models.TaskBlueprint = models.TaskBlueprint.objects.create(**TaskBlueprint_test_data())
- task_blueprint_4: models.TaskBlueprint = models.TaskBlueprint.objects.create(**TaskBlueprint_test_data())
- task_blueprint_5: models.TaskBlueprint = models.TaskBlueprint.objects.create(**TaskBlueprint_test_data())
- task_blueprint_6: models.TaskBlueprint = models.TaskBlueprint.objects.create(**TaskBlueprint_test_data())
+ task_blueprint_2: models.TaskBlueprint = models.TaskBlueprint.objects.create(**TaskBlueprint_test_data(task_draft=task_blueprint_1.draft, scheduling_unit_blueprint=task_blueprint_1.scheduling_unit_blueprint))
+ task_blueprint_3: models.TaskBlueprint = models.TaskBlueprint.objects.create(**TaskBlueprint_test_data(task_draft=task_blueprint_1.draft, scheduling_unit_blueprint=task_blueprint_1.scheduling_unit_blueprint))
+ task_blueprint_4: models.TaskBlueprint = models.TaskBlueprint.objects.create(**TaskBlueprint_test_data(task_draft=task_blueprint_1.draft, scheduling_unit_blueprint=task_blueprint_1.scheduling_unit_blueprint))
+ task_blueprint_5: models.TaskBlueprint = models.TaskBlueprint.objects.create(**TaskBlueprint_test_data(task_draft=task_blueprint_1.draft, scheduling_unit_blueprint=task_blueprint_1.scheduling_unit_blueprint))
+ task_blueprint_6: models.TaskBlueprint = models.TaskBlueprint.objects.create(**TaskBlueprint_test_data(task_draft=task_blueprint_1.draft, scheduling_unit_blueprint=task_blueprint_1.scheduling_unit_blueprint))
# ST1 ---> ST3 ---> ST4
# | |
@@ -661,11 +767,15 @@ class TaskBlueprintTest(unittest.TestCase):
class TaskRelationBlueprintTest(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls) -> None:
+ cls.producer = models.TaskBlueprint.objects.create(**TaskBlueprint_test_data())
+ cls.consumer = models.TaskBlueprint.objects.create(**TaskBlueprint_test_data())
def test_TaskRelationBlueprint_gets_created_with_correct_creation_timestamp(self):
# setup
before = datetime.utcnow()
- entry = models.TaskRelationBlueprint.objects.create(**TaskRelationBlueprint_test_data())
+ entry = models.TaskRelationBlueprint.objects.create(**TaskRelationBlueprint_test_data(producer=self.producer, consumer=self.consumer))
after = datetime.utcnow()
@@ -675,7 +785,7 @@ class TaskRelationBlueprintTest(unittest.TestCase):
def test_TaskRelationBlueprint_update_timestamp_gets_changed_correctly(self):
# setup
- entry = models.TaskRelationBlueprint.objects.create(**TaskRelationBlueprint_test_data())
+ entry = models.TaskRelationBlueprint.objects.create(**TaskRelationBlueprint_test_data(producer=self.producer, consumer=self.consumer))
before = datetime.utcnow()
entry.save()
after = datetime.utcnow()
@@ -686,7 +796,7 @@ class TaskRelationBlueprintTest(unittest.TestCase):
def test_TaskRelationBlueprint_prevents_missing_selection_template(self):
# setup
- test_data = dict(TaskRelationBlueprint_test_data())
+ test_data = dict(TaskRelationBlueprint_test_data(producer=self.producer, consumer=self.consumer))
test_data['selection_template'] = None
# assert
@@ -695,7 +805,7 @@ class TaskRelationBlueprintTest(unittest.TestCase):
def test_TaskRelationBlueprint_prevents_missing_draft(self):
# setup
- test_data = dict(TaskRelationBlueprint_test_data())
+ test_data = dict(TaskRelationBlueprint_test_data(producer=self.producer, consumer=self.consumer))
test_data['draft'] = None
# assert
@@ -704,7 +814,7 @@ class TaskRelationBlueprintTest(unittest.TestCase):
def test_TaskRelationBlueprint_prevents_missing_producer(self):
# setup
- test_data = dict(TaskRelationBlueprint_test_data())
+ test_data = dict(TaskRelationBlueprint_test_data(producer=self.producer, consumer=self.consumer))
test_data['producer'] = None
# assert
@@ -713,7 +823,7 @@ class TaskRelationBlueprintTest(unittest.TestCase):
def test_TaskRelationBlueprint_prevents_missing_consumer(self):
# setup
- test_data = dict(TaskRelationBlueprint_test_data())
+ test_data = dict(TaskRelationBlueprint_test_data(producer=self.producer, consumer=self.consumer))
test_data['consumer'] = None
# assert
@@ -722,7 +832,7 @@ class TaskRelationBlueprintTest(unittest.TestCase):
def test_TaskRelationBlueprint_prevents_missing_input(self):
# setup
- test_data = dict(TaskRelationBlueprint_test_data())
+ test_data = dict(TaskRelationBlueprint_test_data(producer=self.producer, consumer=self.consumer))
test_data['input_role'] = None
# assert
@@ -731,7 +841,7 @@ class TaskRelationBlueprintTest(unittest.TestCase):
def test_TaskRelationBlueprint_prevents_missing_output(self):
# setup
- test_data = dict(TaskRelationBlueprint_test_data())
+ test_data = dict(TaskRelationBlueprint_test_data(producer=self.producer, consumer=self.consumer))
test_data['output_role'] = None
# assert
diff --git a/SAS/TMSS/test/test_utils.py b/SAS/TMSS/test/test_utils.py
index 542ec26b7e50c43c0771615430a2045bf317d18a..627713b67e2e8ae4cea367e680a5dd39f945defc 100644
--- a/SAS/TMSS/test/test_utils.py
+++ b/SAS/TMSS/test/test_utils.py
@@ -111,6 +111,17 @@ class TMSSTestDatabaseInstance(PostgresTestDatabaseInstance):
raise TMSSException("Could not initialize TMSS database with django migrations")
+def minimal_json_schema(title:str="my title", description:str="my description", id:str="http://example.com/foo/bar.json", properties:dict={}, required=[]):
+ return {"$schema": "http://json-schema.org/draft-06/schema#",
+ "$id": id,
+ "title": title,
+ "description": description,
+ "type": "object",
+ "properties": properties,
+ "required": required,
+ "default": {}
+ }
+
class TMSSPostgresTestMixin(PostgresTestMixin):
'''
A common test mixin class from which you can derive to get a freshly setup postgres testing instance with the latest TMSS sql schema.
@@ -123,21 +134,27 @@ class TMSSPostgresTestMixin(PostgresTestMixin):
class TMSSDjangoServerInstance():
''' Creates a running django TMSS server at the requested port with the requested database credentials.
'''
- def __init__(self, db_dbcreds_id: str="TMSS", ldap_dbcreds_id: str="TMSS_LDAP", host: str='127.0.0.1', port: int=8000,
+ def __init__(self, db_dbcreds_id: str="TMSS", ldap_dbcreds_id: str="TMSS_LDAP", host: str='127.0.0.1', port: int=8000, public_host: str=None,
exchange: str=os.environ.get("TMSS_EXCHANGE", DEFAULT_BUSNAME), broker: str=os.environ.get("TMSS_EXCHANGE", DEFAULT_BUSNAME)):
self._db_dbcreds_id = db_dbcreds_id
self._ldap_dbcreds_id = ldap_dbcreds_id
self.host = host
self.port = port
+ self.public_host = public_host or host
self._server_process = None
self._exchange = exchange
self._broker = broker
@property
- def address(self):
+ def host_address(self):
''':returns the address and port of the django server'''
return "%s:%d" % (self.host, self.port)
+ @property
+ def address(self):
+ ''':returns the public address and port of the django server'''
+ return "%s:%d" % (self.public_host, self.port)
+
@property
def url(self):
''':returns the http url to the django server'''
@@ -190,14 +207,14 @@ class TMSSDjangoServerInstance():
self.setup_django()
django.core.management.call_command('runserver',
use_reloader=False,
- addrport=self.address)
+ addrport=self.host_address)
self._server_process = Process(target=_helper_runserver_loop, daemon=True)
self._server_process.start()
# wait for server to be up and running....
# or exit via TimeoutError
- self.check_running_server(timeout=10)
+ self.check_running_server(timeout=30)
def stop(self):
'''
@@ -215,31 +232,29 @@ class TMSSDjangoServerInstance():
def check_running_server(self, timeout: float = 10) -> bool:
'''Check the running django server for a valid response'''
- try:
- import requests, urllib3
- from _datetime import datetime, timedelta
- start = datetime.utcnow()
- while True:
- try:
- response = requests.get(self.url, auth=(self.ldap_dbcreds.user, self.ldap_dbcreds.password), timeout=max(1, timeout/10))
-
- if response.status_code in [200, 401, 403]:
- logger.info("TMSS Django server is up and running at %s with database: %s and LDAP: %s",
- self.url, self.database_dbcreds, self.ldap_dbcreds)
-
- if response.status_code in [401, 403]:
- logger.warning("TMSS Django server at %s could not autenticate with LDAP creds: %s", self.url, self.ldap_dbcreds)
-
- # TODO: logout, otherwise django remembers our login session.
- return True
- except Exception as e:
- time.sleep(0.25)
-
- if datetime.utcnow() - start > timedelta(seconds=timeout):
- raise TimeoutError("Could not get a valid response from the django server at %s" % self.url)
- except Exception as e:
- logger.error(e)
- return False
+ import requests
+ from _datetime import datetime, timedelta
+ start = datetime.utcnow()
+ while True:
+ try:
+ logger.info("Checking if TMSS Django server is up and running at %s with database: %s and LDAP: %s ....",
+ self.url, self.database_dbcreds, self.ldap_dbcreds)
+ response = requests.get(self.url, auth=(self.ldap_dbcreds.user, self.ldap_dbcreds.password), timeout=max(1, timeout/10))
+
+ if response.status_code in [200, 401, 403]:
+ logger.info("TMSS Django server is up and running at %s with database: %s and LDAP: %s",
+ self.url, self.database_dbcreds, self.ldap_dbcreds)
+
+ if response.status_code in [401, 403]:
+ logger.warning("TMSS Django server at %s could not autenticate with LDAP creds: %s", self.url, self.ldap_dbcreds)
+
+ # TODO: logout, otherwise django remembers our login session.
+ return True
+ except Exception as e:
+ time.sleep(0.5)
+
+ if datetime.utcnow() - start > timedelta(seconds=timeout):
+ raise TimeoutError("Could not get a valid response from the django server at %s within %s seconds" % (self.url,timeout))
def __enter__(self):
try:
@@ -256,14 +271,18 @@ class TMSSDjangoServerInstance():
class TMSSTestEnvironment:
'''Create and run a test django TMSS server against a newly created test database and a test ldap server (and cleanup automagically)'''
- def __init__(self, host: str='127.0.0.1', preferred_django_port: int=8000,
- exchange: str=os.environ.get("TMSS_EXCHANGE", DEFAULT_BUSNAME), broker: str=os.environ.get("TMSS_BROKER", DEFAULT_BROKER)):
+ def __init__(self, host: str='127.0.0.1', preferred_django_port: int=8000, public_host: str=None,
+ exchange: str=os.environ.get("TMSS_EXCHANGE", DEFAULT_BUSNAME), broker: str=os.environ.get("TMSS_BROKER", DEFAULT_BROKER),
+ populate_schemas:bool=False, populate_test_data:bool=False):
+ self._populate_schemas = populate_schemas
+ self._populate_test_data = populate_test_data
self.ldap_server = TestLDAPServer(user='test', password='test')
self.database = TMSSTestDatabaseInstance()
self.django_server = TMSSDjangoServerInstance(db_dbcreds_id=self.database.dbcreds_id,
ldap_dbcreds_id=self.ldap_server.dbcreds_id,
host=host,
port=find_free_port(preferred_django_port),
+ public_host=public_host,
exchange=exchange,
broker=broker)
self.client_credentials = TemporaryCredentials(user=self.ldap_server.dbcreds.user,
@@ -279,7 +298,7 @@ class TMSSTestEnvironment:
self.django_server.start()
# store client credentials in the TemporaryCredentials file...
- self.client_credentials.dbcreds.host = self.django_server.host
+ self.client_credentials.dbcreds.host = self.django_server.public_host
self.client_credentials.dbcreds.port = self.django_server.port
self.client_credentials.dbcreds.type = "http"
self.client_credentials.create()
@@ -299,6 +318,12 @@ class TMSSTestEnvironment:
user.is_superuser = True
user.save()
+ if self._populate_schemas or self._populate_test_data:
+ self.populate_schemas()
+
+ if self._populate_test_data:
+ self.populate_test_data()
+
def stop(self):
self.django_server.stop()
self.ldap_server.stop()
@@ -317,6 +342,19 @@ class TMSSTestEnvironment:
def __exit__(self, exc_type, exc_val, exc_tb):
self.stop()
+ def populate_schemas(self):
+ # populate the items that rely on a running REST API server (which cannot be populated via the django model.objects API)
+ from lofar.sas.tmss.client.populate import populate_schemas
+ populate_schemas()
+
+ # the connectors rely on the schemas to be populated first (above)
+ from lofar.sas.tmss.tmss.tmssapp.populate import populate_connectors
+ populate_connectors()
+
+ def populate_test_data(self):
+ from lofar.sas.tmss.tmss.tmssapp.populate import populate_test_data
+ populate_test_data()
+
def create_tmss_client(self):
return TMSSsession.create_from_dbcreds_for_ldap(self.client_credentials.dbcreds_id)
@@ -347,27 +385,36 @@ def main_test_environment():
parser = OptionParser('%prog [options]',
description='setup/run/teardown a full TMSS test environment including a fresh and isolated database, LDAP server and DJANGO REST server.')
- parser.add_option("-H", "--host", dest="host", type="string", default='127.0.0.1',
- help="expose the TMSS Django REST API via this host. [default=%default]")
- parser.add_option("-p", "--port", dest="port", type="int", default=find_free_port(8000),
+
+ group = OptionGroup(parser, 'Network')
+ parser.add_option_group(group)
+ group.add_option("-H", "--host", dest="host", type="string", default='0.0.0.0',
+ help="serve the TMSS Django REST API server via this host. [default=%default]")
+ group.add_option("-p", "--port", dest="port", type="int", default=find_free_port(8000),
help="try to use this port for the DJANGO REST API. If not available, then a random free port is used and logged. [default=%default]")
+ group.add_option("-P", "--public_host", dest="public_host", type="string", default='127.0.0.1',
+ help="expose the TMSS Django REST API via this host. [default=%default]")
+
+ group = OptionGroup(parser, 'Example/Test data')
+ parser.add_option_group(group)
+ group.add_option('-d', '--data', dest='data', action='store_true', help='populate the test-database with test/example data')
+
group = OptionGroup(parser, 'Messaging options')
+ parser.add_option_group(group)
group.add_option('-b', '--broker', dest='broker', type='string', default=DEFAULT_BROKER, help='Address of the message broker, default: %default')
group.add_option('-e', "--exchange", dest="exchange", type="string", default=DEFAULT_BUSNAME, help="Bus or queue where the TMSS messages are published. [default: %default]")
- parser.add_option_group(group)
+
(options, args) = parser.parse_args()
logging.basicConfig(format = '%(asctime)s %(levelname)s %(message)s', level = logging.INFO)
with RATestEnvironment(exchange=options.exchange, broker=options.broker):
- with TMSSTestEnvironment(host=options.host, preferred_django_port=options.port, exchange=options.exchange, broker=options.broker) as instance:
+ with TMSSTestEnvironment(host=options.host, preferred_django_port=options.port, public_host=options.public_host,
+ exchange=options.exchange, broker=options.broker,
+ populate_schemas=True, populate_test_data=options.data) as instance:
from lofar.sas.tmss.services.dynamic_scheduling import create_dynamic_scheduling_service
with create_dynamic_scheduling_service(options.exchange, options.broker):
-
- from lofar.sas.tmss.tmss.tmssapp.populate import populate_test_data
- populate_test_data()
-
# print some nice info for the user to use the test servers...
# use print instead of log for clean lines.
for h in logging.root.handlers:
diff --git a/SAS/TMSS/test/testdata/example_UC1_scheduling_unit.json b/SAS/TMSS/test/testdata/example_UC1_scheduling_unit.json
index 28b908235b5c6600ce359ccc3dcef72f5af4f505..639ad9535ae620604a82c8bdb9752c3a253d5618 100644
--- a/SAS/TMSS/test/testdata/example_UC1_scheduling_unit.json
+++ b/SAS/TMSS/test/testdata/example_UC1_scheduling_unit.json
@@ -10,11 +10,10 @@
"pointing": {
"direction_type": "J2000",
"angle1": 0,
- "angle2": 0,
- "angle3": 0
+ "angle2": 0
}
},
- "specifications_template": "calibrator schema"
+ "specifications_template": "calibrator observation"
},
{
"name": "Pipeline 1",
@@ -63,19 +62,15 @@
"integration_time": 1,
"channels_per_subband": 64
},
- "antenna_set": "HBA_DUAL_INNER",
- "filter": "HBA_110_190",
- "stations": [
- {
- "group": "ALL",
- "min_stations": 1
- }
- ],
+ "antenna_settings": {
+ "antenna_set": "HBA_DUAL_INNER",
+ "filter": "HBA_110_190"
+ },
+ "stations": ["CS001","CS002","CS003"],
"tile_beam": {
"direction_type": "J2000",
"angle1": 42,
- "angle2": 42,
- "angle3": 42
+ "angle2": 42
},
"SAPs": [
{
@@ -83,8 +78,7 @@
"digital_pointing": {
"direction_type": "J2000",
"angle1": 24,
- "angle2": 24,
- "angle3": 24
+ "angle2": 24
},
"subbands": [
349,
@@ -93,7 +87,7 @@
}
]
},
- "specifications_template": "observation schema"
+ "specifications_template": "target observation"
},
{
"name": "Pipeline SAP0",
@@ -153,11 +147,10 @@
"pointing": {
"direction_type": "J2000",
"angle1": 0,
- "angle2": 0,
- "angle3": 0
+ "angle2": 0
}
},
- "specifications_template": "calibrator schema"
+ "specifications_template": "calibrator observation"
},
{
"name": "Pipeline 2",
diff --git a/SAS/TMSS/test/tmss_test_data_django_models.py b/SAS/TMSS/test/tmss_test_data_django_models.py
index f77c2ee61ae5ec505708e8f8143fb1074e478bf7..9bb70d78a1d6fe85ef64c2bf5cc7f92ff4616b16 100644
--- a/SAS/TMSS/test/tmss_test_data_django_models.py
+++ b/SAS/TMSS/test/tmss_test_data_django_models.py
@@ -29,19 +29,17 @@ which is automatically destroyed at the end of the unittest session.
from lofar.sas.tmss.tmss.tmssapp import models
from lofar.common.json_utils import get_default_json_object_for_schema
+from lofar.sas.tmss.test.test_utils import minimal_json_schema
from datetime import datetime, timedelta
import uuid
import json
-def GeneratorTemplate_test_data(name="my_GeneratorTemplate", version:str=None) -> dict:
- if version is None:
- version = str(uuid.uuid4())
+def GeneratorTemplate_test_data(name="my_GeneratorTemplate") -> dict:
return {"name": name,
"description": 'My one observation',
- "version": version,
- "schema": {"mykey": "my value"},
+ "schema": minimal_json_schema(),
"create_function": 'Funky',
"tags": ["TMSS", "TESTING"]}
@@ -50,30 +48,30 @@ def DefaultGeneratorTemplate_test_data(name=None, template=None) -> dict:
'template': template,
'tags':[]}
-def SchedulingUnitTemplate_test_data(name="my_SchedulingUnitTemplate", version:str=None, schema:dict=None) -> dict:
- if version is None:
- version = str(uuid.uuid4())
-
+def SchedulingUnitTemplate_test_data(name="my_SchedulingUnitTemplate", schema:dict=None) -> dict:
if schema is None:
- schema = { "$schema": "https://json-schema.org/draft/2019-09/schema",
- "type": "object",
- "properties": { "foo" : { "type": "string", "default": "bar" } },
- "required": ["foo"],
- "default": {}
- }
+ schema = minimal_json_schema(properties={ "foo" : { "type": "string", "default": "bar" } }, required=["foo"])
return {"name": name,
"description": 'My SchedulingUnitTemplate description',
- "version": version,
"schema": schema,
"tags": ["TMSS", "TESTING"]}
-def SchedulingUnitObservingStrategyTemplate_test_data(name="my_SchedulingUnitObservingStrategyTemplate", version:str=None,
+
+def SchedulingConstraintsTemplate_test_data(name="my_SchedulingConstraintsTemplate", schema:dict=None) -> dict:
+ if schema is None:
+ schema = minimal_json_schema(properties={ "foo" : { "type": "string", "default": "bar" } }, required=["foo"])
+
+ return {"name": name,
+ "description": 'My SchedulingConstraintsTemplate description',
+ "schema": schema,
+ "tags": ["TMSS", "TESTING"]}
+
+
+
+def SchedulingUnitObservingStrategyTemplate_test_data(name="my_SchedulingUnitObservingStrategyTemplate",
scheduling_unit_template:models.SchedulingUnitTemplate=None,
template:dict=None) -> dict:
- if version is None:
- version = str(uuid.uuid4())
-
if scheduling_unit_template is None:
scheduling_unit_template = models.SchedulingUnitTemplate.objects.create(**SchedulingUnitTemplate_test_data())
@@ -82,31 +80,25 @@ def SchedulingUnitObservingStrategyTemplate_test_data(name="my_SchedulingUnitObs
return {"name": name,
"description": 'My SchedulingUnitTemplate description',
- "version": version,
"template": template,
"scheduling_unit_template": scheduling_unit_template,
"tags": ["TMSS", "TESTING"]}
-def TaskTemplate_test_data(name="my TaskTemplate", version:str=None) -> dict:
- if version is None:
- version = str(uuid.uuid4())
+def TaskTemplate_test_data(name="my TaskTemplate", description:str=None, schema:dict=None) -> dict:
+ if schema is None:
+ schema = minimal_json_schema(properties={"mykey": {}})
return {"type": models.TaskType.objects.get(value='observation'),
"validation_code_js":"",
"name": name,
- "description": 'My TaskTemplate description',
- "version": version,
- "schema": {"mykey": "my value"},
+ "description": description or "<no description>",
+ "schema": schema,
"tags": ["TMSS", "TESTING"]}
-def TaskRelationSelectionTemplate_test_data(name="my_TaskRelationSelectionTemplate", version:str=None) -> dict:
- if version is None:
- version = str(uuid.uuid4())
-
+def TaskRelationSelectionTemplate_test_data(name="my_TaskRelationSelectionTemplate") -> dict:
return {"name": name,
"description": 'My TaskRelationSelectionTemplate description',
- "version": version,
- "schema": {"mykey": "my value"},
+ "schema": minimal_json_schema(),
"tags": ["TMSS", "TESTING"]}
def TaskConnectorType_test_data() -> dict:
@@ -123,7 +115,7 @@ def Cycle_test_data() -> dict:
"start": datetime.utcnow().isoformat(),
"stop": datetime.utcnow().isoformat()}
-def Project_test_data(name: str=None, priority_rank: int = 1) -> dict:
+def Project_test_data(name: str=None, priority_rank: int = 1, archive_subdirectory="my_project/") -> dict:
if name is None:
name = 'my_project_' + str(uuid.uuid4())
@@ -136,7 +128,8 @@ def Project_test_data(name: str=None, priority_rank: int = 1) -> dict:
"can_trigger": False,
"private_data": True,
"expert": True,
- "filler": False}
+ "filler": False,
+ "archive_subdirectory": archive_subdirectory}
def ResourceType_test_data() -> dict:
return {
@@ -157,12 +150,15 @@ def SchedulingSet_test_data(name="my_scheduling_set", project: models.Project=No
if project is None:
project = models.Project.objects.create(**Project_test_data())
+ generator_template = models.GeneratorTemplate.objects.create(**GeneratorTemplate_test_data())
+ generator_doc = get_default_json_object_for_schema(generator_template.schema)
+
return {"name": name,
"description": "",
"tags": [],
- "generator_doc": {},
+ "generator_doc": generator_doc,
"project": project,
- "generator_template": models.GeneratorTemplate.objects.create(**GeneratorTemplate_test_data()),
+ "generator_template": generator_template,
"generator_source": None}
def SchedulingUnitDraft_test_data(name="my_scheduling_unit_draft", scheduling_set: models.SchedulingSet=None,
@@ -226,19 +222,25 @@ def TaskRelationDraft_test_data(producer: models.TaskDraft = None, consumer: mod
"output_role": models.TaskConnectorType.objects.create(**TaskConnectorType_test_data()),
"selection_template": models.TaskRelationSelectionTemplate.objects.create(**TaskRelationSelectionTemplate_test_data())}
-def SchedulingUnitBlueprint_test_data(name='my_scheduling_unit_blueprint') -> dict:
+def SchedulingUnitBlueprint_test_data(name='my_scheduling_unit_blueprint', requirements_template: models.SchedulingUnitTemplate=None) -> dict:
+ if requirements_template is None:
+ requirements_template = models.SchedulingUnitTemplate.objects.create(**SchedulingUnitTemplate_test_data())
+
return {"name": name,
"description": "",
"tags": [],
- "requirements_doc": {},
+ "requirements_doc": get_default_json_object_for_schema(requirements_template.schema),
+ "requirements_template": requirements_template,
"do_cancel": False,
- "draft": models.SchedulingUnitDraft.objects.create(**SchedulingUnitDraft_test_data()),
- "requirements_template": models.SchedulingUnitTemplate.objects.create(**SchedulingUnitTemplate_test_data())}
+ "draft": models.SchedulingUnitDraft.objects.create(**SchedulingUnitDraft_test_data()) }
-def TaskBlueprint_test_data(name='my_task_blueprint', task_draft: models.TaskDraft = None) -> dict:
+def TaskBlueprint_test_data(name='my_task_blueprint', task_draft: models.TaskDraft = None, scheduling_unit_blueprint: models.SchedulingUnitBlueprint = None) -> dict:
if task_draft is None:
task_draft = models.TaskDraft.objects.create(**TaskDraft_test_data())
+ if scheduling_unit_blueprint is None:
+ scheduling_unit_blueprint = models.SchedulingUnitBlueprint.objects.create(**SchedulingUnitBlueprint_test_data())
+
return {"name": name,
"description": "",
"tags": [],
@@ -246,7 +248,7 @@ def TaskBlueprint_test_data(name='my_task_blueprint', task_draft: models.TaskDra
"do_cancel": False,
"draft": task_draft,
"specifications_template": task_draft.specifications_template,
- "scheduling_unit_blueprint": models.SchedulingUnitBlueprint.objects.create(**SchedulingUnitBlueprint_test_data())}
+ "scheduling_unit_blueprint": scheduling_unit_blueprint}
def TaskRelationBlueprint_test_data(producer: models.TaskBlueprint = None, consumer: models.TaskBlueprint = None) -> dict:
if producer is None:
@@ -266,17 +268,13 @@ def TaskRelationBlueprint_test_data(producer: models.TaskBlueprint = None, consu
"consumer": consumer}
-def SubtaskTemplate_test_data(schema: object=None, version:str=None) -> dict:
+def SubtaskTemplate_test_data(schema: object=None) -> dict:
if schema is None:
- schema = {}
-
- if version is None:
- version = str(uuid.uuid4())
+ schema = minimal_json_schema()
return {"type": models.SubtaskType.objects.get(value='copy'),
"name": "observation",
"description": 'My one observation',
- "version": version,
"schema": schema,
"realtime": True,
"queue": False,
@@ -307,24 +305,16 @@ def TaskSchedulingRelationBlueprint_test_data(first: models.TaskBlueprint = None
"placement": models.SchedulingRelationPlacement.objects.get(value='after'),
"time_offset":60}
-def DataproductSpecificationsTemplate_test_data(version:str=None) -> dict:
- if version is None:
- version = str(uuid.uuid4())
-
+def DataproductSpecificationsTemplate_test_data() -> dict:
return {"name": "data",
"description": 'My one date',
- "version": version,
- "schema": {"mykey": "my value"},
+ "schema": minimal_json_schema(),
"tags": ["TMSS", "TESTING"]}
-def DataproductFeedbackTemplate_test_data(version:str=None) -> dict:
- if version is None:
- version = str(uuid.uuid4())
-
+def DataproductFeedbackTemplate_test_data() -> dict:
return {"name": "data",
"description": 'My one date',
- "version": version,
- "schema": {"mykey": "my value"},
+ "schema": minimal_json_schema(),
"tags": ["TMSS", "TESTING"]}
def SubtaskOutput_test_data(subtask: models.Subtask=None) -> dict:
@@ -352,7 +342,8 @@ def SubtaskInput_test_data(subtask: models.Subtask=None, producer: models.Subtas
"tags":[]}
def Subtask_test_data(task_blueprint: models.TaskBlueprint=None, subtask_template: models.SubtaskTemplate=None,
- specifications_doc: dict=None, start_time=None, stop_time=None, cluster=None, state=None) -> dict:
+ specifications_doc: dict=None, start_time=None, stop_time=None, cluster=None, state=None,
+ raw_feedback=None) -> dict:
if task_blueprint is None:
task_blueprint = models.TaskBlueprint.objects.create(**TaskBlueprint_test_data())
@@ -371,7 +362,7 @@ def Subtask_test_data(task_blueprint: models.TaskBlueprint=None, subtask_templat
stop_time = datetime.utcnow() + timedelta(minutes=10)
if cluster is None:
- cluster = models.Cluster.objects.create(name="dummy cluster", location="downstairs", tags=[])
+ cluster = models.Cluster.objects.create(name="dummy cluster", location="downstairs", archive_site=True, tags=[])
if state is None:
state = models.SubtaskState.objects.get(value='defining')
@@ -386,13 +377,18 @@ def Subtask_test_data(task_blueprint: models.TaskBlueprint=None, subtask_templat
"do_cancel": datetime.utcnow(),
"priority": 1,
"schedule_method": models.ScheduleMethod.objects.get(value='manual'),
- "cluster": cluster}
+ "cluster": cluster,
+ "raw_feedback": raw_feedback}
def Dataproduct_test_data(producer: models.SubtaskOutput=None,
filename: str="my_file.ext",
directory: str="/data/test-projects",
dataformat: models.Dataformat=None,
- specifications_doc: object=None) -> dict:
+ datatype: models.Datatype=None,
+ specifications_doc: object=None,
+ specifications_template: models.DataproductSpecificationsTemplate=None,
+ feedback_doc: object = None,
+ feedback_template: models.DataproductFeedbackTemplate = None) -> dict:
if producer is None:
producer = models.SubtaskOutput.objects.create(**SubtaskOutput_test_data())
@@ -400,23 +396,36 @@ def Dataproduct_test_data(producer: models.SubtaskOutput=None,
if dataformat is None:
dataformat = models.Dataformat.objects.get(value="MeasurementSet")
+ if datatype is None:
+ datatype = models.Datatype.objects.get(value="visibilities")
+
+ if specifications_template is None:
+ specifications_template = models.DataproductSpecificationsTemplate.objects.create(**DataproductSpecificationsTemplate_test_data())
+
if specifications_doc is None:
- specifications_doc={}
+ specifications_doc = get_default_json_object_for_schema(specifications_template.schema)
+
+ if feedback_template is None:
+ feedback_template = models.DataproductFeedbackTemplate.objects.create(**DataproductFeedbackTemplate_test_data())
+
+ if feedback_doc is None:
+ feedback_doc = get_default_json_object_for_schema(feedback_template.schema)
return {"filename": filename,
"directory": directory,
"dataformat": dataformat,
+ "datatype": datatype,
"deleted_since": None,
"pinned_since": None,
"specifications_doc": specifications_doc,
- "specifications_template": models.DataproductSpecificationsTemplate.objects.create(**DataproductSpecificationsTemplate_test_data()),
+ "specifications_template": specifications_template,
"tags": ["TMSS", "TESTING"],
"producer": producer,
"do_cancel": None,
"expected_size": 1234,
"size": 123,
- "feedback_doc": {},
- "feedback_template": models.DataproductFeedbackTemplate.objects.create(**DataproductFeedbackTemplate_test_data())}
+ "feedback_doc": feedback_doc,
+ "feedback_template": feedback_template}
def AntennaSet_test_data() -> dict:
return {"name": "observation",
@@ -433,14 +442,16 @@ def DataproductTransform_test_data() -> dict:
"identity": True,
"tags": ['tmss', 'testing']}
-def Filesystem_test_data() -> dict:
+def Filesystem_test_data(directory="/") -> dict:
return {"capacity": 1111111111,
"cluster": models.Cluster.objects.create(**Cluster_test_data()),
+ "directory": directory,
"tags": ['tmss', 'testing']}
def Cluster_test_data(name="default cluster") -> dict:
return {"name": name,
"location": "upstairs",
+ "archive_site": True,
"tags": ['tmss', 'testing']}
def DataproductArchiveInfo_test_data() -> dict:
diff --git a/SAS/TMSS/test/tmss_test_data_rest.py b/SAS/TMSS/test/tmss_test_data_rest.py
index 39818fd45d830856c0ef85ef45d1bce7608d5caf..76f23608e242dd7efed4380290729249c999c476 100644
--- a/SAS/TMSS/test/tmss_test_data_rest.py
+++ b/SAS/TMSS/test/tmss_test_data_rest.py
@@ -21,12 +21,14 @@
# the methods below can be used to to HTTP REST calls to the django server and check the results
################################################################################################
-from datetime import datetime
+from datetime import datetime, timedelta
import uuid
import requests
import json
-from lofar.common.json_utils import get_default_json_object_for_schema
from http import HTTPStatus
+from copy import deepcopy
+from lofar.sas.tmss.test.test_utils import minimal_json_schema
+
class TMSSRESTTestDataCreator():
def __init__(self, django_api_url: str, auth: requests.auth.HTTPBasicAuth):
@@ -58,85 +60,106 @@ class TMSSRESTTestDataCreator():
# Because I don't like 'Bad Request' errors, I want more content if it goes wrong
raise Exception("Error during POST request of '%s' result is '%s'" % (url_postfix, result))
return url
-
+
+ def update_schema_from_template(self, model_name:str, template_test_data:dict) -> dict:
+ '''helper method to update the schema subdict in the template_test_data dict with the auto-injected-by-the-backend-properties if needed'''
+ updated_test_data = deepcopy(template_test_data)
+
+ if 'schema' in updated_test_data:
+ if 'name' in template_test_data and 'version' in template_test_data:
+ updated_test_data['schema']['$id'] = "%s/schemas/%s/%s/%s#" % (self.django_api_url,
+ model_name,
+ template_test_data['name'],
+ template_test_data['version'])
+ else:
+ updated_test_data['schema'].pop('$id','')
+
+
+ if 'name' in template_test_data:
+ updated_test_data['schema']['title'] = template_test_data['name']
+ else:
+ updated_test_data['schema'].pop('title','')
+
+ if 'description' in template_test_data:
+ updated_test_data['schema']['description'] = template_test_data['description']
+ else:
+ updated_test_data['schema'].pop('description','')
+
+ if 'version' in template_test_data:
+ updated_test_data['schema']['version'] = template_test_data['version']
+ else:
+ updated_test_data['schema'].pop('version','')
+
+ return updated_test_data
+
+ def update_document_from_template(self, model_name:str, data:dict, document_key:str, template_key:str) -> dict:
+ updated_data = deepcopy(data)
+ updated_data[document_key] = self.update_schema_from_template(model_name, updated_data[document_key])
+ return updated_data
+
#######################################################
# the methods below can be used to create test data
# naming convention is: <django_model_name>()
#######################################################
- def GeneratorTemplate(self, name="generatortemplate", version:str=None) -> dict:
- if version is None:
- version = str(uuid.uuid4())
-
+ def GeneratorTemplate(self, name="generatortemplate") -> dict:
return {"name": name,
"description": 'My one observation',
- "version": version,
- "schema": {"mykey": "my value"},
+ "schema": minimal_json_schema(properties={"foo": {"type": "string", "default": "bar"}}),
"create_function": 'Funky',
"tags": ["TMSS", "TESTING"]}
- def SchedulingUnitTemplate(self, name="schedulingunittemplate1", version:str=None, schema:dict=None) -> dict:
- if version is None:
- version = str(uuid.uuid4())
+ def SchedulingUnitTemplate(self, name="schedulingunittemplate1", schema:dict=None) -> dict:
+ if schema is None:
+ schema = minimal_json_schema(properties={"foo": {"type": "string", "default": "bar"}})
+ return { "name": name,
+ "description": 'My description',
+ "schema": schema,
+ "tags": ["TMSS", "TESTING"]}
+
+
+ def SchedulingConstraintsTemplate(self, name="schedulingcontraintstemplate1", schema:dict=None) -> dict:
if schema is None:
- schema = {"$schema": "https://json-schema.org/draft/2019-09/schema",
- "type": "object",
- "properties": {"foo": {"type": "string", "default": "bar"}},
- "required": ["foo"],
- "default": {}
- }
+ schema = minimal_json_schema(properties={"foo": {"type": "string", "default": "bar"}})
return { "name": name,
"description": 'My description',
- "version": version,
"schema": schema,
"tags": ["TMSS", "TESTING"]}
- def SchedulingUnitObservingStrategyTemplate(self, name="my_SchedulingUnitObservingStrategyTemplate", version:str=None,
+
+ def SchedulingUnitObservingStrategyTemplate(self, name="my_SchedulingUnitObservingStrategyTemplate",
scheduling_unit_template_url=None,
template:dict=None) -> dict:
- if version is None:
- version = str(uuid.uuid4())
-
if scheduling_unit_template_url is None:
scheduling_unit_template_url = self.post_data_and_get_url(self.SchedulingUnitTemplate(), '/scheduling_unit_template/')
if template is None:
- scheduling_unit_template = self.get_response_as_json_object(scheduling_unit_template_url)
- template = get_default_json_object_for_schema(scheduling_unit_template['schema'])
+ template = self.get_response_as_json_object(scheduling_unit_template_url+'/default')
return {"name": name,
"description": 'My SchedulingUnitTemplate description',
- "version": version,
"template": template,
"scheduling_unit_template": scheduling_unit_template_url,
"tags": ["TMSS", "TESTING"]}
- def TaskTemplate(self, name="tasktemplate1", task_type_url: str = None, version: str = None) -> dict:
- if version is None:
- version = str(uuid.uuid4())
-
+ def TaskTemplate(self, name="tasktemplate1", task_type_url: str = None) -> dict:
if task_type_url is None:
task_type_url = self.django_api_url + '/task_type/observation'
return {"name": name,
"description": 'My one observation',
- "version": version,
- "schema": {"mykey": "my value"},
+ "schema": minimal_json_schema(),
"tags": ["TMSS", "TESTING"],
"type": task_type_url,
"validation_code_js": "???"}
- def TaskRelationSelectionTemplate(self, name="taskrelationselectiontemplate1", version:str=None) -> dict:
- if version is None:
- version = str(uuid.uuid4())
-
+ def TaskRelationSelectionTemplate(self, name="taskrelationselectiontemplate1") -> dict:
return {"name": name,
"description": 'My one observation',
- "version": version,
- "schema": {"mykey": "my value"},
+ "schema": minimal_json_schema(),
"tags": ["TMSS", "TESTING"]}
def TaskConnectorType(self, role="correlator", input_of_url=None, output_of_url=None):
@@ -176,7 +199,8 @@ class TMSSRESTTestDataCreator():
"trigger_priority": 1000,
"can_trigger": False,
"private_data": True,
- "cycles": []}
+ "cycles": [],
+ "archive_subdirectory": 'my_project/'}
def ResourceType(self, description="my resource_type description"):
return {
@@ -200,32 +224,40 @@ class TMSSRESTTestDataCreator():
}
- def SchedulingSet(self, name="my_scheduling_set", project_url=None, generator_template_url=None):
+ def SchedulingSet(self, name="my_scheduling_set", project_url=None, generator_template_url=None, generator_doc=None):
if project_url is None:
project_url = self.post_data_and_get_url(self.Project(), '/project/')
if generator_template_url is None:
generator_template_url = self.post_data_and_get_url(self.GeneratorTemplate(), '/generator_template/')
-
+
+ if generator_doc is None:
+ generator_doc = self.get_response_as_json_object(generator_template_url+'/default')
+
return {"name": name,
"description": "This is my scheduling set",
"tags": [],
- "generator_doc": "{}",
+ "generator_doc": generator_doc,
"project": project_url,
"generator_template": generator_template_url,
"generator_source": None,
"scheduling_unit_drafts": []}
- def SchedulingUnitDraft(self, name="my_scheduling_unit_draft", scheduling_set_url=None, template_url=None, requirements_doc=None, observation_strategy_template_url=None):
+ def SchedulingUnitDraft(self, name="my_scheduling_unit_draft", scheduling_set_url=None, template_url=None, scheduling_constraints_template_url=None, requirements_doc=None, scheduling_constraints_doc=None, scheduling_constraints_template=None, observation_strategy_template_url=None):
if scheduling_set_url is None:
scheduling_set_url = self.post_data_and_get_url(self.SchedulingSet(), '/scheduling_set/')
if template_url is None:
template_url = self.post_data_and_get_url(self.SchedulingUnitTemplate(), '/scheduling_unit_template/')
+ if scheduling_constraints_template_url is None:
+ scheduling_constraints_template_url = self.post_data_and_get_url(self.SchedulingConstraintsTemplate(), '/scheduling_constraints_template/')
+
if requirements_doc is None:
- scheduling_unit_template = self.get_response_as_json_object(template_url)
- requirements_doc = get_default_json_object_for_schema(scheduling_unit_template['schema'])
+ requirements_doc = self.get_response_as_json_object(template_url+'/default')
+
+ if scheduling_constraints_doc is None:
+ scheduling_constraints_doc = self.get_response_as_json_object(scheduling_constraints_template_url+'/default')
# if observation_strategy_template_url is None:
# observation_strategy_template_url = self.post_data_and_get_url(self.SchedulingUnitObservingStrategyTemplate(scheduling_unit_template_url=template_url), '/scheduling_unit_observing_strategy_template/')
@@ -235,25 +267,30 @@ class TMSSRESTTestDataCreator():
"tags": [],
"requirements_doc": requirements_doc,
"copy_reason": self.django_api_url + '/copy_reason/template',
- "generator_instance_doc": "{}",
+ "generator_instance_doc": {},
"copies": None,
"scheduling_set": scheduling_set_url,
"requirements_template": template_url,
+ "scheduling_constraints_doc": scheduling_constraints_doc,
+ "scheduling_constraints_template": scheduling_constraints_template_url,
"observation_strategy_template": observation_strategy_template_url,
"scheduling_unit_blueprints": [],
"task_drafts": []}
- def TaskDraft(self, name='my_task_draft', scheduling_unit_draft_url=None, template_url=None):
+ def TaskDraft(self, name='my_task_draft', scheduling_unit_draft_url=None, template_url=None, specifications_doc=None):
if scheduling_unit_draft_url is None:
scheduling_unit_draft_url = self.post_data_and_get_url(self.SchedulingUnitDraft(), '/scheduling_unit_draft/')
if template_url is None:
template_url = self.post_data_and_get_url(self.TaskTemplate(), '/task_template/')
-
+
+ if specifications_doc is None:
+ specifications_doc = self.get_response_as_json_object(template_url+'/default')
+
return {"name": name,
"description": "This is my task draft",
"tags": [],
- "specifications_doc": "{}",
+ "specifications_doc": specifications_doc,
"copy_reason": self.django_api_url + '/copy_reason/template',
"copies": None,
"scheduling_unit_draft": scheduling_unit_draft_url,
@@ -265,16 +302,24 @@ class TMSSRESTTestDataCreator():
'second_to_connect': []}
- def TaskRelationDraft(self, producer_url=None, consumer_url=None, template_url=None, input_role_url=None, output_role_url=None):
+ def TaskRelationDraft(self, producer_url=None, consumer_url=None, template_url=None, input_role_url=None, output_role_url=None, selection_doc=None):
if producer_url is None:
producer_url = self.post_data_and_get_url(self.TaskDraft(), '/task_draft/')
if consumer_url is None:
consumer_url = self.post_data_and_get_url(self.TaskDraft(),'/task_draft/')
-
+
if template_url is None:
- template_url = self.post_data_and_get_url(self.TaskRelationSelectionTemplate(), '/task_relation_selection_template/')
-
+ template_url = self.post_data_and_get_url(self.TaskRelationSelectionTemplate(),
+ '/task_relation_selection_template/')
+
+ if template_url is None:
+ template_url = self.post_data_and_get_url(self.TaskRelationSelectionTemplate(),
+ '/task_relation_selection_template/')
+
+ if selection_doc is None:
+ selection_doc = self.get_response_as_json_object(template_url+'/default')
+
if input_role_url is None:
input_role_url = self.post_data_and_get_url(self.TaskConnectorType(), '/task_connector_type/')
@@ -282,7 +327,7 @@ class TMSSRESTTestDataCreator():
output_role_url = self.post_data_and_get_url(self.TaskConnectorType(), '/task_connector_type/')
return {"tags": [],
- "selection_doc": "{}",
+ "selection_doc": selection_doc,
"dataformat": self.django_api_url + "/dataformat/Beamformed",
"producer": producer_url,
"consumer": consumer_url,
@@ -299,8 +344,7 @@ class TMSSRESTTestDataCreator():
scheduling_unit_draft_url = self.post_data_and_get_url(self.SchedulingUnitDraft(template_url=template_url), '/scheduling_unit_draft/')
if requirements_doc is None:
- scheduling_unit_template = self.get_response_as_json_object(template_url)
- requirements_doc = get_default_json_object_for_schema(scheduling_unit_template['schema'])
+ requirements_doc = self.get_response_as_json_object(template_url+'/default')
return {"name": name,
"description": "This is my run blueprint",
@@ -311,7 +355,7 @@ class TMSSRESTTestDataCreator():
"requirements_template": template_url,
"task_blueprints": []}
- def TaskBlueprint(self, name="my_TaskBlueprint", draft_url=None, template_url=None, scheduling_unit_blueprint_url=None):
+ def TaskBlueprint(self, name="my_TaskBlueprint", draft_url=None, template_url=None, scheduling_unit_blueprint_url=None, specifications_doc=None):
if draft_url is None:
task_draft = self.TaskDraft()
draft_url = self.post_data_and_get_url(task_draft, '/task_draft/')
@@ -319,13 +363,16 @@ class TMSSRESTTestDataCreator():
if template_url is None:
template_url = self.post_data_and_get_url(self.TaskTemplate(), '/task_template/')
+ if specifications_doc is None:
+ specifications_doc = self.get_response_as_json_object(template_url+'/default')
+
if scheduling_unit_blueprint_url is None:
scheduling_unit_blueprint_url = self.post_data_and_get_url(self.SchedulingUnitBlueprint(), '/scheduling_unit_blueprint/')
return {"name": name,
"description": "This is my work request blueprint",
"tags": [],
- "specifications_doc": "{}",
+ "specifications_doc": specifications_doc,
"do_cancel": False,
"draft": draft_url,
"specifications_template": template_url,
@@ -336,7 +383,7 @@ class TMSSRESTTestDataCreator():
'first_to_connect': [],
'second_to_connect': []}
- def TaskRelationBlueprint(self, draft_url=None, template_url=None, input_role_url=None, output_role_url=None, consumer_url=None, producer_url=None):
+ def TaskRelationBlueprint(self, draft_url=None, template_url=None, input_role_url=None, output_role_url=None, consumer_url=None, producer_url=None, selection_doc=None):
if draft_url is None:
draft_url = self.post_data_and_get_url(self.TaskRelationDraft(), '/task_relation_draft/')
@@ -349,6 +396,9 @@ class TMSSRESTTestDataCreator():
if template_url is None:
template_url = self.post_data_and_get_url(self.TaskRelationSelectionTemplate(), '/task_relation_selection_template/')
+ if selection_doc is None:
+ selection_doc = self.get_response_as_json_object(template_url+'/default')
+
if input_role_url is None:
input_role_url = self.post_data_and_get_url(self.TaskConnectorType(), '/task_connector_type/')
@@ -357,7 +407,7 @@ class TMSSRESTTestDataCreator():
# test data
return {"tags": [],
- "selection_doc": "{}",
+ "selection_doc": selection_doc,
"dataformat": self.django_api_url + '/dataformat/MeasurementSet',
"input_role": input_role_url,
"output_role": output_role_url,
@@ -366,12 +416,9 @@ class TMSSRESTTestDataCreator():
"producer": producer_url,
"consumer": consumer_url}
- def SubtaskTemplate(self, name="subtask_template_1", schema=None, subtask_type_url: str=None, version:str=None) -> dict:
- if version is None:
- version = str(uuid.uuid4())
-
+ def SubtaskTemplate(self, name="subtask_template_1", schema=None, subtask_type_url: str=None) -> dict:
if schema is None:
- schema = {}
+ schema = minimal_json_schema()
if subtask_type_url is None:
subtask_type_url = self.django_api_url + '/subtask_type/observation'
@@ -379,7 +426,6 @@ class TMSSRESTTestDataCreator():
return {"type": subtask_type_url,
"name": name,
"description": 'My one observation',
- "version": version,
"schema": schema,
"realtime": True,
"queue": False,
@@ -411,24 +457,16 @@ class TMSSRESTTestDataCreator():
"placement": self.django_api_url + '/scheduling_relation_placement/%s'%placement,
"time_offset":60}
- def DataproductSpecificationsTemplate(self, name="my_DataproductSpecificationsTemplate", version:str=None) -> dict:
- if version is None:
- version = str(uuid.uuid4())
-
+ def DataproductSpecificationsTemplate(self, name="my_DataproductSpecificationsTemplate") -> dict:
return {"name": name,
"description": 'My one date',
- "version": version,
- "schema": {"mykey": "my value"},
+ "schema": minimal_json_schema(),
"tags": ["TMSS", "TESTING"]}
- def DataproductFeedbackTemplate(self, name="my_DataproductFeedbackTemplate", version:str=None) -> dict:
- if version is None:
- version = str(uuid.uuid4())
-
+ def DataproductFeedbackTemplate(self, name="my_DataproductFeedbackTemplate") -> dict:
return {"name": name,
"description": 'My one date',
- "version": version,
- "schema": {"mykey": "my value"},
+ "schema": minimal_json_schema(),
"tags": ["TMSS", "TESTING"]}
def DefaultSubtaskTemplates(self, name=None, template_url=None):
@@ -443,24 +481,36 @@ class TMSSRESTTestDataCreator():
return {"name": name if name else "Cluster %s" % uuid.uuid4(),
"description": 'My one cluster',
"location": "upstairs",
+ "archive_site": False,
"tags": ['tmss', 'testing']}
- def Subtask(self, cluster_url=None, task_blueprint_url=None, specifications_template_url=None, specifications_doc=None, state:str="defining"):
+ def Subtask(self, cluster_url=None, task_blueprint_url=None, specifications_template_url=None, specifications_doc=None, state:str="defining", start_time: datetime=None, stop_time: datetime=None, raw_feedack:str =None):
if cluster_url is None:
cluster_url = self.post_data_and_get_url(self.Cluster(), '/cluster/')
- # if task_blueprint_url is None:
- # task_blueprint = self.TaskBlueprint()
- # task_blueprint_url = self.post_data_and_get_url(task_blueprint, '/task_blueprint/')
+ if task_blueprint_url is None:
+ task_blueprint_url = self.post_data_and_get_url(self.TaskBlueprint(), '/task_blueprint/')
if specifications_template_url is None:
specifications_template_url = self.post_data_and_get_url(self.SubtaskTemplate(), '/subtask_template/')
if specifications_doc is None:
- specifications_doc = requests.get(specifications_template_url + 'default_specification/', auth=self.auth).content.decode('utf-8')
+ specifications_doc = self.get_response_as_json_object(specifications_template_url+'/default')
+
+ if start_time is None:
+ start_time = datetime.utcnow()
+
+ if stop_time is None:
+ stop_time = start_time + timedelta(minutes=60)
+
+ if isinstance(start_time, datetime):
+ start_time = start_time.isoformat()
- return {"start_time": datetime.utcnow().isoformat(),
- "stop_time": datetime.utcnow().isoformat(),
+ if isinstance(stop_time, datetime):
+ stop_time = stop_time.isoformat()
+
+ return {"start_time": start_time,
+ "stop_time": stop_time,
"state": self.django_api_url + '/subtask_state/%s' % (state,),
"specifications_doc": specifications_doc,
"task_blueprint": task_blueprint_url,
@@ -469,7 +519,8 @@ class TMSSRESTTestDataCreator():
"do_cancel": datetime.utcnow().isoformat(),
"priority": 1,
"schedule_method": self.django_api_url + '/schedule_method/manual',
- "cluster": cluster_url}
+ "cluster": cluster_url,
+ "raw_feedback": raw_feedack}
def SubtaskOutput(self, subtask_url=None):
if subtask_url is None:
@@ -479,29 +530,40 @@ class TMSSRESTTestDataCreator():
return {"subtask": subtask_url,
"tags": []}
- def Dataproduct(self, filename="my_filename", directory="/tmp/", specifications_template_url=None, subtask_output_url=None, dataproduct_feedback_template_url=None, dataformat="MeasurementSet"):
+ def Dataproduct(self, filename="my_filename", directory="/tmp/",
+ specifications_doc=None, specifications_template_url=None,
+ subtask_output_url=None,
+ dataproduct_feedback_doc=None, dataproduct_feedback_template_url=None,
+ dataformat="MeasurementSet", datatype="visibilities"):
if specifications_template_url is None:
specifications_template_url = self.post_data_and_get_url(self.SubtaskTemplate(), '/dataproduct_specifications_template/')
+ if specifications_doc is None:
+ specifications_doc = self.get_response_as_json_object(specifications_template_url+'/default')
+
if subtask_output_url is None:
subtask_output_url = self.post_data_and_get_url(self.SubtaskOutput(), '/subtask_output/')
if dataproduct_feedback_template_url is None:
dataproduct_feedback_template_url = self.post_data_and_get_url(self.DataproductFeedbackTemplate(), '/dataproduct_feedback_template/')
-
+
+ if dataproduct_feedback_doc is None:
+ dataproduct_feedback_doc = self.get_response_as_json_object(dataproduct_feedback_template_url+'/default')
+
return {"filename": filename,
"directory": directory,
"dataformat": "%s/dataformat/%s" % (self.django_api_url, dataformat),
+ "datatype": "%s/datatype/%s" % (self.django_api_url, datatype),
"deleted_since": None,
"pinned_since": None,
- "specifications_doc": "{}",
+ "specifications_doc": specifications_doc,
"specifications_template": specifications_template_url,
"tags": ["TMSS", "TESTING"],
"producer": subtask_output_url,
"do_cancel": None,
"expected_size": 1234,
"size": 123,
- "feedback_doc": "{}",
+ "feedback_doc": dataproduct_feedback_doc,
"feedback_template": dataproduct_feedback_template_url
}
@@ -548,7 +610,7 @@ class TMSSRESTTestDataCreator():
"corrupted_since": datetime.utcnow().isoformat(),
"tags": ['tmss', 'testing']}
- def SubtaskInput(self, subtask_url=None, task_relation_blueprint_url=None, dataproduct_urls=None, subtask_output_url=None, task_relation_selection_template_url=None):
+ def SubtaskInput(self, subtask_url=None, task_relation_blueprint_url=None, dataproduct_urls=None, subtask_output_url=None, task_relation_selection_template_url=None, selection_doc=None):
if subtask_url is None:
subtask_url = self.post_data_and_get_url(self.Subtask(), '/subtask/')
@@ -564,12 +626,15 @@ class TMSSRESTTestDataCreator():
if task_relation_selection_template_url is None:
task_relation_selection_template_url = self.post_data_and_get_url(self.TaskRelationSelectionTemplate(), '/task_relation_selection_template/')
-
+
+ if selection_doc is None:
+ selection_doc = self.get_response_as_json_object(task_relation_selection_template_url+'/default')
+
return {"subtask": subtask_url,
"task_relation_blueprint": task_relation_blueprint_url,
"producer": subtask_output_url,
"dataproducts": dataproduct_urls,
- "selection_doc": {},
+ "selection_doc": selection_doc,
"selection_template": task_relation_selection_template_url,
"tags": []}
@@ -581,5 +646,6 @@ class TMSSRESTTestDataCreator():
"description": 'My one filesystem',
"capacity": 1111111111,
"cluster": cluster_url,
+ "directory": '/',
"tags": ['tmss', 'testing']}
diff --git a/SAS/TMSS/test/tmss_test_environment_unittest_setup.py b/SAS/TMSS/test/tmss_test_environment_unittest_setup.py
index 6dc906b94d38860d3bacf11393ea58a5e006a1d4..dc4f72644bf2b40058a6eb6571218f7cf6fd3d89 100644
--- a/SAS/TMSS/test/tmss_test_environment_unittest_setup.py
+++ b/SAS/TMSS/test/tmss_test_environment_unittest_setup.py
@@ -95,8 +95,11 @@ def _call_API_and_assert_expected_response(test_instance, url, call, data, expec
elif isinstance(value, datetime.datetime):
# URL (r_dict[key]) is string but the test_data object (value) is datetime format, convert latter to string format to compare
test_instance.assertEqual(value.isoformat(), r_dict[key])
+ elif isinstance(value, dict):
+ # only look for expected (sub)keys. More key/value pairs in the response dict are allowed.
+ for sub_key, sub_value in value.items():
+ test_instance.assertEqual(sub_value, r_dict[key][sub_key])
else:
-
test_instance.assertEqual(value, r_dict[key])
return r_dict
diff --git a/SAS/XML_generator/src/xmlgen.py b/SAS/XML_generator/src/xmlgen.py
index 0012a42e4514ce172216c438aa1813e237ca41a2..d963cc622b4fdd0004c7ee21f4bd1d2089cd9bad 100755
--- a/SAS/XML_generator/src/xmlgen.py
+++ b/SAS/XML_generator/src/xmlgen.py
@@ -26,7 +26,7 @@
# Last change by : $Author: renting $
# Change date : $Date: 2016-05-18 11:47:57 +0200 (wo, 18 mei 2016) $
-VERSION = "4.0.4"
+VERSION = "4.0.5"
import sys, getopt, time
from xml.sax.saxutils import escape as XMLescape
@@ -51,11 +51,11 @@ IMAGING_PIPELINE_TYPES = ['MSSS', 'standard', 'none']
# MODES = ['Calobs','Calbeam','MultiObs']
PROCESSING = ['Preprocessing', 'Calibration', 'Pulsar', 'Imaging', 'LongBaseline', 'Prefactor', 'PreprocessingNoFlagging', 'none']
CALIBRATION_MODE = ['internal', 'external', 'none']
-ALL_STATIONS = 'CS001,CS002,CS003,CS004,CS005,CS006,CS007,CS011,CS013,CS017,CS021,CS024,CS026,CS028,CS030,CS031,CS032,CS101,CS103,CS201,CS301,CS302,CS401,CS501,RS106,RS205,RS208,RS210,RS305,RS306,RS307,RS310,RS406,RS407,RS409,RS503,RS508,RS509,DE601,DE602,DE603,DE604,DE605,FR606,SE607,UK608,DE609,PL610,PL611,PL612,IE613'
+ALL_STATIONS = 'CS001,CS002,CS003,CS004,CS005,CS006,CS007,CS011,CS013,CS017,CS021,CS024,CS026,CS028,CS030,CS031,CS032,CS101,CS103,CS201,CS301,CS302,CS401,CS501,RS106,RS205,RS208,RS210,RS305,RS306,RS307,RS310,RS406,RS407,RS409,RS503,RS508,RS509,DE601,DE602,DE603,DE604,DE605,FR606,SE607,UK608,DE609,PL610,PL611,PL612,IE613,LV614'
CORE_STATIONS = 'CS001,CS002,CS003,CS004,CS005,CS006,CS007,CS011,CS013,CS017,CS021,CS024,CS026,CS028,CS030,CS031,CS032,CS101,CS103,CS201,CS301,CS302,CS401,CS501'
SUPERTERP_STATIONS = 'CS002,CS003,CS004,CS005,CS006,CS007'
REMOTE_STATIONS = 'RS106,RS205,RS208,RS210,RS305,RS306,RS307,RS310,RS406,RS407,RS409,RS503,RS508,RS509'
-INTERNATIONAL_STATIONS = 'DE601,DE602,DE603,DE604,DE605,FR606,SE607,UK608,DE609,PL610,PL611,PL612,IE613'
+INTERNATIONAL_STATIONS = 'DE601,DE602,DE603,DE604,DE605,FR606,SE607,UK608,DE609,PL610,PL611,PL612,IE613,LV614'
NL_STATIONS = 'CS001,CS002,CS003,CS004,CS005,CS006,CS007,CS011,CS013,CS017,CS021,CS024,CS026,CS028,CS030,CS031,CS032,CS101,CS103,CS201,CS301,CS302,CS401,CS501,RS106,RS205,RS208,RS210,RS305,RS306,RS307,RS310,RS406,RS407,RS409,RS503,RS508,RS509'
DEFAULT_TASKS_PER_NODE = 11
DEFAULT_CORES_PER_TASK = 2
diff --git a/SubSystems/Online_Cobalt/validation/cobalt/network/interface_mtu_settings.test b/SubSystems/Online_Cobalt/validation/cobalt/network/interface_mtu_settings.test
index 792f895ef85738b521f6aacb06e9ea2031f16723..5ad51de7d86b6c3761961abd45e7546bd36c0d6b 100755
--- a/SubSystems/Online_Cobalt/validation/cobalt/network/interface_mtu_settings.test
+++ b/SubSystems/Online_Cobalt/validation/cobalt/network/interface_mtu_settings.test
@@ -10,7 +10,8 @@ for i in {201..213} ; do
# the following interfaces need to be connected
# 10GB03 and 10GB07 are spares and do not need to connected at this moment
for INTERFACE in 10GB01 10GB02 10GB04 10GB05 10GB06 ; do
- RESULT=$(ssh $NODE netstat -i | grep $INTERFACE | awk '{ print $2 }')
+ # Avoid grepping VLANs by not accepting interfaces followed by a . (f.e. 10GB06.2201)
+ RESULT=$(ssh $NODE netstat -i | grep "$INTERFACE[^.]" | awk '{ print $2 }')
if [ $? -eq 0 ] && [ "$RESULT" -eq "9000" ] ; then
echo "$NODE : interface $INTERFACE has correct MTU of $RESULT"
else
diff --git a/SubSystems/Online_Cobalt/validation/cobalt/system/tuned.test b/SubSystems/Online_Cobalt/validation/cobalt/system/tuned.test
index ac26530105bb005bf5fec81b7643017f39bf93cd..97dc42a9dea8d9c06f64afeb56168facbe1b6c8f 100755
--- a/SubSystems/Online_Cobalt/validation/cobalt/system/tuned.test
+++ b/SubSystems/Online_Cobalt/validation/cobalt/system/tuned.test
@@ -3,7 +3,8 @@
# Our own hostname must be resolvable
for i in {201..213} ; do
- ssh cbm$i systemctl status tuned.service | grep "Active: inactive" || exit 1
+ # check for tuned.service to be NOT active, as we then also catch the case in which it is not installed in the first place
+ ssh cbm$i systemctl status tuned.service | grep "Active: active" && exit 1
done
exit 0