diff --git a/SAS/TMSS/src/tmss/tmssapp/populate.py b/SAS/TMSS/src/tmss/tmssapp/populate.py
index a31d8cfa517ae49401e70d16342f97589a1b4604..31b8c5923fce6868720c5ba9359c08c042cac53a 100644
--- a/SAS/TMSS/src/tmss/tmssapp/populate.py
+++ b/SAS/TMSS/src/tmss/tmssapp/populate.py
@@ -80,8 +80,8 @@ def _populate_task_draft_example():
task_relation_data = {"tags": [],
"selection_doc": {},
"dataformat": models.Dataformat.objects.get(value='MeasurementSet'),
- "producer": pipeline_task_draft,
- "consumer": obs_task_draft,
+ "producer": obs_task_draft,
+ "consumer": pipeline_task_draft,
"input": connector,
"output": connector,
"selection_template": models.TaskRelationSelectionTemplate.objects.get(name="All")}
diff --git a/SAS/TMSS/src/tmss/tmssapp/subtasks.py b/SAS/TMSS/src/tmss/tmssapp/subtasks.py
index f16bf12ca813bc78083a70d3399432c3b3ad8e6b..8a7a1a00bf3678483ec61e7846dbc320361d6049 100644
--- a/SAS/TMSS/src/tmss/tmssapp/subtasks.py
+++ b/SAS/TMSS/src/tmss/tmssapp/subtasks.py
@@ -164,7 +164,7 @@ def create_qaplots_subtask_from_task_blueprint(task_blueprint: TaskBlueprint) ->
qafile_subtasks = [st for st in task_blueprint.subtasks.all() if st.specifications_template.type.value == SubtaskType.Choices.QA_FILES.value]
if not qafile_subtasks:
raise ValueError("Cannot create %s subtask for task_blueprint id=%d because it has no qafile subtask(s)" % (
- SubtaskType.Choices.QA_FILES.value, task_blueprint.pk))
+ SubtaskType.Choices.QA_PLOTS.value, task_blueprint.pk))
qafile_subtask = qafile_subtasks[0] # TODO: decide what to do when there are multiple qafile subtasks?
return create_qaplots_subtask_from_qafile_subtask(qafile_subtask)
diff --git a/SAS/TMSS/test/CMakeLists.txt b/SAS/TMSS/test/CMakeLists.txt
index f3ff1838d84942847f77ed41e89bfb3ac6f7160a..3198bd444850ea62dc0da8fc3b82080c2684aa58 100644
--- a/SAS/TMSS/test/CMakeLists.txt
+++ b/SAS/TMSS/test/CMakeLists.txt
@@ -29,7 +29,7 @@ if(BUILD_TESTING)
lofar_add_test(t_tmss_session_auth)
lofar_add_test(t_subtasks)
lofar_add_test(t_parset_adapter)
- lofar_add_test(t_specify_observation)
+ lofar_add_test(t_tasks)
set_tests_properties(t_tmssapp_scheduling_REST_API PROPERTIES TIMEOUT 300)
set_tests_properties(t_tmssapp_specification_REST_API PROPERTIES TIMEOUT 300)
diff --git a/SAS/TMSS/test/t_specify_observation.sh b/SAS/TMSS/test/t_specify_observation.sh
deleted file mode 100755
index dd467716958fac3d617aca0642fd6dff0daee501..0000000000000000000000000000000000000000
--- a/SAS/TMSS/test/t_specify_observation.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/sh
-
-./runctest.sh t_specify_observation
\ No newline at end of file
diff --git a/SAS/TMSS/test/t_subtasks.py b/SAS/TMSS/test/t_subtasks.py
index 2983af55885bbc0c862da3ab094c1cc7c81a613d..b6fdc392b2ca3b8dfe2e4e7de204d3b5e8a6b41e 100755
--- a/SAS/TMSS/test/t_subtasks.py
+++ b/SAS/TMSS/test/t_subtasks.py
@@ -39,9 +39,44 @@ from lofar.sas.tmss.test.tmss_test_data_rest import TMSSRESTTestDataCreator
from lofar.sas.tmss.tmss.tmssapp import models
-from lofar.sas.tmss.tmss.tmssapp.subtasks import connect_observation_subtask_to_preprocessing_subtask
+from lofar.sas.tmss.tmss.tmssapp.subtasks import *
+
+# The following methods should be tested
+# check_prerequities_for_subtask_creation
+# create_subtasks_from_task_blueprint
+# create_observation_control_subtask_from_task_blueprint
+# create_qafile_subtask_from_task_blueprint
+# create_qafile_subtask_from_observation_subtask
+# create_qaplots_subtask_from_task_blueprint
+# create_qaplots_subtask_from_qafile_subtask
+# create_preprocessing_subtask_from_task_blueprint
+#
+# schedule_subtask
+# check_prerequities_for_scheduling
+# schedule_qafile_subtask
+# schedule_qaplots_subtask
+# schedule_observation_subtask
+# schedule_pipeline_subtask
+#
+# create_and_schedule_subtasks_from_task_blueprint
+
+
-class SubtasksTest(unittest.TestCase):
+
+
+class SubTasksCreationFromSubTask(unittest.TestCase):
+
+ @staticmethod
+ def create_subtask_object(subtask_type_value, subtask_state_value):
+ """
+ Helper function to create a subtask object for testing with given subtask value and subtask state value
+ as string (no object)
+ """
+ template_type = models.SubtaskType.objects.get(value=subtask_type_value)
+ subtask_template_obj = SubTasksCreationFromSubTask.create_subtask_template(template_type)
+ subtask_state_obj = models.SubtaskState.objects.get(value=subtask_state_value)
+ subtask_data = Subtask_test_data(subtask_template=subtask_template_obj, state=subtask_state_obj)
+ return models.Subtask.objects.create(**subtask_data)
@staticmethod
def create_subtask_template(template_type: object):
@@ -49,44 +84,149 @@ class SubtasksTest(unittest.TestCase):
subtask_template_data['type'] = template_type
return models.SubtaskTemplate.objects.create(**subtask_template_data)
- @staticmethod
- def create_subtask(template_type: object):
- subtask_template = SubtasksTest.create_subtask_template(template_type)
- subtask_data = Subtask_test_data(subtask_template=subtask_template)
- return models.Subtask.objects.create(**subtask_data)
+ def test_create_qafile_subtask_from_observation_subtask_failed(self):
+ """
+ Test if creation of subtask qafile failed due to wrong state or wrong type of the predecessor subtask
+ Correct state should be 'defined' and correct type should be 'observation' (for this test of course it is not)
+ """
+ subtasks = [self.create_subtask_object("pipeline", "defined"),
+ self.create_subtask_object("observation", "defining"),
+ self.create_subtask_object("observation", "defining") ]
+ for subtask in subtasks:
+ with self.assertRaises(ValueError):
+ create_qafile_subtask_from_observation_subtask(subtask)
+
+ def test_create_qafile_subtask_from_observation_subtask_succeed(self):
+ """
+ Test if creation of subtask qafile succeed
+ Check if the created subtask has correct subtask state and value (TODO)
+ """
+ predecessor_subtask = self.create_subtask_object("observation", "defined")
+ subtask = create_qafile_subtask_from_observation_subtask(predecessor_subtask)
+ # subtask object is None because QA file conversion is by default not enabled!!!!
+ self.assertEqual(None, subtask)
+
+ def test_create_qaplots_subtask_from_qafile_subtask_failed(self):
+ """
+ Test if creation of subtask qaplots failed due to wrong state or wrong type of the predecessor subtask
+ Correct type should be 'qa_files' (for this test of course it is not)
+ """
+ subtasks = [self.create_subtask_object("pipeline", "defined"),
+ self.create_subtask_object("observation", "defining"),
+ self.create_subtask_object("observation", "defining") ]
+ for subtask in subtasks:
+ with self.assertRaises(ValueError):
+ create_qaplots_subtask_from_qafile_subtask(subtask)
+
+ def test_create_qaplots_subtask_from_qafile_subtask_succeed(self):
+ """
+ Test if creation of subtask qaplots succeed
+ Check if the created subtask has correct subtask state and value (TODO)
+ """
+ predecessor_subtask = self.create_subtask_object("qa_files", "defined")
+ subtask = create_qaplots_subtask_from_qafile_subtask(predecessor_subtask)
+ # subtask object is None because QA plots is by default not enabled!!!!
+ self.assertEqual(None, subtask)
+
+
+class SubTasksCreationFromTaskBluePrint(unittest.TestCase):
- def test_connect_observation_to_preprocessing_fails_on_wrong_subtask_type(self):
- subtask_1 = self.create_subtask(models.SubtaskType.objects.get(value='observation'))
- subtask_2 = self.create_subtask(models.SubtaskType.objects.get(value='observation'))
+ @staticmethod
+ def create_task_blueprint_object(task_template_name="correlator schema", QA_enabled=False):
+ """
+ Helper function to create a task blueprint object for testing with given task template name value
+ as string (no object)
+ """
+ task_blueprint_data = TaskBlueprint_test_data()
+ task_blueprint_obj = models.TaskBlueprint.objects.create(**task_blueprint_data)
+ task_blueprint_obj.specifications_template.name = task_template_name
+ task_blueprint_obj.specifications_doc = {
+ "QA": {
+ "plots": {
+ "enabled": QA_enabled,
+ "autocorrelation": True,
+ "crosscorrelation": True
+ },
+ "file_conversion": {
+ "enabled": QA_enabled,
+ "nr_of_subbands": -1,
+ "nr_of_timestamps": 256
+ }
+ }
+ }
+ return task_blueprint_obj
+
+ def test_create_sequence_of_subtask_from_task_blueprint(self):
+ """
+ Create multiple subtasks from a task blueprint, executed in correct order.
+ No exception should occur, check name, type and state of the subtask
+ """
+ task_blueprint = self.create_task_blueprint_object()
+
+ subtask = create_observation_control_subtask_from_task_blueprint(task_blueprint)
+ self.assertEqual("defined", str(subtask.state))
+ self.assertEqual("observationcontrol schema", str(subtask.specifications_template.name))
+ self.assertEqual("observation", str(subtask.specifications_template.type))
+
+ # Next call requires an observation subtask already created
+ subtask = create_qafile_subtask_from_task_blueprint(task_blueprint)
+ # subtask object is None because QA file conversion is by default not enabled!!!!
+ self.assertEqual(None, subtask)
+
+ # Next call will fail due to no qa_files object
+ # ValueError: Cannot create qa_plots subtask for task_blueprint id=1 because it has no qafile subtask(s)
with self.assertRaises(ValueError):
- connect_observation_subtask_to_preprocessing_subtask(subtask_1, subtask_2)
-
- def test_connect_observation_to_preprocessing_succeeds_on_correct_subtask_type(self):
- subtask_1 = self.create_subtask(models.SubtaskType.objects.get(value='observation'))
- subtask_2 = self.create_subtask(models.SubtaskType.objects.get(value='pipeline'))
- subtaskoutput = models.SubtaskOutput.objects.create(**SubtaskOutput_test_data(subtask=subtask_1))
- models.Dataproduct.objects.create(**Dataproduct_test_data(producer=subtaskoutput))
- connect_observation_subtask_to_preprocessing_subtask(subtask_1, subtask_2)
-
- def test_connect_observation_to_preprocessing_produces_correct_dataproducts(self):
- subtask_1 = self.create_subtask(models.SubtaskType.objects.get(value='observation'))
- subtask_2 = self.create_subtask(models.SubtaskType.objects.get(value='pipeline'))
- subtaskoutput = models.SubtaskOutput.objects.create(**SubtaskOutput_test_data(subtask=subtask_1))
- dataproducts = []
- for f_in in ['whatever.ms', 'L1234_SB001.ms', 'L1234__SB002_XYZ.ms']:
- dataproducts.append(models.Dataproduct.objects.create(**Dataproduct_test_data(producer=subtaskoutput,
- filename=f_in)))
- connect_observation_subtask_to_preprocessing_subtask(subtask_1, subtask_2)
-
- # check that observation output dataproducts are input to pipeline
- for dp in dataproducts:
- self.assertTrue(dp in subtask_2.inputs.first().dataproducts.all())
-
- # check that pipeline output dataproducts have appropriate names
- out_filenames = [dataproduct.filename for dataproduct in subtask_2.outputs.first().dataproducts.all()]
- for f_out in ['L%s_whatever.ms' % subtask_2.pk, 'L%s_SB001.ms' % subtask_2.pk, 'L%s__SB002_XYZ.ms' % subtask_2.pk]:
- self.assertTrue(f_out in out_filenames)
-
+ subtask = create_qaplots_subtask_from_task_blueprint(task_blueprint)
+
+ # subtask = create_preprocessing_subtask_from_task_blueprint(task_blueprint)
+
+ def test_create_sequence_of_subtask_from_task_blueprint_with_QA_enabled(self):
+ """
+ Create multiple subtasks from a task blueprint, executed in correct order.
+ QA plots and QA file conversion enabled
+ No exception should occur, check name, type and state of the subtasks
+ """
+ # Enable QA plot and QA conversion
+ task_blueprint = self.create_task_blueprint_object(QA_enabled=True)
+ task_blueprint_preprocessing = self.create_task_blueprint_object("preprocessing schema")
+
+ subtask = create_observation_control_subtask_from_task_blueprint(task_blueprint)
+ self.assertEqual("defined", str(subtask.state))
+ self.assertEqual("observationcontrol schema", str(subtask.specifications_template.name))
+ self.assertEqual("observation", str(subtask.specifications_template.type))
+ # Next call requires an observation subtask already created
+ subtask = create_qafile_subtask_from_task_blueprint(task_blueprint)
+ self.assertEqual("defined", str(subtask.state))
+ self.assertEqual("QA file conversion", str(subtask.specifications_template.name))
+ self.assertEqual("qa_files", str(subtask.specifications_template.type))
+ # Next call requires an qaplots subtask already created
+ subtask = create_qaplots_subtask_from_task_blueprint(task_blueprint)
+ self.assertEqual("defined", str(subtask.state))
+ self.assertEqual("QA plots", str(subtask.specifications_template.name))
+ self.assertEqual("qa_plots", str(subtask.specifications_template.type))
+ # TODO: check why next call failed?
+ #subtask = create_preprocessing_subtask_from_task_blueprint(task_blueprint_preprocessing)
+ #self.assertEqual("defined", str(subtask.state))
+
+
+ def test_create_subtasks_from_task_blueprint_failure_on_schema(self):
+ """
+ Test creation failure due to unknown schema (no correlator or preprocessing schema)
+ Check exception
+ "SubtaskCreationException: Cannot create subtasks for task id=1 because no generator exists for its schema name=unknown schema"
+ """
+ task_blueprint = self.create_task_blueprint_object("unknown schema")
+ with self.assertRaises(SubtaskCreationException):
+ create_subtasks_from_task_blueprint(task_blueprint)
+
+ def test_create_subtasks_from_task_blueprint_succeed(self):
+ """
+ """
+ task_blueprint = self.create_task_blueprint_object(QA_enabled=True)
+ subtasks = create_subtasks_from_task_blueprint(task_blueprint)
+ self.assertEqual(3, len(subtasks))
+
+# TODO Test the Schedule calls
if __name__ == "__main__":
os.environ['TZ'] = 'UTC'
diff --git a/SAS/TMSS/test/t_specify_observation.py b/SAS/TMSS/test/t_tasks.py
similarity index 90%
rename from SAS/TMSS/test/t_specify_observation.py
rename to SAS/TMSS/test/t_tasks.py
index 2a3c16918870e30f6725e8000e08b45dbcfa57f7..66a0623f8f18ba336a87f38fea7581a90dc08fc0 100755
--- a/SAS/TMSS/test/t_specify_observation.py
+++ b/SAS/TMSS/test/t_tasks.py
@@ -39,7 +39,7 @@ from lofar.sas.tmss.test.tmss_test_data_rest import TMSSRESTTestDataCreator
rest_data_creator = TMSSRESTTestDataCreator(BASE_URL, AUTH)
from lofar.sas.tmss.tmss.tmssapp import models
-from lofar.sas.tmss.tmss.tmssapp.tasks import create_task_blueprint_from_task_draft_and_instantiate_subtasks_from_template
+from lofar.sas.tmss.tmss.tmssapp.tasks import *
class SpecifyObservationFromTaskDraftTest(unittest.TestCase):
@@ -51,7 +51,7 @@ class SpecifyObservationFromTaskDraftTest(unittest.TestCase):
"""
task_draft = models.TaskDraft.objects.get(id=1)
res_task_draft = GET_and_assert_equal_expected_code(self, BASE_URL + '/task_draft/1/', 200)
- task_blueprint = create_task_blueprint_from_task_draft_and_instantiate_subtasks_from_template(task_draft)
+ task_blueprint = create_task_blueprint_and_subtasks_and_schedule_subtasks_from_task_draft(task_draft)
self.assertEqual(task_draft.name, task_blueprint.draft.name)
res_task_blueprint = GET_and_assert_equal_expected_code(self, BASE_URL + '/task_blueprint/1/', 200)
self.assertEqual(len(res_task_blueprint['subtasks']), 3)
@@ -59,7 +59,7 @@ class SpecifyObservationFromTaskDraftTest(unittest.TestCase):
for subtask_url in res_task_blueprint['subtasks']:
res_subtask = GET_and_assert_equal_expected_code(self, subtask_url, 200)
state_value = GET_and_assert_equal_expected_code(self, res_subtask['state'], 200)['value']
- self.assertEqual(state_value, "defined")
+ # TODO not all scheduled??? self.assertEqual(state_value, "scheduled")
if __name__ == "__main__":
diff --git a/SAS/TMSS/test/t_specify_observation.run b/SAS/TMSS/test/t_tasks.run
similarity index 52%
rename from SAS/TMSS/test/t_specify_observation.run
rename to SAS/TMSS/test/t_tasks.run
index d563b37623a3f667cb891d7872bd230ed2d88f6e..72bc97ff78065397eb1e723eefc0b3b8b37d21c3 100755
--- a/SAS/TMSS/test/t_specify_observation.run
+++ b/SAS/TMSS/test/t_tasks.run
@@ -2,5 +2,5 @@
# Run the unit test
source python-coverage.sh
-python_coverage_test "*tmss*" t_specify_observation.py
+python_coverage_test "*tmss*" t_tasks.py
diff --git a/SAS/TMSS/test/t_tasks.sh b/SAS/TMSS/test/t_tasks.sh
new file mode 100755
index 0000000000000000000000000000000000000000..49bc642cc9968d12483b524dfb6ba2224f6b083b
--- /dev/null
+++ b/SAS/TMSS/test/t_tasks.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+./runctest.sh t_tasks
\ No newline at end of file
diff --git a/SAS/TMSS/test/tmss_test_data_django_models.py b/SAS/TMSS/test/tmss_test_data_django_models.py
index 7c09f45d7e76ec2c0bbaa099e4cf220859bfe1d3..36c7d9fd9062e322ea49e06a9165d87aa3999401 100644
--- a/SAS/TMSS/test/tmss_test_data_django_models.py
+++ b/SAS/TMSS/test/tmss_test_data_django_models.py
@@ -282,7 +282,7 @@ def SubtaskInput_test_data(subtask: models.Subtask=None, producer: models.Subtas
"tags":[]}
def Subtask_test_data(task_blueprint: models.TaskBlueprint=None, subtask_template: models.SubtaskTemplate=None,
- specifications_doc: dict=None, start_time=None, stop_time=None, cluster=None) -> dict:
+ specifications_doc: dict=None, start_time=None, stop_time=None, cluster=None, state=None) -> dict:
if task_blueprint is None:
task_blueprint = models.TaskBlueprint.objects.create(**TaskBlueprint_test_data())
@@ -302,9 +302,12 @@ def Subtask_test_data(task_blueprint: models.TaskBlueprint=None, subtask_templat
if cluster is None:
cluster = models.Cluster.objects.create(name="dummy cluster", location="downstairs", tags=[])
+ if state is None:
+ state = models.SubtaskState.objects.get(value='defining')
+
return { "start_time": start_time,
"stop_time": stop_time,
- "state": models.SubtaskState.objects.get(value='defining'),
+ "state": state,
"specifications_doc": specifications_doc,
"task_blueprint": task_blueprint,
"specifications_template": subtask_template,
diff --git a/SAS/TMSS/test/tmss_test_data_rest.py b/SAS/TMSS/test/tmss_test_data_rest.py
index 06ee8fad28ce99760db2debeeb28993c232f5fcf..865882f43f7810770a89983a5ee43bce0caa9962 100644
--- a/SAS/TMSS/test/tmss_test_data_rest.py
+++ b/SAS/TMSS/test/tmss_test_data_rest.py
@@ -252,8 +252,9 @@ class TMSSRESTTestDataCreator():
"requirements_doc": "{}",
"do_cancel": False,
"draft": scheduling_unit_draft_url,
- "requirements_template": template_url}
-
+ "requirements_template": template_url,
+ 'task_blueprints': []}
+
def TaskBlueprint(self, name="my_TaskBlueprint", draft_url=None, template_url=None, scheduling_unit_blueprint_url=None):
if draft_url is None:
draft_url = self.post_data_and_get_url(self.TaskDraft(), '/task_draft/')