diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/migrations/0001_initial.py b/SAS/TMSS/backend/src/tmss/tmssapp/migrations/0001_initial.py index 2df67c9b37f610d3abe48ac11aeaa440843794f5..e3df916b8f9d6cb69c3e4c4d442f52d5e178c2a5 100644 --- a/SAS/TMSS/backend/src/tmss/tmssapp/migrations/0001_initial.py +++ b/SAS/TMSS/backend/src/tmss/tmssapp/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 3.0.9 on 2021-02-22 09:32 +# Generated by Django 3.0.9 on 2021-02-25 14:10 from django.conf import settings import django.contrib.postgres.fields @@ -1107,7 +1107,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='subtask', name='task_blueprint', - field=models.ForeignKey(help_text='Task Blueprint to which this Subtask belongs.', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='subtasks', to='tmssapp.TaskBlueprint'), + field=models.ManyToManyField(help_text='Task Blueprint to which this Subtask belongs.', null=True, related_name='subtasks', to='tmssapp.TaskBlueprint'), ), migrations.AddConstraint( model_name='schedulingunittemplate', @@ -1151,7 +1151,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='schedulingunitblueprint', name='draft', - field=models.ForeignKey(help_text='Scheduling Unit Draft which this run instantiates.', on_delete=django.db.models.deletion.PROTECT, related_name='scheduling_unit_blueprints', to='tmssapp.SchedulingUnitDraft'), + field=models.ForeignKey(help_text='Scheduling Unit Draft which this run instantiates.', on_delete=django.db.models.deletion.CASCADE, related_name='scheduling_unit_blueprints', to='tmssapp.SchedulingUnitDraft'), ), migrations.AddField( model_name='schedulingunitblueprint', diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/models/scheduling.py b/SAS/TMSS/backend/src/tmss/tmssapp/models/scheduling.py index 9535b3c3d9732a2ae062b3186717697a5f853cd4..5d651ab5ebf0e0be93898d158eb1201bf02c3b11 100644 --- a/SAS/TMSS/backend/src/tmss/tmssapp/models/scheduling.py +++ b/SAS/TMSS/backend/src/tmss/tmssapp/models/scheduling.py @@ -147,7 +147,7 @@ class Subtask(BasicCommon): stop_time = DateTimeField(null=True, help_text='Stop this subtask at the specified time (NULLable).') state = ForeignKey('SubtaskState', null=False, on_delete=PROTECT, related_name='task_states', help_text='Subtask state (see Subtask State Machine).') specifications_doc = JSONField(help_text='Final specifications, as input for the controller.') - task_blueprint = ForeignKey('TaskBlueprint', related_name='subtasks', null=True, on_delete=SET_NULL, help_text='Task Blueprint to which this Subtask belongs.') + task_blueprints = ManyToManyField('TaskBlueprint', related_name='subtasks', null=True, help_text='Task Blueprint to which this Subtask belongs.') specifications_template = ForeignKey('SubtaskTemplate', null=False, on_delete=PROTECT, help_text='Schema used for specifications_doc.') do_cancel = DateTimeField(null=True, help_text='Timestamp when the subtask has been ordered to cancel (NULLable).') cluster = ForeignKey('Cluster', null=True, on_delete=PROTECT, help_text='Where the Subtask is scheduled to run (NULLable).') @@ -340,6 +340,7 @@ class SubtaskInput(BasicCommon): class SubtaskOutput(BasicCommon): subtask = ForeignKey('Subtask', null=False, on_delete=CASCADE, related_name='outputs', help_text='Subtask to which this output specification refers.') + task_blueprint = ForeignKey('TaskBlueprint', null=False, on_delete=CASCADE, related_name='outputs', help_text='Task to which this output specification refers.') class SAP(BasicCommon): diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/subtasks.py b/SAS/TMSS/backend/src/tmss/tmssapp/subtasks.py index 2e62394fc92e667ddb385b580c6a2ba879a0d941..a242101c40860dbf1a76f7af4479ca5c9d856ff3 100644 --- a/SAS/TMSS/backend/src/tmss/tmssapp/subtasks.py +++ b/SAS/TMSS/backend/src/tmss/tmssapp/subtasks.py @@ -253,16 +253,18 @@ def create_observation_control_subtask_from_task_blueprint(task_blueprint: TaskB "stop_time": None, "state": SubtaskState.objects.get(value=SubtaskState.Choices.DEFINING.value), "specifications_doc": specifications_doc, - "task_blueprint": task_blueprint, + #"task_blueprint": task_blueprint, # ManyToMany, so use set()! "specifications_template": subtask_template, "tags": [], "cluster": Cluster.objects.get(name=cluster_name) } subtask = Subtask.objects.create(**subtask_data) + subtask.task_blueprints.set([task_blueprint]) # step 2: create and link subtask input/output # an observation has no input, it just produces output data - subtask_output = SubtaskOutput.objects.create(subtask=subtask) + subtask_output = SubtaskOutput.objects.create(subtask=subtask, + task_blueprint=task_blueprint) # step 3: set state to DEFINED subtask.state = SubtaskState.objects.get(value=SubtaskState.Choices.DEFINED.value) @@ -286,7 +288,8 @@ def create_qafile_subtask_from_observation_subtask(observation_subtask: Subtask) https://support.astron.nl/confluence/display/TMSS/Specification+Flow ''' # step 0: check pre-requisites - check_prerequities_for_subtask_creation(observation_subtask.task_blueprint) + for tb in observation_subtask.task_blueprints.all(): + check_prerequities_for_subtask_creation(tb) if observation_subtask.specifications_template.type.value != SubtaskType.Choices.OBSERVATION.value: raise ValueError("Cannot create %s subtask for subtask id=%d type=%s because it is not an %s" % ( @@ -314,11 +317,12 @@ def create_qafile_subtask_from_observation_subtask(observation_subtask: Subtask) qafile_subtask_data = { "start_time": None, "stop_time": None, "state": SubtaskState.objects.get(value=SubtaskState.Choices.DEFINING.value), - "task_blueprint": observation_subtask.task_blueprint, + #"task_blueprint": observation_subtask.task_blueprint, # ManyToMany, use set() "specifications_template": qafile_subtask_template, "specifications_doc": qafile_subtask_spec, "cluster": observation_subtask.cluster} qafile_subtask = Subtask.objects.create(**qafile_subtask_data) + qafile_subtask.task_blueprints.set(observation_subtask.task_blueprints.all()) # step 2: create and link subtask input/output selection_template = TaskRelationSelectionTemplate.objects.get(name="all") @@ -327,7 +331,8 @@ def create_qafile_subtask_from_observation_subtask(observation_subtask: Subtask) producer=observation_subtask.outputs.first(), # TODO: determine proper producer based on spec in task_relation_blueprint selection_doc=selection_doc, selection_template=selection_template) - qafile_subtask_output = SubtaskOutput.objects.create(subtask=qafile_subtask) + qafile_subtask_output = SubtaskOutput.objects.create(subtask=qafile_subtask, + task_blueprint=observation_subtask.task_blueprints.first()) # todo: first? # step 3: set state to DEFINED qafile_subtask.state = SubtaskState.objects.get(value=SubtaskState.Choices.DEFINED.value) @@ -352,7 +357,8 @@ def create_qaplots_subtask_from_qafile_subtask(qafile_subtask: Subtask) -> Subta https://support.astron.nl/confluence/display/TMSS/Specification+Flow ''' # step 0: check pre-requisites - check_prerequities_for_subtask_creation(qafile_subtask.task_blueprint) + for tb in qafile_subtask.task_blueprints.all(): + check_prerequities_for_subtask_creation(tb) if qafile_subtask.specifications_template.type.value != SubtaskType.Choices.QA_FILES.value: raise ValueError("Cannot create %s subtask for subtask id=%d type=%s because it is not an %s" % ( @@ -376,11 +382,12 @@ def create_qaplots_subtask_from_qafile_subtask(qafile_subtask: Subtask) -> Subta qaplots_subtask_data = { "start_time": None, "stop_time": None, "state": SubtaskState.objects.get(value=SubtaskState.Choices.DEFINING.value), - "task_blueprint": qafile_subtask.task_blueprint, + #"task_blueprint": qafile_subtask.task_blueprint, "specifications_template": qaplots_subtask_template, "specifications_doc": qaplots_subtask_spec_doc, "cluster": qafile_subtask.cluster} qaplots_subtask = Subtask.objects.create(**qaplots_subtask_data) + qaplots_subtask.task_blueprints.set(qafile_subtask.task_blueprints.all()) # step 2: create and link subtask input/output selection_template = TaskRelationSelectionTemplate.objects.get(name="all") @@ -389,7 +396,8 @@ def create_qaplots_subtask_from_qafile_subtask(qafile_subtask: Subtask) -> Subta producer=qafile_subtask.outputs.first(), selection_doc=selection_doc, selection_template=selection_template) - qaplots_subtask_output = SubtaskOutput.objects.create(subtask=qaplots_subtask) + qaplots_subtask_output = SubtaskOutput.objects.create(subtask=qaplots_subtask, + task_blueprint=qafile_subtask.task_blueprints.first()) # todo: first? # step 3: set state to DEFINED qaplots_subtask.state = SubtaskState.objects.get(value=SubtaskState.Choices.DEFINED.value) @@ -421,11 +429,12 @@ def create_preprocessing_subtask_from_task_blueprint(task_blueprint: TaskBluepri subtask_data = { "start_time": None, "stop_time": None, "state": SubtaskState.objects.get(value=SubtaskState.Choices.DEFINING.value), - "task_blueprint": task_blueprint, + #"task_blueprint": task_blueprint, # ManyToMany, so use set()! "specifications_template": subtask_template, "specifications_doc": subtask_specs, "cluster": Cluster.objects.get(name=cluster_name) } subtask = Subtask.objects.create(**subtask_data) + subtask.task_blueprints.set([task_blueprint]) # step 2: create and link subtask input/output for task_relation_blueprint in task_blueprint.produced_by.all(): @@ -439,7 +448,8 @@ def create_preprocessing_subtask_from_task_blueprint(task_blueprint: TaskBluepri producer=predecessor_subtask_output, selection_doc=task_relation_blueprint.selection_doc, selection_template=task_relation_blueprint.selection_template) - subtask_output = SubtaskOutput.objects.create(subtask=subtask) + subtask_output = SubtaskOutput.objects.create(subtask=subtask, + task_blueprint=task_blueprint) # step 3: set state to DEFINED subtask.state = SubtaskState.objects.get(value=SubtaskState.Choices.DEFINED.value) @@ -465,11 +475,12 @@ def create_ingest_subtask_from_task_blueprint(task_blueprint: TaskBlueprint) -> subtask_data = {"start_time": None, "stop_time": None, "state": SubtaskState.objects.get(value=SubtaskState.Choices.DEFINING.value), - "task_blueprint": task_blueprint, + #"task_blueprint": task_blueprint, # ManyToMany, so use set()! "specifications_template": subtask_template, "specifications_doc": subtask_specs, "cluster": Cluster.objects.get(name=cluster_name)} subtask = Subtask.objects.create(**subtask_data) + subtask.task_blueprints.set([task_blueprint]) # step 2: create and link subtask input for task_relation_blueprint in task_blueprint.produced_by.all(): @@ -770,17 +781,18 @@ def get_station_groups(subtask): :return: station_groups which is a list of dict. { station_list, max_nr_missing } """ station_groups = [] - if 'calibrator' in subtask.task_blueprint.specifications_template.name.lower(): - # Calibrator requires related Target Task Observation for some specifications - target_task_blueprint = get_related_target_observation_task_blueprint(subtask.task_blueprint) - if target_task_blueprint is None: - raise SubtaskException("Cannot retrieve related target observation of task_blueprint %d (subtask %d)" % - (subtask.task_blueprint.id, subtask.id)) - if "station_groups" in target_task_blueprint.specifications_doc.keys(): - station_groups = target_task_blueprint.specifications_doc["station_groups"] - else: - if "station_groups" in subtask.task_blueprint.specifications_doc.keys(): - station_groups = subtask.task_blueprint.specifications_doc["station_groups"] + for task_blueprint in subtask.task_blueprints.all(): + if 'calibrator' in task_blueprint.specifications_template.name.lower(): + # Calibrator requires related Target Task Observation for some specifications + target_task_blueprint = get_related_target_observation_task_blueprint(task_blueprint) + if target_task_blueprint is None: + raise SubtaskException("Cannot retrieve related target observation of task_blueprint %d (subtask %d)" % + (task_blueprint.id, subtask.id)) + if "station_groups" in target_task_blueprint.specifications_doc.keys(): + station_groups = target_task_blueprint.specifications_doc["station_groups"] + else: + if "station_groups" in task_blueprint.specifications_doc.keys(): + station_groups = task_blueprint.specifications_doc["station_groups"] return station_groups @@ -1164,7 +1176,8 @@ def schedule_ingest_subtask(ingest_subtask: Subtask): ingest_subtask_input.dataproducts.set(input_dataproducts) # define output and create output dataproducts. - ingest_subtask_output = SubtaskOutput.objects.create(subtask=ingest_subtask) + ingest_subtask_output = SubtaskOutput.objects.create(subtask=ingest_subtask, + task_blueprint=ingest_subtask.task_blueprints.first()) # todo: first? # prepare identifiers in bulk for each output_dataproduct dp_gids = [SIPidentifier(source="TMSS") for _ in input_dataproducts] @@ -1370,14 +1383,15 @@ def get_observation_task_specification_with_check_for_calibrator(subtask): :param: subtask object :return: task_spec: the specifications_doc of the blue print task which is allways a target observation """ - if 'calibrator' in subtask.task_blueprint.specifications_template.name.lower(): - # Calibrator requires related Target Task Observation for some specifications - target_task_blueprint = get_related_target_observation_task_blueprint(subtask.task_blueprint) - if target_task_blueprint is None: - raise SubtaskCreationException("Cannot retrieve specifications for subtask id=%d because no related target observation is found " % subtask.pk) - task_spec = target_task_blueprint.specifications_doc - logger.info("Using specifications for calibrator observation (id=%s) from target observation task_blueprint id=%s", - subtask.task_blueprint.id, target_task_blueprint.id) - else: - task_spec = subtask.task_blueprint.specifications_doc - return task_spec + for task_blueprint in subtask.task_blueprints.all(): + if 'calibrator' in task_blueprint.specifications_template.name.lower(): + # Calibrator requires related Target Task Observation for some specifications + target_task_blueprint = get_related_target_observation_task_blueprint(task_blueprint) + if target_task_blueprint is None: + raise SubtaskCreationException("Cannot retrieve specifications for subtask id=%d because no related target observation is found " % subtask.pk) + task_spec = target_task_blueprint.specifications_doc + logger.info("Using specifications for calibrator observation (id=%s) from target observation task_blueprint id=%s", + task_blueprint.id, target_task_blueprint.id) + else: + task_spec = task_blueprint.specifications_doc + return task_spec diff --git a/SAS/TMSS/backend/test/t_tasks.py b/SAS/TMSS/backend/test/t_tasks.py index 1ecf416c17a35c8cb36a7bba2006e25c6490d328..1a07a4fcbb3bfb9f7423988e7cd0147f4ccb0c3f 100755 --- a/SAS/TMSS/backend/test/t_tasks.py +++ b/SAS/TMSS/backend/test/t_tasks.py @@ -262,9 +262,10 @@ class TaskBlueprintStateTest(unittest.TestCase): task_blueprint_data = TaskBlueprint_test_data(name="Task Blueprint With One Subtask") task_blueprint = models.TaskBlueprint.objects.create(**task_blueprint_data) # Create pipeline subtask related to taskblueprint - subtask_data = Subtask_test_data(task_blueprint, state=models.SubtaskState.objects.get(value="defined"), + subtask_data = Subtask_test_data(state=models.SubtaskState.objects.get(value="defined"), subtask_template=models.SubtaskTemplate.objects.get(name='pipeline control')) subtask_pipe = models.Subtask.objects.create(**subtask_data) + subtask_pipe.task_blueprints.set([task_blueprint]) # Do the actual test for test_item in test_table: @@ -331,12 +332,14 @@ class TaskBlueprintStateTest(unittest.TestCase): task_blueprint_data = TaskBlueprint_test_data(name="Task Blueprint With Subtasks") task_blueprint = models.TaskBlueprint.objects.create(**task_blueprint_data) # Create observation and qa subtask related to taskblueprint - subtask_data = Subtask_test_data(task_blueprint, state=models.SubtaskState.objects.get(value="defined"), + subtask_data = Subtask_test_data(state=models.SubtaskState.objects.get(value="defined"), subtask_template=models.SubtaskTemplate.objects.get(name='observation control')) subtask_obs = models.Subtask.objects.create(**subtask_data) - subtask_data = Subtask_test_data(task_blueprint, state=models.SubtaskState.objects.get(value="defined"), + subtask_obs.task_blueprints.set([task_blueprint]) + subtask_data = Subtask_test_data(state=models.SubtaskState.objects.get(value="defined"), subtask_template=models.SubtaskTemplate.objects.get(name='QA file conversion')) subtask_qa = models.Subtask.objects.create(**subtask_data) + subtask_qa.task_blueprints.set([task_blueprint]) # Do the actual test for test_item in test_table: @@ -372,14 +375,19 @@ class TaskBlueprintStateTest(unittest.TestCase): task_blueprint_data = TaskBlueprint_test_data(name="Task Blueprint With Subtasks") task_blueprint = models.TaskBlueprint.objects.create(**task_blueprint_data) # Create observation and qa subtasks related to taskblueprint - subtask_data = Subtask_test_data(task_blueprint, state=models.SubtaskState.objects.get(value="defined"), + subtask_data = Subtask_test_data(state=models.SubtaskState.objects.get(value="defined"), subtask_template=models.SubtaskTemplate.objects.get(name='observation control')) subtask_obs1 = models.Subtask.objects.create(**subtask_data) + subtask_obs1.task_blueprints.set([task_blueprint]) subtask_obs2 = models.Subtask.objects.create(**subtask_data) - subtask_data = Subtask_test_data(task_blueprint, state=models.SubtaskState.objects.get(value="defined"), + subtask_obs2.task_blueprints.set([task_blueprint]) + + subtask_data = Subtask_test_data(state=models.SubtaskState.objects.get(value="defined"), subtask_template=models.SubtaskTemplate.objects.get(name='QA file conversion')) subtask_qa1 = models.Subtask.objects.create(**subtask_data) + subtask_qa1.task_blueprints.set([task_blueprint]) subtask_qa2 = models.Subtask.objects.create(**subtask_data) + subtask_qa2.task_blueprints.set([task_blueprint]) # Do the actual test for test_item in test_table: @@ -416,12 +424,13 @@ class SchedulingUnitBlueprintStateTest(unittest.TestCase): # Create observation task task_data = TaskBlueprint_test_data(name="Task Observation "+str(uuid.uuid4()), scheduling_unit_blueprint=schedulingunit_blueprint) task_obs = models.TaskBlueprint.objects.create(**task_data) - subtask_data = Subtask_test_data(task_obs, state=models.SubtaskState.objects.get(value="defined"), + subtask_data = Subtask_test_data(state=models.SubtaskState.objects.get(value="defined"), subtask_template=models.SubtaskTemplate.objects.get(name='observation control')) if "observation" in skip_create_subtask: subtask_obs = None else: subtask_obs = models.Subtask.objects.create(**subtask_data) + subtask_obs.task_blueprints.set([task_obs]) # Create pipeline task task_data = TaskBlueprint_test_data(name="Task Pipeline", scheduling_unit_blueprint=schedulingunit_blueprint) @@ -429,13 +438,13 @@ class SchedulingUnitBlueprintStateTest(unittest.TestCase): # Need to change the default template type (observation) to pipeline task_pipe.specifications_template = models.TaskTemplate.objects.get(type=models.TaskType.Choices.PIPELINE.value) task_pipe.save() - subtask_data = Subtask_test_data(task_pipe, - state=models.SubtaskState.objects.get(value="defined"), + subtask_data = Subtask_test_data(state=models.SubtaskState.objects.get(value="defined"), subtask_template=models.SubtaskTemplate.objects.get(name='pipeline control')) if "pipeline" in skip_create_subtask: subtask_pipe = None else: subtask_pipe = models.Subtask.objects.create(**subtask_data) + subtask_pipe.task_blueprints.set([task_pipe]) # Create ingest task # Because there is no taskTemplate object for ingest by default I have to create one @@ -447,13 +456,13 @@ class SchedulingUnitBlueprintStateTest(unittest.TestCase): task_ingest.save() # There is no template defined for ingest yet ...but I can use pipeline control, only the template type matters # ....should become other thing in future but for this test does not matter - subtask_data = Subtask_test_data(task_ingest, - state=models.SubtaskState.objects.get(value="defined"), + subtask_data = Subtask_test_data(state=models.SubtaskState.objects.get(value="defined"), subtask_template=models.SubtaskTemplate.objects.get(name='pipeline control')) if "ingest" in skip_create_subtask: subtask_ingest = None else: subtask_ingest = models.Subtask.objects.create(**subtask_data) + subtask_ingest.task_blueprints.set([task_ingest]) return {"observation": {"task": task_obs, "subtask": subtask_obs}, "pipeline": {"task": task_pipe, "subtask": subtask_pipe}, diff --git a/SAS/TMSS/backend/test/tmss_test_data_django_models.py b/SAS/TMSS/backend/test/tmss_test_data_django_models.py index 7d3e065be36e697dd96a80777a6a9c9044fce46d..616298e418636d097ac59dfdd4c0ff5acb80d0a0 100644 --- a/SAS/TMSS/backend/test/tmss_test_data_django_models.py +++ b/SAS/TMSS/backend/test/tmss_test_data_django_models.py @@ -388,13 +388,10 @@ def SubtaskInput_test_data(subtask: models.Subtask=None, producer: models.Subtas "selection_template": selection_template, "tags":[]} -def Subtask_test_data(task_blueprint: models.TaskBlueprint=None, subtask_template: models.SubtaskTemplate=None, +def Subtask_test_data(subtask_template: models.SubtaskTemplate=None, specifications_doc: dict=None, start_time=None, stop_time=None, cluster=None, state=None, raw_feedback=None) -> dict: - if task_blueprint is None: - task_blueprint = models.TaskBlueprint.objects.create(**TaskBlueprint_test_data()) - if subtask_template is None: subtask_template = models.SubtaskTemplate.objects.create(**SubtaskTemplate_test_data()) @@ -418,7 +415,7 @@ def Subtask_test_data(task_blueprint: models.TaskBlueprint=None, subtask_templat "stop_time": stop_time, "state": state, "specifications_doc": specifications_doc, - "task_blueprint": task_blueprint, + #"task_blueprint": task_blueprint, # ManyToMany, use set() "specifications_template": subtask_template, "tags": ["TMSS", "TESTING"], "do_cancel": datetime.utcnow(), diff --git a/SAS/TMSS/backend/test/tmss_test_data_rest.py b/SAS/TMSS/backend/test/tmss_test_data_rest.py index b3357b2bf9d0b83160e1245e5109f153c44f7348..e485e8f47ce746c0b63e08193e2dd61c501d6eb8 100644 --- a/SAS/TMSS/backend/test/tmss_test_data_rest.py +++ b/SAS/TMSS/backend/test/tmss_test_data_rest.py @@ -656,11 +656,16 @@ class TMSSRESTTestDataCreator(): self._subtask_url = self.post_data_and_get_url(self.Subtask(), '/subtask/') return self._subtask_url - def SubtaskOutput(self, subtask_url=None): + def SubtaskOutput(self, subtask_url=None, task_blueprint_url=None): + if subtask_url is None: subtask_url = self.cached_subtask_url + if task_blueprint_url is None: + task_blueprint_url = self.cached_task_blueprint_url + return {"subtask": subtask_url, + "task_blueprint": task_blueprint_url, "tags": []} @property