From 7ecb974fffe32ffd5947180b270ae09ad3d276fc Mon Sep 17 00:00:00 2001
From: Jorrit Schaap <schaap@astron.nl>
Date: Thu, 2 Sep 2021 21:35:17 +0200
Subject: [PATCH] TMSS-917: now that we reverted to a single parent
TaskBlueprint for a Subtask there is no need anymore for a SubtaskOutput to
have a task_blueprint reference.
---
.../scheduling/lib/dynamic_scheduling.py | 2 +-
.../tmss/tmssapp/migrations/0001_initial.py | 7 +-----
.../src/tmss/tmssapp/models/scheduling.py | 1 -
SAS/TMSS/backend/src/tmss/tmssapp/subtasks.py | 25 +++++++------------
.../test/t_tmssapp_scheduling_REST_API.py | 20 ---------------
.../test/t_tmssapp_scheduling_django_API.py | 21 +++++++---------
SAS/TMSS/backend/test/test_environment.py | 8 +++---
.../test/tmss_test_data_django_models.py | 6 +----
SAS/TMSS/backend/test/tmss_test_data_rest.py | 6 +----
9 files changed, 26 insertions(+), 70 deletions(-)
diff --git a/SAS/TMSS/backend/services/scheduling/lib/dynamic_scheduling.py b/SAS/TMSS/backend/services/scheduling/lib/dynamic_scheduling.py
index 296c28b0f2d..bbf6e5d542b 100644
--- a/SAS/TMSS/backend/services/scheduling/lib/dynamic_scheduling.py
+++ b/SAS/TMSS/backend/services/scheduling/lib/dynamic_scheduling.py
@@ -293,7 +293,7 @@ def create_dynamic_scheduling_service(exchange: str=DEFAULT_BUSNAME, broker: str
def get_dynamically_schedulable_scheduling_units() -> [models.SchedulingUnitBlueprint]:
'''get a list of all dynamically schedulable scheduling_units'''
defined_independend_subtasks = models.Subtask.independent_subtasks().filter(state__value='defined')
- defined_independend_subtask_ids = defined_independend_subtasks.values('task_blueprints__scheduling_unit_blueprint_id').distinct().all()
+ defined_independend_subtask_ids = defined_independend_subtasks.values('task_blueprint__scheduling_unit_blueprint_id').distinct().all()
scheduling_units = models.SchedulingUnitBlueprint.objects.filter(id__in=defined_independend_subtask_ids) \
.filter(scheduling_constraints_template__isnull=False).all()
return [su for su in scheduling_units if su.status == 'schedulable']
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/migrations/0001_initial.py b/SAS/TMSS/backend/src/tmss/tmssapp/migrations/0001_initial.py
index 8942a477ae8..b39c19f3e09 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/migrations/0001_initial.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/migrations/0001_initial.py
@@ -1,4 +1,4 @@
-# Generated by Django 3.0.9 on 2021-09-02 15:48
+# Generated by Django 3.0.9 on 2021-09-02 19:01
from django.conf import settings
import django.contrib.auth.models
@@ -1170,11 +1170,6 @@ class Migration(migrations.Migration):
name='subtask',
field=models.ForeignKey(help_text='Subtask to which this output specification refers.', on_delete=django.db.models.deletion.CASCADE, related_name='outputs', to='tmssapp.Subtask'),
),
- migrations.AddField(
- model_name='subtaskoutput',
- name='task_blueprint',
- field=models.ForeignKey(help_text='Task to which this output specification refers.', on_delete=django.db.models.deletion.CASCADE, related_name='outputs', to='tmssapp.TaskBlueprint'),
- ),
migrations.AddField(
model_name='subtaskinput',
name='dataproducts',
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/models/scheduling.py b/SAS/TMSS/backend/src/tmss/tmssapp/models/scheduling.py
index 72eabea2299..60806b9b48e 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/models/scheduling.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/models/scheduling.py
@@ -386,7 +386,6 @@ class SubtaskInput(BasicCommon, TemplateSchemaMixin):
class SubtaskOutput(BasicCommon):
subtask = ForeignKey('Subtask', null=False, on_delete=CASCADE, related_name='outputs', help_text='Subtask to which this output specification refers.')
- task_blueprint = ForeignKey('TaskBlueprint', null=False, on_delete=CASCADE, related_name='outputs', help_text='Task to which this output specification refers.')
class SAP(BasicCommon, TemplateSchemaMixin):
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/subtasks.py b/SAS/TMSS/backend/src/tmss/tmssapp/subtasks.py
index b389b09bb52..3def90606c9 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/subtasks.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/subtasks.py
@@ -500,8 +500,7 @@ def create_observation_control_subtask_from_task_blueprint(task_blueprint: TaskB
# step 2: create and link subtask input/output
# an observation has no input, it just produces output data
- subtask_output = SubtaskOutput.objects.create(subtask=subtask,
- task_blueprint=task_blueprint)
+ subtask_output = SubtaskOutput.objects.create(subtask=subtask)
# step 3: set state to DEFINED
subtask.state = SubtaskState.objects.get(value=SubtaskState.Choices.DEFINED.value)
@@ -566,8 +565,7 @@ def create_qafile_subtask_from_observation_subtask(observation_subtask: Subtask)
selection_doc=selection_doc,
selection_template=selection_template)
- qafile_subtask_output = SubtaskOutput.objects.create(subtask=qafile_subtask,
- task_blueprint=observation_subtask.task_blueprint)
+ qafile_subtask_output = SubtaskOutput.objects.create(subtask=qafile_subtask)
# step 3: set state to DEFINED
qafile_subtask.state = SubtaskState.objects.get(value=SubtaskState.Choices.DEFINED.value)
@@ -643,8 +641,7 @@ def create_qaplots_subtask_from_qafile_subtask(qafile_subtask: Subtask) -> Subta
selection_doc=selection_doc,
selection_template=selection_template)
- qaplots_subtask_output = SubtaskOutput.objects.create(subtask=qaplots_subtask,
- task_blueprint=qafile_subtask.task_blueprint)
+ qaplots_subtask_output = SubtaskOutput.objects.create(subtask=qaplots_subtask)
# step 3: set state to DEFINED
qaplots_subtask.state = SubtaskState.objects.get(value=SubtaskState.Choices.DEFINED.value)
@@ -695,14 +692,12 @@ def create_pipeline_subtask_from_task_blueprint(task_blueprint: TaskBlueprint, s
predecessor_observation_subtasks = [st for st in producing_task_blueprint.subtasks.order_by('id').all() if st.specifications_template.type.value == SubtaskType.Choices.OBSERVATION.value]
for predecessor_obs_subtask in predecessor_observation_subtasks:
for predecessor_subtask_output in predecessor_obs_subtask.outputs.all():
- if predecessor_subtask_output.task_blueprint == task_relation_blueprint.producer:
- subtask_input = SubtaskInput.objects.create(subtask=subtask,
- producer=predecessor_subtask_output,
- selection_doc=task_relation_blueprint.selection_doc,
- selection_template=task_relation_blueprint.selection_template)
+ subtask_input = SubtaskInput.objects.create(subtask=subtask,
+ producer=predecessor_subtask_output,
+ selection_doc=task_relation_blueprint.selection_doc,
+ selection_template=task_relation_blueprint.selection_template)
- subtask_output = SubtaskOutput.objects.create(subtask=subtask,
- task_blueprint=task_blueprint)
+ subtask_output = SubtaskOutput.objects.create(subtask=subtask)
# step 3: set state to DEFINED
subtask.state = SubtaskState.objects.get(value=SubtaskState.Choices.DEFINED.value)
@@ -1800,9 +1795,7 @@ def schedule_ingest_subtask(ingest_subtask: Subtask):
ingest_subtask_input.dataproducts.set(input_dataproducts)
# define output and create output dataproducts.
- tb = ingest_subtask_input.producer.task_blueprint # output dataproducts are linked to the same task as its input dataproduct
- ingest_subtask_output = SubtaskOutput.objects.create(subtask=ingest_subtask,
- task_blueprint=tb)
+ ingest_subtask_output = SubtaskOutput.objects.create(subtask=ingest_subtask)
# prepare identifiers in bulk for each output_dataproduct
dp_gids = [SIPidentifier(source="TMSS") for _ in input_dataproducts]
diff --git a/SAS/TMSS/backend/test/t_tmssapp_scheduling_REST_API.py b/SAS/TMSS/backend/test/t_tmssapp_scheduling_REST_API.py
index d9fa22bf3c0..165cbf1e71e 100755
--- a/SAS/TMSS/backend/test/t_tmssapp_scheduling_REST_API.py
+++ b/SAS/TMSS/backend/test/t_tmssapp_scheduling_REST_API.py
@@ -343,26 +343,6 @@ class SubtaskTestCase(unittest.TestCase):
# DELETE and check it's gone
DELETE_and_assert_gone(self, url)
- def test_subtask_PROTECT_behavior_on_state_choice_deleted(self):
- st_test_data = test_data_creator.Subtask(cluster_url=self.cluster_url, task_blueprint_url=self.task_blueprint_url, specifications_template_url=self.specifications_template_url)
-
- # create dependency that is safe to delete (enums are not populated / re-established between tests)
- state_data = {'value': 'defining'}
- POST_and_assert_expected_response(self, BASE_URL + '/subtask_state/', state_data, 201, state_data)
- state_url = BASE_URL + '/subtask_state/kickme'
-
- # POST new item and verify
- test_data = dict(st_test_data)
- test_data['state'] = state_url
- url = POST_and_assert_expected_response(self, BASE_URL + '/subtask/', test_data, 201, test_data)['url']
- GET_OK_and_assert_equal_expected_response(self, url, test_data)
-
- # Try to DELETE dependency, verify that was not successful
- # Unfortunately we don't get a nice error in json, but a Django debug page on error 500...
- response = requests.delete(state_url, auth=AUTH)
- self.assertEqual(500, response.status_code)
- self.assertTrue("ProtectedError" in str(response.content))
- GET_OK_and_assert_equal_expected_response(self, state_url, state_data)
def test_subtask_SET_NULL_behavior_on_task_blueprint_deleted(self):
# make new task_blueprint_url instance, but reuse related data for speed
diff --git a/SAS/TMSS/backend/test/t_tmssapp_scheduling_django_API.py b/SAS/TMSS/backend/test/t_tmssapp_scheduling_django_API.py
index 4006b45507b..8208d564f31 100755
--- a/SAS/TMSS/backend/test/t_tmssapp_scheduling_django_API.py
+++ b/SAS/TMSS/backend/test/t_tmssapp_scheduling_django_API.py
@@ -132,7 +132,6 @@ class SubtaskOutputTest(unittest.TestCase):
# setup
test_data = dict(SubtaskOutput_test_data())
test_data['subtask'] = None
- test_data['task_blueprint'] = None
# assert
with self.assertRaises(IntegrityError):
@@ -233,7 +232,7 @@ class SubtaskTest(unittest.TestCase):
subtask1:models.Subtask = models.Subtask.objects.create(**Subtask_test_data(task_blueprint=self.task_blueprint))
subtask2:models.Subtask = models.Subtask.objects.create(**Subtask_test_data(task_blueprint=self.task_blueprint))
- output1 = models.SubtaskOutput.objects.create(subtask=subtask1, task_blueprint=self.task_blueprint)
+ output1 = models.SubtaskOutput.objects.create(subtask=subtask1)
models.SubtaskInput.objects.create(**SubtaskInput_test_data(subtask=subtask2, producer=output1))
self.assertEqual(subtask1, subtask2.predecessors.all()[0])
@@ -251,12 +250,12 @@ class SubtaskTest(unittest.TestCase):
# | |
# ST2 - -> ST5 ---> ST6
- output1 = models.SubtaskOutput.objects.create(subtask=subtask1, task_blueprint=self.task_blueprint)
- output2 = models.SubtaskOutput.objects.create(subtask=subtask2, task_blueprint=self.task_blueprint)
- output3 = models.SubtaskOutput.objects.create(subtask=subtask3, task_blueprint=self.task_blueprint)
- output4 = models.SubtaskOutput.objects.create(subtask=subtask4, task_blueprint=self.task_blueprint)
- output5 = models.SubtaskOutput.objects.create(subtask=subtask5, task_blueprint=self.task_blueprint)
- output6 = models.SubtaskOutput.objects.create(subtask=subtask6, task_blueprint=self.task_blueprint)
+ output1 = models.SubtaskOutput.objects.create(subtask=subtask1)
+ output2 = models.SubtaskOutput.objects.create(subtask=subtask2)
+ output3 = models.SubtaskOutput.objects.create(subtask=subtask3)
+ output4 = models.SubtaskOutput.objects.create(subtask=subtask4)
+ output5 = models.SubtaskOutput.objects.create(subtask=subtask5)
+ output6 = models.SubtaskOutput.objects.create(subtask=subtask6)
models.SubtaskInput.objects.create(**SubtaskInput_test_data(subtask=subtask3, producer=output1))
models.SubtaskInput.objects.create(**SubtaskInput_test_data(subtask=subtask3, producer=output2))
@@ -278,8 +277,7 @@ class SubtaskTest(unittest.TestCase):
def test_Subtask_transformed_dataproducts(self):
# setup
subtask1:models.Subtask = models.Subtask.objects.create(**Subtask_test_data())
- output1:models.SubtaskOutput = models.SubtaskOutput.objects.create(subtask=subtask1,
- task_blueprint=self.task_blueprint)
+ output1:models.SubtaskOutput = models.SubtaskOutput.objects.create(subtask=subtask1)
output1_dp:models.Dataproduct = models.Dataproduct.objects.create(**Dataproduct_test_data(producer=output1))
subtask2:models.Subtask = models.Subtask.objects.create(**Subtask_test_data())
@@ -287,8 +285,7 @@ class SubtaskTest(unittest.TestCase):
input2_dp = output1_dp
input2.dataproducts.set([input2_dp])
input2.save()
- output2:models.SubtaskOutput = models.SubtaskOutput.objects.create(subtask=subtask2,
- task_blueprint=self.task_blueprint)
+ output2:models.SubtaskOutput = models.SubtaskOutput.objects.create(subtask=subtask2)
output2_dp:models.Dataproduct = models.Dataproduct.objects.create(**Dataproduct_test_data(producer=output2))
models.DataproductTransform.objects.create(input=input2_dp, output=output2_dp, identity=True)
diff --git a/SAS/TMSS/backend/test/test_environment.py b/SAS/TMSS/backend/test/test_environment.py
index 183ded21b40..5654fadc6e2 100644
--- a/SAS/TMSS/backend/test/test_environment.py
+++ b/SAS/TMSS/backend/test/test_environment.py
@@ -671,7 +671,7 @@ def create_scheduling_unit_blueprint_simulator(scheduling_unit_blueprint_id: int
self.create_output_dataproducts = create_output_dataproducts
def need_to_handle(self, subtask: models.Subtask) -> bool:
- if self.scheduling_unit_blueprint_id not in [tb.scheduling_unit_blueprint.id for tb in subtask.task_blueprints.all()]:
+ if self.scheduling_unit_blueprint_id != subtask.task_blueprint.scheduling_unit_blueprint.id:
return False
if subtask.specifications_template.type.value == models.SubtaskType.Choices.OBSERVATION.value and not self.handle_observations:
@@ -712,7 +712,7 @@ def create_scheduling_unit_blueprint_simulator(scheduling_unit_blueprint_id: int
super().start_handling()
# trick: trigger any already scheduled subtasks, cascading in events simulating the run
- scheduled_subtasks = models.Subtask.objects.filter(task_blueprints__scheduling_unit_blueprint_id=self.scheduling_unit_blueprint_id).filter(state__value=models.SubtaskState.Choices.SCHEDULED.value).all()
+ scheduled_subtasks = models.Subtask.objects.filter(task_blueprint__scheduling_unit_blueprint_id=self.scheduling_unit_blueprint_id).filter(state__value=models.SubtaskState.Choices.SCHEDULED.value).all()
for subtask in scheduled_subtasks:
self.onSubTaskStatusChanged(subtask.id, "scheduled")
@@ -802,7 +802,7 @@ def create_scheduling_unit_blueprint_simulator(scheduling_unit_blueprint_id: int
output_dp.feedback_doc = feedback_doc
output_dp.save()
elif subtask.specifications_template.type.value == models.SubtaskType.Choices.INGEST.value:
- project_name = subtask.task_blueprints.first().draft.scheduling_unit_draft.scheduling_set.project.name # todo: support for multiple projects needs to be picked up in TMSS-689
+ project_name = subtask.project.name # todo: support for multiple projects needs to be picked up in TMSS-689
for output_dp in subtask.output_dataproducts:
try:
@@ -834,7 +834,7 @@ def create_scheduling_unit_blueprint_simulator(scheduling_unit_blueprint_id: int
if subtask.specifications_template.type.value == 'ingest':
logger.info("subtask id=%d is an ingest task which requires permission in order to be scheduled", subtask.id)
- if self.auto_grant_ingest_permission and any([tb.scheduling_unit_blueprint.ingest_permission_required for tb in subtask.task_blueprints.all()]):
+ if self.auto_grant_ingest_permission and subtask.task_blueprint.scheduling_unit_blueprint.ingest_permission_required:
# just granting the permission triggers the scheduling_service to check and schedulable ingest subtasks,
# resulting in a scheduled ingest subtask.
logger.info("granting ingest subtask id=%d ingest_permission", subtask.id)
diff --git a/SAS/TMSS/backend/test/tmss_test_data_django_models.py b/SAS/TMSS/backend/test/tmss_test_data_django_models.py
index 60553c4230c..a2942421cd8 100644
--- a/SAS/TMSS/backend/test/tmss_test_data_django_models.py
+++ b/SAS/TMSS/backend/test/tmss_test_data_django_models.py
@@ -374,15 +374,11 @@ def DataproductFeedbackTemplate_test_data() -> dict:
"schema": minimal_json_schema(),
"tags": ["TMSS", "TESTING"]}
-def SubtaskOutput_test_data(subtask: models.Subtask=None, task_blueprint: models.TaskBlueprint=None) -> dict:
+def SubtaskOutput_test_data(subtask: models.Subtask=None) -> dict:
if subtask is None:
subtask = models.Subtask.objects.create(**Subtask_test_data())
- if task_blueprint is None:
- task_blueprint = models. TaskBlueprint.objects.create(**TaskBlueprint_test_data(()))
-
return {"subtask": subtask,
- "task_blueprint": task_blueprint,
"tags":[]}
def SubtaskInput_test_data(subtask: models.Subtask=None, producer: models.SubtaskOutput=None, selection_doc=None, selection_template: models.TaskRelationSelectionTemplate=None) -> dict:
diff --git a/SAS/TMSS/backend/test/tmss_test_data_rest.py b/SAS/TMSS/backend/test/tmss_test_data_rest.py
index 47b33d858b0..6d550ac2d05 100644
--- a/SAS/TMSS/backend/test/tmss_test_data_rest.py
+++ b/SAS/TMSS/backend/test/tmss_test_data_rest.py
@@ -688,16 +688,12 @@ class TMSSRESTTestDataCreator():
self._subtask_url = self.post_data_and_get_url(self.Subtask(), '/subtask/')
return self._subtask_url
- def SubtaskOutput(self, subtask_url=None, task_blueprint_url=None):
+ def SubtaskOutput(self, subtask_url=None):
if subtask_url is None:
subtask_url = self.cached_subtask_url
- if task_blueprint_url is None:
- task_blueprint_url = self.cached_task_blueprint_url
-
return {"subtask": subtask_url,
- "task_blueprint": task_blueprint_url,
"tags": []}
@property
--
GitLab