TMSS-1124: created test to prove bug TMSS-1124

SAS/TMSS/backend/test/t_scheduling.py

@@ -657,13 +657,6 @@ class SubtaskInputOutputTest(unittest.TestCase):
     dataproducts are assigned
     """
 
-    def setUp(self) -> None:
-        # make sure we're allowed to schedule
-        setting = Setting.objects.get(name='dynamic_scheduling_enabled')
-        setting.value = True
-        setting.save()
-
-
     def test_specifications_doc_meets_selection_doc(self):
         # empty selection matches all
         self.assertTrue(specifications_doc_meets_selection_doc({'something else': 'target0'}, {}))
@@ -711,6 +704,118 @@ class SubtaskInputOutputTest(unittest.TestCase):
         self.assertEqual(set(pipe_in1.dataproducts.all()), {dp1_1, dp1_3})
         self.assertEqual(set(pipe_in2.dataproducts.all()), {dp2_2})
 
+    @mock.patch("lofar.sas.tmss.tmss.tmssapp.subtasks.assign_or_unassign_resources")
+    def test_bug_TMSS_1124_ingest_cannot_be_scheduled_due_to_unfinished_subtask_input_output_implementation(self, assign_resources_mock):
+        '''
+        See Jira: https://support.astron.nl/jira/browse/TMSS-1124
+        This bug seemed to indicate that something is wrong in scheduling ingest tasks.
+        The root-cause is the fact that currently conversion from task_relations into SubtaskInput/Subtaskoutputs
+        does not filter for datatype/dataformat.
+        In this initial version of this test we prove that the bug exists, and then we fix it in a later commit.
+        '''
+        # Setup: create a SchedulingUnitDraft/Blueprint with a BF Obs, Pulp, and Ingest
+        parent_set = models.SchedulingSet.objects.create(**SchedulingSet_test_data())
+        su_draft = models.SchedulingUnitDraft.objects.create(scheduling_set=parent_set,
+                                                             specifications_template=models.SchedulingUnitTemplate.objects.get(name='scheduling unit', version=1))
+
+        obs_task_template = models.TaskTemplate.objects.get(name='beamforming observation', version=1)
+        obs_specifications_doc = obs_task_template.get_default_json_document_for_schema()
+        obs_specifications_doc['SAPs'][0]['subbands'] = [0,1,2,3]
+        obs_specifications_doc['beamformers'][0]['flys eye']['enabled']= True
+        obs_task_draft = models.TaskDraft.objects.create(name='obs',
+                                                         scheduling_unit_draft=su_draft,
+                                                         specifications_template=obs_task_template,
+                                                         specifications_doc=obs_specifications_doc)
+
+        pulp_task_template = models.TaskTemplate.objects.get(name='pulsar pipeline', version=1)
+        pulp_task_draft = models.TaskDraft.objects.create(name='pulp',
+                                                          scheduling_unit_draft=su_draft,
+                                                          specifications_template=pulp_task_template,
+                                                          specifications_doc=pulp_task_template.get_default_json_document_for_schema())
+
+        # connect the obs and the pulp pipeline
+        # it's a single relation of beamformed timeseries data
+        trsel_template = models.TaskRelationSelectionTemplate.objects.get(name='all', version=1)
+        trsel_obs_pulp_draft = models.TaskRelationDraft.objects.create(producer=obs_task_draft,
+                                                                       consumer=pulp_task_draft,
+                                                                       output_role=obs_task_template.connector_types.get(iotype__value=models.IOType.Choices.OUTPUT.value,
+                                                                                                                         dataformat__value=models.Dataformat.Choices.BEAMFORMED.value,
+                                                                                                                         datatype__value=models.Datatype.Choices.TIME_SERIES.value,
+                                                                                                                         role__value=models.Role.Choices.BEAMFORMER.value),
+                                                                       input_role=pulp_task_template.connector_types.get(iotype__value=models.IOType.Choices.INPUT.value,
+                                                                                                                         dataformat__value=models.Dataformat.Choices.BEAMFORMED.value,
+                                                                                                                         datatype__value=models.Datatype.Choices.TIME_SERIES.value,
+                                                                                                                         role__value=models.Role.Choices.BEAMFORMER.value),
+                                                                       selection_template=trsel_template,
+                                                                       selection_doc=trsel_template.get_default_json_document_for_schema())
+
+        ingest_task_template = models.TaskTemplate.objects.get(name='ingest', version=1)
+        ingest_task_draft = models.TaskDraft.objects.create(name='ingest',
+                                                            scheduling_unit_draft=su_draft,
+                                                            specifications_template=ingest_task_template,
+                                                            specifications_doc=ingest_task_template.get_default_json_document_for_schema())
+
+        # connect the pulsar pipeline and ingest
+        # there are two connections, 1) pulp summary, 2) pulsar profile
+        # this should result in two connections at SubtaskInput/SubtaskOutput level, but it does not (which is the bug)
+        trsel_pulp_ingest_draft1 = models.TaskRelationDraft.objects.create(producer=pulp_task_draft,
+                                                                           consumer=ingest_task_draft,
+                                                                           output_role=pulp_task_template.connector_types.get(iotype__value=models.IOType.Choices.OUTPUT.value,
+                                                                                                                              dataformat__value=models.Dataformat.Choices.PULP_SUMMARY.value,
+                                                                                                                              datatype__value=models.Datatype.Choices.QUALITY.value,
+                                                                                                                              role__value=models.Role.Choices.ANY.value),
+                                                                           input_role=ingest_task_template.connector_types.get(iotype__value=models.IOType.Choices.INPUT.value,
+                                                                                                                               dataformat__value=models.Dataformat.Choices.PULP_SUMMARY.value,
+                                                                                                                               datatype__value=models.Datatype.Choices.QUALITY.value,
+                                                                                                                               role__value=models.Role.Choices.ANY.value),
+                                                                           selection_template=trsel_template,
+                                                                           selection_doc=trsel_template.get_default_json_document_for_schema())
+
+
+        trsel_pulp_ingest_draft2 = models.TaskRelationDraft.objects.create(producer=pulp_task_draft,
+                                                                           consumer=ingest_task_draft,
+                                                                           output_role=pulp_task_template.connector_types.get(iotype__value=models.IOType.Choices.OUTPUT.value,
+                                                                                                                              dataformat__value=models.Dataformat.Choices.PULP_ANALYSIS.value,
+                                                                                                                              datatype__value=models.Datatype.Choices.PULSAR_PROFILE.value,
+                                                                                                                              role__value=models.Role.Choices.ANY.value),
+                                                                           input_role=ingest_task_template.connector_types.get(iotype__value=models.IOType.Choices.INPUT.value,
+                                                                                                                               dataformat__value=models.Dataformat.Choices.PULP_ANALYSIS.value,
+                                                                                                                               datatype__value=models.Datatype.Choices.PULSAR_PROFILE.value,
+                                                                                                                               role__value=models.Role.Choices.ANY.value),
+                                                                           selection_template=trsel_template,
+                                                                           selection_doc=trsel_template.get_default_json_document_for_schema())
+
+        # convert to blueprint, ingest_permission_required = False
+        su_blueprint = create_scheduling_unit_blueprint_and_tasks_and_subtasks_from_scheduling_unit_draft(su_draft)
+        su_blueprint.ingest_permission_required = False
+        su_blueprint.save()
+
+        # fetch the created subtasks
+        obs_subtak = models.Subtask.objects.get(task_blueprint__scheduling_unit_blueprint=su_blueprint, specifications_template__type__value='observation')
+        pulp_subtak = models.Subtask.objects.get(task_blueprint__scheduling_unit_blueprint=su_blueprint, specifications_template__type__value='pipeline')
+        ingest_subtak = models.Subtask.objects.get(task_blueprint__scheduling_unit_blueprint=su_blueprint, specifications_template__type__value='ingest')
+
+        # scheduling the observation should succeed and creates the obs output dataproducts
+        schedule_subtask(obs_subtak)
+        set_subtask_state_following_allowed_transitions(obs_subtak, 'finished')
+
+        # scheduling the pipeline should succeed and creates the pulp output dataproducts
+        schedule_subtask(pulp_subtak)
+        set_subtask_state_following_allowed_transitions(pulp_subtak, 'finished')
+
+        # prove the bug: scheduling the ingest subtask should raise
+        # with an error message that a dataproduct (in the input) is duplicate.
+        # The error seems to indicate that something is wrong in scheduling or in the ingest subtask.
+        # But that is not the case.
+        # The root-cause is the fact that the two task_relations between the pipeline and
+        # ingest are ignoring the datatype and dataformat when converting them into SubtaskInput/Subtaskoutputs.
+        with self.assertRaises(SubtaskSchedulingException) as context:
+            schedule_subtask(ingest_subtak)
+        self.assertTrue('Key (directory, filename)=(LTA, L%s_summaryCS.tar) already exists'%(pulp_subtak.id,) in str(context.exception))
+
+
+
+
 
 class SAPTest(unittest.TestCase):
     """