TMSS-1218: bigfixes and refer to observation in pipeline parset

SAS/TMSS/backend/src/tmss/tmssapp/adapters/parset.py

@@ -310,7 +310,7 @@ def _convert_to_parset_dict_for_observationcontrol_schema(subtask: models.Subtas
     # ----------------------------
 
     parset = dict() # parameterset has no proper assignment operators, so take detour via dict...
-    parset["Observation.ObsID"] = subtask.pk
+    parset["Observation.ObsID"] = subtask.pk   # todo: need to check for primary subtask?
     parset["Observation.momID"] = 0 # Needed by MACScheduler
     parset["Observation.otdbID"] = 0 # Needed by MACScheduler; should/can this be the same as subtask.pk?
     parset["Observation.tmssID"] = subtask.pk
@@ -428,7 +428,7 @@ def _common_parset_dict_for_pipeline_schemas(subtask: models.Subtask) -> dict:
 
     # General
     parset["prefix"] = "LOFAR."
-    parset["Observation.ObsID"] = subtask.pk
+    parset["Observation.ObsID"] = subtask.related_primary_observation_subtask.pk
     parset["Observation.momID"] = 0 # Needed by MACScheduler
     parset["Observation.otdbID"] = subtask.pk # HACK: the pipeline uses otdbID as the sasID. our tmssID>2000000 to prevent clashes. TODO: replace all otdbID's by sasID.
     parset["Observation.tmssID"] = subtask.pk

SAS/TMSS/backend/src/tmss/tmssapp/adapters/sip.py

@@ -133,7 +133,7 @@ def add_subbtask_and_input_dataproducts_to_sip(subtask: Subtask, sip: siplib.Sip
             for dataproduct in input.dataproducts.all():
                 sip_dataproduct = create_sip_representation_for_dataproduct(dataproduct)
                 sip.add_related_dataproduct(sip_dataproduct, return_xml=False)
-    sip.add_parset(parset_sip_identifier, contents=convert_to_parset(subtask))
+    sip.add_parset(parset_sip_identifier, contents=str(convert_to_parset(subtask)))
 
 @lru_cache(maxsize=10, typed=False)
 def create_sip_representation_for_subtask(subtask: Subtask, parset_sip_identifier=None):
@@ -217,21 +217,12 @@ def create_sip_representation_for_subtask(subtask: Subtask, parset_sip_identifie
         if not sourcedata_identifiers:
             raise TMSSException("There seems to be no subtask input associated to your pipeline subtask id %s. Please define what data the pipeline processed." % subtask.id)
 
-        # determine the observation this pipeline belongs to and set correct identifier for reference
-        # todo: verify that the primary observation subtask SIP identifier is equivalent to the old 'SAS VIC Tree Id'
-        def recursively_search_for_primary_predecessor_observation_subtask(subtask):
-            primary_observation_control_subtasks = subtask.predecessors.filter(specifications_template__name='observation control', primary=True)
-            if primary_observation_control_subtasks.count() > 0:
-                observation_subtask = primary_observation_control_subtasks.first()
-                return get_siplib_identifier(observation_subtask.global_identifier, "Subtask id=%s" % observation_subtask.id, name=observation_subtask.task_blueprint.short_description)
-            else:
-                for predecessor in subtask.predecessor_subtasks:
-                    return recursively_search_for_primary_predecessor_observation_subtask(predecessor)
-
-        observation_identifier = recursively_search_for_primary_predecessor_observation_subtask(subtask)
-        if observation_identifier is None:
+        observation_subtask = subtask.related_primary_observation_subtask
+        if observation_subtask is None:
             raise TMSSException('Could not identify primary predecessor observation subtask')
-        process_map.observation_identifier = observation_identifier
+        else:
+            process_map.observation_identifier = get_siplib_identifier(observation_subtask.global_identifier, "Subtask id=%s" % observation_subtask.id, name=observation_subtask.task_blueprint.short_description)
+
         pipeline_map = siplib.PipelineMap(
                 name=subtask.task_blueprint.name,
                 version='unknown',  # todo from subtask.specifications_doc? from feedback (we have feedback and storagewriter versions there, not pipeline version or sth)?

SAS/TMSS/backend/src/tmss/tmssapp/models/scheduling.py

@@ -153,7 +153,7 @@ class Subtask(BasicCommon, ProjectPropertyMixin, TemplateSchemaMixin):
     error_reason = CharField(null=True, max_length=200, help_text='Reason why the Subtask went to error.')
     raw_feedback = CharField(null=True, max_length=1048576, help_text='The raw feedback for this Subtask')
     global_identifier = OneToOneField('SIPidentifier', null=False, editable=False, on_delete=PROTECT, help_text='The global unique identifier for LTA SIP.')
-    global_parset_identifier = OneToOneField('SIPidentifier', related_name='related_subtask', null=True, editable=False, on_delete=PROTECT, help_text='The global unique identifier of this Subtask\'s parset for LTA SIP.')
+    global_parset_identifier = OneToOneField('SIPidentifier', related_name='related_subtask', null=False, editable=False, on_delete=PROTECT, help_text='The global unique identifier of this Subtask\'s parset for LTA SIP.')
     path_to_project = 'task_blueprint__scheduling_unit_blueprint__draft__scheduling_set__project'
     obsolete_since = DateTimeField(null=True, help_text='When this subtask was marked obsolete, or NULL if not obsolete (NULLable).')
 
@@ -235,6 +235,20 @@ class Subtask(BasicCommon, ProjectPropertyMixin, TemplateSchemaMixin):
                                                     "INNER JOIN tmssapp_subtaskinput as st_input on st_input.producer_id = st_output.id\n"
                                                     "WHERE st_input.subtask_id = %s", params=[self.id]))
 
+    @property
+    def related_primary_observation_subtask(self) -> 'Subtask' or None:
+        if self.specifications_template.name == 'observation control' and self.primary is True:
+            return self
+        else:
+            def _recursively_search_for_primary_predecessor_observation_subtask(subtask):
+                primary_observation_control_subtasks = subtask.predecessors.filter(specifications_template__name='observation control', primary=True)
+                if primary_observation_control_subtasks.count() > 0:
+                    return primary_observation_control_subtasks.first()
+                else:
+                    for predecessor in subtask.predecessors:
+                        return _recursively_search_for_primary_predecessor_observation_subtask(predecessor)
+            return _recursively_search_for_primary_predecessor_observation_subtask(self)
+
     @property
     def input_dataproducts(self) -> QuerySet:
         '''return the input dataproducts(s) as queryset (over which you can perform extended queries, or return via the serializers/viewsets)