TMSS-307: and TMSS-573: added a method and test to reprocess raw_feedback....

TMSS-307: and TMSS-573: added a method and test to reprocess raw_feedback. Handy feature for support.

SAS/TMSS/backend/src/tmss/tmssapp/adapters/feedback.py

@@ -169,3 +169,19 @@ def process_feedback_for_subtask_and_set_to_finished_if_complete(subtask: Subtas
     return subtask
 
 
+def reprocess_raw_feedback_for_subtask_and_set_to_finished_if_complete(subtask: Subtask) -> Subtask:
+    """
+    Reprocesses the stored raw feedback from a subtask (which has been provided by Cobalt or pipelines) and translate it to
+    json documents for the individual dataproducts.
+    """
+    subtask = process_feedback_into_subtask_dataproducts(subtask, parameterset.fromString(subtask.raw_feedback))
+
+    # if complete, set subtask state to finished
+    if subtask.is_feedback_complete:
+        logger.info("Feedback for subtask id=%s is complete. Setting state to 'finished'", subtask.id)
+        subtask.state = SubtaskState.objects.get(value='finished')
+        subtask.save()
+
+    return subtask
+
+

SAS/TMSS/backend/src/tmss/tmssapp/viewsets/scheduling.py

@@ -303,6 +303,20 @@ class SubtaskViewSet(LOFARViewSet):
         serializer = self.get_serializer(updated_subtask)
         return RestResponse(serializer.data)
 
+
+    @swagger_auto_schema(responses={200: 'The updated (maybe) finished version of this subtask after reprocessing the stored raw_feedback.',
+                                    403: 'forbidden',
+                                    500: 'The feedback for this subtask could not be reprocessed'},
+                         operation_description="Reprocess the raw_feedback in the subtask into json feedback per dataproduct. Sets the subtask to finished if all dataproducts are processed.")
+    @action(methods=['get'], detail=True, url_path='reprocess_raw_feedback_for_subtask_and_set_to_finished_if_complete')
+    def reprocess_raw_feedback_for_subtask_and_set_to_finished_if_complete(self, request, pk=None):
+        from lofar.sas.tmss.tmss.tmssapp.adapters.feedback import reprocess_raw_feedback_for_subtask_and_set_to_finished_if_complete
+        subtask = get_object_or_404(models.Subtask, pk=pk)
+        updated_subtask = reprocess_raw_feedback_for_subtask_and_set_to_finished_if_complete(subtask)
+        serializer = self.get_serializer(updated_subtask)
+        return RestResponse(serializer.data)
+
+
     @swagger_auto_schema(responses={200: 'Get progress of this subtask ranging from 0.0 when just started up to 1.0 when finished.',
                                     403: 'forbidden',
                                     500: 'Could not compute the progress'},

SAS/TMSS/backend/test/t_adapter.py

@@ -47,7 +47,7 @@ from lofar.sas.tmss.tmss.exceptions import SubtaskInvalidStateException
 from lofar.sas.tmss.tmss.tmssapp.adapters.parset import convert_to_parset
 from lofar.common.json_utils import get_default_json_object_for_schema
 from lofar.sas.tmss.tmss.tmssapp.adapters.sip import generate_sip_for_dataproduct
-from lofar.sas.tmss.tmss.tmssapp.adapters.feedback import append_to_subtask_raw_feedback, process_feedback_into_subtask_dataproducts, process_feedback_for_subtask_and_set_to_finished_if_complete
+from lofar.sas.tmss.tmss.tmssapp.adapters.feedback import append_to_subtask_raw_feedback, process_feedback_into_subtask_dataproducts, process_feedback_for_subtask_and_set_to_finished_if_complete, reprocess_raw_feedback_for_subtask_and_set_to_finished_if_complete
 from lofar.lta.sip import constants
 from lofar.parameterset import parameterset
 
@@ -274,7 +274,48 @@ _isCobalt=T
         # assert raw_feedback stored
         self.assertEqual(self.feedback_pipe_incomplete.strip(), subtask_pipe.raw_feedback.strip())
 
+    def test_generate_dataproduct_feedback_from_subtask_feedback_and_set_finished_after_reprocessing(self):
+        subtask_data = Subtask_test_data(state=models.SubtaskState.objects.get(value='finishing'),
+                                         subtask_template=models.SubtaskTemplate.objects.get(name='observation control'))
+        subtask_obs:models.Subtask = models.Subtask.objects.create(**subtask_data)
+        subtask_obs_output = models.SubtaskOutput.objects.create(**SubtaskOutput_test_data(subtask=subtask_obs))
+
+        subtask_data = Subtask_test_data(state=models.SubtaskState.objects.get(value='finishing'),
+                                         subtask_template=models.SubtaskTemplate.objects.get(name='pipeline control'))
+        subtask_pipe: models.Subtask = models.Subtask.objects.create(**subtask_data)
+        subtask_pipe_output = models.SubtaskOutput.objects.create(**SubtaskOutput_test_data(subtask=subtask_pipe))
 
+        empty_feedback_template = models.DataproductFeedbackTemplate.objects.get(name='empty')
+        dataproduct_obs_out1:models.Dataproduct = models.Dataproduct.objects.create(**Dataproduct_test_data(filename='L220133_SAP000_SB000_uv.MS', producer=subtask_obs_output, feedback_template=empty_feedback_template))
+        dataproduct_obs_out2: models.Dataproduct = models.Dataproduct.objects.create(**Dataproduct_test_data(filename='L220133_SAP000_SB001_uv.MS', producer=subtask_obs_output, feedback_template=empty_feedback_template))
+        dataproduct_pipe_out1: models.Dataproduct = models.Dataproduct.objects.create(**Dataproduct_test_data(filename='L99307_SB000_uv.dppp.MS', producer=subtask_pipe_output, feedback_template=empty_feedback_template))
+        dataproduct_pipe_out2: models.Dataproduct = models.Dataproduct.objects.create(**Dataproduct_test_data(filename='L99307_SB001_uv.dppp.MS', producer=subtask_pipe_output, feedback_template=empty_feedback_template))
+        models.DataproductTransform.objects.create(input=dataproduct_obs_out1, output=dataproduct_pipe_out1, identity=True)
+        models.DataproductTransform.objects.create(input=dataproduct_obs_out2, output=dataproduct_pipe_out2, identity=True)
+
+        # assert dataproducts have no feedback docs before conversion
+        for dataproduct in [dataproduct_obs_out1, dataproduct_obs_out2, dataproduct_pipe_out1, dataproduct_pipe_out2]:
+            self.assertNotIn('percentage_written', dataproduct.feedback_doc)
+
+        process_feedback_for_subtask_and_set_to_finished_if_complete(subtask_obs, self.feedback_obs_complete)
+        process_feedback_for_subtask_and_set_to_finished_if_complete(subtask_pipe, self.feedback_pipe_incomplete) # <--- Note: test for incomplete feedback
+
+        # assert not in FINISHED state
+        self.assertEqual(models.SubtaskState.objects.get(value='finishing'), subtask_pipe.state)
+        self.assertFalse(subtask_pipe.is_feedback_complete)
+
+        # assert raw_feedback stored
+        self.assertEqual(self.feedback_pipe_incomplete.strip(), subtask_pipe.raw_feedback.strip())
+
+        # update the complete raw_feedback (for example by a support engineer)
+        subtask_pipe.raw_feedback = self.feedback_pipe_complete
+        subtask_pipe.save()
+
+        reprocess_raw_feedback_for_subtask_and_set_to_finished_if_complete(subtask_pipe)
+
+        # Now it should be finished and complete
+        self.assertEqual(models.SubtaskState.objects.get(value='finished'), subtask_pipe.state)
+        self.assertTrue(subtask_pipe.is_feedback_complete)
 
     def test_generate_dataproduct_feedback_from_subtask_feedback_and_set_finished(self):
         subtask_data = Subtask_test_data(state=models.SubtaskState.objects.get(value='finishing'),