TMSS-307: Debugging

SAS/TMSS/src/tmss/tmssapp/adapters/feedback.py

@@ -49,7 +49,7 @@ def check_feedback_is_complete(raw_feedback):
 
 
 def process_subtask_feedback(subtask:Subtask):
-    logger.info('############# Now processing feedback of subtask id=%s type=%s' % (subtask.id, subtask.specification_template.type.value))
+    logger.info('Now processing feedback of subtask id=%s type=%s' % (subtask.id, subtask.specifications_template.type.value))
     feedback_dict = parse_feedback(subtask.raw_feedback)
 
     dataproduct_feedback_docs = {}
@@ -72,7 +72,7 @@ def process_subtask_feedback(subtask:Subtask):
 
             # determine corresponding TMSS dataproduct
             dataproduct = Dataproduct.objects.get(filename=feedback_dict[dpkey+'.filename'])
-            logger.info('############# Found dataproduct %s' % dataproduct.filename)
+            logger.debug('Found dataproduct %s' % dataproduct.filename)
 
             # derive values or collect for different subtask types
             storagewriter = feedback_dict[dpkey + '.storageWriter'].lower()
@@ -86,23 +86,23 @@ def process_subtask_feedback(subtask:Subtask):
                 duration = (subtask.stop_time - subtask.start_time).total_seconds()
                 antennaset = subtask.specifications_doc['stations']['antenna_set']
                 stationlist = subtask.specifications_doc['stations']['station_list']
-                antennafield = subtask.specifications_doc['stations']['antenna_set']  # todo: compute per station <- meaning?
-                antennatype = subtask.specifications_doc['stations']['antenna_set']   # todo: compute per field <- meaning?
-                pointing = subtask.specifications_doc['stations']['digital_pointings'][0]['pointing'] # todo: extract for this dataproduct <- index by what?
+                antennafield = subtask.specifications_doc['stations']['antenna_set'].split('_')[0]  # todo: compute per station <- meaning?
+                antennatype = subtask.specifications_doc['stations']['antenna_set'].split('_')[0]   # todo: compute per field <- meaning?
+                pointing = subtask.specifications_doc['stations']['digital_pointings'][int(feedback_dict[dpkey+'.SAP'])]['pointing'] # todo: extract for this dataproduct <- correctly indexed by SAP?
             else:
                 input_dataproduct = DataproductTransform.objects.get(output=dataproduct).input
-                logger.info('############# Found input dataproduct %s' % input_dataproduct.filename)
+                logger.debug('Found input dataproduct %s' % input_dataproduct.filename)
                 subbands = input_dataproduct.feedback_doc["frequency"]['subbands']
-                duration = feedback_dict[dpkey + '.duration']
+                duration = float(feedback_dict[dpkey + '.duration'])
                 antennaset = input_dataproduct.feedback_doc["antennas"]['set']
-                stationlist = input_dataproduct.feedback_doc["antennas"]['fields']['station']
-                antennafield = input_dataproduct.feedback_doc["antennas"]['fields']['field']
-                antennatype = input_dataproduct.feedback_doc["antennas"]['field']['type']
+                stationlist = input_dataproduct.feedback_doc["antennas"]['fields'][0]['station']
+                antennafield = input_dataproduct.feedback_doc["antennas"]['fields'][0]['field']
+                antennatype = input_dataproduct.feedback_doc["antennas"]['fields'][0]['type']
                 pointing = input_dataproduct.feedback_doc["target"]['pointing']
 
             # add feedback doc to dataproduct
             dataproduct.feedback_doc={
-                "percentage_written": feedback_dict[dpkey+'.percentageWritten'],
+                "percentage_written": int(feedback_dict[dpkey+'.percentageWritten']),
                 "frequency": {
                     "subbands": subbands,
                     "central_frequencies": [float(feedback_dict[dpkey+'.centralFrequency'])],
@@ -112,7 +112,7 @@ def process_subtask_feedback(subtask:Subtask):
                 "time": {
                     "start_time": feedback_dict[dpkey+'.startTime'],
                     "duration": duration,
-                    "sample_width": feedback_dict[dpkey+'.integrationInterval'],
+                    "sample_width": float(feedback_dict[dpkey+'.integrationInterval']),
                 },
                 "antennas": {
                     "set": antennaset,
@@ -132,12 +132,13 @@ def process_subtask_feedback(subtask:Subtask):
                     "type": "float",                                # fixed
                     "bits": 32,                                     # fixed
                     "writer": storagewriter,
-                    "writer_version": feedback_dict[dpkey + '.storageWriterVersion']
+                    "writer_version": feedback_dict[dpkey + '.storageWriterVersion'],
+                    "complex": True                                 # fixed
                 }
             }
             i += 1
             dataproduct.save()
-            logger.info('############# saved %s %s' % (dataproduct.filename, dataproduct.feedback_doc))
+            logger.debug('Saved %s %s' % (dataproduct.filename, dataproduct.feedback_doc))
 
 
 def generate_dataproduct_feedback_from_subtask_feedback_and_set_finished(subtask:Subtask):

SAS/TMSS/src/tmss/tmssapp/schemas/dataproduct-feedback.json

@@ -183,7 +183,7 @@
           }
         }
       },
-      "required": [ "antenna_set", "fields" ]
+      "required": [ "fields" ]
     },
     "target": {
       "title": "Target",

SAS/TMSS/test/t_adapter.py

@@ -210,7 +210,7 @@ feedback_version=03.01.00
                                          subtask_template=models.SubtaskTemplate.objects.get(name='observation control'))
         subtask_obs:models.Subtask = models.Subtask.objects.create(**subtask_data)
 
-        subtask_data = Subtask_test_data(raw_feedback=self.feedback_obs_complete,
+        subtask_data = Subtask_test_data(raw_feedback=self.feedback_pipe_complete,
                                              state=models.SubtaskState.objects.get(value='finishing'),
                                              subtask_template=models.SubtaskTemplate.objects.get(name='pipeline control'))
         subtask_pipe: models.Subtask = models.Subtask.objects.create(**subtask_data)
@@ -229,8 +229,9 @@ feedback_version=03.01.00
         generate_dataproduct_feedback_from_subtask_feedback_and_set_finished(subtask_obs)
         generate_dataproduct_feedback_from_subtask_feedback_and_set_finished(subtask_pipe)
 
-        # assert dataproduct feedback docs have feedback after conversion
+        # reload dataproducts and assert dataproduct feedback docs have feedback after conversion
         for dataproduct in [dataproduct_obs_out1, dataproduct_obs_out2, dataproduct_pipe_out1, dataproduct_pipe_out2]:
+            dataproduct.refresh_from_db()
             self.assertIsNotNone(dataproduct.feedback_doc)
             self.assertIn('percentage_written', dataproduct.feedback_doc)