diff --git a/SAS/TMSS/src/tmss/tmssapp/adapters/feedback.py b/SAS/TMSS/src/tmss/tmssapp/adapters/feedback.py index 48449e2b9f6805654509d76d42baf1610c02831e..80cbd05270e81e24342610e8a92cc0e50c4dd253 100644 --- a/SAS/TMSS/src/tmss/tmssapp/adapters/feedback.py +++ b/SAS/TMSS/src/tmss/tmssapp/adapters/feedback.py @@ -49,7 +49,7 @@ def check_feedback_is_complete(raw_feedback): def process_subtask_feedback(subtask:Subtask): - logger.info('############# Now processing feedback of subtask id=%s type=%s' % (subtask.id, subtask.specification_template.type.value)) + logger.info('Now processing feedback of subtask id=%s type=%s' % (subtask.id, subtask.specifications_template.type.value)) feedback_dict = parse_feedback(subtask.raw_feedback) dataproduct_feedback_docs = {} @@ -72,7 +72,7 @@ def process_subtask_feedback(subtask:Subtask): # determine corresponding TMSS dataproduct dataproduct = Dataproduct.objects.get(filename=feedback_dict[dpkey+'.filename']) - logger.info('############# Found dataproduct %s' % dataproduct.filename) + logger.debug('Found dataproduct %s' % dataproduct.filename) # derive values or collect for different subtask types storagewriter = feedback_dict[dpkey + '.storageWriter'].lower() @@ -86,23 +86,23 @@ def process_subtask_feedback(subtask:Subtask): duration = (subtask.stop_time - subtask.start_time).total_seconds() antennaset = subtask.specifications_doc['stations']['antenna_set'] stationlist = subtask.specifications_doc['stations']['station_list'] - antennafield = subtask.specifications_doc['stations']['antenna_set'] # todo: compute per station <- meaning? - antennatype = subtask.specifications_doc['stations']['antenna_set'] # todo: compute per field <- meaning? - pointing = subtask.specifications_doc['stations']['digital_pointings'][0]['pointing'] # todo: extract for this dataproduct <- index by what? + antennafield = subtask.specifications_doc['stations']['antenna_set'].split('_')[0] # todo: compute per station <- meaning? + antennatype = subtask.specifications_doc['stations']['antenna_set'].split('_')[0] # todo: compute per field <- meaning? + pointing = subtask.specifications_doc['stations']['digital_pointings'][int(feedback_dict[dpkey+'.SAP'])]['pointing'] # todo: extract for this dataproduct <- correctly indexed by SAP? else: input_dataproduct = DataproductTransform.objects.get(output=dataproduct).input - logger.info('############# Found input dataproduct %s' % input_dataproduct.filename) + logger.debug('Found input dataproduct %s' % input_dataproduct.filename) subbands = input_dataproduct.feedback_doc["frequency"]['subbands'] - duration = feedback_dict[dpkey + '.duration'] + duration = float(feedback_dict[dpkey + '.duration']) antennaset = input_dataproduct.feedback_doc["antennas"]['set'] - stationlist = input_dataproduct.feedback_doc["antennas"]['fields']['station'] - antennafield = input_dataproduct.feedback_doc["antennas"]['fields']['field'] - antennatype = input_dataproduct.feedback_doc["antennas"]['field']['type'] + stationlist = input_dataproduct.feedback_doc["antennas"]['fields'][0]['station'] + antennafield = input_dataproduct.feedback_doc["antennas"]['fields'][0]['field'] + antennatype = input_dataproduct.feedback_doc["antennas"]['fields'][0]['type'] pointing = input_dataproduct.feedback_doc["target"]['pointing'] # add feedback doc to dataproduct dataproduct.feedback_doc={ - "percentage_written": feedback_dict[dpkey+'.percentageWritten'], + "percentage_written": int(feedback_dict[dpkey+'.percentageWritten']), "frequency": { "subbands": subbands, "central_frequencies": [float(feedback_dict[dpkey+'.centralFrequency'])], @@ -112,7 +112,7 @@ def process_subtask_feedback(subtask:Subtask): "time": { "start_time": feedback_dict[dpkey+'.startTime'], "duration": duration, - "sample_width": feedback_dict[dpkey+'.integrationInterval'], + "sample_width": float(feedback_dict[dpkey+'.integrationInterval']), }, "antennas": { "set": antennaset, @@ -132,12 +132,13 @@ def process_subtask_feedback(subtask:Subtask): "type": "float", # fixed "bits": 32, # fixed "writer": storagewriter, - "writer_version": feedback_dict[dpkey + '.storageWriterVersion'] + "writer_version": feedback_dict[dpkey + '.storageWriterVersion'], + "complex": True # fixed } } i += 1 dataproduct.save() - logger.info('############# saved %s %s' % (dataproduct.filename, dataproduct.feedback_doc)) + logger.debug('Saved %s %s' % (dataproduct.filename, dataproduct.feedback_doc)) def generate_dataproduct_feedback_from_subtask_feedback_and_set_finished(subtask:Subtask): diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/dataproduct-feedback.json b/SAS/TMSS/src/tmss/tmssapp/schemas/dataproduct-feedback.json index e236ff624dc17c8139c9ee3c350885070fb86354..313f738868197d5cf1a1bd414b631d7ccaee1f1a 100644 --- a/SAS/TMSS/src/tmss/tmssapp/schemas/dataproduct-feedback.json +++ b/SAS/TMSS/src/tmss/tmssapp/schemas/dataproduct-feedback.json @@ -183,7 +183,7 @@ } } }, - "required": [ "antenna_set", "fields" ] + "required": [ "fields" ] }, "target": { "title": "Target", diff --git a/SAS/TMSS/test/t_adapter.py b/SAS/TMSS/test/t_adapter.py index c588548ee495c2cb6c091a82707f866a0ed91458..4a7428ce1a224630289cc99aed89501458719bb9 100755 --- a/SAS/TMSS/test/t_adapter.py +++ b/SAS/TMSS/test/t_adapter.py @@ -210,7 +210,7 @@ feedback_version=03.01.00 subtask_template=models.SubtaskTemplate.objects.get(name='observation control')) subtask_obs:models.Subtask = models.Subtask.objects.create(**subtask_data) - subtask_data = Subtask_test_data(raw_feedback=self.feedback_obs_complete, + subtask_data = Subtask_test_data(raw_feedback=self.feedback_pipe_complete, state=models.SubtaskState.objects.get(value='finishing'), subtask_template=models.SubtaskTemplate.objects.get(name='pipeline control')) subtask_pipe: models.Subtask = models.Subtask.objects.create(**subtask_data) @@ -229,8 +229,9 @@ feedback_version=03.01.00 generate_dataproduct_feedback_from_subtask_feedback_and_set_finished(subtask_obs) generate_dataproduct_feedback_from_subtask_feedback_and_set_finished(subtask_pipe) - # assert dataproduct feedback docs have feedback after conversion + # reload dataproducts and assert dataproduct feedback docs have feedback after conversion for dataproduct in [dataproduct_obs_out1, dataproduct_obs_out2, dataproduct_pipe_out1, dataproduct_pipe_out2]: + dataproduct.refresh_from_db() self.assertIsNotNone(dataproduct.feedback_doc) self.assertIn('percentage_written', dataproduct.feedback_doc)