Skip to content
Snippets Groups Projects
Commit fa996f8e authored by Jörn Künsemöller's avatar Jörn Künsemöller
Browse files

TMSS-307: Debugging

parent 225c3b6f
No related branches found
No related tags found
1 merge request!226Resolve TMSS-307
...@@ -49,7 +49,7 @@ def check_feedback_is_complete(raw_feedback): ...@@ -49,7 +49,7 @@ def check_feedback_is_complete(raw_feedback):
def process_subtask_feedback(subtask:Subtask): def process_subtask_feedback(subtask:Subtask):
logger.info('############# Now processing feedback of subtask id=%s type=%s' % (subtask.id, subtask.specification_template.type.value)) logger.info('Now processing feedback of subtask id=%s type=%s' % (subtask.id, subtask.specifications_template.type.value))
feedback_dict = parse_feedback(subtask.raw_feedback) feedback_dict = parse_feedback(subtask.raw_feedback)
dataproduct_feedback_docs = {} dataproduct_feedback_docs = {}
...@@ -72,7 +72,7 @@ def process_subtask_feedback(subtask:Subtask): ...@@ -72,7 +72,7 @@ def process_subtask_feedback(subtask:Subtask):
# determine corresponding TMSS dataproduct # determine corresponding TMSS dataproduct
dataproduct = Dataproduct.objects.get(filename=feedback_dict[dpkey+'.filename']) dataproduct = Dataproduct.objects.get(filename=feedback_dict[dpkey+'.filename'])
logger.info('############# Found dataproduct %s' % dataproduct.filename) logger.debug('Found dataproduct %s' % dataproduct.filename)
# derive values or collect for different subtask types # derive values or collect for different subtask types
storagewriter = feedback_dict[dpkey + '.storageWriter'].lower() storagewriter = feedback_dict[dpkey + '.storageWriter'].lower()
...@@ -86,23 +86,23 @@ def process_subtask_feedback(subtask:Subtask): ...@@ -86,23 +86,23 @@ def process_subtask_feedback(subtask:Subtask):
duration = (subtask.stop_time - subtask.start_time).total_seconds() duration = (subtask.stop_time - subtask.start_time).total_seconds()
antennaset = subtask.specifications_doc['stations']['antenna_set'] antennaset = subtask.specifications_doc['stations']['antenna_set']
stationlist = subtask.specifications_doc['stations']['station_list'] stationlist = subtask.specifications_doc['stations']['station_list']
antennafield = subtask.specifications_doc['stations']['antenna_set'] # todo: compute per station <- meaning? antennafield = subtask.specifications_doc['stations']['antenna_set'].split('_')[0] # todo: compute per station <- meaning?
antennatype = subtask.specifications_doc['stations']['antenna_set'] # todo: compute per field <- meaning? antennatype = subtask.specifications_doc['stations']['antenna_set'].split('_')[0] # todo: compute per field <- meaning?
pointing = subtask.specifications_doc['stations']['digital_pointings'][0]['pointing'] # todo: extract for this dataproduct <- index by what? pointing = subtask.specifications_doc['stations']['digital_pointings'][int(feedback_dict[dpkey+'.SAP'])]['pointing'] # todo: extract for this dataproduct <- correctly indexed by SAP?
else: else:
input_dataproduct = DataproductTransform.objects.get(output=dataproduct).input input_dataproduct = DataproductTransform.objects.get(output=dataproduct).input
logger.info('############# Found input dataproduct %s' % input_dataproduct.filename) logger.debug('Found input dataproduct %s' % input_dataproduct.filename)
subbands = input_dataproduct.feedback_doc["frequency"]['subbands'] subbands = input_dataproduct.feedback_doc["frequency"]['subbands']
duration = feedback_dict[dpkey + '.duration'] duration = float(feedback_dict[dpkey + '.duration'])
antennaset = input_dataproduct.feedback_doc["antennas"]['set'] antennaset = input_dataproduct.feedback_doc["antennas"]['set']
stationlist = input_dataproduct.feedback_doc["antennas"]['fields']['station'] stationlist = input_dataproduct.feedback_doc["antennas"]['fields'][0]['station']
antennafield = input_dataproduct.feedback_doc["antennas"]['fields']['field'] antennafield = input_dataproduct.feedback_doc["antennas"]['fields'][0]['field']
antennatype = input_dataproduct.feedback_doc["antennas"]['field']['type'] antennatype = input_dataproduct.feedback_doc["antennas"]['fields'][0]['type']
pointing = input_dataproduct.feedback_doc["target"]['pointing'] pointing = input_dataproduct.feedback_doc["target"]['pointing']
# add feedback doc to dataproduct # add feedback doc to dataproduct
dataproduct.feedback_doc={ dataproduct.feedback_doc={
"percentage_written": feedback_dict[dpkey+'.percentageWritten'], "percentage_written": int(feedback_dict[dpkey+'.percentageWritten']),
"frequency": { "frequency": {
"subbands": subbands, "subbands": subbands,
"central_frequencies": [float(feedback_dict[dpkey+'.centralFrequency'])], "central_frequencies": [float(feedback_dict[dpkey+'.centralFrequency'])],
...@@ -112,7 +112,7 @@ def process_subtask_feedback(subtask:Subtask): ...@@ -112,7 +112,7 @@ def process_subtask_feedback(subtask:Subtask):
"time": { "time": {
"start_time": feedback_dict[dpkey+'.startTime'], "start_time": feedback_dict[dpkey+'.startTime'],
"duration": duration, "duration": duration,
"sample_width": feedback_dict[dpkey+'.integrationInterval'], "sample_width": float(feedback_dict[dpkey+'.integrationInterval']),
}, },
"antennas": { "antennas": {
"set": antennaset, "set": antennaset,
...@@ -132,12 +132,13 @@ def process_subtask_feedback(subtask:Subtask): ...@@ -132,12 +132,13 @@ def process_subtask_feedback(subtask:Subtask):
"type": "float", # fixed "type": "float", # fixed
"bits": 32, # fixed "bits": 32, # fixed
"writer": storagewriter, "writer": storagewriter,
"writer_version": feedback_dict[dpkey + '.storageWriterVersion'] "writer_version": feedback_dict[dpkey + '.storageWriterVersion'],
"complex": True # fixed
} }
} }
i += 1 i += 1
dataproduct.save() dataproduct.save()
logger.info('############# saved %s %s' % (dataproduct.filename, dataproduct.feedback_doc)) logger.debug('Saved %s %s' % (dataproduct.filename, dataproduct.feedback_doc))
def generate_dataproduct_feedback_from_subtask_feedback_and_set_finished(subtask:Subtask): def generate_dataproduct_feedback_from_subtask_feedback_and_set_finished(subtask:Subtask):
......
...@@ -183,7 +183,7 @@ ...@@ -183,7 +183,7 @@
} }
} }
}, },
"required": [ "antenna_set", "fields" ] "required": [ "fields" ]
}, },
"target": { "target": {
"title": "Target", "title": "Target",
......
...@@ -210,7 +210,7 @@ feedback_version=03.01.00 ...@@ -210,7 +210,7 @@ feedback_version=03.01.00
subtask_template=models.SubtaskTemplate.objects.get(name='observation control')) subtask_template=models.SubtaskTemplate.objects.get(name='observation control'))
subtask_obs:models.Subtask = models.Subtask.objects.create(**subtask_data) subtask_obs:models.Subtask = models.Subtask.objects.create(**subtask_data)
subtask_data = Subtask_test_data(raw_feedback=self.feedback_obs_complete, subtask_data = Subtask_test_data(raw_feedback=self.feedback_pipe_complete,
state=models.SubtaskState.objects.get(value='finishing'), state=models.SubtaskState.objects.get(value='finishing'),
subtask_template=models.SubtaskTemplate.objects.get(name='pipeline control')) subtask_template=models.SubtaskTemplate.objects.get(name='pipeline control'))
subtask_pipe: models.Subtask = models.Subtask.objects.create(**subtask_data) subtask_pipe: models.Subtask = models.Subtask.objects.create(**subtask_data)
...@@ -229,8 +229,9 @@ feedback_version=03.01.00 ...@@ -229,8 +229,9 @@ feedback_version=03.01.00
generate_dataproduct_feedback_from_subtask_feedback_and_set_finished(subtask_obs) generate_dataproduct_feedback_from_subtask_feedback_and_set_finished(subtask_obs)
generate_dataproduct_feedback_from_subtask_feedback_and_set_finished(subtask_pipe) generate_dataproduct_feedback_from_subtask_feedback_and_set_finished(subtask_pipe)
# assert dataproduct feedback docs have feedback after conversion # reload dataproducts and assert dataproduct feedback docs have feedback after conversion
for dataproduct in [dataproduct_obs_out1, dataproduct_obs_out2, dataproduct_pipe_out1, dataproduct_pipe_out2]: for dataproduct in [dataproduct_obs_out1, dataproduct_obs_out2, dataproduct_pipe_out1, dataproduct_pipe_out2]:
dataproduct.refresh_from_db()
self.assertIsNotNone(dataproduct.feedback_doc) self.assertIsNotNone(dataproduct.feedback_doc)
self.assertIn('percentage_written', dataproduct.feedback_doc) self.assertIn('percentage_written', dataproduct.feedback_doc)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment