diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/adapters/sip.py b/SAS/TMSS/backend/src/tmss/tmssapp/adapters/sip.py
index 1ed040a79cc936a9b4fcca4ee9c392e0c3cf6254..47772d5a4ec45504390cc655f9f3a3b10548fbb9 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/adapters/sip.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/adapters/sip.py
@@ -292,14 +292,18 @@ def create_sip_representation_for_dataproduct(dataproduct: Dataproduct):
Datatype.Choices.TIME_SERIES.value: constants.DATAPRODUCTTYPE_BEAM_FORMED_DATA,
Datatype.Choices.INSTRUMENT_MODEL.value : constants.DATAPRODUCTTYPE_INSTRUMENT_MODEL,
Datatype.Choices.IMAGE.value : constants.DATAPRODUCTTYPE_SKY_IMAGE,
- Datatype.Choices.QUALITY.value : constants.DATAPRODUCTTYPE_UNKNOWN}
+ Datatype.Choices.QUALITY.value : constants.DATAPRODUCTTYPE_UNKNOWN,
+ # Datatype.Choices.???.value : constants.DATAPRODUCTTYPE_PULSAR_PIPELINE_SUMMARY_OUTPUT, # todo: we have an extra type for the summary in the XSD, but not in TMSS. Check if we need to add that (example SIPs apparently use the non-summary type for both flavors.)
+ Datatype.Choices.PULSAR_PROFILE.value : constants.DATAPRODUCTTYPE_PULSAR_PIPELINE_OUTPUT}
# Note: this is for the fileformat property present on all dataproduct flavors,
# dataproduct classes are differentiated in addition to that below
fileformat_map = {Dataformat.Choices.MEASUREMENTSET.value: constants.FILEFORMATTYPE_AIPS___CASA,
Dataformat.Choices.BEAMFORMED.value: constants.FILEFORMATTYPE_HDF5,
Dataformat.Choices.QA_HDF5.value: constants.FILEFORMATTYPE_HDF5,
- Dataformat.Choices.QA_PLOTS.value: constants.FILEFORMATTYPE_UNDOCUMENTED}
+ Dataformat.Choices.QA_PLOTS.value: constants.FILEFORMATTYPE_UNDOCUMENTED,
+ Dataformat.Choices.PULP_SUMMARY.value: constants.FILEFORMATTYPE_PULP,
+ Dataformat.Choices.PULP_ANALYSIS.value: constants.FILEFORMATTYPE_PULP}
storage_writer_map = {"dysco": constants.STORAGEWRITERTYPE_DYSCOSTORAGEMANAGER,
"unknown": constants.STORAGEWRITERTYPE_UNKNOWN,
@@ -310,13 +314,13 @@ def create_sip_representation_for_dataproduct(dataproduct: Dataproduct):
dataproduct_type = type_map[dataproduct.datatype.value]
except Exception as err:
dataproduct_type = constants.DATAPRODUCTTYPE_UNKNOWN
- logger.warning("Could not determine the type of dataproduct id %s (%s). Falling back to %s" % (dataproduct.id, err, dataproduct_type))
+ logger.warning("Could not determine the dataproduct type (datatype) of dataproduct id %s (%s). Falling back to %s" % (dataproduct.id, err, dataproduct_type))
try:
dataproduct_fileformat = fileformat_map[dataproduct.dataformat.value] # todo same as with type? Why is this not with the data? Why is this so different from the LTA datamodel?
except Exception as err:
dataproduct_fileformat = constants.FILEFORMATTYPE_UNDOCUMENTED
- logger.warning("Could not determine the type of dataproduct id %s (%s). Falling back to %s" % (dataproduct.id, err, dataproduct_fileformat))
+ logger.warning("Could not determine the fileformat type (dataformat) of dataproduct id %s (%s). Falling back to %s" % (dataproduct.id, err, dataproduct_fileformat))
dataproduct_map = siplib.DataProductMap(type=dataproduct_type,
identifier=get_siplib_identifier(dataproduct.global_identifier, "Dataproduct %s" % dataproduct.id),
@@ -373,7 +377,7 @@ def create_sip_representation_for_dataproduct(dataproduct: Dataproduct):
)
# IncoherentStokesBeam
- elif 'coherent' in dataproduct.specifications_doc and dataproduct.specifications_doc['coherent'] is False:
+ elif 'coherent' in dataproduct.specifications_doc and dataproduct.specifications_doc['coherent'] is False: # todo: should this be read from feedback_doc now?
beams.append(
siplib.IncoherentStokesBeam(arraybeam_map=beam_map)
)
@@ -400,6 +404,38 @@ def create_sip_representation_for_dataproduct(dataproduct: Dataproduct):
dataproduct_map,
beams
)
+
+ elif dataproduct.dataformat.value == Dataformat.Choices.PULP_SUMMARY.value:
+ sip_dataproduct = siplib.PulpSummaryDataProduct(
+ dataproduct_map,
+ filecontent=dataproduct.feedback_doc['files'],
+ datatype="SummaryCoherentStokes" if dataproduct.feedback_doc['target']['coherent'] else "SummaryIncoherentStokes" # todo: correct? what about complex voltages?
+ )
+
+ elif dataproduct.dataformat.value == Dataformat.Choices.PULP_ANALYSIS.value:
+ field = dataproduct.feedback_doc['antennas']['fields'][0] # todo: can we have more than one here?
+ sip_dataproduct = siplib.PulpDataProduct(
+ dataproduct_map,
+ filecontent=dataproduct.feedback_doc['files'],
+ datatype="CoherentStokes" if dataproduct.feedback_doc['target']['coherent'] else "IncoherentStokes", # todo: correct? what about complex voltages?
+ arraybeam=siplib.FlysEyeBeam(
+ siplib.ArrayBeamMap(subarraypointing_identifier=get_siplib_identifier(dataproduct.sap.global_identifier, "SAP %s" % dataproduct.sap.id),
+ beamnumber=dataproduct.specifications_doc.get('identifiers', {}).get('tab_index', 0), # todo: verify: correct?
+ dispersionmeasure=0, # fixed
+ numberofsubbands=len(dataproduct.feedback_doc['frequency']['subbands']),
+ stationsubbands=dataproduct.feedback_doc['frequency']['subbands'],
+ samplingtime=dataproduct.feedback_doc['time']['sample_width'],
+ samplingtimeunit="s",
+ centralfrequencies=dataproduct.feedback_doc['frequency']['central_frequencies'],
+ centralfrequencies_unit="Hz",
+ channelwidth_frequency=dataproduct.feedback_doc['frequency']['channel_width'],
+ channelwidth_frequencyunit="Hz",
+ channelspersubband=dataproduct.feedback_doc['frequency']['channels_per_subband'],
+ stokes=dataproduct.feedback_doc['samples']['polarisations']),
+ station=siplib.Station.preconfigured(field['station'], [field['field']])
+ )
+ )
+
# todo: distinguish and create other dataproduct types. Probably most of these can be filled in over time as needed,
# but they are not required for UC1. Here are stubs to start from for the other types the LTA supports:
# elif dataproduct.dataformat.value == Dataformat.Choices.<???>.value: # todo
@@ -496,6 +532,7 @@ def create_sip_representation_for_dataproduct(dataproduct: Dataproduct):
else:
logger.warning('Could not identify corresponding SIP dataproduct flavor for dataproduct=%s, dataformat=%s. Adding as SimpleDataproduct to SIP.' % (dataproduct, dataproduct.dataformat))
sip_dataproduct = siplib.SimpleDataProduct(dataproduct_map)
+ # todo: do we have to include related dataproducts in the SIP or does the LTA make that connection for us?
return sip_dataproduct
diff --git a/SAS/TMSS/backend/test/t_adapter.py b/SAS/TMSS/backend/test/t_adapter.py
index 396d77eeb72a6f4498ded1b4646a253a7fbb63b7..c230c3ed384fb058db42adf0df8cada37a46dfa1 100755
--- a/SAS/TMSS/backend/test/t_adapter.py
+++ b/SAS/TMSS/backend/test/t_adapter.py
@@ -350,6 +350,92 @@ class SIPadapterTest(unittest.TestCase):
# assert we get a flyseye beam if we have a single antenna field
self.assertIn(str('<arrayBeam xsi:type="sip:FlysEyeBeam">'), sip.get_prettyxml())
+ def test_simple_sip_generate_from_dataproduct_pulp(self):
+ """
+ Test if SIP is generated successfully when subtask, dataproduct and SAP objects are created
+ Check some value in the SIP (xml) output
+ Check that the SIP identifiers are in SIP (xml) output
+ Check the number of SIP identifiers are increased with 3
+ Check that all SIP identifiers are unique
+ """
+ subtask_template = models.SubtaskTemplate.objects.get(name='observation control')
+ specifications_doc = get_default_json_object_for_schema(subtask_template.schema)
+ specifications_doc['stations']['filter'] = "HBA_110_190"
+ feedback_template = models.DataproductFeedbackTemplate.objects.get(name='feedback')
+ # feedback_doc = get_default_json_object_for_schema(feedback_template.schema) # todo <- fix the default generator, for some reason it does not produce valid json here...
+ feedback_doc = {'percentage_written': 100, 'frequency': {'subbands': [152], 'central_frequencies': [33593750.0], 'channel_width': 3051.7578125, 'channels_per_subband': 64}, 'time': {'start_time': '2013-02-16T17:00:00', 'duration': 5.02732992172, 'sample_width': 2.00278016}, 'antennas': {'set': 'HBA_DUAL', 'fields': [{'type': 'HBA', 'field': 'HBA0', 'station': 'CS001'}, {'type': 'HBA', 'field': 'HBA1', 'station': 'CS001'}]}, 'target': {'pointing': {'angle1': 0, 'angle2': 0, 'direction_type': 'J2000'}, 'coherent': True}, 'samples': {'polarisations': ['XX', 'XY', 'YX', 'YY'], 'type': 'float', 'bits': 32, 'writer': 'standard', 'writer_version': '2.2.0', 'complex': True}, 'files': ['stokes/SAP0/CS003HBA1/L773569_SAP000_B005_S0_P000_bf.h5', 'stokes/SAP0/RS106HBA/L773569_SAP000_B046_S0_P000_bf.h5'], '$schema': 'http://127.0.0.1:8001/api/schemas/dataproductfeedbacktemplate/feedback/1#'}
+ for dp in specifications_doc['stations']['digital_pointings']:
+ dp['subbands'] = list(range(8))
+ # Create SubTask(output)
+ subtask_data = Subtask_test_data(subtask_template=subtask_template, specifications_doc=specifications_doc)
+ subtask: models.Subtask = models.Subtask.objects.create(**subtask_data)
+ subtask.task_blueprints.set([models.TaskBlueprint.objects.create(**TaskBlueprint_test_data())])
+ subtask_output = models.SubtaskOutput.objects.create(**SubtaskOutput_test_data(subtask=subtask))
+ # Create Dataproduct
+ dataproduct: models.Dataproduct = models.Dataproduct.objects.create(**Dataproduct_test_data(feedback_doc=feedback_doc, producer=subtask_output,
+ dataformat=models.Dataformat.objects.get(value="pulp analysis"),
+ datatype=models.Datatype.objects.get(value="pulsar profile")))
+
+ # Create SAP
+ sap_template = models.SAPTemplate.objects.get(name="SAP")
+ specifications_doc = get_default_json_object_for_schema(sap_template.schema)
+ sap = models.SAP.objects.create(specifications_doc=specifications_doc, specifications_template=sap_template)
+ sap.save()
+
+ dataproduct.sap = sap
+ dataproduct.save()
+
+ # PULP ANALYSIS
+
+ sip = generate_sip_for_dataproduct(dataproduct)
+
+ # double-check that SIP contains values from feedback and specifications docs
+ self.assertIn(str(feedback_doc['frequency']['channel_width']), sip.get_prettyxml())
+ self.assertIn(constants.FILTERSELECTIONTYPE_110_190_MHZ, sip.get_prettyxml())
+ for pol in feedback_doc['samples']['polarisations']:
+ self.assertIn(str(pol), sip.get_prettyxml())
+
+ self.assertIn(str(subtask.global_identifier.unique_identifier), sip.get_prettyxml())
+ self.assertIn(str(dataproduct.global_identifier.unique_identifier), sip.get_prettyxml())
+ self.assertIn(str(sap.global_identifier.unique_identifier), sip.get_prettyxml())
+
+ # assert that a pulp analysis dataproduct in TMSS creates a PulpDataProduct in the SIP
+ self.assertIn(str('<dataProduct xsi:type="sip:PulpDataProduct">'), sip.get_prettyxml())
+
+ # assert beam type
+ self.assertIn(str('FlysEyeBeam'), sip.get_prettyxml())
+
+ # assert datatype
+ self.assertIn(str('<dataType>CoherentStokes</dataType>'), sip.get_prettyxml())
+
+ # assert fileformat
+ self.assertIn(str('<fileFormat>PULP</fileFormat>'), sip.get_prettyxml())
+
+ # alter dataproduct, recreate sip
+ dataproduct.feedback_doc['target']['coherent'] = False
+ dataproduct.save()
+ sip = generate_sip_for_dataproduct(dataproduct)
+
+ # assert datatype reflects change of coherent flag
+ self.assertIn(str('<dataType>IncoherentStokes</dataType>'), sip.get_prettyxml())
+
+ # PULP SUMMARY
+
+ # alter dataproduct, recreate sip
+ dataproduct.dataformat = models.Dataformat.objects.get(value="pulp summary")
+ dataproduct.feedback_doc['$schema'] = 'http://127.0.0.1:8001/api/schemas/dataproductfeedbacktemplate/pulp summary/1#'
+ dataproduct.save()
+ sip = generate_sip_for_dataproduct(dataproduct)
+
+ # assert datatype reflects change of dataformat
+ self.assertIn(str('<dataType>SummaryIncoherentStokes</dataType>'), sip.get_prettyxml())
+
+ # assert that a pulp summary dataproduct in TMSS creates a PulpSummaryDataProduct in the SIP
+ self.assertIn(str('<dataProduct xsi:type="sip:PulpSummaryDataProduct">'), sip.get_prettyxml())
+
+ # assert fileformat
+ self.assertIn(str('<fileFormat>PULP</fileFormat>'), sip.get_prettyxml())
+
class ProjectReportTest(unittest.TestCase):
def setUp(self):