From 65889b3272f1b0f927dca5fe2c22500fd6850737 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rn=20K=C3=BCnsem=C3=B6ller?=
<jkuensem@physik.uni-bielefeld.de>
Date: Tue, 2 Nov 2021 22:20:10 +0100
Subject: [PATCH] TMSS-1144: add more realistic, complex test case for sip
generation
---
SAS/TMSS/backend/test/t_adapter.py | 67 ++++++++++++++++++++++++++++++
1 file changed, 67 insertions(+)
diff --git a/SAS/TMSS/backend/test/t_adapter.py b/SAS/TMSS/backend/test/t_adapter.py
index 0bdb68bde7e..be02b92c61a 100755
--- a/SAS/TMSS/backend/test/t_adapter.py
+++ b/SAS/TMSS/backend/test/t_adapter.py
@@ -449,6 +449,73 @@ class SIPadapterTest(unittest.TestCase):
# assert fileformat
self.assertIn(str('<fileFormat>PULP</fileFormat>'), sip.get_prettyxml())
+
+ def test_sip_generate_with_related_dataproducts(self):
+ """
+ Test if a SIP is generated successfully when it is pipeline output.
+ Assert that related intermediate dataproducts from the observation are referenced in the SIP.
+ Produce a bunch of SIPs with a larger number of related dataproducts to make sure this performs well enough
+ and finishes within test time.
+ """
+
+ # create obs subtask
+ subtask_template = models.SubtaskTemplate.objects.get(name='observation control')
+ specifications_doc = get_default_json_object_for_schema(subtask_template.schema)
+ specifications_doc['stations']['filter'] = "HBA_210_250"
+ feedback_template = models.DataproductFeedbackTemplate.objects.get(name='feedback')
+ # feedback_doc = get_default_json_object_for_schema(feedback_template.schema) # todo <- fix the default generator, for some reason it does not produce valid json here...
+ feedback_doc = {'percentage_written': 100, 'frequency': {'subbands': [156], 'central_frequencies': [33593750.0], 'channel_width': 6103.515625, 'channels_per_subband': 32}, 'time': {'start_time': '2013-02-16T17:00:00', 'duration': 5.02732992172, 'sample_width': 2.00278016}, 'antennas': {'set': 'HBA_DUAL', 'fields': [{'type': 'HBA', 'field': 'HBA0', 'station': 'CS001'}, {'type': 'HBA', 'field': 'HBA1', 'station': 'CS001'}]}, 'target': {'pointing': {'angle1': 0, 'angle2': 0, 'direction_type': 'J2000', "target": "target1"}}, 'samples': {'polarisations': ['XX', 'XY', 'YX', 'YY'], 'type': 'float', 'bits': 32, 'writer': 'standard', 'writer_version': '2.2.0', 'complex': True}, '$schema': 'http://127.0.0.1:8001/api/schemas/dataproductfeedbacktemplate/feedback/1#'}
+ for dp in specifications_doc['stations']['digital_pointings']:
+ dp['subbands'] = list(range(8))
+ # Create SubTask(output)
+ subtask_data = Subtask_test_data(subtask_template=subtask_template, specifications_doc=specifications_doc, task_blueprint=models.TaskBlueprint.objects.create(**TaskBlueprint_test_data()))
+ subtask:models.Subtask = models.Subtask.objects.create(**subtask_data)
+ subtask_output = models.SubtaskOutput.objects.create(**SubtaskOutput_test_data(subtask=subtask))
+
+ # Create SAP
+ sap_template = models.SAPTemplate.objects.get(name="SAP")
+ specifications_doc = get_default_json_object_for_schema(sap_template.schema)
+ sap = models.SAP.objects.create(specifications_doc=specifications_doc, specifications_template=sap_template)
+ sap.save()
+
+ # Create a bunch of related Dataproducts (observation output, pipeline input)
+ related_dataproducts = []
+ for i in range(244):
+ dataproduct: models.Dataproduct = models.Dataproduct.objects.create(**Dataproduct_test_data(filename='my_related_dataproduct_%s' % i, feedback_doc=feedback_doc, producer=subtask_output,
+ dataformat=models.Dataformat.objects.get(value="Beamformed"),
+ datatype=models.Datatype.objects.get(value="time series")))
+
+ dataproduct.sap = sap
+ dataproduct.save()
+ related_dataproducts.append(dataproduct)
+
+ # create pipeline subtask
+ subtask_template = models.SubtaskTemplate.objects.get(name='preprocessing pipeline')
+ specifications_doc = get_default_json_object_for_schema(subtask_template.schema)
+ feedback_template = models.DataproductFeedbackTemplate.objects.get(name='feedback')
+ # feedback_doc = get_default_json_object_for_schema(feedback_template.schema) # todo <- fix the default generator, for some reason it does not produce valid json here...
+ feedback_doc = {'percentage_written': 100, 'frequency': {'subbands': [156], 'central_frequencies': [33593750.0], 'channel_width': 6103.515625, 'channels_per_subband': 32}, 'time': {'start_time': '2013-02-16T17:00:00', 'duration': 5.02732992172, 'sample_width': 2.00278016}, 'antennas': {'set': 'HBA_DUAL', 'fields': [{'type': 'HBA', 'field': 'HBA0', 'station': 'CS001'}, {'type': 'HBA', 'field': 'HBA1', 'station': 'CS001'}]}, 'target': {'pointing': {'angle1': 0, 'angle2': 0, 'direction_type': 'J2000', "target": "target1"}}, 'samples': {'polarisations': ['XX', 'XY', 'YX', 'YY'], 'type': 'float', 'bits': 32, 'writer': 'standard', 'writer_version': '2.2.0', 'complex': True}, '$schema': 'http://127.0.0.1:8001/api/schemas/dataproductfeedbacktemplate/feedback/1#'}
+
+ # Create SubTask(in/output)
+ subtask_data = Subtask_test_data(subtask_template=subtask_template, specifications_doc=specifications_doc, task_blueprint=models.TaskBlueprint.objects.create(**TaskBlueprint_test_data()))
+ subtask:models.Subtask = models.Subtask.objects.create(**subtask_data)
+ subtask_input = models.SubtaskInput.objects.create(**SubtaskInput_test_data(subtask=subtask, producer=subtask_output))
+ subtask_input.dataproducts.set(related_dataproducts)
+ subtask_output = models.SubtaskOutput.objects.create(**SubtaskOutput_test_data(subtask=subtask))
+
+ # create a bunch of main dataproducts (pipeline output) and their SIPs (we create several to test performance):
+ for i in range(10):
+ dataproduct: models.Dataproduct = models.Dataproduct.objects.create(**Dataproduct_test_data(filename='my_main_dataproduct_%s' % i, feedback_doc=feedback_doc, producer=subtask_output,
+ dataformat=models.Dataformat.objects.get(value="Beamformed"),
+ datatype=models.Datatype.objects.get(value="time series")))
+
+ dataproduct.sap = sap
+ dataproduct.save()
+
+ sip = generate_sip_for_dataproduct(dataproduct)
+ self.assertIn(str('my_related_dataproduct_42'), sip.get_prettyxml())
+ self.assertIn(str(f'my_main_dataproduct_{i}'), sip.get_prettyxml())
+ self.assertNotIn(str(f'my_main_dataproduct_{i+1}'), sip.get_prettyxml())
class CycleReportTest(unittest.TestCase):
--
GitLab