diff --git a/.gitattributes b/.gitattributes
index aa8685e37c563ab1cb688798ff3bc645693b6bbf..233a119f558b778b0966ea88871080e5b86f92f7 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -5122,6 +5122,7 @@ SAS/ResourceAssignment/ResourceAssignmentEstimator/test/__init__.py -text
SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.in_beam_observation -text
SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.in_calibration_pipeline -text
SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.in_calibration_pipeline_predecessor_558022 -text
+SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.in_imaging_pipeline -text
SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.in_interferometer_observation -text
SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.in_long_baseline_pipeline -text
SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.in_long_baseline_pipeline_predecessor_556601 -text
@@ -5135,6 +5136,7 @@ SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_est
SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.in_pulsar_pipeline -text
SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_beam_observation -text
SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_calibration_pipeline -text
+SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_imaging_pipeline -text
SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_interferometer_observation -text
SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_long_baseline_observation -text
SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_long_baseline_pipeline -text
@@ -5332,7 +5334,8 @@ SAS/TriggerEmailService/Server/bin/CMakeLists.txt -text
SAS/TriggerEmailService/Server/bin/TiggerEmailService -text
SAS/TriggerEmailService/Server/bin/TiggerEmailService.ini -text
SAS/TriggerEmailService/Server/lib/CMakeLists.txt -text
-SAS/TriggerEmailService/Server/lib/TiggerEmailService.py -text
+SAS/TriggerEmailService/Server/lib/Templates.py -text
+SAS/TriggerEmailService/Server/lib/TriggerEmailService.py -text
SAS/TriggerEmailService/Server/lib/__init__.py -text
SAS/TriggerEmailService/Server/test/CMakeLists.txt -text
SAS/TriggerEmailService/Server/test/t_TriggerEmailService.py -text
@@ -5414,6 +5417,7 @@ SAS/XML_generator/test/test_regression.in_data/txt/old_pulsar_pipe_test.txt -tex
SAS/XML_generator/test/test_regression.in_data/txt/test_LB.txt -text
SAS/XML_generator/test/test_regression.in_data/txt/test_input.txt -text
SAS/XML_generator/test/test_regression.in_data/txt/test_input_cep4.txt -text
+SAS/XML_generator/test/test_regression.in_data/txt/test_input_commensal_obs_DRAGNET.txt -text
SAS/XML_generator/test/test_regression.in_data/txt/test_input_long_baseline_pipeline.txt -text
SAS/XML_generator/test/test_regression.in_data/xml/20150713_4C17.31.xml -text
SAS/XML_generator/test/test_regression.in_data/xml/20150731_G46_run1_HBA.xml -text
diff --git a/SAS/DataManagement/Cleanup/CleanupService/service.py b/SAS/DataManagement/Cleanup/CleanupService/service.py
index c60b5a99634a60d53a1e95f214071ff93e01bb28..b94224dd8f09fe96b04f57f138ab9fb45086b94c 100644
--- a/SAS/DataManagement/Cleanup/CleanupService/service.py
+++ b/SAS/DataManagement/Cleanup/CleanupService/service.py
@@ -252,7 +252,7 @@ class CleanupHandler(MessageHandlerInterface):
#update resource claim
radbrpc = self.path_resolver.radbrpc
storage_resources = radbrpc.getResources(resource_types='storage')
- cep4_storage_resource = next(x for x in storage_resources if 'cep4' in x['name'])
+ cep4_storage_resource = next(x for x in storage_resources if 'CEP4' in x['name'])
task = radbrpc.getTask(otdb_id=otdb_id)
if task:
claims = radbrpc.getResourceClaims(task_ids=task['id'], resource_type='storage')
diff --git a/SAS/DataManagement/StorageQueryService/cache.py b/SAS/DataManagement/StorageQueryService/cache.py
index cae145a8ead0738a4ceaaa167f705ebe441d75f5..f42179bfcafbf1f62ff9caea4bb047aa425ea7d2 100644
--- a/SAS/DataManagement/StorageQueryService/cache.py
+++ b/SAS/DataManagement/StorageQueryService/cache.py
@@ -291,7 +291,7 @@ class CacheManager:
#get the total used space, and update the resource availability in the radb
radbrpc = self.disk_usage.path_resolver.radbrpc
storage_resources = radbrpc.getResources(resource_types='storage', include_availability=True)
- cep4_storage_resource = next(x for x in storage_resources if 'cep4' in x['name'])
+ cep4_storage_resource = next(x for x in storage_resources if 'CEP4' in x['name'])
total_capacity = cep4_storage_resource.get('total_capacity')
used_capacity = projects_du_result.get('disk_usage')
diff --git a/SAS/ResourceAssignment/RAtoOTDBTaskSpecificationPropagator/lib/translator.py b/SAS/ResourceAssignment/RAtoOTDBTaskSpecificationPropagator/lib/translator.py
index 07bd18123f27ed2914a359409f7ab2239678273a..0733eb319b5ad52fae22416a2d9fcd8ae71c94e9 100755
--- a/SAS/ResourceAssignment/RAtoOTDBTaskSpecificationPropagator/lib/translator.py
+++ b/SAS/ResourceAssignment/RAtoOTDBTaskSpecificationPropagator/lib/translator.py
@@ -98,6 +98,7 @@ class RAtoOTDBTranslator():
result = {}
# Split storage_property_list in items wrt observation and wrt pipeline.
+ total_nr_files = 0
obs_max_uv_sap_nr = -1
obs_next_sb_nrs_per_sap = []
obs_locations = [None] * 1024 # the easy way and 1024 is enough even in 4 bit mode
@@ -131,20 +132,22 @@ class RAtoOTDBTranslator():
prop['resource_name']) + '/uv'
obs_filenames[sb_nr] = "L%d_SAP%03d_SB%03d_uv.MS" % (otdb_id, sap_nr, sb_nr)
obs_skip.append("0") # what's this for?
+ total_nr_files += 1
obs_next_sb_nrs_per_sap[sap_nr] = sb_nr + 1
if obs_max_uv_sap_nr >= 0:
- total_nr_files = obs_next_sb_nrs_per_sap[-1]
- if total_nr_files < 1:
+ if total_nr_files == 0:
raise Exception('CreateCorrelated: skipping obs parset filenames: total_nr_files = %d' % total_nr_files)
logger.info('CreateCorrelated: total_nr_files = %d', total_nr_files)
- obs_locations = obs_locations[ : total_nr_files] # truncate trailing None init values
- obs_filenames = obs_filenames[ : total_nr_files] # idem
- if None in obs_locations or None in obs_filenames:
- logger.error('CreateCorrelated: skipping obs parset filenames: None in locations = %s', obs_locations)
- logger.error('CreateCorrelated: skipping obs parset filenames: None in filenames = %s', obs_filenames)
- raise Exception('CreateCorrelated: skipping obs parset filenames: None in locations and/or filenames')
+ obs_locations = [loc for loc in obs_locations if loc is not None] # strip unused init values
+ obs_filenames = [fn for fn in obs_filenames if fn is not None] # idem
+ if len(obs_locations) != total_nr_files or len(obs_filenames) != total_nr_files:
+ # If the total nr_of_uv_files in a SAP does not correspond to the start_sb_nr
+ # props of this and the next SAP, entries have been overwritten. Bail in that case.
+ logger.error('CreateCorrelated: skipping obs parset filenames: len(obs_locations) = %d and/or len(obs_filenames) = %d not equal to total_nr_files = %d',
+ len(obs_locations), len(obs_filenames), total_nr_files)
+ raise Exception('CreateCorrelated: skipping obs parset filenames: unexpected nr of locations and/or filenames vs total_nr_files')
result[PREFIX + 'DataProducts.%s_Correlated.locations' % (io_type)] = '[' + to_csv_string(obs_locations) + ']'
result[PREFIX + 'DataProducts.%s_Correlated.filenames' % (io_type)] = '[' + to_csv_string(obs_filenames) + ']'
@@ -227,7 +230,7 @@ class RAtoOTDBTranslator():
nr_cs_stokes = sap['properties']['nr_of_cs_stokes'] # the 'cs_stokes' term here can also mean cv XXYY
nr_parts = sap['properties']['nr_of_cs_files'] / nr_cs_stokes # in this prop's claim!
nparts_tab = nr_parts_per_tab_per_sap[sap_nr] # alias for readability; this is also per stokes
- while nr_parts > 0: # loops over tab nrs
+ while nr_parts > 0:
tab_nr = next_tab_part_nrs_per_sap[sap_nr] / nparts_tab
tab_part_nr = next_tab_part_nrs_per_sap[sap_nr] % nparts_tab
nparts_remain = min(nr_parts, nparts_tab - tab_part_nr) # nr parts left before we go to the next tab
@@ -369,7 +372,7 @@ class RAtoOTDBTranslator():
if 'start_sb_nr' in prop:
sb_nr = prop['start_sb_nr']
- otdb_id = sap['properties']['im_otdb_id']
+ otdb_id = prop['im_otdb_id']
for _ in xrange(prop['nr_of_im_files']):
locations.append(self.locationPath(cluster, project_name, otdb_id,
prop['resource_name']) + '/im')
@@ -396,7 +399,7 @@ class RAtoOTDBTranslator():
if 'nr_of_img_files' not in prop:
continue
- otdb_id = sap['properties']['img_otdb_id']
+ otdb_id = prop['img_otdb_id']
for _ in xrange(prop['nr_of_img_files']):
locations.append(self.locationPath(cluster, project_name, otdb_id,
prop['resource_name']) + '/img')
@@ -423,7 +426,7 @@ class RAtoOTDBTranslator():
if 'nr_of_pulp_files' not in prop:
continue
- otdb_id = sap['properties']['pulp_otdb_id']
+ otdb_id = prop['pulp_otdb_id']
for _ in xrange(prop['nr_of_pulp_files']):
locations.append(self.locationPath(cluster, project_name, otdb_id,
prop['resource_name']) + '/pulp')
diff --git a/SAS/ResourceAssignment/ResourceAssigner/lib/assignment.py b/SAS/ResourceAssignment/ResourceAssigner/lib/assignment.py
index ff8e286f506258dae9aac07d305808192f18f130..a03295e59ea78cd84ce1451fe27522d10444c77e 100755
--- a/SAS/ResourceAssignment/ResourceAssigner/lib/assignment.py
+++ b/SAS/ResourceAssignment/ResourceAssigner/lib/assignment.py
@@ -234,7 +234,7 @@ class ResourceAssigner():
startTime, endTime, str(mainParset), clusterName)
except Exception as e:
logger.error(str(e))
- result = dict() # handled next
+ result = {'inserted': False} # handled next
if not result['inserted']:
logger.error('could not insert specification and task: result = %s', result)
return
@@ -289,7 +289,7 @@ class ResourceAssigner():
estimates = estimates['estimates']
if not all(est_val > 0 for est in estimates for est_val in est['resource_types'].values()):
- # Avoid div by 0 and inf looping from estimate < 0 later on.
+ # Avoid div by 0 and inf looping from estimate <= 0 later on.
raise Exception("at least one of the estimates is not a positive number")
except Exception as e:
logger.error(str(e))
@@ -399,6 +399,9 @@ class ResourceAssigner():
predecessor_task['mom_id'], predecessor_task['otdb_id'], task['mom_id'], task['otdb_id'])
self.radbrpc.insertTaskPredecessor(task['id'], predecessor_task['id'])
else:
+ # Occurs when setting a pipeline to prescheduled while a predecessor has e.g. never been beyond approved,
+ # which is in principle valid. The link in the radb will be made later via processSuccessors() below.
+ # Alternatively, a predecessor could have been deleted.
logger.warning('could not find predecessor task with mom_id=%s in radb for task otdb_id=%s', predecessor_mom_id, task['otdb_id'])
@@ -423,6 +426,9 @@ class ResourceAssigner():
self.radbrpc.insertTaskPredecessor(successor_task['id'], task['id'])
movePipelineAfterItsPredecessors(successor_task, self.radbrpc)
else:
+ # Occurs when settings a obs or task to prescheduled while a successor has e.g. not yet been beyond approved,
+ # which is quite normal. The link in the radb will be made later via processPredecessors() above.
+ # Alternatively, a successor could have been deleted.
logger.warning('could not find successor task with mom_id=%s in radb for task otdb_id=%s', successor_mom_id, task['otdb_id'])
def getMaxPredecessorEndTime(self, specification_tree):
@@ -701,7 +707,7 @@ class ResourceAssigner():
def getProperties(self, db_resource_prop_types, files_dict, io_type):
""" Return list of properties in claim format converted from files_dict.
- E.g. files_dict: {'cs': {'sap_nr': 2, ..., 'properties': {'nr_of_uv_files': 123, ...}}, ...}
+ E.g. files_dict: {'cs': [ {'sap_nr': 2, ..., 'properties': {'nr_of_uv_files': 123, ...}}, {...} ], 'is': ...}
"""
if files_dict is None:
return []
@@ -709,19 +715,20 @@ class ResourceAssigner():
logger.info('getProperties: processing %s_files: %s', io_type, files_dict)
properties = []
- for dptype, dptype_dict in files_dict.items():
- sap_nr = dptype_dict.get('sap_nr') # only with obs output and obs successor input
+ for dptype in files_dict:
+ for dptype_dict in files_dict[dptype]:
+ sap_nr = dptype_dict.get('sap_nr') # only with obs output and obs successor input
- for prop_type_name, prop_value in dptype_dict['properties'].items():
- rc_property_type_id = db_resource_prop_types.get(prop_type_name)
- if rc_property_type_id is None:
- logger.error('getProperties: ignoring unknown prop type: %s', prop_type_name)
- continue
+ for prop_type_name, prop_value in dptype_dict['properties'].items():
+ rc_property_type_id = db_resource_prop_types.get(prop_type_name)
+ if rc_property_type_id is None:
+ logger.error('getProperties: ignoring unknown prop type: %s', prop_type_name)
+ continue
- prop = {'type': rc_property_type_id, 'value': prop_value, 'io_type': io_type}
- if sap_nr is not None:
- prop['sap_nr'] = sap_nr
- properties.append(prop)
+ prop = {'type': rc_property_type_id, 'value': prop_value, 'io_type': io_type}
+ if sap_nr is not None:
+ prop['sap_nr'] = sap_nr
+ properties.append(prop)
return properties
diff --git a/SAS/ResourceAssignment/ResourceAssigner/test/t_resourceassigner.py b/SAS/ResourceAssignment/ResourceAssigner/test/t_resourceassigner.py
index cdb8344f108ec8d29a780e853fe6e758478f9048..9fae3f23edb2d186a67ae290378ab947836f9627 100755
--- a/SAS/ResourceAssignment/ResourceAssigner/test/t_resourceassigner.py
+++ b/SAS/ResourceAssignment/ResourceAssigner/test/t_resourceassigner.py
@@ -189,8 +189,6 @@ class ResourceAssignerTest(unittest.TestCase):
unknown_resource_type_name = "fuel"
unknown_resource_type_otdb_id = 123489
- resource_unknown_property = 'identifications'
-
rerpc_status = 0
rerpc_needed_claim_for_bandwidth_size = 2
rerpc_needed_claim_for_bandwidth = {
@@ -213,7 +211,6 @@ class ResourceAssignerTest(unittest.TestCase):
'sap_nr': 1,
'properties': {
'nr_of_uv_files': 81,
- resource_unknown_property: -1
}
},
{
@@ -234,12 +231,12 @@ class ResourceAssignerTest(unittest.TestCase):
'pipeline': {
'errors': [],
'estimates': [{
- 'resource_types': {'bandwidth': 2, 'storage': 2}, # per sb
+ 'resource_types': {'bandwidth': 2, 'storage': 2},
'resource_count': 2, 'root_resource_group': 'CEP4',
'output_files': {
- 'uv': {'identifications': []},
- 'saps': [{'sap_nr': 0,
- 'properties': {'uv_file_size': 1073741824, 'nr_of_uv_files': 1, 'start_sb_nr': 0}}]
+ 'uv': [{'sap_nr': 0, 'identifications': [],
+ 'properties': {'uv_file_size': 1073741824, 'nr_of_uv_files': 1, 'start_sb_nr': 0}
+ }]
}
}]
}
@@ -257,41 +254,21 @@ class ResourceAssignerTest(unittest.TestCase):
str(resources_with_errors_otdb_id): {
'pipeline': {
'estimates': [{
- 'resource_types': {
- 'bandwidth': {
- 'total_size': 19021319494
- },
- 'storage': {
- 'total_size': 713299481024,
- 'output_files': {
- 'uv': {
- 'nr_of_uv_files': 481,
- 'uv_file_size': 1482951104
- },
- 'saps': [{
- 'sap_nr': 0,
- 'properties': {
- 'nr_of_uv_files': 319
- }
- },
- {
- 'sap_nr': 1,
- 'properties': {
- 'nr_of_uv_files': 81
- }
- },
- {
- 'sap_nr': 2,
- 'properties': {
- 'nr_of_uv_files': 81
- }
- }]
- }
- },
- },
+ 'resource_types': {'bandwidth': 19021319494, 'storage': 713299481024},
+ 'output_files': {
+ 'uv': [{'sap_nr': 0,
+ 'properties': {'nr_of_uv_files': 319, 'uv_file_size': 1482951104}
+ },
+ {'sap_nr': 1,
+ 'properties': {'nr_of_uv_files': 81, 'uv_file_size': 1482951104}
+ },
+ {'sap_nr': 2,
+ 'properties': {'nr_of_uv_files': 81, 'uv_file_size': 1482951104}
+ }]
+ }
}],
'errors': [resource_error1, resource_error2]
- },
+ }
},
str(unknown_resource_type_otdb_id): {
'pipeline': {
@@ -1962,27 +1939,11 @@ class ResourceAssignerTest(unittest.TestCase):
found = True
return found
- # def test_do_assignment_should_log_start_of_claim_resources(self):
- # self.resourceAssigner.doAssignment(self.specification_tree)
- #
- # self.logger_mock.info.assert_any_call('claimResources: task %s needed_resources=%s' %
- # (self.task, self.rerpc_replymessage[str(str(self.otdb_id))]))
-
- # def test_do_assigment_should_log_when_claiming_unknown_resource_type(self):
- # self.specification_tree["otdb_id"] = self.unknown_resource_type_otdb_id
- #
- # self.resourceAssigner.doAssignment(self.specification_tree)
- #
- # self.logger_mock.warn.assert_any_call('claimResources: unknown resource_type:%s' %
- # self.unknown_resource_type_name)
-
- # def test_do_assignment_logs_claimable_resources_in_specification(self):
- # self.resourceAssigner.doAssignment(self.specification_tree)
- #
- # self.logger_mock.info.assert_any_call('claimResources: processing resource_type: %s contents: %s'
- # % ('storage', self.rerpc_needed_claim_for_storage))
- # self.logger_mock.info.assert_any_call('claimResources: processing resource_type: %s contents: %s'
- # % ('bandwidth', self.rerpc_needed_claim_for_bandwidth))
+ def test_do_assignment_should_log_estimator_reply(self):
+ self.resourceAssigner.doAssignment(self.specification_tree)
+
+ self.logger_mock.info.assert_any_call('doAssignment: Resource Estimator reply = %s',
+ self.rerpc_replymessage)
def test_do_assignment_logs_created_claim_per_needed_resource_type(self):
self.resourceAssigner.doAssignment(self.specification_tree)
@@ -2039,25 +2000,12 @@ class ResourceAssignerTest(unittest.TestCase):
self.logger_mock.info.assert_any_call('doAssignment: %d claims were inserted in the radb' % 2)
- def test_do_assignment_logs_unknown_property_on_needed_resources(self):
- self.resourceAssigner.doAssignment(self.specification_tree)
-
- self.logger_mock.warn.assert_any_call('makeProperties: unknown prop_type: %s', self.resource_unknown_property)
+ def test_do_assignment_logs_when_it_was_unable_to_claim_all_resources(self):
+ self.rarpc_mock.insertResourceClaims.return_value = {'ids': []}
- # def test_do_assignment_logs_multiple_properties_on_needed_resource(self):
- # self.resourceAssigner.doAssignment(self.specification_tree)
- #
- # self.logger_mock.info.assert_any_call(
- # 'claimResources: processing resource_type: %s subdict_name: \'%s\' subdict_contents: %s' %
- # ("storage", "output_files", self.rerpc_needed_claim_for_storage_output_files))
+ self.resourceAssigner.doAssignment(self.specification_tree)
- # def test_do_assignment_logs_when_it_was_unable_to_claim_all_resources(self):
- # self.rarpc_mock.insertResourceClaims.return_value = {'ids': []}
- #
- # self.resourceAssigner.doAssignment(self.specification_tree)
- #
- # self.logger_mock.warning.assert_any_call(
- # 'doAssignment: No claims could be made. Setting task %s status to error' % self.task_id)
+ self.logger_mock.error.assert_any_call('doAssignment: too few claims were inserted in the radb')
def test_do_assignment_updates_task_when_it_was_unable_to_claim_all_resources(self):
self.rarpc_mock.insertResourceClaims.return_value = {'ids': []}
@@ -2076,14 +2024,6 @@ class ResourceAssignerTest(unittest.TestCase):
self.assertBusNotificationAndLogging(content, subject)
- # def test_do_assignment_logs_when_it_was_unable_to_claim_some_resources(self):
- # self.rarpc_mock.insertResourceClaims.return_value = {'ids': [1]}
- #
- # self.resourceAssigner.doAssignment(self.specification_tree)
- #
- # self.logger_mock.warning.assert_any_call(
- # 'doAssignment: Not all claims could be inserted. Setting task %s status to conflict' % self.task_id)
-
def test_do_assignment_updates_task_when_it_was_unable_to_claim_some_resources(self):
self.rarpc_mock.insertResourceClaims.return_value = {'ids': [1]}
@@ -2101,16 +2041,16 @@ class ResourceAssignerTest(unittest.TestCase):
self.assertBusNotificationAndLogging(content, subject)
- # def test_do_assignment_logs_when_there_are_conflicting_claims(self):
- # conflicting_claims = [{}]
- #
- # self.rarpc_mock.getResourceClaims.return_value = conflicting_claims
- #
- # self.resourceAssigner.doAssignment(self.specification_tree)
- #
- # self.logger_mock.warning.assert_any_call(
- # 'doAssignment: %s conflicting claims detected. Task cannot be scheduled. %s' %
- # (len(conflicting_claims), conflicting_claims))
+ def test_do_assignment_logs_when_there_are_conflicting_claims(self):
+ conflicting_claims = [{}]
+
+ self.rarpc_mock.getResourceClaims.return_value = conflicting_claims
+
+ self.resourceAssigner.doAssignment(self.specification_tree)
+
+ self.logger_mock.error.assert_any_call(
+ 'doAssignment: Task cannot be scheduled, because of %d conflicting claims: %s' %
+ (len(conflicting_claims), conflicting_claims))
def test_do_assignment_notifies_bus_when_there_are_conflicting_claims(self):
content = {'radb_id': self.task_id, 'otdb_id': self.task_otdb_id, 'mom_id': self.task_mom_id}
diff --git a/SAS/ResourceAssignment/ResourceAssigner/test/t_resourceassigner.run b/SAS/ResourceAssignment/ResourceAssigner/test/t_resourceassigner.run
index 9417586b550145c2d7cdd6ef1578a33f99798b35..636e663a0917a66c8581ca569fd658744b662c80 100755
--- a/SAS/ResourceAssignment/ResourceAssigner/test/t_resourceassigner.run
+++ b/SAS/ResourceAssignment/ResourceAssigner/test/t_resourceassigner.run
@@ -2,5 +2,5 @@
# Run the unit test
source python-coverage.sh
-python_coverage_test "resourceassigner*" t_resourceassigner.py
+python_coverage_test "ResourceAssigner*" t_resourceassigner.py
diff --git a/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/base_pipeline_estimator.py b/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/base_pipeline_estimator.py
index 6be9a9823e6bd1aa4a2ecf96c9f018561d70e3ad..acc981ab46c5554c6d5c6e72dc3c26d9643ef424 100644
--- a/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/base_pipeline_estimator.py
+++ b/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/base_pipeline_estimator.py
@@ -50,7 +50,7 @@ class BasePipelineResourceEstimator(BaseResourceEstimator):
def _getOutputIdentification(self, identifications):
""" For pipeline output, there must be exactly 1 (non-duplicate) identification string per
- data product type. (How can you otherwise refer to it unambiguously?) (Observation output can have 1 per sap.)
+ data product type. (How can you otherwise refer to it unambiguously?) (Observation output can have 1 per SAP.)
"""
if len(set(identifications)) != 1: # make set to filter any duplicates
if not identifications:
diff --git a/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/base_resource_estimator.py b/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/base_resource_estimator.py
index f88a4d4c1e43112b1e57caac6ce351b3126372b7..ee83de541ee2d444bf184038f65e0882ef72fd02 100644
--- a/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/base_resource_estimator.py
+++ b/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/base_resource_estimator.py
@@ -23,6 +23,7 @@
""" Base class for Resource Estimators
"""
import logging
+import copy
import pprint
from datetime import datetime
from lofar.common.datetimeutils import totalSeconds, parseDatetime
@@ -61,46 +62,56 @@ class BaseResourceEstimator(object):
def _calculate(self, parset, predecessor_estimates=[]):
raise NotImplementedError('calculate() in base class is called. Please implement calculate() in your subclass')
- def _extract_sap_nr(self, identification):
- """ Return sap nr as int from identification or None if
- no int xxx in '.SAPxxx' in identification.
+ def _add_predecessor_output(self, input_files, predecessor_estimate, identification, dptype):
+ """ Add copy of an element under the dptype key in predecessor_estimate
+ to input_files if it matches identification, or else return False.
+ But see comment below on resource_count collapsing to convert resource_count > 1 to pipelines.
"""
- for s in identification.split('.'): # Find the SAP number, if present
- if 'SAP' not in s:
+ if 'output_files' not in predecessor_estimate or \
+ dptype not in predecessor_estimate['output_files']:
+ return False
+
+ for dt_values in predecessor_estimate['output_files'][dptype]:
+ if dt_values['identification'] != identification:
continue
- try:
- return int(s[3:])
- except:
- pass
-
- return None
-
- def _filterInputs(self, input_files, identifications):
- """ Return copy of input_files with only parts of input_files covered by identifications.
- There may not be duplicates in the identifications iterable.
-
- Example argument values:
- 'input_files': {
- 'uv': {'identification': 'mom.G777955.B15.1.CPC.uv.dps', # or w/ obs e.g.: 'mom.G777955.B15.1.C.SAP000.uv.dps'
- 'uv_file_size': 1073741824, 'nr_of_uv_files': 42, <uv specific key-values>},
- 'im': {'identification': 'mom.G777955.B15.1.CPC.inst.dps',
- 'im_file_size': 1000, 'nr_of_im_files': 42, <im specific key-values>}
- }
- There may also be a key-value pair 'sap_nr': N, typically for observation.
- 'identifications': ['mom.G777955.B2.1.C.SAP002.uv.dps', ...] (or without SAPxxx in it, as shown in the input_files example).
+
+ logger.info('Found predecessor output identification matching %s', identification)
+ if dptype not in input_files:
+ input_files[dptype] = []
+ input_files[dptype].append(copy.deepcopy(dt_values))
+
+ # Observation estimates have resource_count > 1 to be able to assign each output to another resource,
+ # but that is not supported atm for pipelines. We only use input params to produce parset filenames etc,
+ # but not to reserve resources (not covered by resource count). Collapse to implied resource_count of 1.
+ input_files[dptype][-1]['properties']['nr_of_' + dptype + '_files'] *= predecessor_estimate['resource_count']
+ return True
+
+ return False
+
+ def get_inputs_from_predecessors(self, predecessor_estimates, identifications, dptype):
+ """ Return copy of parts with dptype in predecessor_estimates matching identifications
+ If any of any of identifications could not be found, the empty dict is returned.
+ dptype is one of the observation/pipeline data product types, e.g. 'uv', 'cs', 'pulp', ...
+ No duplicates in the identifications iterable!
+
+ See the calibration and observation pipeline estimators for parameter value examples.
"""
- output_files = {}
- logger.info('parsing input for identifications: %s' % (identifications,))
+ input_files = {}
+ logger.info('get_inputs_from_predecessors: parsing predecessor output for identifications: %s', identifications)
for identification in identifications:
- sap_nr = self._extract_sap_nr(identification)
- for data_type, data_properties in input_files.items():
- if identification == data_properties['identification']:
- logger.info('Found input identification matching %s' % (identification,))
- output_files[data_type] = dict(data_properties) # shallow copy is enough to avoid unintended changes
-
- logger.info('_filterInputs: filtered down to: \n' + pprint.pformat(output_files))
- return output_files
+ found = False
+ for estimate in predecessor_estimates:
+ if self._add_predecessor_output(input_files, estimate, identification, dptype):
+ found = True
+ break
+ if not found:
+ logger.warn('get_inputs_from_predecessors: failed to find predecessor output matching %s', identification)
+ return {}
+
+ logger.info('get_inputs_from_predecessors: filtered predecessor output for dptype=' + dptype +
+ ' down to: \n' + pprint.pformat(input_files))
+ return input_files
def verify_and_estimate(self, parset, predecessor_estimates=[]):
""" Create estimates for an observation or pipeline step based on its parset and,
diff --git a/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/calibration_pipeline.py b/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/calibration_pipeline.py
index be115be587b3b31bcf3e77c8d5229075e1faa837..30a791c91fd8c78bbfd453664a0ab6892cb35908 100644
--- a/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/calibration_pipeline.py
+++ b/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/calibration_pipeline.py
@@ -54,40 +54,78 @@ class CalibrationPipelineResourceEstimator(BasePipelineResourceEstimator):
PIPELINE + 'DPPP.demixer.timestep')
def _calculate(self, parset, predecessor_estimates):
- """ Estimate for Calibration Pipeline. Also gets used for Averaging Pipeline
+ """ Estimator for calibration pipeline step. Also used for averaging pipeline step.
calculates: datasize (number of files, file size), bandwidth
predecessor_estimates looks something like (see also output format in observation.py and (other) pipelines):
[{
- 'resource_types': {'bandwidth': 35791395, 'storage': 1073741824}, # per sb
+ 'resource_types': {'bandwidth': 286331153, 'storage': 1073741824}, # per 'uv' dict
'resource_count': 20, 'root_resource_group': 'CEP4',
'output_files': {
'uv': [{'sap_nr': 2, 'identification': 'mom.G777955.B2.1.C.SAP002.uv.dps',
- 'properties': {'uv_file_size': 1073741824, 'nr_of_uv_files': 1, 'start_sb_nr': 0}}]
+ 'properties': {'uv_file_size': 1073741824, 'nr_of_uv_files': 1, 'start_sb_nr': 0}},
+ {'sap_nr': 3, 'identification': 'mom.G777955.B2.1.C.SAP003.uv.dps',
+ 'properties': {'uv_file_size': 1073741824, 'nr_of_uv_files': 1, 'start_sb_nr': 20}}
+ ]
}
},
- <optionally more estimates>
+ <optionally more estimates>
]
- The reply is something along the lines of: (assumes task duration of 30 s)
- [{
- 'resource_types': {'bandwidth': 2236995, 'storage': 67109864}, # for each uv+im output data prod (thus the total is times the resource_count val)
- 'resource_count': 20, 'root_resource_group': 'CEP4',
- 'input_files': {
- 'uv': {'sap_nr': 2, 'identification': 'mom.G777955.B2.1.C.SAP002.uv.dps', # w/ sap only if predecessor is an observation
- 'properties': {'uv_file_size': 1073741824, 'nr_of_uv_files': 1, 'start_sb_nr': 0}},
- 'im': {'identification': ..., # no 'im' as input if predecessor is an observation
- 'properties': {'im_file_size': 1000, 'nr_of_im_files': 1, 'start_sb_nr': 0}}
- },
- 'output_files': {
- 'uv': {'identification': 'mom.G777956.B2.1.CPC.uv.dps',
- 'properties': {'uv_file_size': 67108864, 'nr_of_uv_files': 1, 'start_sb_nr': 0}},
- 'im': {'identification': 'mom.G777956.B2.1.CPC.inst.dps',
- 'properties': {'im_file_size': 1000, 'nr_of_im_files': 1, 'start_sb_nr': 0}}
- }
- },
- <optionally more estimates>
+ The reply is something along the lines of the example below
+ (assumes task duration of 30 s, and typically 'im' is not in both input_files and output_files)
+ {
+ 'errors': [],
+ 'estimates': [
+ {
+ 'resource_types': {'bandwidth': 2236995 * 20, 'storage': 67109864 * 20},
+ 'resource_count': 1, 'root_resource_group': 'CEP4',
+
+ # input resources not (yet) allocated: bandwidth only, but coupled to specific storage resource
+ 'input_files': {
+ 'uv': [{'sap_nr': 2, 'identification': 'mom.G777955.B2.1.C.SAP002.uv.dps', # w/ sap only if predecessor is an observation
+ 'properties': {'uv_file_size': 1073741824, 'nr_of_uv_files': 20, 'start_sb_nr': 0}},
+ {'sap_nr': 3, 'identification': 'mom.G777955.B2.1.C.SAP003.uv.dps', # idem, >1 input SAP possible for e.g. the pulsar pipeline
+ 'properties': {'uv_file_size': 1073741824, 'nr_of_uv_files': 20, 'start_sb_nr': 20}}
+ ]
+ 'im': [{'identification': ..., # 'im' example; no 'im' input if predecessor is an observation as above
+ 'properties': {'im_file_size': 1000, 'nr_of_im_files': 20, 'start_sb_nr': 0}}]
+ },
+ 'output_files': {
+ 'uv': [{'identification': 'mom.G777956.B2.1.CPC.uv.dps',
+ 'properties': {'uv_file_size': 67108864, 'nr_of_uv_files': 40, 'start_sb_nr': 0}}],
+ 'im': [{'identification': 'mom.G777956.B2.1.CPC.inst.dps',
+ 'properties': {'im_file_size': 1000, 'nr_of_im_files': 20, 'start_sb_nr': 0}}]
+ }
+ },
+ <optionally more estimates>
]
+ }
+
+ The estimates key has a list to support observations with >1 data product type that can have
+ different properties (typically CS/IS (per SAP), but always used for observations w/ >1 SAP).
+ This is the reason that observations have an estimate per SAP and per data product type:
+ their resource_types values may be different. This is needed to employ resource_count > 1.
+ See the observation estimator for an extensive observation data product type example.
+
+ For each estimate, the total output_files resources to be claimed is resource_count * resources_types.
+ Thus resource_types is a total across all output_files content. The idea is to keep this
+ singular per data product type (inner list len 1), but for pipelines this is not possible atm.
+
+ Note that atm input_files resources are not included or claimed.
+ However, input_files properties must be added to resource claims to later generate the parset.
+ This caveat must be fixed at some point, but until then, we cannot have input_files-only estimates.
+ (After it is fixed, we should not have that either; it makes no sense.)
+
+ For pipelines we don't support output to multiple storage areas atm, so resource_count is 1.
+ We still have to deal with input_files from an observation with >1 SAP (used for the pulsar pipeline).
+ For this case, we generate 1 estimate, but use a list per data product type (e.g. 'uv': [...]).
+ Also, we may need multiple data product types in one pipeline estimate, but there the reason
+ is that e.g. 'uv' and 'im' file(s) belong together, so we must produce one estimate per pair,
+ (but again, it's a pipeline so atm it is collapsed to a single estimate, i.e. resource_count 1).
+ The inner data product type list can be removed once pipelines also use resource_count > 1.
+
+ Some RA_Services design aspects work well. Others fail to capture the underlying concepts close enough, hence inelegance.
"""
logger.debug("start estimate '{}'".format(self.name))
logger.info('parset: %s ' % parset)
@@ -104,20 +142,35 @@ class CalibrationPipelineResourceEstimator(BasePipelineResourceEstimator):
not parset.getBool(DATAPRODUCTS + 'Output_Correlated.enabled'):
logger.error('Input_Correlated or Output_Correlated is not enabled')
result['errors'].append('Input_Correlated or Output_Correlated is not enabled')
- if result['errors']:
- return result
duration = self._getDuration(parset.getString('Observation.startTime'),
parset.getString('Observation.stopTime'))
- # NOTE: input bandwidth is not included in the resulting estimate atm.
- # Proper input bandwidth est has limited use and is tricky, because of pipeline duration est, tmp files, multiple passes, nr nodes and caching, ...
- #input_cluster_uv = parset.getString(DATAPRODUCTS + 'Input_Correlated.storageClusterName')
-
input_idents_uv = parset.getStringVector(DATAPRODUCTS + 'Input_Correlated.identifications')
+ input_files = self.get_inputs_from_predecessors(predecessor_estimates, input_idents_uv, 'uv')
+ if not input_files:
+ logger.error('Missing uv dataproducts in predecessor output_files')
+ result['errors'].append('Missing uv dataproducts in predecessor output_files')
+
have_im_input = parset.getBool(DATAPRODUCTS + 'Input_InstrumentModel.enabled')
if have_im_input:
input_idents_im = parset.getStringVector(DATAPRODUCTS + 'Input_InstrumentModel.identifications')
+ input_files_im = self.get_inputs_from_predecessors(predecessor_estimates, input_idents_im, 'im')
+ if not input_files_im:
+ logger.error('Input_InstrumentModel enabled, but missing \'im\' dataproducts in predecessor output_files')
+ result['errors'].append('Input_InstrumentModel enabled, but missing \'im\' dataproducts in predecessor output_files')
+ input_files['im'] = input_files_im['im']
+
+ if result['errors']:
+ return result
+
+ estimate = {'input_files': input_files}
+
+ # NOTE: input bandwidth is not included in the resulting estimate atm.
+ # Proper input bandwidth estimation has limited use atm and is tricky, because of pipeline duration est, tmp files,
+ # multiple passes, nr nodes and caching, but for sure also because bandwidth must be tied to *predecessor* storage!
+ #input_cluster_uv = parset.getString(DATAPRODUCTS + 'Input_Correlated.storageClusterName')
+
output_ident_uv = self._getOutputIdentification( parset.getStringVector(DATAPRODUCTS + 'Output_Correlated.identifications') )
output_cluster_uv = parset.getString(DATAPRODUCTS + 'Output_Correlated.storageClusterName')
have_im_output = parset.getBool(DATAPRODUCTS + 'Output_InstrumentModel.enabled')
@@ -129,74 +182,62 @@ class CalibrationPipelineResourceEstimator(BasePipelineResourceEstimator):
logger.warn('storageClusterName differs between uv: \'%s\' and im: \'%s\': to be packed in 1 estimate, so ignoring \'im\' storageClusterName',
output_cluster_uv, output_cluster_im)
- for pred_est in predecessor_estimates:
- pred_output_files = pred_est.get('output_files')
- if pred_output_files is None:
- continue
-
- idents = input_idents_uv
- if have_im_input:
- if 'im' in pred_output_files:
- idents.extend(input_idents_im)
- else:
- logger.warn('Input_InstrumentModel enabled, but missing \'im\' dataproducts in predecessor output_files')
- input_files = self._filterInputs(pred_output_files, idents)
- if not input_files:
- continue
-
- if 'uv' not in input_files:
- logger.error('Missing uv dataproducts in predecessor output_files')
- result['errors'].append('Missing uv dataproducts in predecessor output_files')
- continue
-
- estimate = {'input_files': input_files}
-
- nr_input_files = input_files['uv']['properties']['nr_of_uv_files']
- uv_input_file_size = input_files['uv']['properties']['uv_file_size']
- start_sb_nr = input_files['uv']['properties']['start_sb_nr']
-
- # TODO: This output file size calculation comes from the (old) Scheduler without explaining comments.
- # The reason why it isn't a simple div, is that parts of the metadata are not reduced in size (and casacore storage mgrs).
- # With reduction_factor 1, computed output size increases by 53%. Casacore storage mgrs may change size, but that much?!?
- # If you can figure out what/how, please fix this calculation. Avoid unnamed magic values and document why!
- logger.debug("calculate correlated data size")
- new_size = uv_input_file_size / float(reduction_factor)
- uv_output_file_size = int(new_size + new_size / 64.0 * (1.0 + reduction_factor) + new_size / 2.0)
-
- nr_output_files = nr_input_files # pure 'map' (bijective) operation, no split or reduce
- logger.info("correlated_uv: {} files {} bytes each".format(nr_output_files, uv_output_file_size))
- estimate['output_files'] = {'uv': {'identification': output_ident_uv,
- 'properties': {'nr_of_uv_files': nr_output_files, 'uv_file_size': uv_output_file_size, 'start_sb_nr': start_sb_nr}}}
- data_size = uv_output_file_size
-
- # If instrument model output is needed, add it to the same estimate,
- # since it must be written to the same storage as the uv output (same nr_output_files).
- if have_im_output:
- logger.info("calculate instrument-model data size")
- im_file_size = 1000 # TODO: 1 kB was hardcoded in the Scheduler
-
- logger.info("correlated_im: {} files {} bytes each".format(nr_output_files, im_file_size))
- estimate['output_files']['im'] = {'identification': output_ident_im,
- 'properties': {'nr_of_im_files': nr_output_files, 'im_file_size': im_file_size, 'start_sb_nr': start_sb_nr}}
- # FIXME I don't think this is technically correct, as the IM files are sometimes created and used, just not a required export?
- # Need to split averaging pipeline and calibration pipeline
- data_size += im_file_size
-
- data_size *= nr_output_files # bytes
- if data_size:
- bandwidth = int(ceil(8 * data_size / duration)) # bits/second
- estimate['resource_types'] = {'bandwidth': bandwidth, 'storage': data_size}
- estimate['resource_count'] = pred_est['resource_count']
- estimate['root_resource_group'] = output_cluster_uv
- else:
- logger.error('An estimate of zero was calculated!')
- result['errors'].append('An estimate of zero was calculated!')
-
- result['estimates'].append(estimate)
-
- if not result['estimates'] and not result['errors']:
- logger.error('calibration / averaging pipeline estimator produced no estimates')
- result['errors'].append('calibration / averaging pipeline estimator produced no estimates')
+ # Observations can have multiple output estimates, but atm pipelines do not.
+ # (Reason: incomplete info avail and effective assigner claim merging is harder)
+ # As long as this is the case, try to do a best effort to map any predecessor (obs or pipeline) to single estimate output.
+ nr_input_files = sum([uv_dict['properties']['nr_of_uv_files'] for uv_dict in input_files['uv']])
+
+ # Assume all uv file sizes are the same size as in dict 0. For uv data, we never had pipelines with >1 dict,
+ # but this could be meaningful when averaging multiple SAPs in 1 go (and no further processing steps).
+ # (Never done, since subsequent pipeline steps must then also work on all SAPs. But averaging could be the last step.)
+ # The potential other case is >1 dict from different obs with different file sizes.
+ # In general, this requires >1 output est dict, which the estimate fmt allows, but atm is only used for observations.
+ uv_input_file_size = input_files['uv'][0]['properties']['uv_file_size']
+
+ # For start_sb_nr, take the minimum of all start_sb_nr values.
+ # This fails when the input identifications has a sparse SAP list, but that was never supported either.
+ # A likely setup where this could happen is LOTAAS+pulp, but pulp has no equivalent to start_sb[g]_nr,
+ # so solve here and in the pulsar pipeline pragmatically each.
+ start_sb_nr = min([uv_dict['properties']['start_sb_nr'] for uv_dict in input_files['uv']])
+
+ # TODO: This output file size calculation comes from the (old) Scheduler without explaining comments.
+ # The reason why it isn't a simple div, is that parts of the metadata are not reduced in size (and casacore storage mgrs).
+ # With reduction_factor 1, computed output size increases by 53%. Casacore storage mgrs may change size, but that much?!?
+ # If you can figure out what/how, please fix this calculation. Avoid unnamed magic values and document why!
+ logger.debug("calculate correlated data size")
+ new_size = uv_input_file_size / float(reduction_factor)
+ uv_output_file_size = int(new_size + new_size / 64.0 * (1.0 + reduction_factor) + new_size / 2.0)
+
+ nr_output_files = nr_input_files # pure 'map' (bijective) operation, no split or reduce
+ logger.info("correlated_uv: {} files of {} bytes each".format(nr_output_files, uv_output_file_size))
+ estimate['output_files'] = {'uv': [{'identification': output_ident_uv,
+ 'properties': {'nr_of_uv_files': nr_output_files, 'uv_file_size': uv_output_file_size, 'start_sb_nr': start_sb_nr}}]}
+ data_size = uv_output_file_size
+
+ # If instrument model output is needed, add it to the same estimate,
+ # since it must be written to the same storage as the uv output (same nr_output_files).
+ if have_im_output:
+ logger.info("calculate instrument-model data size")
+ im_file_size = 1000 # TODO: 1 kB was hardcoded in the Scheduler
+
+ logger.info("correlated_im: {} files {} bytes each".format(nr_output_files, im_file_size))
+ estimate['output_files']['im'] = [{'identification': output_ident_im,
+ 'properties': {'nr_of_im_files': nr_output_files, 'im_file_size': im_file_size, 'start_sb_nr': start_sb_nr}}]
+ # FIXME I don't think this is technically correct, as the IM files are sometimes created and used, just not a required export?
+ # Need to split averaging pipeline and calibration pipeline
+ data_size += im_file_size
+
+ data_size *= nr_output_files # bytes
+ if data_size:
+ bandwidth = int(ceil(8 * data_size / duration)) # bits/second
+ estimate['resource_types'] = {'bandwidth': bandwidth, 'storage': data_size}
+ estimate['resource_count'] = 1
+ estimate['root_resource_group'] = output_cluster_uv
+ else:
+ logger.error('An estimate of zero was calculated!')
+ result['errors'].append('An estimate of zero was calculated!')
+
+ result['estimates'].append(estimate)
return result
diff --git a/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/image_pipeline.py b/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/image_pipeline.py
index 810be6a8839e869688392a6c4f31d0bbcb645f24..960c38330fe3940bce7dd7700b587bc0f2e8abef 100644
--- a/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/image_pipeline.py
+++ b/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/image_pipeline.py
@@ -23,7 +23,6 @@
import logging
from math import ceil
from base_pipeline_estimator import BasePipelineResourceEstimator
-from lofar.parameterset import parameterset
logger = logging.getLogger(__name__)
@@ -51,7 +50,7 @@ class ImagePipelineResourceEstimator(BasePipelineResourceEstimator):
PIPELINE + 'Imaging.subbands_per_image')
def _calculate(self, parset, predecessor_estimates):
- """ Estimate for Imaging Pipeline. Also gets used for MSSS Imaging Pipeline
+ """ Estimate for imaging pipeline step. Also used for MSSS imaging pipeline.
calculates: datasize (number of files, file size), bandwidth
For a predecessor_estimates example, see the calibration/averaging
@@ -70,74 +69,61 @@ class ImagePipelineResourceEstimator(BasePipelineResourceEstimator):
slices_per_image = parset.getInt(PIPELINE + 'Imaging.slices_per_image', 0) #TODO, should these have defaults?
subbands_per_image = parset.getInt(PIPELINE + 'Imaging.subbands_per_image', 0)
if slices_per_image < 1 or subbands_per_image < 1:
- logger.error('slices_per_image or subbands_per_image are not valid')
- result['errors'].append('slices_per_image or subbands_per_image are not valid')
+ logger.error('slices_per_image or subbands_per_image is not valid')
+ result['errors'].append('slices_per_image or subbands_per_image is not valid')
if not parset.getBool(DATAPRODUCTS + 'Output_SkyImage.enabled'):
logger.error('Output_SkyImage is not enabled')
result['errors'].append('Output_SkyImage is not enabled')
- if result['errors']:
- return result
duration = self._getDuration(parset.getString('Observation.startTime'),
parset.getString('Observation.stopTime'))
+ input_idents_uv = parset.getStringVector(DATAPRODUCTS + 'Input_Correlated.identifications')
+ input_files = self.get_inputs_from_predecessors(predecessor_estimates, input_idents_uv, 'uv')
+ if not input_files:
+ logger.error('Missing uv dataproducts in predecessor output_files')
+ result['errors'].append('Missing uv dataproducts in predecessor output_files')
+
+ if result['errors']:
+ return result
+
+ estimate = {'input_files': input_files}
+
# NOTE: input bandwidth is not included in the resulting estimate atm.
# Proper input bandwidth est has limited use and is tricky, because of pipeline duration est, tmp files, multiple passes, nr nodes and caching, ...
#input_cluster_uv = parset.getString(DATAPRODUCTS + 'Input_Correlated.storageClusterName')
- input_idents_uv = parset.getStringVector(DATAPRODUCTS + 'Input_Correlated.identifications')
output_ident_img = self._getOutputIdentification( parset.getStringVector(DATAPRODUCTS + 'Output_SkyImage.identifications') )
output_cluster_img = parset.getString(DATAPRODUCTS + 'Output_SkyImage.storageClusterName')
- for pred_est in predecessor_estimates:
- pred_output_files = pred_est.get('output_files')
- if pred_output_files is None:
- continue
-
- input_files = self._filterInputs(pred_output_files, input_idents_uv)
- if not input_files:
- continue
-
- if 'uv' not in input_files:
- logger.error('Missing uv dataproducts in predecessor output_files')
- result['errors'].append('Missing uv dataproducts in predecessor output_files')
- continue
-
- estimate = {'input_files': input_files}
-
- total_nr_input_subbands = nr_uv_files * pred_est['resource_count']
- if total_nr_input_subbands % (subbands_per_image * slices_per_image) > 0:
- logger.error('slices_per_image and subbands_per_image not a multiple of number of inputs')
- result['errors'].append('slices_per_image and subbands_per_image not a multiple of number of inputs')
- continue
- nr_images = total_nr_input_subbands / (subbands_per_image * slices_per_image)
-
- logger.debug("calculate sky image data size")
- nr_uv_files = input_files['uv']['properties']['nr_of_uv_files']
- uv_file_size = input_files['uv']['properties']['uv_file_size'] # same for each uv data product across all SAPs
- img_file_size = 1000 # TODO: 1 kB was hardcoded in the Scheduler
-
- logger.info("sky_images: {} files {} bytes each".format(nr_images, img_file_size))
- estimate['output_files'] = {'img': {'identification': output_ident_img,
- 'properties': {'nr_of_img_files': 1, # also see estimate['resource_count'] below
- 'img_file_size': img_file_size}}}
-
- # count total data size
- data_size = img_file_size # bytes
- if data_size:
- bandwidth = int(ceil(8 * data_size / duration)) # bits/second
- estimate['resource_types'] = {'bandwidth': bandwidth, 'storage': data_size}
- estimate['resource_count'] = nr_images # as such, potentially different resources can be allocated for each output
- estimate['root_resource_group'] = output_cluster_img
- else:
- logger.error('An estimate of zero was calculated!')
- result['errors'].append('An estimate of zero was calculated!')
-
- result['estimates'].append(estimate)
-
- if not result['estimates'] and not result['errors']:
- logger.error('imaging pipeline estimator produced no estimates')
- result['errors'].append('imaging pipeline estimator produced no estimates')
+ # See the calibration pipeline estimator for why this is done in this way atm.
+ nr_input_subbands = sum([uv_dict['properties']['nr_of_uv_files'] for uv_dict in input_files['uv']])
+ uv_file_size = input_files['uv'][0]['properties']['uv_file_size']
+ if nr_input_subbands % (subbands_per_image * slices_per_image) > 0:
+ logger.error('slices_per_image and subbands_per_image not a multiple of number of inputs')
+ result['errors'].append('slices_per_image and subbands_per_image not a multiple of number of inputs')
+ nr_images = nr_input_subbands / (subbands_per_image * slices_per_image)
+
+ logger.debug("calculate sky image data size")
+ img_file_size = 1000 # TODO: 1 kB was hardcoded in the Scheduler
+
+ logger.info("sky_images: {} files {} bytes each".format(nr_images, img_file_size))
+ estimate['output_files'] = {'img': [{'identification': output_ident_img,
+ 'properties': {'nr_of_img_files': nr_images,
+ 'img_file_size': img_file_size}}]}
+
+ # count total data size
+ data_size = nr_images * img_file_size # bytes
+ if data_size:
+ bandwidth = int(ceil(8 * data_size / duration)) # bits/second
+ estimate['resource_types'] = {'bandwidth': bandwidth, 'storage': data_size}
+ estimate['resource_count'] = 1
+ estimate['root_resource_group'] = output_cluster_img
+ else:
+ logger.error('An estimate of zero was calculated!')
+ result['errors'].append('An estimate of zero was calculated!')
+
+ result['estimates'].append(estimate)
return result
diff --git a/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/longbaseline_pipeline.py b/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/longbaseline_pipeline.py
index b898f42c43714597320efc656d03ae3fc240b479..e74b8f8efa486bfaa9960d2974836058025f6511 100644
--- a/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/longbaseline_pipeline.py
+++ b/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/longbaseline_pipeline.py
@@ -51,7 +51,7 @@ class LongBaselinePipelineResourceEstimator(BasePipelineResourceEstimator):
def _calculate(self, parset, predecessor_estimates):
- """ Estimate for Long Baseline Pipeline
+ """ Estimator for long baseline pipeline step.
calculates: datasize (number of files, file size), bandwidth
For a predecessor_estimates example, see the calibration/averaging
@@ -68,78 +68,69 @@ class LongBaselinePipelineResourceEstimator(BasePipelineResourceEstimator):
subbandgroups_per_ms = parset.getInt(PIPELINE + 'LongBaseline.subbandgroups_per_ms', 0) #TODO, should these have defaults?
subbands_per_subbandgroup = parset.getInt(PIPELINE + 'LongBaseline.subbands_per_subbandgroup', 0)
- if not subbandgroups_per_ms or not subbands_per_subbandgroup:
- logger.error('subbandgroups_per_ms or subbands_per_subbandgroup are not valid')
- result['errors'].append('subbandgroups_per_ms or subbands_per_subbandgroup are not valid')
+ if subbandgroups_per_ms < 1 or subbands_per_subbandgroup < 1:
+ logger.error('subbandgroups_per_ms or subbands_per_subbandgroup is not valid')
+ result['errors'].append('subbandgroups_per_ms or subbands_per_subbandgroup is not valid')
if not parset.getBool(DATAPRODUCTS + 'Output_Correlated.enabled'):
logger.error('Output_Correlated is not enabled')
result['errors'].append('Output_Correlated is not enabled')
- if result['errors']:
- return result
duration = self._getDuration(parset.getString('Observation.startTime'),
parset.getString('Observation.stopTime'))
+ input_idents_uv = parset.getStringVector(DATAPRODUCTS + 'Input_Correlated.identifications')
+ input_files = self.get_inputs_from_predecessors(predecessor_estimates, input_idents_uv, 'uv')
+ if not input_files:
+ logger.error('Missing uv dataproducts in predecessor output_files')
+ result['errors'].append('Missing uv dataproducts in predecessor output_files')
+
+ if result['errors']:
+ return result
+
+ estimate = {'input_files': input_files}
+
# NOTE: input bandwidth is not included in the resulting estimate atm.
# Proper input bandwidth est has limited use and is tricky, because of pipeline duration est, tmp files, multiple passes, nr nodes and caching, ...
#input_cluster_uv = parset.getString(DATAPRODUCTS + 'Input_Correlated.storageClusterName')
- input_idents_uv = parset.getStringVector(DATAPRODUCTS + 'Input_Correlated.identifications')
output_ident_uv = self._getOutputIdentification( parset.getStringVector(DATAPRODUCTS + 'Output_Correlated.identifications') )
output_cluster_uv = parset.getString(DATAPRODUCTS + 'Output_Correlated.storageClusterName')
- for pred_est in predecessor_estimates:
- pred_output_files = pred_est.get('output_files')
- if pred_output_files is None:
- continue
-
- input_files = self._filterInputs(pred_output_files, input_idents_uv)
- if not input_files:
- continue
-
- if 'uv' not in input_files:
- logger.error('Missing uv dataproducts in predecessor output_files')
- result['errors'].append('Missing uv dataproducts in predecessor output_files')
- continue
-
- estimate = {'input_files': input_files}
-
- nr_input_files = input_files['uv']['properties']['nr_of_uv_files']
- uv_input_file_size = input_files['uv']['properties']['uv_file_size'] # same for each uv data product across all SAPs
- start_sb_nr = input_files['uv']['properties']['start_sb_nr']
-
- if nr_input_files % (subbands_per_subbandgroup * subbandgroups_per_ms) > 0:
- logger.error('subbandgroups_per_ms and subbands_per_subbandgroup not a multiple of number of inputs')
- result['errors'].append('subbandgroups_per_ms and subbands_per_subbandgroup not a multiple of number of inputs')
- continue
- total_nr_input_files = nr_input_files * pred_est['resource_count']
- nr_output_files = total_nr_input_files / (subbands_per_subbandgroup * subbandgroups_per_ms)
-
- logger.debug("calculate correlated data size")
- uv_output_file_size = 1000 # TODO: 1 kB was hardcoded in the Scheduler
- start_sbg_nr = start_sb_nr / (subbands_per_subbandgroup * subbandgroups_per_ms)
-
- logger.info("correlated_uv: {} files {} bytes each".format(nr_output_files, uv_output_file_size))
- estimate['output_files'] = {'uv': {'identification': output_ident_uv,
- 'properties': {'nr_of_uv_files': 1, # also see estimate['resource_count'] below
- 'uv_file_size': uv_output_file_size,
- 'start_sbg_nr': start_sbg_nr}}}
-
- # count total data size
- data_size = nr_output_files * uv_output_file_size # bytes
- if data_size:
- bandwidth = int(ceil(8 * data_size / duration)) # bits/second
- estimate['resource_types'] = {'bandwidth': bandwidth, 'storage': data_size}
- estimate['resource_count'] = nr_output_files # as such, potentially different resources can be allocated for each output
- estimate['root_resource_group'] = output_cluster_uv
- else:
- logger.error('An estimate of zero was calculated!')
- result['errors'].append('An estimate of zero was calculated!')
-
- result['estimates'].append(estimate)
-
- if not result['estimates'] and not result['errors']:
- logger.error('long baseline estimator produced no estimates')
- result['errors'].append('long baseline pipeline estimator produced no estimates')
+ # See the calibration pipeline estimator for why this is done in this way atm.
+ nr_input_files = sum([uv_dict['properties']['nr_of_uv_files'] for uv_dict in input_files['uv']])
+ uv_input_file_size = input_files['uv'][0]['properties']['uv_file_size']
+ start_sb_nr = min([uv_dict['properties']['start_sb_nr'] for uv_dict in input_files['uv']])
+
+ if nr_input_files % (subbands_per_subbandgroup * subbandgroups_per_ms) > 0:
+ logger.error('subbandgroups_per_ms and subbands_per_subbandgroup not a multiple of number of inputs')
+ result['errors'].append('subbandgroups_per_ms and subbands_per_subbandgroup not a multiple of number of inputs')
+ nr_output_files = nr_input_files / (subbands_per_subbandgroup * subbandgroups_per_ms)
+
+ logger.debug("calculate correlated data size")
+ uv_output_file_size = 1000 # TODO: 1 kB was hardcoded in the Scheduler
+
+ # Computing start_sbg_nr in the same way as nr_output_files may not always work out as perhaps originally intended,
+ # e.g. if this is SAP 1, while SAP 0 had a different nr of subbands, but for filenames it doesn't really matter and we cannot easily do a lot better.
+ start_sbg_nr = start_sb_nr / (subbands_per_subbandgroup * subbandgroups_per_ms)
+
+ logger.info("correlated_uv: {} files {} bytes each".format(nr_output_files, uv_output_file_size))
+ estimate['output_files'] = {'uv': [{'identification': output_ident_uv,
+ 'properties': {'nr_of_uv_files': nr_output_files,
+ 'uv_file_size': uv_output_file_size,
+ 'start_sbg_nr': start_sbg_nr}}]}
+
+ # count total data size
+ data_size = nr_output_files * uv_output_file_size # bytes
+ if data_size:
+ bandwidth = int(ceil(8 * data_size / duration)) # bits/second
+ estimate['resource_types'] = {'bandwidth': bandwidth, 'storage': data_size}
+ estimate['resource_count'] = 1
+ estimate['root_resource_group'] = output_cluster_uv
+ else:
+ logger.error('An estimate of zero was calculated!')
+ result['errors'].append('An estimate of zero was calculated!')
+
+ result['estimates'].append(estimate)
return result
+
diff --git a/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/observation.py b/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/observation.py
index 259aa9e6aadb46414be4e03d014df930497a306b..f0f4753c2ad0a4ff7edcf5ea2197b6ba3636ab7a 100644
--- a/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/observation.py
+++ b/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/observation.py
@@ -69,50 +69,51 @@ class ObservationResourceEstimator(BaseResourceEstimator):
The predecessor_estimates arg is just to implement the same interface as pipelines. Observations have no predecessor.
The following return value example is for an obs duration of 240.0 s and 3 data product types for 2 clusters.
- NOTE: 'nr_of_XX_files' is for that estimate, times the 'resource_count' (hence each estimates contains 1 SAP).
- 'nr_of_cs_parts' is for a CS TAB (per stokes component) in that SAP, not per estimate.
- More examples at scu001:/opt/lofar/var/log/raestimatorservice.log
+ NOTE: 'nr_of_XX_files' is for that SAP estimate. The total is thus times the 'resource_count'.
+ 'nr_of_cs_parts' is for a full CS TAB (per stokes component) in that SAP; not per estimate, which may still describe one part.
+
+ See the calibration pipeline estimator for some explanation on why parts of this format are needed atm. It also has input_files.
{
'errors': [],
'estimates': [{
'resource_types': {'bandwidth': 35791395, 'storage': 1073741824}, # for each uv output data prod (thus the total is times the resource_count val)
'resource_count': 20, 'root_resource_group': 'CEP4',
'output_files': {
- 'uv': {'sap_nr': 0, 'identification': 'mom.G777955.B2.1.C.SAP000.uv.dps',
- 'properties': {'uv_file_size': 1073741824, 'nr_of_uv_files': 1, 'start_sb_nr': 0}}
+ 'uv': [{'sap_nr': 0, 'identification': 'mom.G777955.B2.1.C.SAP000.uv.dps',
+ 'properties': {'uv_file_size': 1073741824, 'nr_of_uv_files': 1, 'start_sb_nr': 0}}]
}
}, {'resource_types': {'bandwidth': 35791395, 'storage': 1073741824}, # idem
'resource_count': 60, 'root_resource_group': 'CEP4',
'output_files': {
- 'uv': {'sap_nr': 1, 'identification': 'mom.G777955.B2.1.C.SAP001.uv.dps',
- 'properties': {'uv_file_size': 1073741824, 'nr_of_uv_files': 1, 'start_sb_nr': 20}}
+ 'uv': [{'sap_nr': 1, 'identification': 'mom.G777955.B2.1.C.SAP001.uv.dps',
+ 'properties': {'uv_file_size': 1073741824, 'nr_of_uv_files': 1, 'start_sb_nr': 20}}]
}
}, {'resource_types': {'bandwidth': 35791395, 'storage': 1073741824}, # idem
'resource_count': 20, 'root_resource_group': 'CEP4',
'output_files': {
- 'uv': {'sap_nr': 2, 'identification': 'mom.G777955.B2.1.C.SAP002.uv.dps',
- 'properties': {'uv_file_size': 1073741824, 'nr_of_uv_files': 1, 'start_sb_nr': 80}}
+ 'uv': [{'sap_nr': 2, 'identification': 'mom.G777955.B2.1.C.SAP002.uv.dps',
+ 'properties': {'uv_file_size': 1073741824, 'nr_of_uv_files': 1, 'start_sb_nr': 80}}]
}
}, {'resource_types': {'bandwidth': 71582789, 'storage': 2147483648}, # for each quad (4 stokes) of cs output tab part (thus the total is times the resource_count val)
'resource_count': 34, 'root_resource_group': 'DRAGNET',
'output_files': {
- 'cs': {'sap_nr': 0, 'identification': 'mom.G777955.B2.1.C.SAP000.cs.dps',
- 'properties': {'cs_file_size': 536870912, 'nr_of_cs_files': 4, 'nr_of_cs_stokes': 4,
- 'nr_of_cs_parts': 2}} # parts per tab for this sap
+ 'cs': [{'sap_nr': 0, 'identification': 'mom.G777955.B2.1.C.SAP000.cs.dps',
+ 'properties': {'cs_file_size': 536870912, 'nr_of_cs_files': 4, 'nr_of_cs_stokes': 4,
+ 'nr_of_cs_parts': 2}}] # parts per tab for this sap
}
}, {'resource_types': {'bandwidth': 71582789, 'storage': 2147483648}, # idem
'resource_count': 6, 'root_resource_group': 'DRAGNET',
'output_files': {
- 'cs': {'sap_nr': 1, 'identification': 'mom.G777955.B2.1.C.SAP001.cs.dps',
- 'properties': {'cs_file_size': 536870912, 'nr_of_cs_files': 4, 'nr_of_cs_stokes': 4,
- 'nr_of_cs_parts': 1, 'is_tab_nr': 0}} # parts per tab for this sap
+ 'cs': [{'sap_nr': 1, 'identification': 'mom.G777955.B2.1.C.SAP001.cs.dps',
+ 'properties': {'cs_file_size': 536870912, 'nr_of_cs_files': 4, 'nr_of_cs_stokes': 4,
+ 'nr_of_cs_parts': 1, 'is_tab_nr': 0}}] # parts per tab for this sap
}
}, {'resource_types': {'bandwidth': 17895698, 'storage': 536870912}, # for each 'is' output tab part (thus the total is times the resource_count val)
'resource_count': 1, 'root_resource_group': 'DRAGNET',
'output_files': {
- 'is': {'sap_nr': 1, 'identification': 'mom.G777955.B2.1.C.SAP001.is.dps',
- 'properties': {'is_file_size': 536870912, 'nr_of_is_files': 1, 'nr_of_is_stokes': 1,
- 'is_tab_nr': 0}} # IS can have >1 parts, but atm max 1 IS TAB per SAP
+ 'is': [{'sap_nr': 1, 'identification': 'mom.G777955.B2.1.C.SAP001.is.dps',
+ 'properties': {'is_file_size': 536870912, 'nr_of_is_files': 1, 'nr_of_is_stokes': 1,
+ 'is_tab_nr': 0}}] # IS can have >1 parts, but atm max 1 IS TAB per SAP
}
}]
}
@@ -214,9 +215,9 @@ class ObservationResourceEstimator(BaseResourceEstimator):
est = {'resource_types': {'bandwidth': bandwidth, 'storage': file_size},
'resource_count': nr_subbands,
'root_resource_group': root_resource_group,
- 'output_files': {'uv': {'sap_nr': sap_nr, 'identification': sap_idents[sap_nr],
- 'properties': {'uv_file_size': file_size, 'nr_of_uv_files': 1, # i.e. total nr_of_uv_files is resource_count times 1
- 'start_sb_nr': total_files}}}}
+ 'output_files': {'uv': [{'sap_nr': sap_nr, 'identification': sap_idents[sap_nr],
+ 'properties': {'uv_file_size': file_size, 'nr_of_uv_files': 1, # i.e. total nr_of_uv_files is resource_count times 1
+ 'start_sb_nr': total_files}}]}}
total_files += nr_subbands
estimates.append(est)
@@ -311,11 +312,11 @@ class ObservationResourceEstimator(BaseResourceEstimator):
est = {'resource_types': {'storage': storage, 'bandwidth': bandwidth},
'resource_count': nr_coherent_tabs * nr_parts_per_tab,
'root_resource_group': root_resource_group,
- 'output_files': {'cs': {'sap_nr': sap_nr, 'identification': sap_idents[sap_nr],
- 'properties': {'cs_file_size': file_size, 'nr_of_cs_files': nr_coherent,
- 'nr_of_cs_stokes': nr_coherent, 'nr_of_cs_parts': nr_parts_per_tab}}}}
+ 'output_files': {'cs': [{'sap_nr': sap_nr, 'identification': sap_idents[sap_nr],
+ 'properties': {'cs_file_size': file_size, 'nr_of_cs_files': nr_coherent,
+ 'nr_of_cs_stokes': nr_coherent, 'nr_of_cs_parts': nr_parts_per_tab}}]}}
if is_tab_nr != -1: # translator to filenames needs to know: it may not have all CS+IS info in one claim
- est['output_files']['cs']['properties']['is_tab_nr'] = is_tab_nr
+ est['output_files']['cs'][0]['properties']['is_tab_nr'] = is_tab_nr
estimates.append(est)
logger.debug("Coherent Stokes data estimates: {}".format(estimates))
@@ -397,9 +398,9 @@ class ObservationResourceEstimator(BaseResourceEstimator):
est = {'resource_types': {'storage': storage, 'bandwidth': bandwidth},
'resource_count': nr_incoherent_tabs * nr_parts_per_tab,
'root_resource_group': root_resource_group,
- 'output_files': {'is': {'sap_nr': sap_nr, 'identification': sap_idents[sap_nr],
- 'properties': {'is_file_size': file_size, 'nr_of_is_files': nr_incoherent,
- 'nr_of_is_stokes': nr_incoherent, 'is_tab_nr': is_tab_nr}}}}
+ 'output_files': {'is': [{'sap_nr': sap_nr, 'identification': sap_idents[sap_nr],
+ 'properties': {'is_file_size': file_size, 'nr_of_is_files': nr_incoherent,
+ 'nr_of_is_stokes': nr_incoherent, 'is_tab_nr': is_tab_nr}}]}}
estimates.append(est)
logger.debug("Incoherent Stokes data estimates: {}".format(estimates))
@@ -431,6 +432,20 @@ class ObservationResourceEstimator(BaseResourceEstimator):
logger.info("number of virtual stations = {}".format(nr_virtual_stations))
return nr_virtual_stations
+ def _extract_sap_nr(self, identification):
+ """ Return sap nr as int from identification or None if
+ no int xxx in '.SAPxxx.' in identification.
+ """
+ for s in identification.split('.'): # Find the SAP number, if present
+ if 'SAP' not in s:
+ continue
+ try:
+ return int(s[3:])
+ except:
+ pass
+
+ return None
+
def _sap_identifications(self, identifications, nr_saps):
""" Return list with identifications' identification for sap i at index i,
or '' at index i if no such identification for sap i.
diff --git a/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/pulsar_pipeline.py b/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/pulsar_pipeline.py
index 82dd7e52515d81710cb4d21914f002bc618f592a..05220b949ba9607227d1e38c5eb6a5769864f043 100644
--- a/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/pulsar_pipeline.py
+++ b/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/pulsar_pipeline.py
@@ -68,74 +68,75 @@ class PulsarPipelineResourceEstimator(BasePipelineResourceEstimator):
if not parset.getBool(DATAPRODUCTS + 'Output_Pulsar.enabled'):
logger.error('Output_Pulsar is not enabled')
result['errors'].append('Output_Pulsar is not enabled')
- if result['errors']:
- return result
duration = self._getDuration(parset.getString('Observation.startTime'),
parset.getString('Observation.stopTime'))
+ # The current XML generator can produce a pulsar pipeline step that operates on 1 SAP,
+ # however, pulsar astronomers produce an XML that works on all SAPs (CS+IS) of an obs.
+ # This is regular and going on for years, so we need to support multi-SAP pulp input.
+ # Note that when selecting obs SAP nr > 0 or a sparse SAP nr range, TAB nrs in pulp filenames do not
+ # match TAB nrs in obs filenames, because there is no pulp equiv of start_sb[g]_nr. Nobody cares.
+ # (Then there is the possibility of multi-obs pulp input. As-is that may turn out to work as well.)
+ input_files = {}
+ input_idents_cs = parset.getStringVector(DATAPRODUCTS + 'Input_CoherentStokes.identifications')
+ input_files_cs = self.get_inputs_from_predecessors(predecessor_estimates, input_idents_cs, 'cs')
+ if input_files_cs:
+ input_files['cs'] = input_files_cs['cs']
+ input_idents_is = parset.getStringVector(DATAPRODUCTS + 'Input_IncoherentStokes.identifications')
+ input_files_is = self.get_inputs_from_predecessors(predecessor_estimates, input_idents_is, 'is')
+ if input_files_is:
+ input_files['is'] = input_files_is['is']
+ if not input_files:
+ logger.error('Missing \'cs\' or \'is\' dataproducts in predecessor output_files')
+ result['errors'].append('Missing \'cs\' or \'is\' dataproducts in predecessor output_files')
+
+ if result['errors']:
+ return result
+
+ estimate = {'input_files': input_files}
+
# NOTE: input bandwidth is not included in the resulting estimate atm.
# Proper input bandwidth est has limited use and is tricky, because of pipeline duration est, tmp files, multiple passes, nr nodes and caching, ...
#input_cluster_cs = parset.getString(DATAPRODUCTS + 'Input_CoherentStokes.storageClusterName')
#input_cluster_is = parset.getString(DATAPRODUCTS + 'Input_IncoherentStokes.storageClusterName')
- input_idents_cs = parset.getStringVector(DATAPRODUCTS + 'Input_CoherentStokes.identifications')
- input_idents_is = parset.getStringVector(DATAPRODUCTS + 'Input_IncoherentStokes.identifications')
output_ident_pulp = self._getOutputIdentification( parset.getStringVector(DATAPRODUCTS + 'Output_Pulsar.identifications') )
output_cluster_pulp = parset.getString(DATAPRODUCTS + 'Output_Pulsar.storageClusterName')
- for pred_est in predecessor_estimates:
- pred_output_files = pred_est.get('output_files')
- if pred_output_files is None:
- continue
-
- idents = []
- if 'cs' in pred_output_files:
- idents.extend(input_idents_cs)
- if 'is' in pred_output_files:
- idents.extend(input_idents_is)
- input_files = self._filterInputs(pred_output_files, idents)
- if not input_files:
- continue
-
- if not 'cs' in input_files and not 'is' in input_files:
- logger.error('Missing both CS and IS dataproducts in input_files')
- result['errors'].append('Missing both CS and IS dataproducts in input_files')
-
- estimate = {'input_files': input_files}
-
- # It seems 1 pulp data product is produced per beam, also with stokes IQUV or with complex voltages (XXYY).
- nr_output_files = 0
- if 'cs' in input_files:
- nr_output_files += input_files['cs']['properties']['nr_of_cs_files'] / \
- input_files['cs']['properties']['nr_of_cs_stokes']
- if 'is' in input_files:
- nr_output_files += input_files['is']['properties']['nr_of_is_files'] / \
- input_files['is']['properties']['nr_of_is_stokes']
-
- logger.debug("calculate pulp data size")
- pulp_file_size = 1000 # TODO: 1 kB was hardcoded in the Scheduler
-
- logger.info("pulsar_pipeline pulp: {} files {} bytes each".format(nr_output_files, pulp_file_size))
- estimate['output_files'] = {'pulp': {'identification': output_ident_pulp,
- 'properties': {'nr_of_pulp_files': nr_output_files, # times pred_est['resource_count'] in total
- 'pulp_file_size': pulp_file_size}}}
-
- # count total data size
- data_size = nr_output_files * pulp_file_size
- if data_size > 0:
- bandwidth = int(ceil(8 * data_size / duration)) # bits/second
- estimate['resource_types'] = {'bandwidth': bandwidth, 'storage': data_size}
- estimate['resource_count'] = pred_est['resource_count']
- estimate['root_resource_group'] = output_cluster_pulp
- else:
- logger.error('An estimate of zero was calculated!')
- result['errors'].append('An estimate of zero was calculated!')
-
- result['estimates'].append(estimate)
-
- if not result['estimates'] and not result['errors']:
- logger.error('pulsar pipeline estimator produced no estimates')
- result['errors'].append('pulsar pipeline estimator produced no estimates')
+ # The pulsar pipeline ('pulp') produces 1 data product per tied-array beam, it seems also for complex voltages (XXYY) and stokes IQUV(?).
+ # For XXYY it really needs all 4 components at once. For IQUV this is less important, but atm we treat it the same (1 obs output estimate).
+ # Note that it also produces 1 additional "summary" data product per data product *type* (i.e. 1 for 'cs' and/or 1 for 'is'),
+ # but the RA_Services sub-system does not know about it. Adding support may be a waste of time(?).
+ # Currently, RO controlled pulp grabs all inputs given some project name/id(?) and obs id, not from rotspservice generated parset parts.
+ nr_input_files = 0
+ if 'cs' in input_files:
+ nr_input_files += sum([cs_dict['properties']['nr_of_cs_files'] / \
+ cs_dict['properties']['nr_of_cs_stokes'] for cs_dict in input_files['cs']])
+ if 'is' in input_files:
+ nr_input_files += sum([is_dict['properties']['nr_of_is_files'] / \
+ is_dict['properties']['nr_of_is_stokes'] for is_dict in input_files['is']])
+
+ logger.debug("calculate pulp data size")
+ pulp_file_size = 1000 # TODO: 1 kB was hardcoded in the Scheduler
+
+ logger.info("pulsar_pipeline pulp: {} files {} bytes each".format(nr_input_files, pulp_file_size))
+ estimate['output_files'] = {'pulp': [{'identification': output_ident_pulp,
+ 'properties': {'nr_of_pulp_files': nr_input_files,
+ 'pulp_file_size': pulp_file_size}}]}
+
+ # count total data size
+ data_size = nr_input_files * pulp_file_size
+ if data_size > 0:
+ bandwidth = int(ceil(8 * data_size / duration)) # bits/second
+ estimate['resource_types'] = {'bandwidth': bandwidth, 'storage': data_size}
+ estimate['resource_count'] = 1
+ estimate['root_resource_group'] = output_cluster_pulp
+ else:
+ logger.error('An estimate of zero was calculated!')
+ result['errors'].append('An estimate of zero was calculated!')
+
+ result['estimates'].append(estimate)
return result
+
diff --git a/SAS/ResourceAssignment/ResourceAssignmentEstimator/service.py b/SAS/ResourceAssignment/ResourceAssignmentEstimator/service.py
index a72044e4b6166dc9413b27d47a6c12f51bd93ea4..ad62ad399684d23aa78ccaef48705c71aeae46fe 100644
--- a/SAS/ResourceAssignment/ResourceAssignmentEstimator/service.py
+++ b/SAS/ResourceAssignment/ResourceAssignmentEstimator/service.py
@@ -56,10 +56,12 @@ class ResourceEstimatorHandler(MessageHandlerInterface):
if 'storage' in est['resource_types']:
# We only need to do output files, it will be someone else's input
output_files = est.get('output_files')
- if output_files is not None:
- for data_type in output_files:
- output_files[data_type]['properties'][data_type + '_otdb_id'] = otdb_id
- logger.info('add_id: added %s to properties of data type %s' % (otdb_id, data_type))
+ if output_files is None:
+ continue
+ for dptype in output_files:
+ for dptype_dict in output_files[dptype]:
+ dptype_dict['properties'][dptype + '_otdb_id'] = otdb_id
+ logger.info('add_id: added %s to properties of data type %s' % (otdb_id, dptype))
return task_estimate
@@ -88,10 +90,10 @@ class ResourceEstimatorHandler(MessageHandlerInterface):
for branch_otdb_id, branch_estimate in branch_estimates.items():
logger.info('Looking at predecessor %s' % branch_otdb_id)
estimates = branch_estimate.values()[0]['estimates']
- if any(['uv' in est['output_files'] and 'im' not in est['output_files'] for est in estimates]): # Not a calibrator pipeline
+ if any(['uv' in est['output_files'] and 'im' not in est['output_files'] for est in estimates if 'output_files' in est]): # Not a calibrator pipeline
logger.info('found %s as the target of pipeline %s' % (branch_otdb_id, otdb_id))
predecessor_estimates.extend(estimates)
- elif any(['im' in est['output_files'] for est in estimates]):
+ elif any(['im' in est['output_files'] for est in estimates if 'output_files' in est]):
logger.info('found %s as the calibrator of pipeline %s' % (branch_otdb_id, otdb_id))
predecessor_estimates.extend(estimates)
return {str(otdb_id): self.add_id(self.calibration_pipeline.verify_and_estimate(parset, predecessor_estimates), otdb_id)}
diff --git a/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/CMakeLists.txt b/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/CMakeLists.txt
index 5027001655b0c311a509f980afec9165a3c09cb7..69b91228a3ad9a1e0b6b9490c9844f7a2297bce2 100644
--- a/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/CMakeLists.txt
+++ b/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/CMakeLists.txt
@@ -1,9 +1,8 @@
# $Id: CMakeLists.txt $
include(LofarCTest)
-
#lofar_add_test(t_cobaltblocksize)
-lofar_add_test(t_resource_estimator) # to be re-enabled when estimator pipeline code is fixed by Alexander
+lofar_add_test(t_resource_estimator)
set(_py_files
__init__.py
diff --git a/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.in_imaging_pipeline b/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.in_imaging_pipeline
new file mode 100644
index 0000000000000000000000000000000000000000..9001f539b1bd22d8a2c9a3350847be901f194558
--- /dev/null
+++ b/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.in_imaging_pipeline
@@ -0,0 +1,54 @@
+{
+ 'Observation.DataProducts.Output_InstrumentModel.enabled': False,
+ 'Observation.DataProducts.Output_CoherentStokes.storageClusterName': '',
+ 'Observation.stopTime': '2016-12-09 16:01:25',
+ 'Observation.VirtualInstrument.stationList': [
+ ],
+ 'Observation.DataProducts.Input_CoherentStokes.enabled': False,
+ 'Observation.DataProducts.Output_CoherentStokes.enabled': False,
+ 'Observation.DataProducts.Output_Correlated.storageClusterName': '',
+ 'Observation.DataProducts.Input_Correlated.identifications': [
+ 'mom.G732487.B0.1.C.SAP000.uv.dps'
+ ],
+ 'Observation.DataProducts.Output_SkyImage.storageClusterName': 'CEP4',
+ 'Observation.DataProducts.Output_InstrumentModel.identifications': [
+ ],
+ 'Observation.antennaSet': 'LBA_INNER',
+ 'Observation.DataProducts.Output_Pulsar.identifications': [
+ ],
+ 'Observation.nrBitsPerSample': '16',
+ 'Observation.DataProducts.Output_IncoherentStokes.enabled': False,
+ 'Observation.DataProducts.Input_IncoherentStokes.enabled': False,
+ 'Observation.DataProducts.Input_Correlated.enabled': True,
+ 'Observation.DataProducts.Output_Pulsar.enabled': False,
+ 'Observation.DataProducts.Output_Correlated.identifications': [
+ ],
+ 'Observation.DataProducts.Input_CoherentStokes.identifications': [
+ ],
+ 'Observation.DataProducts.Output_InstrumentModel.storageClusterName': '',
+ 'Observation.DataProducts.Input_InstrumentModel.enabled': False,
+ 'Observation.DataProducts.Output_SkyImage.enabled': True,
+ 'Version.number': '33385',
+ 'Observation.DataProducts.Output_Pulsar.storageClusterName': '',
+ 'Observation.momID': '737233',
+ 'Observation.DataProducts.Input_InstrumentModel.identifications': [
+ ],
+ 'Observation.DataProducts.Output_SkyImage.identifications': [
+ 'mom.G732488.B0.PI0.dps'
+ ],
+ 'Observation.ObservationControl.PythonControl.Imaging.slices_per_image': 3,
+ 'Observation.ObservationControl.PythonControl.Imaging.subbands_per_image': 4,
+ 'Observation.startTime': '2016-12-09 07:21:25',
+ 'Observation.nrBeams': '0',
+ 'Observation.Scheduler.taskDuration': '31200',
+ 'Observation.DataProducts.Output_IncoherentStokes.identifications': [
+ ],
+ 'Observation.DataProducts.Input_IncoherentStokes.identifications': [
+ ],
+ 'Observation.DataProducts.Output_CoherentStokes.identifications': [
+ ],
+ 'Observation.DataProducts.Output_Correlated.enabled': False,
+ 'Observation.DataProducts.Output_IncoherentStokes.storageClusterName': '',
+ 'Observation.sampleClock': '200'
+}
+
diff --git a/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_beam_observation b/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_beam_observation
index acc45aa829bf27e874b18de9036bcf7abf366ce3..4d8f13c4d8d0ca4a471c0d8b244a8f54e899cbe8 100644
--- a/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_beam_observation
+++ b/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_beam_observation
@@ -1 +1 @@
-"{'1': {'observation': {'errors': [], 'estimates': [{'root_resource_group': 'CEP4', 'resource_types': {'bandwidth': 169869312, 'storage': 76441190400}, 'resource_count': 73, 'output_files': {'cs': {'cs_otdb_id': 1, 'identifications': ['mom.G735371.LOTAAS-P1296B-SAP0.1296.SAP0.obs.cs', 'mom.G735371.LOTAAS-P1296B-SAP0.1296.SAP0.obs.cs', 'mom.G735371.LOTAAS-P1296B-SAP1.1296.SAP1.obs.cs', 'mom.G735371.LOTAAS-P1296B-SAP1.1296.SAP1.obs.cs', 'mom.G735371.LOTAAS-P1296B-SAP2.1296.SAP2.obs.cs', 'mom.G735371.LOTAAS-P1296B-SAP2.1296.SAP2.obs.cs'], 'nr_of_cs_stokes': 1}, 'saps': [{'sap_nr': 0, 'properties': {'cs_file_size': 76441190400, 'nr_of_cs_parts': 1, 'is_tab_nr': 12, 'nr_of_cs_files': 1}}]}}, {'root_resource_group': 'CEP4', 'resource_types': {'bandwidth': 169869312, 'storage': 76441190400}, 'resource_count': 73, 'output_files': {'cs': {'cs_otdb_id': 1, 'identifications': ['mom.G735371.LOTAAS-P1296B-SAP0.1296.SAP0.obs.cs', 'mom.G735371.LOTAAS-P1296B-SAP0.1296.SAP0.obs.cs', 'mom.G735371.LOTAAS-P1296B-SAP1.1296.SAP1.obs.cs', 'mom.G735371.LOTAAS-P1296B-SAP1.1296.SAP1.obs.cs', 'mom.G735371.LOTAAS-P1296B-SAP2.1296.SAP2.obs.cs', 'mom.G735371.LOTAAS-P1296B-SAP2.1296.SAP2.obs.cs'], 'nr_of_cs_stokes': 1}, 'saps': [{'sap_nr': 1, 'properties': {'cs_file_size': 76441190400, 'nr_of_cs_parts': 1, 'is_tab_nr': 12, 'nr_of_cs_files': 1}}]}}, {'root_resource_group': 'CEP4', 'resource_types': {'bandwidth': 169869312, 'storage': 76441190400}, 'resource_count': 73, 'output_files': {'cs': {'cs_otdb_id': 1, 'identifications': ['mom.G735371.LOTAAS-P1296B-SAP0.1296.SAP0.obs.cs', 'mom.G735371.LOTAAS-P1296B-SAP0.1296.SAP0.obs.cs', 'mom.G735371.LOTAAS-P1296B-SAP1.1296.SAP1.obs.cs', 'mom.G735371.LOTAAS-P1296B-SAP1.1296.SAP1.obs.cs', 'mom.G735371.LOTAAS-P1296B-SAP2.1296.SAP2.obs.cs', 'mom.G735371.LOTAAS-P1296B-SAP2.1296.SAP2.obs.cs'], 'nr_of_cs_stokes': 1}, 'saps': [{'sap_nr': 2, 'properties': {'cs_file_size': 76441190400, 'nr_of_cs_parts': 1, 'is_tab_nr': 12, 'nr_of_cs_files': 1}}]}}, {'root_resource_group': 'CEP4', 'resource_types': {'bandwidth': 169869312, 'storage': 76441190400}, 'resource_count': 1, 'output_files': {'is': {'identifications': ['mom.G735371.LOTAAS-P1296B-SAP0.1296.SAP0.obs.is', 'mom.G735371.LOTAAS-P1296B-SAP0.1296.SAP0.obs.is', 'mom.G735371.LOTAAS-P1296B-SAP1.1296.SAP1.obs.is', 'mom.G735371.LOTAAS-P1296B-SAP1.1296.SAP1.obs.is', 'mom.G735371.LOTAAS-P1296B-SAP2.1296.SAP2.obs.is', 'mom.G735371.LOTAAS-P1296B-SAP2.1296.SAP2.obs.is'], 'is_otdb_id': 1, 'nr_of_is_stokes': 1}, 'saps': [{'sap_nr': 0, 'properties': {'is_file_size': 76441190400, 'nr_of_is_files': 1, 'is_tab_nr': 12}}]}}, {'root_resource_group': 'CEP4', 'resource_types': {'bandwidth': 169869312, 'storage': 76441190400}, 'resource_count': 1, 'output_files': {'is': {'identifications': ['mom.G735371.LOTAAS-P1296B-SAP0.1296.SAP0.obs.is', 'mom.G735371.LOTAAS-P1296B-SAP0.1296.SAP0.obs.is', 'mom.G735371.LOTAAS-P1296B-SAP1.1296.SAP1.obs.is', 'mom.G735371.LOTAAS-P1296B-SAP1.1296.SAP1.obs.is', 'mom.G735371.LOTAAS-P1296B-SAP2.1296.SAP2.obs.is', 'mom.G735371.LOTAAS-P1296B-SAP2.1296.SAP2.obs.is'], 'is_otdb_id': 1, 'nr_of_is_stokes': 1}, 'saps': [{'sap_nr': 1, 'properties': {'is_file_size': 76441190400, 'nr_of_is_files': 1, 'is_tab_nr': 12}}]}}, {'root_resource_group': 'CEP4', 'resource_types': {'bandwidth': 169869312, 'storage': 76441190400}, 'resource_count': 1, 'output_files': {'is': {'identifications': ['mom.G735371.LOTAAS-P1296B-SAP0.1296.SAP0.obs.is', 'mom.G735371.LOTAAS-P1296B-SAP0.1296.SAP0.obs.is', 'mom.G735371.LOTAAS-P1296B-SAP1.1296.SAP1.obs.is', 'mom.G735371.LOTAAS-P1296B-SAP1.1296.SAP1.obs.is', 'mom.G735371.LOTAAS-P1296B-SAP2.1296.SAP2.obs.is', 'mom.G735371.LOTAAS-P1296B-SAP2.1296.SAP2.obs.is'], 'is_otdb_id': 1, 'nr_of_is_stokes': 1}, 'saps': [{'sap_nr': 2, 'properties': {'is_file_size': 76441190400, 'nr_of_is_files': 1, 'is_tab_nr': 12}}]}}, {'root_resource_group': 'CS004', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS004RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3888}, 'resource_count': 1}, {'root_resource_group': 'CS004RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3888}, 'resource_count': 1}, {'root_resource_group': 'CS005', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS005RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3888}, 'resource_count': 1}, {'root_resource_group': 'CS005RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3888}, 'resource_count': 1}, {'root_resource_group': 'CS003', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS003RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3888}, 'resource_count': 1}, {'root_resource_group': 'CS003RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3888}, 'resource_count': 1}, {'root_resource_group': 'CS002', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS002RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3888}, 'resource_count': 1}, {'root_resource_group': 'CS002RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3888}, 'resource_count': 1}, {'root_resource_group': 'CS007', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS007RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3888}, 'resource_count': 1}, {'root_resource_group': 'CS007RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3888}, 'resource_count': 1}, {'root_resource_group': 'CS006', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS006RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3888}, 'resource_count': 1}, {'root_resource_group': 'CS006RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3888}, 'resource_count': 1}]}}}"
+"{'1': {'observation': {'errors': [], 'estimates': [{'root_resource_group': 'CEP4', 'resource_types': {'bandwidth': 169869312, 'storage': 76441190400}, 'resource_count': 73, 'output_files': {'cs': [{'identification': 'mom.G735371.LOTAAS-P1296B-SAP0.1296.SAP0.obs.cs', 'sap_nr': 0, 'properties': {'nr_of_cs_stokes': 1, 'cs_otdb_id': 1, 'cs_file_size': 76441190400, 'nr_of_cs_parts': 1, 'is_tab_nr': 12, 'nr_of_cs_files': 1}}]}}, {'root_resource_group': 'CEP4', 'resource_types': {'bandwidth': 169869312, 'storage': 76441190400}, 'resource_count': 73, 'output_files': {'cs': [{'identification': 'mom.G735371.LOTAAS-P1296B-SAP1.1296.SAP1.obs.cs', 'sap_nr': 1, 'properties': {'nr_of_cs_stokes': 1, 'cs_otdb_id': 1, 'cs_file_size': 76441190400, 'nr_of_cs_parts': 1, 'is_tab_nr': 12, 'nr_of_cs_files': 1}}]}}, {'root_resource_group': 'CEP4', 'resource_types': {'bandwidth': 169869312, 'storage': 76441190400}, 'resource_count': 73, 'output_files': {'cs': [{'identification': 'mom.G735371.LOTAAS-P1296B-SAP2.1296.SAP2.obs.cs', 'sap_nr': 2, 'properties': {'nr_of_cs_stokes': 1, 'cs_otdb_id': 1, 'cs_file_size': 76441190400, 'nr_of_cs_parts': 1, 'is_tab_nr': 12, 'nr_of_cs_files': 1}}]}}, {'root_resource_group': 'CEP4', 'resource_types': {'bandwidth': 169869312, 'storage': 76441190400}, 'resource_count': 1, 'output_files': {'is': [{'identification': 'mom.G735371.LOTAAS-P1296B-SAP0.1296.SAP0.obs.is', 'sap_nr': 0, 'properties': {'is_tab_nr': 12, 'is_file_size': 76441190400, 'nr_of_is_files': 1, 'nr_of_is_stokes': 1, 'is_otdb_id': 1}}]}}, {'root_resource_group': 'CEP4', 'resource_types': {'bandwidth': 169869312, 'storage': 76441190400}, 'resource_count': 1, 'output_files': {'is': [{'identification': 'mom.G735371.LOTAAS-P1296B-SAP1.1296.SAP1.obs.is', 'sap_nr': 1, 'properties': {'is_tab_nr': 12, 'is_file_size': 76441190400, 'nr_of_is_files': 1, 'nr_of_is_stokes': 1, 'is_otdb_id': 1}}]}}, {'root_resource_group': 'CEP4', 'resource_types': {'bandwidth': 169869312, 'storage': 76441190400}, 'resource_count': 1, 'output_files': {'is': [{'identification': 'mom.G735371.LOTAAS-P1296B-SAP2.1296.SAP2.obs.is', 'sap_nr': 2, 'properties': {'is_tab_nr': 12, 'is_file_size': 76441190400, 'nr_of_is_files': 1, 'nr_of_is_stokes': 1, 'is_otdb_id': 1}}]}}, {'root_resource_group': 'CS004', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS004RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3888}, 'resource_count': 1}, {'root_resource_group': 'CS004RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3888}, 'resource_count': 1}, {'root_resource_group': 'CS005', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS005RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3888}, 'resource_count': 1}, {'root_resource_group': 'CS005RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3888}, 'resource_count': 1}, {'root_resource_group': 'CS003', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS003RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3888}, 'resource_count': 1}, {'root_resource_group': 'CS003RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3888}, 'resource_count': 1}, {'root_resource_group': 'CS002', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS002RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3888}, 'resource_count': 1}, {'root_resource_group': 'CS002RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3888}, 'resource_count': 1}, {'root_resource_group': 'CS007', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS007RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3888}, 'resource_count': 1}, {'root_resource_group': 'CS007RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3888}, 'resource_count': 1}, {'root_resource_group': 'CS006', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS006RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3888}, 'resource_count': 1}, {'root_resource_group': 'CS006RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3888}, 'resource_count': 1}]}}}"
diff --git a/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_calibration_pipeline b/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_calibration_pipeline
index 09369fffcd477941ab58846b809e68ec35a32f5b..5fa5fe9ba3fb45a0d52a566a46b0a282a7cb157d 100644
--- a/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_calibration_pipeline
+++ b/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_calibration_pipeline
@@ -1 +1 @@
-"{'1': {'pipeline': {'bandwidth': {'total_size': 308768086}, 'errors': [], 'storage': {'total_size': 171289095360, 'output_files': {'uv': {'start_sb_nr': 0, 'identifications': ['mom.G732487.B0.1.CPC.uv.dps'], 'nr_of_uv_files': 120, 'uv_file_size': 1427408128, 'uv_otdb_id': 1}, 'im': {'im_otdb_id': 1, 'identifications': ['mom.G732487.B0.1.CPC.inst.dps'], 'start_sb_nr': 0, 'im_file_size': 1000, 'nr_of_im_files': 120}}, 'input_files': {'uv': {'start_sb_nr': 0, 'identifications': ['mom.G732487.B0.1.C.SAP000.uv.dps'], 'nr_of_uv_files': 120, 'uv_file_size': 3617984960, 'uv_otdb_id': 2}, 'saps': [{'sap_nr': 0, 'properties': {'nr_of_uv_files': 120, 'start_sb_nr': 0}}]}}}}}"
+"{'1': {'pipeline': {'errors': [], 'estimates': [{'root_resource_group': 'CEP4', 'output_files': {'uv': [{'identification': 'mom.G732487.B0.1.CPC.uv.dps', 'properties': {'start_sb_nr': 0, 'nr_of_uv_files': 120, 'uv_file_size': 1427408128, 'uv_otdb_id': 1}}], 'im': [{'identification': 'mom.G732487.B0.1.CPC.inst.dps', 'properties': {'im_otdb_id': 1, 'start_sb_nr': 0, 'im_file_size': 1000, 'nr_of_im_files': 120}}]}, 'resource_types': {'bandwidth': 308768086, 'storage': 171289095360}, 'resource_count': 1, 'input_files': {'uv': [{'identification': 'mom.G732487.B0.1.C.SAP000.uv.dps', 'sap_nr': 0, 'properties': {'nr_of_uv_files': 120, 'start_sb_nr': 0, 'uv_file_size': 3617984960, 'uv_otdb_id': 2}}]}}]}}}"
diff --git a/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_imaging_pipeline b/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_imaging_pipeline
new file mode 100644
index 0000000000000000000000000000000000000000..6b51700eb530c195a572b8b2524305bdb64db0ca
--- /dev/null
+++ b/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_imaging_pipeline
@@ -0,0 +1 @@
+"{'1': {'pipeline': {'errors': [], 'estimates': [{'root_resource_group': 'CEP4', 'output_files': {'img': [{'identification': 'mom.G732488.B0.PI0.dps', 'properties': {'img_file_size': 1000, 'img_otdb_id': 1, 'nr_of_img_files': 10}}]}, 'resource_types': {'bandwidth': 3, 'storage': 10000}, 'resource_count': 1, 'input_files': {'uv': [{'identification': 'mom.G732487.B0.1.C.SAP000.uv.dps', 'sap_nr': 0, 'properties': {'nr_of_uv_files': 120, 'start_sb_nr': 0, 'uv_file_size': 3617984960, 'uv_otdb_id': 2}}]}}]}}}"
diff --git a/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_interferometer_observation b/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_interferometer_observation
index bcdb60141b79a8c34fd651ef576eac5f7ddcbfd5..a80d9cf8aaf2eb9dd7bbee9f826c3a67f508468d 100644
--- a/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_interferometer_observation
+++ b/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_interferometer_observation
@@ -1 +1 @@
-"{'1': {'observation': {'errors': [], 'estimates': [{'root_resource_group': 'CEP4', 'resource_types': {'bandwidth': 45688942, 'storage': 164480190400}, 'resource_count': 1, 'output_files': {'uv': {'identifications': ['mom.G737227.B1.1.T.SAP000.uv.dps', 'mom.G737227.B1.1.T.SAP001.uv.dps', 'mom.G737227.B1.1.T.SAP002.uv.dps'], 'uv_otdb_id': 1}, 'saps': [{'sap_nr': 0, 'properties': {'start_sb_nr': 0, 'nr_of_uv_files': 1, 'uv_file_size': 164480190400}}]}}, {'root_resource_group': 'CEP4', 'resource_types': {'bandwidth': 45688942, 'storage': 164480190400}, 'resource_count': 243, 'output_files': {'uv': {'identifications': ['mom.G737227.B1.1.T.SAP000.uv.dps', 'mom.G737227.B1.1.T.SAP001.uv.dps', 'mom.G737227.B1.1.T.SAP002.uv.dps'], 'uv_otdb_id': 1}, 'saps': [{'sap_nr': 1, 'properties': {'start_sb_nr': 1, 'nr_of_uv_files': 1, 'uv_file_size': 164480190400}}]}}, {'root_resource_group': 'CEP4', 'resource_types': {'bandwidth': 45688942, 'storage': 164480190400}, 'resource_count': 243, 'output_files': {'uv': {'identifications': ['mom.G737227.B1.1.T.SAP000.uv.dps', 'mom.G737227.B1.1.T.SAP001.uv.dps', 'mom.G737227.B1.1.T.SAP002.uv.dps'], 'uv_otdb_id': 1}, 'saps': [{'sap_nr': 2, 'properties': {'start_sb_nr': 244, 'nr_of_uv_files': 1, 'uv_file_size': 164480190400}}]}}, {'root_resource_group': 'CS001', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS001RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS001RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS002', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS002RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS002RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS003', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS003RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS003RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS004', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS004RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS004RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS005', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS005RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS005RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS006', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS006RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS006RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS007', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS007RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS007RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS011', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS011RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS011RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS013', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS013RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS013RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS017', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS017RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS017RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS021', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS021RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS021RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS024', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS024RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS024RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS026', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS026RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS026RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS028', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS028RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS028RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS030', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS030RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS030RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS031', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS031RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS031RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS032', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS032RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS032RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS101', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS101RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS101RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS103', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS103RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS103RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS201', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS201RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS201RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS301', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS301RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS301RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS302', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS302RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS302RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS401', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS401RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS401RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS501', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS501RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS501RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'DE602', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111'}, 'resource_count': 1}, {'root_resource_group': 'DE602RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'DE603', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111'}, 'resource_count': 1}, {'root_resource_group': 'DE603RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'DE605', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111'}, 'resource_count': 1}, {'root_resource_group': 'DE605RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'DE609', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111'}, 'resource_count': 1}, {'root_resource_group': 'DE609RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'FR606', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111'}, 'resource_count': 1}, {'root_resource_group': 'FR606RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'PL610', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111'}, 'resource_count': 1}, {'root_resource_group': 'PL610RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'PL611', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111'}, 'resource_count': 1}, {'root_resource_group': 'PL611RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'PL612', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111'}, 'resource_count': 1}, {'root_resource_group': 'PL612RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'RS106', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'RS106RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'RS205', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'RS205RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'RS208', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'RS208RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'RS210', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'RS210RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'RS305', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'RS305RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'RS306', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'RS306RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'RS307', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'RS307RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'RS310', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'RS310RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'RS406', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'RS406RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'RS407', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'RS407RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'RS409', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'RS409RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'RS503', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'RS503RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'RS508', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'RS508RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'RS509', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'RS509RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'SE607', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111'}, 'resource_count': 1}, {'root_resource_group': 'SE607RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'UK608', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111'}, 'resource_count': 1}, {'root_resource_group': 'UK608RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}]}}}"
+"{'1': {'observation': {'errors': [], 'estimates': [{'root_resource_group': 'CEP4', 'resource_types': {'bandwidth': 45688942, 'storage': 164480190400}, 'resource_count': 1, 'output_files': {'uv': [{'identification': 'mom.G737227.B1.1.T.SAP000.uv.dps', 'sap_nr': 0, 'properties': {'start_sb_nr': 0, 'nr_of_uv_files': 1, 'uv_file_size': 164480190400, 'uv_otdb_id': 1}}]}}, {'root_resource_group': 'CEP4', 'resource_types': {'bandwidth': 45688942, 'storage': 164480190400}, 'resource_count': 243, 'output_files': {'uv': [{'identification': 'mom.G737227.B1.1.T.SAP001.uv.dps', 'sap_nr': 1, 'properties': {'start_sb_nr': 1, 'nr_of_uv_files': 1, 'uv_file_size': 164480190400, 'uv_otdb_id': 1}}]}}, {'root_resource_group': 'CEP4', 'resource_types': {'bandwidth': 45688942, 'storage': 164480190400}, 'resource_count': 243, 'output_files': {'uv': [{'identification': 'mom.G737227.B1.1.T.SAP002.uv.dps', 'sap_nr': 2, 'properties': {'start_sb_nr': 244, 'nr_of_uv_files': 1, 'uv_file_size': 164480190400, 'uv_otdb_id': 1}}]}}, {'root_resource_group': 'CS001', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS001RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS001RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS002', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS002RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS002RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS003', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS003RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS003RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS004', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS004RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS004RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS005', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS005RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS005RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS006', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS006RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS006RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS007', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS007RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS007RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS011', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS011RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS011RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS013', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS013RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS013RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS017', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS017RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS017RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS021', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS021RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS021RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS024', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS024RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS024RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS026', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS026RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS026RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS028', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS028RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS028RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS030', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS030RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS030RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS031', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS031RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS031RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS032', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS032RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS032RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS101', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS101RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS101RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS103', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS103RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS103RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS201', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS201RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS201RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS301', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS301RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS301RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS302', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS302RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS302RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS401', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS401RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS401RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS501', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'CS501RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'CS501RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'DE602', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111'}, 'resource_count': 1}, {'root_resource_group': 'DE602RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'DE603', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111'}, 'resource_count': 1}, {'root_resource_group': 'DE603RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'DE605', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111'}, 'resource_count': 1}, {'root_resource_group': 'DE605RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'DE609', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111'}, 'resource_count': 1}, {'root_resource_group': 'DE609RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'FR606', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111'}, 'resource_count': 1}, {'root_resource_group': 'FR606RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'PL610', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111'}, 'resource_count': 1}, {'root_resource_group': 'PL610RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'PL611', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111'}, 'resource_count': 1}, {'root_resource_group': 'PL611RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'PL612', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111'}, 'resource_count': 1}, {'root_resource_group': 'PL612RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'RS106', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'RS106RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'RS205', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'RS205RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'RS208', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'RS208RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'RS210', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'RS210RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'RS305', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'RS305RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'RS306', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'RS306RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'RS307', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'RS307RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'RS310', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'RS310RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'RS406', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'RS406RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'RS407', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'RS407RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'RS409', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'RS409RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'RS503', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'RS503RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'RS508', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'RS508RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'RS509', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111100'}, 'resource_count': 1}, {'root_resource_group': 'RS509RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'SE607', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111'}, 'resource_count': 1}, {'root_resource_group': 'SE607RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}, {'root_resource_group': 'UK608', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111'}, 'resource_count': 1}, {'root_resource_group': 'UK608RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3896}, 'resource_count': 1}]}}}"
diff --git a/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_long_baseline_pipeline b/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_long_baseline_pipeline
index 3c1e91974271296491464e835b8ffb2cb5d6a098..70cfb84749bb094223426f3abcebcccf51bab2dc 100644
--- a/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_long_baseline_pipeline
+++ b/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_long_baseline_pipeline
@@ -1 +1 @@
-"{'1': {'pipeline': {'bandwidth': {'total_size': 3}, 'errors': [], 'storage': {'total_size': 1000, 'output_files': {'uv': {'identifications': ['mom.G724024.B0.1.LBP27.uv.dps'], 'nr_of_uv_files': 1, 'start_sbg_nr': 26, 'uv_file_size': 1000, 'uv_otdb_id': 1}}, 'input_files': {'uv': {'nr_of_uv_files': 8, 'identifications': ['mom.G724024.B0.1.PTLB27.uv.dps'], 'start_sb_nr': 209, 'uv_file_size': 11130001, 'uv_otdb_id': 2}}}}}}"
+"{'1': {'pipeline': {'errors': [], 'estimates': [{'root_resource_group': 'CEP4', 'output_files': {'uv': [{'identification': 'mom.G724024.B0.1.LBP27.uv.dps', 'properties': {'start_sbg_nr': 26, 'nr_of_uv_files': 1, 'uv_file_size': 1000, 'uv_otdb_id': 1}}]}, 'resource_types': {'bandwidth': 3, 'storage': 1000}, 'resource_count': 1, 'input_files': {'uv': [{'identification': 'mom.G724024.B0.1.PTLB27.uv.dps', 'properties': {'nr_of_uv_files': 8, 'start_sb_nr': 209, 'uv_file_size': 11130001, 'uv_otdb_id': 2}}]}}]}}}"
diff --git a/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_maintenance_reservation b/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_maintenance_reservation
index 544aeb31f05d354b92f1e446066e6b87e064777e..8dcbf12fef951e1f753be29261aa248a4aeada42 100644
--- a/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_maintenance_reservation
+++ b/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_maintenance_reservation
@@ -1 +1 @@
-"{'1': {'reservation': {'errors': [], 'estimates': [{'root_resource_group': 'CS004', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111'}, 'resource_count': 1}, {'root_resource_group': 'CS004RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3904}, 'resource_count': 1}, {'root_resource_group': 'CS004RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3904}, 'resource_count': 1}, {'root_resource_group': 'RS005', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111'}, 'resource_count': 1}, {'root_resource_group': 'RS005RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3904}, 'resource_count': 1}, {'root_resource_group': 'DE603', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111'}, 'resource_count': 1}, {'root_resource_group': 'DE603RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3904}, 'resource_count': 1}]}}}"
+"{'1': {'reservation': {'errors': [], 'estimates': [{'root_resource_group': 'CS004', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111'}, 'resource_count': 1}, {'root_resource_group': 'CS004RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3904}, 'resource_count': 1}, {'root_resource_group': 'CS004RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3904}, 'resource_count': 1}, {'root_resource_group': 'RS005', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111'}, 'resource_count': 1}, {'root_resource_group': 'RS005RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3904}, 'resource_count': 1}, {'root_resource_group': 'DE603', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111'}, 'resource_count': 1}, {'root_resource_group': 'DE603RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3904}, 'resource_count': 1}]}}}"
diff --git a/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_preprocessing_pipeline b/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_preprocessing_pipeline
index caebc87725cf6f9a099b0434b6a3aefb46817750..96e0a44e1b26c8a4359123d6b797e2edf60def10 100644
--- a/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_preprocessing_pipeline
+++ b/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_preprocessing_pipeline
@@ -1 +1 @@
-"{'1': {'pipeline': {'bandwidth': {'total_size': 4043306604}, 'errors': [], 'storage': {'total_size': 15768895753674, 'output_files': {'uv': {'start_sb_nr': 244, 'identifications': ['mom.G737227.B1.1.PT2.uv.dps'], 'nr_of_uv_files': 243, 'uv_file_size': 64892575118, 'uv_otdb_id': 1}}, 'input_files': {'uv': {'start_sb_nr': 244, 'identifications': ['mom.G737227.B1.1.T.SAP000.uv.dps', 'mom.G737227.B1.1.T.SAP001.uv.dps', 'mom.G737227.B1.1.T.SAP002.uv.dps'], 'nr_of_uv_files': 243, 'uv_file_size': 164480190400, 'uv_otdb_id': 2}, 'saps': [{'sap_nr': 2, 'properties': {'nr_of_uv_files': 243, 'start_sb_nr': 244}}]}}}}}"
+"{'1': {'pipeline': {'errors': [], 'estimates': [{'root_resource_group': 'CEP4', 'output_files': {'uv': [{'identification': 'mom.G737227.B1.1.PT2.uv.dps', 'properties': {'start_sb_nr': 244, 'nr_of_uv_files': 243, 'uv_file_size': 64892575118, 'uv_otdb_id': 1}}]}, 'resource_types': {'bandwidth': 4043306604, 'storage': 15768895753674}, 'resource_count': 1, 'input_files': {'uv': [{'identification': 'mom.G737227.B1.1.T.SAP002.uv.dps', 'sap_nr': 2, 'properties': {'nr_of_uv_files': 243, 'start_sb_nr': 244, 'uv_file_size': 164480190400, 'uv_otdb_id': 2}}]}}]}}}"
diff --git a/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_project_reservation b/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_project_reservation
index 544aeb31f05d354b92f1e446066e6b87e064777e..8dcbf12fef951e1f753be29261aa248a4aeada42 100644
--- a/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_project_reservation
+++ b/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_project_reservation
@@ -1 +1 @@
-"{'1': {'reservation': {'errors': [], 'estimates': [{'root_resource_group': 'CS004', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111'}, 'resource_count': 1}, {'root_resource_group': 'CS004RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3904}, 'resource_count': 1}, {'root_resource_group': 'CS004RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3904}, 'resource_count': 1}, {'root_resource_group': 'RS005', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111'}, 'resource_count': 1}, {'root_resource_group': 'RS005RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3904}, 'resource_count': 1}, {'root_resource_group': 'DE603', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111'}, 'resource_count': 1}, {'root_resource_group': 'DE603RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3904}, 'resource_count': 1}]}}}"
+"{'1': {'reservation': {'errors': [], 'estimates': [{'root_resource_group': 'CS004', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111'}, 'resource_count': 1}, {'root_resource_group': 'CS004RSP0', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3904}, 'resource_count': 1}, {'root_resource_group': 'CS004RSP1', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3904}, 'resource_count': 1}, {'root_resource_group': 'RS005', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111'}, 'resource_count': 1}, {'root_resource_group': 'RS005RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3904}, 'resource_count': 1}, {'root_resource_group': 'DE603', 'resource_types': {'rcu': '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111'}, 'resource_count': 1}, {'root_resource_group': 'DE603RSP', 'resource_types': {'bandwidth': 3000000000, 'rsp': 3904}, 'resource_count': 1}]}}}"
diff --git a/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_pulsar_observation b/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_pulsar_observation
index ae9821918feeebd2cc15e7553e039503d5b64ee7..cdb01d1f434579650faa7a43f1689f145f2a227a 100644
--- a/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_pulsar_observation
+++ b/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/data_sets/t_resource_estimator.out_pulsar_observation
@@ -1 +1 @@
-"{'1': {'pipeline': {'bandwidth': {'total_size': 24}, 'errors': [], 'storage': {'total_size': 74000, 'output_files': {'pulp': {'pulp_otdb_id': 1, 'nr_of_pulp_files': 74, 'pulp_file_size': 1000, 'identifications': ['mom.G735371.LOTAAS-P1296B.1296.pipe.dps']}}, 'input_files': {'cs': {'cs_file_size': 76441190400, 'identifications': ['mom.G735371.LOTAAS-P1296B-SAP0.1296.SAP0.obs.cs', 'mom.G735371.LOTAAS-P1296B-SAP0.1296.SAP0.obs.cs', 'mom.G735371.LOTAAS-P1296B-SAP1.1296.SAP1.obs.cs', 'mom.G735371.LOTAAS-P1296B-SAP1.1296.SAP1.obs.cs', 'mom.G735371.LOTAAS-P1296B-SAP2.1296.SAP2.obs.cs', 'mom.G735371.LOTAAS-P1296B-SAP2.1296.SAP2.obs.cs'], 'nr_of_cs_stokes': 1, 'cs_otdb_id': 2, 'nr_of_cs_files': 73}, 'is': {'identifications': ['mom.G735371.LOTAAS-P1296B-SAP0.1296.SAP0.obs.is', 'mom.G735371.LOTAAS-P1296B-SAP0.1296.SAP0.obs.is', 'mom.G735371.LOTAAS-P1296B-SAP1.1296.SAP1.obs.is', 'mom.G735371.LOTAAS-P1296B-SAP1.1296.SAP1.obs.is', 'mom.G735371.LOTAAS-P1296B-SAP2.1296.SAP2.obs.is', 'mom.G735371.LOTAAS-P1296B-SAP2.1296.SAP2.obs.is'], 'is_file_size': 76441190400, 'nr_of_is_files': 1, 'nr_of_is_stokes': 1, 'is_otdb_id': 2}, 'saps': [{'sap_nr': 0, 'properties': {'nr_of_is_files': 1, 'nr_of_tabs': 74, 'is_tab_nr': 12, 'nr_of_cs_files': 73}}, {'sap_nr': 1, 'properties': {'nr_of_is_files': 1, 'nr_of_tabs': 74, 'is_tab_nr': 12, 'nr_of_cs_files': 73}}, {'sap_nr': 2, 'properties': {'nr_of_is_files': 1, 'nr_of_tabs': 74, 'is_tab_nr': 12, 'nr_of_cs_files': 73}}, {'sap_nr': 0, 'properties': {'nr_of_is_files': 1, 'nr_of_tabs': 74, 'is_tab_nr': 12, 'nr_of_cs_files': 73}}, {'sap_nr': 1, 'properties': {'nr_of_is_files': 1, 'nr_of_tabs': 74, 'is_tab_nr': 12, 'nr_of_cs_files': 73}}, {'sap_nr': 2, 'properties': {'nr_of_is_files': 1, 'nr_of_tabs': 74, 'is_tab_nr': 12, 'nr_of_cs_files': 73}}]}}}}}"
+"{'1': {'pipeline': {'errors': [], 'estimates': [{'root_resource_group': 'CEP4', 'output_files': {'pulp': [{'identification': 'mom.G735371.LOTAAS-P1296B.1296.pipe.dps', 'properties': {'pulp_otdb_id': 1, 'nr_of_pulp_files': 222, 'pulp_file_size': 1000}}]}, 'resource_types': {'bandwidth': 71, 'storage': 222000}, 'resource_count': 1, 'input_files': {'cs': [{'identification': 'mom.G735371.LOTAAS-P1296B-SAP0.1296.SAP0.obs.cs', 'sap_nr': 0, 'properties': {'nr_of_cs_stokes': 1, 'cs_otdb_id': 2, 'cs_file_size': 76441190400, 'nr_of_cs_parts': 1, 'is_tab_nr': 12, 'nr_of_cs_files': 73}}, {'identification': 'mom.G735371.LOTAAS-P1296B-SAP1.1296.SAP1.obs.cs', 'sap_nr': 1, 'properties': {'nr_of_cs_stokes': 1, 'cs_otdb_id': 2, 'cs_file_size': 76441190400, 'nr_of_cs_parts': 1, 'is_tab_nr': 12, 'nr_of_cs_files': 73}}, {'identification': 'mom.G735371.LOTAAS-P1296B-SAP2.1296.SAP2.obs.cs', 'sap_nr': 2, 'properties': {'nr_of_cs_stokes': 1, 'cs_otdb_id': 2, 'cs_file_size': 76441190400, 'nr_of_cs_parts': 1, 'is_tab_nr': 12, 'nr_of_cs_files': 73}}], 'is': [{'identification': 'mom.G735371.LOTAAS-P1296B-SAP0.1296.SAP0.obs.is', 'sap_nr': 0, 'properties': {'is_file_size': 76441190400, 'nr_of_is_stokes': 1, 'nr_of_is_files': 1, 'is_tab_nr': 12, 'is_otdb_id': 2}}, {'identification': 'mom.G735371.LOTAAS-P1296B-SAP1.1296.SAP1.obs.is', 'sap_nr': 1, 'properties': {'is_file_size': 76441190400, 'nr_of_is_stokes': 1, 'nr_of_is_files': 1, 'is_tab_nr': 12, 'is_otdb_id': 2}}, {'identification': 'mom.G735371.LOTAAS-P1296B-SAP2.1296.SAP2.obs.is', 'sap_nr': 2, 'properties': {'is_file_size': 76441190400, 'nr_of_is_stokes': 1, 'nr_of_is_files': 1, 'is_tab_nr': 12, 'is_otdb_id': 2}}]}}]}}}"
diff --git a/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/t_resource_estimator.py b/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/t_resource_estimator.py
index 792c7baf82129492a45f6828c63af9c4820a826f..7e2f94acb8d5cc914d331a05a52f1f53544818ce 100755
--- a/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/t_resource_estimator.py
+++ b/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/t_resource_estimator.py
@@ -98,170 +98,207 @@ class TestEstimationsAgainstGoldenOutput(unittest.TestCase):
self.assertEqual(len(error_messages), 0, "\nThe uut reported errors:\n" + '\n- '.join(error_messages))
self.assertEqual(self.get_datastructure_as_string(estimation), golden_estimation)
-# TODO!!! uncomment after Alexander fixed pipeline estimates:
- # # ------------------------------------------------------------------------------------------------------------------
- # # Test estimation for pipelines
- #
- # def test_estimate_for_pulsar_pipeline(self):
- # """ Verify estimation for a pulsar pipeline specification tree agains the golden output. Pulsar pipelines need a
- # beamformer observation as their predecessor. """
- # # Arrange
- # data_set_filepath = os.path.join(self.data_sets_dir, 't_resource_estimator.in_pulsar_pipeline')
- # golden_output_filepath = os.path.join(self.data_sets_dir, 't_resource_estimator.out_pulsar_observation')
- # task_type = 'pipeline'
- # task_subtype = 'pulsar pipeline'
- # specification_tree = self.get_specification_tree(data_set_filepath, task_type, task_subtype)
- #
- # # Pulsar pipelines need a beamformer observation as their predecessor
- # predecessor_data_set_filepath = os.path.join(self.data_sets_dir, 't_resource_estimator.in_beam_observation')
- # predecessor_task_type = 'observation'
- # predecessor_task_subtype = 'bfmeasurement'
- # self.add_predecessor_to_specification_tree(predecessor_data_set_filepath,
- # predecessor_task_type,
- # predecessor_task_subtype,
- # specification_tree['predecessors'])
- #
- # uut = ResourceEstimatorHandler()
- # golden_estimation = self.get_golden_estimate(golden_output_filepath,
- # uut._get_estimated_resources,
- # specification_tree)
- #
- # # Act
- # estimation = uut.handle_message({'specification_tree': specification_tree})
- #
- # # Assert
- # error_messages = self.get_uut_errors(estimation)
- # self.assertEqual(len(error_messages), 0, "\nThe uut reported errors:\n" + '\n- '.join(error_messages))
- # self.assertEqual(self.get_datastructure_as_string(estimation), golden_estimation)
- #
- # def test_estimate_for_preprocessing_pipeline(self):
- # """ Verify estimation for a preprocessing pipeline specification tree against the golden output. Preprocessing
- # pipelines need an interferometer observation as their predecessor. """
- # # Arrange
- # data_set_filepath = os.path.join(self.data_sets_dir, 't_resource_estimator.in_preprocessing_pipeline')
- # golden_output_filepath = os.path.join(self.data_sets_dir, 't_resource_estimator.out_preprocessing_pipeline')
- # task_type = 'pipeline'
- # task_subtype = 'averaging pipeline'
- # specification_tree = self.get_specification_tree(data_set_filepath, task_type, task_subtype)
- #
- # # Pipelines need a predecessor so give it one
- # predecessor_data_set_filepath = os.path.join(self.data_sets_dir,
- # 't_resource_estimator.in_interferometer_observation')
- # predecessor_task_type = 'observation'
- # predecessor_task_subtype = 'interferometer'
- # self.add_predecessor_to_specification_tree(predecessor_data_set_filepath,
- # predecessor_task_type,
- # predecessor_task_subtype,
- # specification_tree['predecessors'])
- #
- # uut = ResourceEstimatorHandler()
- # golden_estimation = self.get_golden_estimate(golden_output_filepath,
- # uut._get_estimated_resources,
- # specification_tree)
- #
- # # Act
- # estimation = uut.handle_message({'specification_tree': specification_tree})
- #
- # # Assert
- # error_messages = self.get_uut_errors(estimation)
- # self.assertEqual(len(error_messages), 0, "\nThe uut reported errors:\n" + '\n- '.join(error_messages))
- # self.assertEqual(self.get_datastructure_as_string(estimation), golden_estimation)
- #
- # def test_estimate_for_calibration_pipeline(self):
- # """ Verify estimation for a calibration pipeline specification tree against the golden output """
- # # Arrange
- # data_set_filepath = os.path.join(self.data_sets_dir, 't_resource_estimator.in_calibration_pipeline')
- # golden_output_filepath = os.path.join(self.data_sets_dir, 't_resource_estimator.out_calibration_pipeline')
- # task_type = 'pipeline'
- # task_subtype = 'calibration pipeline'
- # specification_tree = self.get_specification_tree(data_set_filepath, task_type, task_subtype)
- #
- # self.add_predecessor_to_specification_tree(os.path.join(self.data_sets_dir,
- # 't_resource_estimator.in_calibration_pipeline_predecessor_558022'),
- # 'observation',
- # 'bfmeasurement',
- # specification_tree['predecessors'])
- #
- # uut = ResourceEstimatorHandler()
- # golden_estimation = self.get_golden_estimate(golden_output_filepath,
- # uut._get_estimated_resources,
- # specification_tree)
- #
- # # Act
- # estimation = uut.handle_message({'specification_tree': specification_tree})
- #
- # # Assert
- # error_messages = self.get_uut_errors(estimation)
- # self.assertEqual(len(error_messages), 0, "\nThe uut reported errors:\n" + '\n- '.join(error_messages))
- # self.assertEqual(self.get_datastructure_as_string(estimation), golden_estimation)
- #
- # def test_estimate_for_long_baseline_pipeline(self):
- # """ Verify estimation for a long baseline pipeline specification tree against the golden output """
- # # Arrange
- # data_set_filepath = os.path.join(self.data_sets_dir, 't_resource_estimator.in_long_baseline_pipeline')
- # golden_output_filepath = os.path.join(self.data_sets_dir, 't_resource_estimator.out_long_baseline_pipeline')
- # task_type = 'pipeline'
- # task_subtype = 'long baseline pipeline'
- # specification_tree = self.get_specification_tree(data_set_filepath, task_type, task_subtype)
- #
- # tree = specification_tree
- # predecessor_tree = self.get_specification_tree(
- # os.path.join(
- # self.data_sets_dir,
- # 't_resource_estimator.in_long_baseline_pipeline_predecessor_556601'),
- # 'pipeline',
- # 'averaging pipeline')
- # tree['predecessors'].append(predecessor_tree)
- #
- # tree = predecessor_tree
- # predecessor_tree = self.get_specification_tree(
- # os.path.join(
- # self.data_sets_dir,
- # 't_resource_estimator.in_long_baseline_pipeline_predecessor_556601_556429'),
- # 'pipeline',
- # 'calibration pipeline')
- # tree['predecessors'].append(predecessor_tree)
- #
- # tree = predecessor_tree
- # predecessor_tree_branch_a = self.get_specification_tree(
- # os.path.join(
- # self.data_sets_dir,
- # 't_resource_estimator.in_long_baseline_pipeline_predecessor_556601_556429_556373'),
- # 'pipeline',
- # 'calibration pipeline')
- # tree['predecessors'].append(predecessor_tree_branch_a)
- #
- # predecessor_tree_branch_b = self.get_specification_tree(
- # os.path.join(
- # self.data_sets_dir,
- # 't_resource_estimator.in_long_baseline_pipeline_predecessor_556601_556429_556375'),
- # 'observation',
- # 'bfmeasurement')
- # tree['predecessors'].append(predecessor_tree_branch_b)
- #
- # predecessor_tree = self.get_specification_tree(
- # os.path.join(
- # self.data_sets_dir,
- # 't_resource_estimator.in_long_baseline_pipeline_predecessor_556601_556429_xxxxxx_556371'),
- # 'observation',
- # 'bfmeasurement')
- # predecessor_tree_branch_a['predecessors'].append(predecessor_tree)
- # predecessor_tree_branch_b['predecessors'].append(predecessor_tree)
- #
- # uut = ResourceEstimatorHandler()
- # golden_estimation = self.get_golden_estimate(golden_output_filepath,
- # uut._get_estimated_resources,
- # specification_tree)
- #
- # # Act
- # estimation = uut.handle_message({'specification_tree': specification_tree})
- #
- # # Assert
- # error_messages = self.get_uut_errors(estimation)
- # self.assertEqual(len(error_messages), 0, "\nThe uut reported errors:\n" + '\n- '.join(error_messages))
- # self.assertEqual(self.get_datastructure_as_string(estimation), golden_estimation)
- #
- # # ------------------------------------------------------------------------------------------------------------------
+ # ------------------------------------------------------------------------------------------------------------------
+ # Test estimation for pipelines
+
+ @mock.patch('lofar.sas.resourceassignment.resourceassignmentestimator.resource_estimators.reservation.AntennaSetsParser')
+ @mock.patch('lofar.sas.resourceassignment.resourceassignmentestimator.resource_estimators.observation.AntennaSetsParser')
+ def test_estimate_for_pulsar_pipeline(self, mock_asp, mock_notusedhere):
+ """ Verify estimation for a pulsar pipeline specification tree agains the golden output. Pulsar pipelines need a
+ beamformer observation as their predecessor. """
+ # Arrange
+ data_set_filepath = os.path.join(self.data_sets_dir, 't_resource_estimator.in_pulsar_pipeline')
+ golden_output_filepath = os.path.join(self.data_sets_dir, 't_resource_estimator.out_pulsar_observation')
+ task_type = 'pipeline'
+ task_subtype = 'pulsar pipeline'
+ specification_tree = self.get_specification_tree(data_set_filepath, task_type, task_subtype)
+
+ # Pulsar pipelines need a beamformer observation as their predecessor
+ predecessor_data_set_filepath = os.path.join(self.data_sets_dir, 't_resource_estimator.in_beam_observation')
+ predecessor_task_type = 'observation'
+ predecessor_task_subtype = 'bfmeasurement'
+ self.add_predecessor_to_specification_tree(predecessor_data_set_filepath,
+ predecessor_task_type,
+ predecessor_task_subtype,
+ specification_tree['predecessors'])
+
+ uut = ResourceEstimatorHandler()
+ golden_estimation = self.get_golden_estimate(golden_output_filepath,
+ uut._get_estimated_resources,
+ specification_tree)
+
+ # Act
+ estimation = uut.handle_message({'specification_tree': specification_tree})
+
+ # Assert
+ error_messages = self.get_uut_errors(estimation)
+ self.assertEqual(len(error_messages), 0, "\nThe uut reported errors:\n" + '\n- '.join(error_messages))
+ self.assertEqual(self.get_datastructure_as_string(estimation), golden_estimation)
+
+ @mock.patch('lofar.sas.resourceassignment.resourceassignmentestimator.resource_estimators.reservation.AntennaSetsParser')
+ @mock.patch('lofar.sas.resourceassignment.resourceassignmentestimator.resource_estimators.observation.AntennaSetsParser')
+ def test_estimate_for_preprocessing_pipeline(self, mock_asp, mock_notusedhere):
+ """ Verify estimation for a preprocessing pipeline specification tree against the golden output. Preprocessing
+ pipelines need an interferometer observation as their predecessor. """
+ # Arrange
+ data_set_filepath = os.path.join(self.data_sets_dir, 't_resource_estimator.in_preprocessing_pipeline')
+ golden_output_filepath = os.path.join(self.data_sets_dir, 't_resource_estimator.out_preprocessing_pipeline')
+ task_type = 'pipeline'
+ task_subtype = 'averaging pipeline'
+ specification_tree = self.get_specification_tree(data_set_filepath, task_type, task_subtype)
+
+ # Pipelines need a predecessor so give it one
+ predecessor_data_set_filepath = os.path.join(self.data_sets_dir,
+ 't_resource_estimator.in_interferometer_observation')
+ predecessor_task_type = 'observation'
+ predecessor_task_subtype = 'interferometer'
+ self.add_predecessor_to_specification_tree(predecessor_data_set_filepath,
+ predecessor_task_type,
+ predecessor_task_subtype,
+ specification_tree['predecessors'])
+
+ uut = ResourceEstimatorHandler()
+ golden_estimation = self.get_golden_estimate(golden_output_filepath,
+ uut._get_estimated_resources,
+ specification_tree)
+
+ # Act
+ estimation = uut.handle_message({'specification_tree': specification_tree})
+
+ # Assert
+ error_messages = self.get_uut_errors(estimation)
+ self.assertEqual(len(error_messages), 0, "\nThe uut reported errors:\n" + '\n- '.join(error_messages))
+ self.assertEqual(self.get_datastructure_as_string(estimation), golden_estimation)
+
+ @mock.patch('lofar.sas.resourceassignment.resourceassignmentestimator.resource_estimators.reservation.AntennaSetsParser')
+ @mock.patch('lofar.sas.resourceassignment.resourceassignmentestimator.resource_estimators.observation.AntennaSetsParser')
+ def test_estimate_for_calibration_pipeline(self, mock_asp, mock_notusedhere):
+ """ Verify estimation for a calibration pipeline specification tree against the golden output """
+ # Arrange
+ data_set_filepath = os.path.join(self.data_sets_dir, 't_resource_estimator.in_calibration_pipeline')
+ golden_output_filepath = os.path.join(self.data_sets_dir, 't_resource_estimator.out_calibration_pipeline')
+ task_type = 'pipeline'
+ task_subtype = 'calibration pipeline'
+ specification_tree = self.get_specification_tree(data_set_filepath, task_type, task_subtype)
+
+ self.add_predecessor_to_specification_tree(os.path.join(self.data_sets_dir,
+ 't_resource_estimator.in_calibration_pipeline_predecessor_558022'), # predecessor also used for imaging pipeline test
+ 'observation',
+ 'bfmeasurement',
+ specification_tree['predecessors'])
+
+ uut = ResourceEstimatorHandler()
+ golden_estimation = self.get_golden_estimate(golden_output_filepath,
+ uut._get_estimated_resources,
+ specification_tree)
+
+ # Act
+ estimation = uut.handle_message({'specification_tree': specification_tree})
+
+ # Assert
+ error_messages = self.get_uut_errors(estimation)
+ self.assertEqual(len(error_messages), 0, "\nThe uut reported errors:\n" + '\n- '.join(error_messages))
+ self.assertEqual(self.get_datastructure_as_string(estimation), golden_estimation)
+
+ @mock.patch('lofar.sas.resourceassignment.resourceassignmentestimator.resource_estimators.reservation.AntennaSetsParser')
+ @mock.patch('lofar.sas.resourceassignment.resourceassignmentestimator.resource_estimators.observation.AntennaSetsParser')
+ def test_estimate_for_imaging_pipeline(self, mock_asp, mock_notusedhere):
+ """ Verify estimation for an imaging pipeline specification tree against the golden output """
+ # Arrange
+ data_set_filepath = os.path.join(self.data_sets_dir, 't_resource_estimator.in_imaging_pipeline')
+ golden_output_filepath = os.path.join(self.data_sets_dir, 't_resource_estimator.out_imaging_pipeline')
+ task_type = 'pipeline'
+ task_subtype = 'imaging pipeline'
+ specification_tree = self.get_specification_tree(data_set_filepath, task_type, task_subtype)
+
+ self.add_predecessor_to_specification_tree(os.path.join(self.data_sets_dir,
+ 't_resource_estimator.in_calibration_pipeline_predecessor_558022'), # predecessor also used for calibration pipeline test
+ 'observation',
+ 'bfmeasurements',
+ specification_tree['predecessors'])
+
+ uut = ResourceEstimatorHandler()
+ golden_estimation = self.get_golden_estimate(golden_output_filepath,
+ uut._get_estimated_resources,
+ specification_tree)
+
+ # Act
+ estimation = uut.handle_message({'specification_tree': specification_tree})
+
+ # Assert
+ error_messages = self.get_uut_errors(estimation)
+ self.assertEqual(len(error_messages), 0, "\nThe uut reported errors:\n" + '\n- '.join(error_messages))
+ self.assertEqual(self.get_datastructure_as_string(estimation), golden_estimation)
+
+ @mock.patch('lofar.sas.resourceassignment.resourceassignmentestimator.resource_estimators.reservation.AntennaSetsParser')
+ @mock.patch('lofar.sas.resourceassignment.resourceassignmentestimator.resource_estimators.observation.AntennaSetsParser')
+ def test_estimate_for_long_baseline_pipeline(self, mock_asp, mock_notusedhere):
+ """ Verify estimation for a long baseline pipeline specification tree against the golden output """
+ # Arrange
+ data_set_filepath = os.path.join(self.data_sets_dir, 't_resource_estimator.in_long_baseline_pipeline')
+ golden_output_filepath = os.path.join(self.data_sets_dir, 't_resource_estimator.out_long_baseline_pipeline')
+ task_type = 'pipeline'
+ task_subtype = 'long baseline pipeline'
+ specification_tree = self.get_specification_tree(data_set_filepath, task_type, task_subtype)
+
+ tree = specification_tree
+ predecessor_tree = self.get_specification_tree(
+ os.path.join(
+ self.data_sets_dir,
+ 't_resource_estimator.in_long_baseline_pipeline_predecessor_556601'),
+ 'pipeline',
+ 'averaging pipeline')
+ tree['predecessors'].append(predecessor_tree)
+
+ tree = predecessor_tree
+ predecessor_tree = self.get_specification_tree(
+ os.path.join(
+ self.data_sets_dir,
+ 't_resource_estimator.in_long_baseline_pipeline_predecessor_556601_556429'),
+ 'pipeline',
+ 'calibration pipeline')
+ tree['predecessors'].append(predecessor_tree)
+
+ tree = predecessor_tree
+ predecessor_tree_branch_a = self.get_specification_tree(
+ os.path.join(
+ self.data_sets_dir,
+ 't_resource_estimator.in_long_baseline_pipeline_predecessor_556601_556429_556373'),
+ 'pipeline',
+ 'calibration pipeline')
+ tree['predecessors'].append(predecessor_tree_branch_a)
+
+ predecessor_tree_branch_b = self.get_specification_tree(
+ os.path.join(
+ self.data_sets_dir,
+ 't_resource_estimator.in_long_baseline_pipeline_predecessor_556601_556429_556375'),
+ 'observation',
+ 'bfmeasurement')
+ tree['predecessors'].append(predecessor_tree_branch_b)
+
+ predecessor_tree = self.get_specification_tree(
+ os.path.join(
+ self.data_sets_dir,
+ 't_resource_estimator.in_long_baseline_pipeline_predecessor_556601_556429_xxxxxx_556371'),
+ 'observation',
+ 'bfmeasurement')
+ predecessor_tree_branch_a['predecessors'].append(predecessor_tree)
+ predecessor_tree_branch_b['predecessors'].append(predecessor_tree)
+
+ uut = ResourceEstimatorHandler()
+ golden_estimation = self.get_golden_estimate(golden_output_filepath,
+ uut._get_estimated_resources,
+ specification_tree)
+
+ # Act
+ estimation = uut.handle_message({'specification_tree': specification_tree})
+
+ # Assert
+ error_messages = self.get_uut_errors(estimation)
+ self.assertEqual(len(error_messages), 0, "\nThe uut reported errors:\n" + '\n- '.join(error_messages))
+ self.assertEqual(self.get_datastructure_as_string(estimation), golden_estimation)
+
+ # ------------------------------------------------------------------------------------------------------------------
# Test estimation for reservations
@@ -419,4 +456,4 @@ class TestEstimationsAgainstGoldenOutput(unittest.TestCase):
f.close()
if __name__ == '__main__':
- unittest.main()
\ No newline at end of file
+ unittest.main()
diff --git a/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/testset.py b/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/testset.py
index b75c8ad545c15198b5ae94fdcff951f6161f1ffb..d5b5773f253bb95b2103ce870812f7e3beec1093 100644
--- a/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/testset.py
+++ b/SAS/ResourceAssignment/ResourceAssignmentEstimator/test/testset.py
@@ -222,8 +222,8 @@ class TestSet(object):
checkset = """
# for image-pipeline
dp.output.skyimage.enabled= true
- dp.output.skyimage.slices_per_image= 1
- dp.output.skyimage.subbands_per_image= 2
+ dp.output.skyimage.slices_per_image= 3
+ dp.output.skyimage.subbands_per_image= 4
"""
self.check_set.import_string(checkset)
self.check_set.import_string(checkset)
diff --git a/SAS/TriggerEmailService/Server/CMakeLists.txt b/SAS/TriggerEmailService/Server/CMakeLists.txt
index 34183d934b189d06ee69ae363e66cade90df8320..2214438256bdb92369552a28193f46f76d7c5dee 100644
--- a/SAS/TriggerEmailService/Server/CMakeLists.txt
+++ b/SAS/TriggerEmailService/Server/CMakeLists.txt
@@ -1,4 +1,4 @@
-lofar_package(TriggerEmailServiceServer 1.0 DEPENDS TriggerEmailServiceCommon OTDB_Services PyMessaging PyCommon)
+lofar_package(TriggerEmailServiceServer 1.0 DEPENDS TriggerEmailServiceCommon OTDB_Services MoMQueryService PyMessaging PyCommon)
lofar_find_package(Python 2.7 REQUIRED)
diff --git a/SAS/TriggerEmailService/Server/lib/CMakeLists.txt b/SAS/TriggerEmailService/Server/lib/CMakeLists.txt
index dd738a6eac0ec56f1d64c0eb1c2df30f5651066b..b6714652cc8643712b69c0a3572e02c5c1fa2d2a 100644
--- a/SAS/TriggerEmailService/Server/lib/CMakeLists.txt
+++ b/SAS/TriggerEmailService/Server/lib/CMakeLists.txt
@@ -2,7 +2,8 @@ include(PythonInstall)
set(_py_files
__init__.py
- TiggerEmailService.py
+ TriggerEmailService.py
+ Templates.py
)
-python_install(${_py_files} DESTINATION lofar/sas/TiggerEmailService)
+python_install(${_py_files} DESTINATION lofar/sas/TriggerEmailService)
diff --git a/SAS/TriggerEmailService/Server/lib/Templates.py b/SAS/TriggerEmailService/Server/lib/Templates.py
new file mode 100644
index 0000000000000000000000000000000000000000..2f025a2e8777b7a9548f1f30d4f9c807d3111df0
--- /dev/null
+++ b/SAS/TriggerEmailService/Server/lib/Templates.py
@@ -0,0 +1,65 @@
+RECEIVED_TEMPLATE_SUBJECT = """%(PROJECTNAME)s trigger %(TRIGGERID)s received"""
+ACCEPTED_TEMPLATE_SUBJECT = """%(PROJECTNAME)s trigger %(TRIGGERID)s completed"""
+REJECTED_TEMPLATE_SUBJECT = """%(PROJECTNAME)s trigger %(TRIGGERID)s cancelled"""
+ABORTED_TEMPLATE_SUBJECT = """%(PROJECTNAME)s trigger %(TRIGGERID)s aborted"""
+FINISHED_TEMPLATE_SUBJECT = """%(PROJECTNAME)s trigger %(TRIGGERID)s completed"""
+
+RECEIVED_TEMPLATE_BODY = """Dear PI/Project Contact,
+
+We have received a trigger for your project %(PROJECTNAME)s with ID %(TRIGGERID)s. Please find the received trigger file attached.
+
+The requested time to schedule the trigger is between %(STARTTIME)s and %(ENDTIME)s. The system will now determine if your trigger can be scheduled within the restrictions specified in the trigger.
+
+kind regards,
+
+LOFAR Science Operations & Support [ sos@astron.nl ]"""
+ACCEPTED_TEMPLATE_BODY = """Dear PI/Project Contact,
+
+This is a follow up message for the trigger for your project %(PROJECTNAME)s with ID %(TRIGGERID)s.
+
+The triggered observations have completed.
+
+The observation IDs are %(OBSSASID)s, the MoM ID is %(OBSMOMID)s.
+
+To follow the progress and check the details, please check the observation in MoM %(MOMLINK)s.
+
+kind regards,
+
+LOFAR Science Operations & Support [ sos@astron.nl ]"""
+REJECTED_TEMPLATE_BODY = """Dear PI/Project Contact,
+
+This is a follow up message for the trigger for your project %(PROJECTNAME)s with ID %(TRIGGERID)s.
+
+We are sorry to inform you that your trigger could not be scheduled at this time. This is either because no resources are available at the requested time and duration, or because you have used up all your triggers. This trigger will not count towards your quota.
+
+kind regards,
+
+LOFAR Science Operations & Support [ sos@astron.nl ]"""
+ABORTED_TEMPLATE_BODY = """Dear PI/Project Contact,
+
+This is a follow up message for the trigger for your project %(PROJECTNAME)s with ID %(TRIGGERID)s.
+
+We are sorry to inform you that the scheduled observation has aborted.
+
+The observation IDs are %(OBSSASID)s, the MoM ID is %(OBSMOMID)s.
+
+Science Support and Operations will be in contact with you about the further process and to see if any of the data can still be used. You can find the observation in MoM %(MOMLINK)s.
+
+kind regards,
+
+LOFAR Science Operations & Support [ sos@astron.nl ]"""
+FINISHED_TEMPLATE_BODY = """Dear PI/Project Contact,
+
+This is a follow up message for the trigger for your project %(PROJECTNAME)s with ID %(TRIGGERID)s.
+
+The triggered observations have completed.
+
+The observation IDs are %(OBSSASID)s, the MoM ID is %(OBSMOMID)s.
+
+To follow the progress and check the details, please check the observation in MoM %(MOMLINK)s.
+
+Science Operations and Support will contact you about the further process in processing and ingesting the data.
+
+kind regards,
+
+LOFAR Science Operations & Support [ sos@astron.nl ]"""
diff --git a/SAS/TriggerEmailService/Server/lib/TiggerEmailService.py b/SAS/TriggerEmailService/Server/lib/TiggerEmailService.py
deleted file mode 100644
index 3cf57744e1ec71062abc270595681470d4ac0486..0000000000000000000000000000000000000000
--- a/SAS/TriggerEmailService/Server/lib/TiggerEmailService.py
+++ /dev/null
@@ -1,75 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright (C) 2017
-# ASTRON (Netherlands Institute for Radio Astronomy)
-# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-#
-# This file is part of the LOFAR software suite.
-# The LOFAR software suite is free software: you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# The LOFAR software suite is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-
-import logging
-from lofar.sas.otdb.OTDBBusListener import OTDBBusListener
-from lofar.common.util import waitForInterrupt
-from lofar.messaging.messagebus import AbstractBusListener
-from lofar.sas.TriggerEmailService.common.config import DEFAULT_OTDB_NOTIFICATION_BUSNAME, DEFAULT_OTDB_NOTIFICATION_SUBJECT
-from lofar.sas.TriggerEmailService.common.config import DEFAULT_TRIGGER_NOTIFICATION_BUSNAME, DEFAULT_TRIGGER_NOTIFICATION_SUBJECT
-
-logger = logging.getLogger(__name__)
-
-
-class OTDBTriggerListener(OTDBBusListener):
- def __init__(self, busname=DEFAULT_OTDB_NOTIFICATION_BUSNAME, subject=DEFAULT_OTDB_NOTIFICATION_SUBJECT, broker=None, **kwargs):
- address = "%s/%s" % (busname, subject)
- super(OTDBTriggerListener, self).__init__(address, broker, **kwargs)
-
- def onObservationAborted(self, treeId, modificationTime):
- pass
-
- def onObservationFinished(self, treeId, modificationTime):
- pass
-
- def onObservationConflict(self, treeId, modificationTime):
- pass
-
- def onObservationApproved(self, treeId, modificationTime):
- pass
-
-
-class TriggerNotificationListener(AbstractBusListener):
- def __init__(self, busname=DEFAULT_TRIGGER_NOTIFICATION_BUSNAME, subject=DEFAULT_TRIGGER_NOTIFICATION_SUBJECT, broker=None, **kwargs):
- """
- TriggerNotificationListener listens on the lofar trigger message bus and emails when trigger gets submitted.
- :param address: valid Qpid address (default: lofar.otdb.status)
- :param broker: valid Qpid broker host (default: None, which means localhost)
- additional parameters in kwargs:
- options= <dict> Dictionary of options passed to QPID
- exclusive= <bool> Create an exclusive binding so no other services can consume duplicate messages (default: False)
- numthreads= <int> Number of parallel threads processing messages (default: 1)
- verbose= <bool> Output extra logging over stdout (default: False)
- """
- address = "%s/%s" % (busname, subject)
- super(TriggerNotificationListener, self).__init__(address, broker, **kwargs)
-
- def _handleMessage(self, msg):
- pass
-
-
-def main():
- with OTDBTriggerListener():
- with TriggerNotificationListener():
- waitForInterrupt()
-
-if __name__ == '__main__':
- main()
-
diff --git a/SAS/TriggerEmailService/Server/lib/TriggerEmailService.py b/SAS/TriggerEmailService/Server/lib/TriggerEmailService.py
new file mode 100644
index 0000000000000000000000000000000000000000..73caf4708535f2b634a9543e52b36c53d5152b28
--- /dev/null
+++ b/SAS/TriggerEmailService/Server/lib/TriggerEmailService.py
@@ -0,0 +1,267 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2017
+# ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+import os
+import smtplib
+import logging
+
+from lofar.sas.TriggerEmailService.Templates import ABORTED_TEMPLATE_BODY, ABORTED_TEMPLATE_SUBJECT
+from lofar.sas.TriggerEmailService.Templates import ACCEPTED_TEMPLATE_BODY, ACCEPTED_TEMPLATE_SUBJECT
+from lofar.sas.TriggerEmailService.Templates import FINISHED_TEMPLATE_BODY, FINISHED_TEMPLATE_SUBJECT
+from lofar.sas.TriggerEmailService.Templates import REJECTED_TEMPLATE_BODY, REJECTED_TEMPLATE_SUBJECT
+from lofar.sas.TriggerEmailService.Templates import RECEIVED_TEMPLATE_BODY, RECEIVED_TEMPLATE_SUBJECT
+
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+
+from datetime import timedelta, datetime
+from lofar.sas.otdb.OTDBBusListener import OTDBBusListener
+from lofar.common.util import waitForInterrupt
+from lofar.messaging.messagebus import AbstractBusListener
+from lofar.sas.TriggerEmailService.common.config import DEFAULT_OTDB_NOTIFICATION_BUSNAME
+from lofar.sas.TriggerEmailService.common.config import DEFAULT_OTDB_NOTIFICATION_SUBJECT
+from lofar.sas.TriggerEmailService.common.config import DEFAULT_TRIGGER_NOTIFICATION_BUSNAME
+from lofar.sas.TriggerEmailService.common.config import DEFAULT_TRIGGER_NOTIFICATION_SUBJECT
+from lofar.mom.momqueryservice.momqueryrpc import MoMQueryRPC
+from lxml import etree
+from StringIO import StringIO
+from re import findall
+
+logger = logging.getLogger(__name__)
+
+
+def email(recipients, subject, body, attachment):
+ if "LOFARENV" in os.environ:
+ lofar_environment = os.environ['LOFARENV']
+
+ if lofar_environment == "PRODUCTION":
+ recipients.append("sos@astron.nl")
+ recipients.append("operator@astron.nl")
+
+ sender = "Trigger Email Service <no-reply@astron.nl>"
+ commaspace = ', '
+
+ msg = MIMEMultipart(body)
+ msg["Subject"] = subject
+ msg["From"] = sender
+ msg["To"] = commaspace.join(recipients)
+
+ if attachment:
+ txt = MIMEText(attachment)
+ msg.attach(txt)
+
+ s = smtplib.SMTP('localhost')
+ s.sendmail(sender, recipients, msg.as_string())
+ s.quit()
+
+
+class OTDBTriggerListener(OTDBBusListener):
+ def __init__(self, momquery_rpc=MoMQueryRPC(), busname=DEFAULT_OTDB_NOTIFICATION_BUSNAME,
+ subject=DEFAULT_OTDB_NOTIFICATION_SUBJECT, broker=None, **kwargs):
+ address = "%s/%s" % (busname, subject)
+ super(OTDBTriggerListener, self).__init__(address, broker, **kwargs)
+
+ self.mom_rpc_client = momquery_rpc
+
+ def start_listening(self, **kwargs):
+ self.mom_rpc_client.open()
+ super(OTDBTriggerListener, self).start_listening(kwargs)
+
+ def stop_listening(self):
+ self.mom_rpc_client.close()
+ super(OTDBTriggerListener, self).stop_listening()
+
+ def onObservationAborted(self, otdb_id, _):
+ self.when_trigger_send_email(otdb_id, ABORTED_TEMPLATE_SUBJECT, ABORTED_TEMPLATE_BODY)
+
+ def onObservationScheduled(self, otdb_id, _):
+ self.when_trigger_send_email(otdb_id, ACCEPTED_TEMPLATE_SUBJECT, ACCEPTED_TEMPLATE_BODY)
+
+ def onObservationFinished(self, otdb_id, _):
+ self.when_trigger_send_email(otdb_id, FINISHED_TEMPLATE_SUBJECT, FINISHED_TEMPLATE_BODY)
+
+ def onObservationConflict(self, otdb_id, _):
+ self.when_trigger_send_email(otdb_id, REJECTED_TEMPLATE_SUBJECT, REJECTED_TEMPLATE_BODY)
+
+ def when_trigger_send_email(self, otdb_id, template_subject, template_body):
+ mom_id = self.mom_rpc_client.getMoMIdsForOTDBIds(otdb_id)[otdb_id]
+ trigger_id = self.mom_rpc_client.get_trigger_id(mom_id)
+
+ if trigger_id:
+ self._send_email(otdb_id, mom_id, trigger_id, template_subject, template_body)
+
+ def _send_email(self, otdb_id, mom_id, trigger_id, template_subject, template_body):
+ subject, body = self._fill_template(otdb_id, mom_id, trigger_id, template_subject, template_body)
+ recipients = self._get_recipients(mom_id)
+
+ email(recipients, subject, body, None)
+
+ def _fill_template(self, otdb_id, mom_id, trigger_id, template_subject, template_body):
+ project = self.mom_rpc_client.getProject(mom_id)
+
+ data = {
+ "PROJECTNAME": project["name"], "TRIGGERID": trigger_id, "OBSSASID": otdb_id, "OBSMOMID": mom_id,
+ "MOMLINK": "https://lofar.astron.nl/mom3/user/main/list/setUpProjectList.do"
+ }
+
+ subject = template_subject % data
+ body = template_body % data
+
+ return subject, body
+
+ def _get_recipients(self, mom_id):
+ recipients = []
+
+ emails = self.mom_rpc_client.get_project_details(mom_id)
+ for k, v in emails.items():
+ recipients.append(v)
+
+ return recipients
+
+
+class TriggerNotificationListener(AbstractBusListener):
+ def __init__(self, momquery_rpc=MoMQueryRPC(), busname=DEFAULT_TRIGGER_NOTIFICATION_BUSNAME,
+ subject=DEFAULT_TRIGGER_NOTIFICATION_SUBJECT, broker=None, **kwargs):
+ """
+ TriggerNotificationListener listens on the lofar trigger message bus and emails when trigger gets submitted.
+ :param address: valid Qpid address (default: lofar.otdb.status)
+ :param broker: valid Qpid broker host (default: None, which means localhost)
+ additional parameters in kwargs:
+ options= <dict> Dictionary of options passed to QPID
+ exclusive= <bool> Create an exclusive binding so no other services can consume duplicate
+ messages (default: False)
+ numthreads= <int> Number of parallel threads processing messages (default: 1)
+ verbose= <bool> Output extra logging over stdout (default: False)
+ """
+ address = "%s/%s" % (busname, subject)
+ super(TriggerNotificationListener, self).__init__(address, broker, **kwargs)
+
+ self.mom_rpc_client = momquery_rpc
+
+ def _handleMessage(self, msg):
+ trigger_id = msg.content['trigger_id']
+ project_name = msg.content['project']
+ trigger_xml = msg.content['metadata']
+ start_time, stop_time = self._get_observation_start_stop_times(trigger_xml)
+
+ mom_id = self._get_mom_id(project_name)
+
+ if mom_id:
+ subject, body = self._fill_template(trigger_id, project_name, start_time, stop_time,
+ RECEIVED_TEMPLATE_SUBJECT, RECEIVED_TEMPLATE_BODY)
+ recipients = self._get_recipients(mom_id)
+
+ email(recipients, subject, body, trigger_xml)
+ else:
+ logger.error("Trigger got entered for a non existing project: %s", project_name)
+
+ def _get_mom_id(self, project_name):
+ # todo add function to momqueryserivce for it (get mom2id for project name)
+ mom_id = None
+
+ projects = self.mom_rpc_client.getProjects()
+ for project in projects:
+ if project["name"] == project_name:
+ mom_id = project["mom2id"]
+
+ return mom_id
+
+ def _get_recipients(self, mom_id):
+
+ recipients = []
+
+ emails = self.mom_rpc_client.get_project_details(mom_id)
+
+ for k, v in emails.items():
+ recipients.append(v)
+
+ return recipients
+
+ def _get_observation_start_stop_times(self, trigger_xml):
+ # for now we work with duration to get stop time
+ doc = etree.parse(StringIO(trigger_xml.encode('UTF-8')))
+
+ start_times = doc.getroot().findall('specification/activity/observation/timeWindowSpecification/startTime')
+
+ start_time = datetime.strptime(start_times[0].text, '%Y-%m-%dT%H:%M:%S')
+
+ durations = doc.getroot().findall(
+ 'specification/activity/observation/timeWindowSpecification/duration/duration')
+
+ duration = durations[0].text
+
+ duration_seconds = self._iso8601_duration_as_seconds(duration)
+
+ stop_time = start_time + timedelta(seconds=duration_seconds)
+
+ return start_time, stop_time
+
+ def start_listening(self, **kwargs):
+ self.mom_rpc_client.open()
+ super(TriggerNotificationListener, self).start_listening(kwargs)
+
+ def stop_listening(self):
+ self.mom_rpc_client.close()
+ super(TriggerNotificationListener, self).stop_listening()
+
+ def _fill_template(self, trigger_id, project_name, start_time, stop_time, template_subject, template_body):
+ data = {
+ "PROJECTNAME": project_name, "TRIGGERID": trigger_id, "STARTTIME": start_time, "ENDTIME": stop_time
+ }
+
+ subject = template_subject % data
+ body = template_body % data
+
+ return subject, body
+
+ def _iso8601_duration_as_seconds(self, duration):
+ if duration[0] != 'P':
+ raise ValueError('Not an ISO 8601 Duration string')
+ seconds = 0
+ for i, item in enumerate(duration.split('T')):
+ for number, unit in findall('(?P<number>\d+)(?P<period>S|M|H|D|W|Y)', item):
+ number = int(number)
+ this = 0
+ if unit == 'Y':
+ this = number * 31557600 # 365.25
+ elif unit == 'W':
+ this = number * 604800
+ elif unit == 'D':
+ this = number * 86400
+ elif unit == 'H':
+ this = number * 3600
+ elif unit == 'M':
+ # ambiguity ellivated with index i
+ if i == 0:
+ this = number * 2678400 # assume 30 days
+ else:
+ this = number * 60
+ elif unit == 'S':
+ this = number
+ seconds += this
+ return seconds
+
+
+def main():
+ with OTDBTriggerListener():
+ with TriggerNotificationListener():
+ waitForInterrupt()
+
+if __name__ == '__main__':
+ main()
diff --git a/SAS/TriggerEmailService/Server/test/t_TriggerEmailService.py b/SAS/TriggerEmailService/Server/test/t_TriggerEmailService.py
index eec009ad1ed34eff5910799d2b860c51a80e833b..f789b369afbadbb1c87d0f061d3b29f6b9d04835 100755
--- a/SAS/TriggerEmailService/Server/test/t_TriggerEmailService.py
+++ b/SAS/TriggerEmailService/Server/test/t_TriggerEmailService.py
@@ -18,12 +18,812 @@
# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
import unittest
+import mock
+import os
+from lofar.sas.TriggerEmailService.TriggerEmailService import OTDBTriggerListener, TriggerNotificationListener, email
-class TestTriggerEmailService(unittest.TestCase):
+class TestEmailing(unittest.TestCase):
def setUp(self):
pass
+ @mock.patch.dict(os.environ, {'LOFARENV': 'PRODUCTION'})
+ @mock.patch("smtplib.SMTP")
+ def test_email_should_send_to_sos_and_operator_when_in_production(self, smtplib_mock):
+ smtp_mock = mock.MagicMock()
+ smtplib_mock.return_value = smtp_mock
+
+ email([], "", "", "<xml></xml>")
+
+ self.assertIn("sos@astron.nl", smtp_mock.sendmail.call_args[0][1])
+ self.assertIn("operator@astron.nl", smtp_mock.sendmail.call_args[0][1])
+
+ @mock.patch.dict(os.environ, {'LOFARENV': 'TEST'})
+ @mock.patch("smtplib.SMTP")
+ def test_email_should_not_send_to_sos_and_operator_when_in_test(self, smtplib_mock):
+ smtp_mock = mock.MagicMock()
+ smtplib_mock.return_value = smtp_mock
+
+ email([], "", "", "<xml></xml>")
+
+ self.assertNotIn("sos@astron.nl", smtp_mock.sendmail.call_args[0][1])
+ self.assertNotIn("operator@astron.nl", smtp_mock.sendmail.call_args[0][1])
+
+
+class TestOTDBTriggerListener(unittest.TestCase):
+ project_name = "test_lofar"
+ trigger_id = 1
+ obs_sas_id = 22
+ obs_mom_id = 44
+ mom_link = "https://lofar.astron.nl/mom3/user/main/list/setUpProjectList.do"
+
+ def setUp(self):
+ self.momqueryrpc_mock = mock.MagicMock()
+ self.momqueryrpc_mock.getProject.return_value = {"name": self.project_name}
+ self.momqueryrpc_mock.get_project_details.return_value = {
+ "author_email": "author@example.com", "pi_email": "pi@example.com"
+ }
+
+ self.momqueryrpc_mock.get_trigger_id.return_value = self.trigger_id
+ self.momqueryrpc_mock.getMoMIdsForOTDBIds.return_value = {self.obs_sas_id: self.obs_mom_id}
+
+ email_patcher = mock.patch('lofar.sas.TriggerEmailService.TriggerEmailService.email')
+ self.addCleanup(email_patcher.stop)
+ self.email_mock = email_patcher.start()
+
+ @mock.patch('lofar.sas.otdb.OTDBBusListener.OTDBBusListener.start_listening')
+ def test_start_listening_should_open_momquery_rpc(self, super_mock):
+ listener = OTDBTriggerListener(self.momqueryrpc_mock)
+
+ listener.start_listening()
+
+ self.momqueryrpc_mock.open.assert_called()
+ super_mock.assert_called()
+
+ @mock.patch('lofar.sas.otdb.OTDBBusListener.OTDBBusListener.stop_listening')
+ def test_stop_listening_should_close_momquery_rpc(self, super_mock):
+ listener = OTDBTriggerListener(self.momqueryrpc_mock)
+
+ listener.stop_listening()
+
+ self.momqueryrpc_mock.close.assert_called()
+ super_mock.assert_called()
+
+ # Aborted
+
+ def test_onObservationAborted_should_not_email_when_its_not_a_trigger(self):
+ self.momqueryrpc_mock.get_trigger_id.return_value = None
+
+ listener = OTDBTriggerListener(self.momqueryrpc_mock)
+
+ listener.onObservationAborted(self.obs_sas_id, None)
+
+ self.email_mock.assert_not_called()
+
+ def test_onObservationAborted_should_email_when_its_a_trigger(self):
+ listener = OTDBTriggerListener(self.momqueryrpc_mock)
+
+ listener.onObservationAborted(self.obs_sas_id, None)
+
+ self.email_mock.assert_called()
+
+ def test_onObservationAborted_should_set_correct_subject(self):
+ listener = OTDBTriggerListener(self.momqueryrpc_mock)
+
+ listener.onObservationAborted(self.obs_sas_id, None)
+
+ self.assertIn(str(self.trigger_id), self.email_mock.call_args[0][1])
+ self.assertIn(self.project_name, self.email_mock.call_args[0][1])
+
+ def test_onObservationAborted_should_set_correct_body(self):
+ listener = OTDBTriggerListener(self.momqueryrpc_mock)
+
+ listener.onObservationAborted(self.obs_sas_id, None)
+
+ self.assertIn(str(self.trigger_id), self.email_mock.call_args[0][2])
+ self.assertIn(self.project_name, self.email_mock.call_args[0][2])
+ self.assertIn(str(self.obs_sas_id), self.email_mock.call_args[0][2])
+ self.assertIn(str(self.obs_mom_id), self.email_mock.call_args[0][2])
+ self.assertIn(self.mom_link, self.email_mock.call_args[0][2])
+
+ # Scheduled
+
+ def test_onObservationScheduled_should_not_email_when_its_not_a_trigger(self):
+ self.momqueryrpc_mock.get_trigger_id.return_value = None
+
+ listener = OTDBTriggerListener(self.momqueryrpc_mock)
+
+ listener.onObservationScheduled(self.obs_sas_id, None)
+
+ self.email_mock.assert_not_called()
+
+ def test_onObservationScheduled_should_email_when_its_a_trigger(self):
+ listener = OTDBTriggerListener(self.momqueryrpc_mock)
+
+ listener.onObservationScheduled(self.obs_sas_id, None)
+
+ self.email_mock.assert_called()
+
+ def test_onObservationScheduled_should_set_correct_subject(self):
+ listener = OTDBTriggerListener(self.momqueryrpc_mock)
+
+ listener.onObservationScheduled(self.obs_sas_id, None)
+
+ self.assertIn(str(self.trigger_id), self.email_mock.call_args[0][1])
+ self.assertIn(self.project_name, self.email_mock.call_args[0][1])
+
+ def test_onObservationScheduled_should_set_correct_body(self):
+ listener = OTDBTriggerListener(self.momqueryrpc_mock)
+
+ listener.onObservationScheduled(self.obs_sas_id, None)
+
+ self.assertIn(str(self.trigger_id), self.email_mock.call_args[0][2])
+ self.assertIn(self.project_name, self.email_mock.call_args[0][2])
+ self.assertIn(str(self.obs_sas_id), self.email_mock.call_args[0][2])
+ self.assertIn(str(self.obs_mom_id), self.email_mock.call_args[0][2])
+ self.assertIn(self.mom_link, self.email_mock.call_args[0][2])
+
+ # Finished
+
+ def test_onObservationFinished_should_not_email_when_its_not_a_trigger(self):
+ self.momqueryrpc_mock.get_trigger_id.return_value = None
+
+ listener = OTDBTriggerListener(self.momqueryrpc_mock)
+
+ listener.onObservationFinished(self.obs_sas_id, None)
+
+ self.email_mock.assert_not_called()
+
+ def test_onObservationFinished_should_email_when_its_a_trigger(self):
+ listener = OTDBTriggerListener(self.momqueryrpc_mock)
+
+ listener.onObservationFinished(self.obs_sas_id, None)
+
+ self.email_mock.assert_called()
+
+ def test_onObservationFinished_should_set_correct_subject(self):
+ listener = OTDBTriggerListener(self.momqueryrpc_mock)
+
+ listener.onObservationFinished(self.obs_sas_id, None)
+
+ self.assertIn(str(self.trigger_id), self.email_mock.call_args[0][1])
+ self.assertIn(self.project_name, self.email_mock.call_args[0][1])
+
+ def test_onObservationFinished_should_set_correct_body(self):
+ listener = OTDBTriggerListener(self.momqueryrpc_mock)
+
+ listener.onObservationFinished(self.obs_sas_id, None)
+
+ self.assertIn(str(self.trigger_id), self.email_mock.call_args[0][2])
+ self.assertIn(self.project_name, self.email_mock.call_args[0][2])
+ self.assertIn(str(self.obs_sas_id), self.email_mock.call_args[0][2])
+ self.assertIn(str(self.obs_mom_id), self.email_mock.call_args[0][2])
+ self.assertIn(self.mom_link, self.email_mock.call_args[0][2])
+
+ #### Conflict
+
+ def test_onObservationConflict_should_not_email_when_its_not_a_trigger(self):
+ self.momqueryrpc_mock.get_trigger_id.return_value = None
+
+ listener = OTDBTriggerListener(self.momqueryrpc_mock)
+
+ listener.onObservationConflict(self.obs_sas_id, None)
+
+ self.email_mock.assert_not_called()
+
+ def test_onObservationConflict_should_email_when_its_a_trigger(self):
+ listener = OTDBTriggerListener(self.momqueryrpc_mock)
+
+ listener.onObservationConflict(self.obs_sas_id, None)
+
+ self.email_mock.assert_called()
+
+ def test_onObservationConflict_should_set_correct_subject(self):
+ listener = OTDBTriggerListener(self.momqueryrpc_mock)
+
+ listener.onObservationConflict(self.obs_sas_id, None)
+
+ self.assertIn(str(self.trigger_id), self.email_mock.call_args[0][1])
+ self.assertIn(self.project_name, self.email_mock.call_args[0][1])
+
+ def test_onObservationConflict_should_set_correct_body(self):
+ listener = OTDBTriggerListener(self.momqueryrpc_mock)
+
+ listener.onObservationConflict(self.obs_sas_id, None)
+
+ self.assertIn(str(self.trigger_id), self.email_mock.call_args[0][2])
+ self.assertIn(self.project_name, self.email_mock.call_args[0][2])
+
+
+class TestTriggerNotificationListener(unittest.TestCase):
+ project_name = "test_lofar"
+ project_mom_id = 33
+ trigger_id = 1
+ obs_sas_id = 22
+ obs_mom_id = 44
+ start_time = "2016-11-23 15:21:44"
+ stop_time = "2016-11-23 16:21:44"
+ mom_link = "https://lofar.astron.nl/mom3/user/main/list/setUpProjectList.do"
+ xml = """<?xml version="1.0" encoding="UTF-8"?>
+<trigger:trigger xsi:schemaLocation="http://www.astron.nl/LofarTrigger LofarTrigger.xsd"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:spec="http://www.astron.nl/LofarSpecification"
+ xmlns:trigger="http://www.astron.nl/LofarTrigger" xmlns:base="http://www.astron.nl/LofarBase">
+ <version>version</version>
+ <name>name</name>
+ <description>description</description>
+ <projectReference>
+ <ProjectCode>test-lofar</ProjectCode>
+ </projectReference>
+ <contactInformation>
+ <name>Sander ter Veen</name>
+ <email>veen@astron.nl</email>
+ <phoneNumber>711</phoneNumber>
+ <affiliation>ASTRON</affiliation>
+ </contactInformation>
+ <userName>veen</userName>
+ <comment>comment</comment>
+ <event>
+ <identification>none</identification>
+ <description>none</description>
+ <type>VOEvent</type>
+ </event>
+ <specification>
+ <version>2.20</version>
+ <projectReference>
+ <ProjectCode>test-lofar</ProjectCode>
+ </projectReference>
+ <userName>veen</userName>
+ <comment>comment</comment>
+ <generatorName>Jan David Mol</generatorName>
+ <generatorVersion>0.0</generatorVersion>
+
+ <!-- folders -->
+ <container>
+ <temporaryIdentifier>
+ <source>0</source>
+ <identifier>100</identifier>
+ </temporaryIdentifier>
+ <addToExistingContainer>false</addToExistingContainer>
+ <folder>
+ <name>TARGET_A</name>
+ <description>First target</description>
+ <topology>0</topology>
+ </folder>
+ </container>
+ <container>
+ <temporaryIdentifier>
+ <source>0</source>
+ <identifier>101</identifier>
+ </temporaryIdentifier>
+ <addToExistingContainer>false</addToExistingContainer>
+ <folder>
+ <name>AARTFAAC-TRIGGERED</name>
+ <description>Triggered observation by AARTFAAC (Preprocessing)</description>
+ <topology>0</topology>
+ </folder>
+ </container>
+
+ <!-- observation -->
+ <activity>
+ <temporaryIdentifier>
+ <source>0</source>
+ <identifier>200</identifier>
+ <description>0</description>
+ </temporaryIdentifier>
+ <observation>
+ <name>Target/1/TO</name>
+ <description>Target/1/TO (Target Observation)</description>
+ <instrument>Beam Observation</instrument>
+ <defaultTemplate>BeamObservation</defaultTemplate>
+ <tbbPiggybackAllowed>true</tbbPiggybackAllowed>
+ <aartfaacPiggybackAllowed>true</aartfaacPiggybackAllowed>
+ <correlatedData>true</correlatedData>
+ <coherentStokesData>false</coherentStokesData>
+ <incoherentStokesData>false</incoherentStokesData>
+ <antenna>LBA Outer</antenna>
+ <clock units="MHz">200</clock>
+ <instrumentFilter>30-90 MHz</instrumentFilter>
+ <integrationInterval>2.0</integrationInterval>
+ <channelsPerSubband>64</channelsPerSubband>
+ <bypassPff>false</bypassPff>
+ <enableSuperterp>false</enableSuperterp>
+ <numberOfBitsPerSample>8</numberOfBitsPerSample>
+ <stationSelectionSpecification>
+ <stationSelection>
+ <stationSet>Custom</stationSet>
+ <stations>
+ <station><name>CS001</name></station>
+ <station><name>CS002</name></station>
+ <station><name>CS003</name></station>
+ <station><name>CS004</name></station>
+ <station><name>CS005</name></station>
+ <station><name>CS006</name></station>
+ <station><name>CS007</name></station>
+ <station><name>CS011</name></station>
+ <station><name>CS013</name></station>
+ <station><name>CS017</name></station>
+ <station><name>CS021</name></station>
+ <station><name>CS024</name></station>
+ <station><name>CS026</name></station>
+ <station><name>CS028</name></station>
+ <station><name>CS030</name></station>
+ <station><name>CS031</name></station>
+ <station><name>CS032</name></station>
+ <station><name>CS101</name></station>
+ <station><name>CS103</name></station>
+ <station><name>CS201</name></station>
+ <station><name>CS301</name></station>
+ <station><name>CS302</name></station>
+ <station><name>CS401</name></station>
+ <station><name>CS501</name></station>
+ <station><name>RS106</name></station>
+ <station><name>RS205</name></station>
+ <station><name>RS208</name></station>
+ <station><name>RS210</name></station>
+ <station><name>RS305</name></station>
+ <station><name>RS306</name></station>
+ <station><name>RS307</name></station>
+ <station><name>RS310</name></station>
+ <station><name>RS406</name></station>
+ <station><name>RS407</name></station>
+ <station><name>RS409</name></station>
+ <station><name>RS503</name></station>
+ <station><name>RS508</name></station>
+ <station><name>RS509</name></station>
+ </stations>
+ </stationSelection>
+ </stationSelectionSpecification>
+ <timeWindowSpecification>
+ <timeFrame>UT</timeFrame>
+ <startTime>2016-11-23T15:21:44</startTime>
+ <duration>
+ <duration>PT3600S</duration>
+ </duration>
+ </timeWindowSpecification>
+ </observation>
+ <status>approved</status>
+ <qualityOfService>LATENCY</qualityOfService>
+ <priority>10</priority>
+ <triggerId>
+ <source>0</source>
+ <identifier>0</identifier>
+ </triggerId>
+ </activity>
+
+ <!-- SAP 0 -->
+ <activity>
+ <temporaryIdentifier>
+ <source>0</source>
+ <identifier>300</identifier>
+ <description>0</description>
+ </temporaryIdentifier>
+ <measurement xsi:type="base:BeamMeasurement">
+ <name>Target</name>
+ <description>Target</description>
+ <ra>204.648425</ra>
+ <dec>-0.172222222222</dec>
+ <equinox>J2000</equinox>
+ <subbandsSpecification>
+ <subbands>160..399</subbands>
+ </subbandsSpecification>
+ <measurementType>Target</measurementType>
+ </measurement>
+
+ <status>approved</status>
+ <qualityOfService>LATENCY</qualityOfService>
+ <priority>10</priority>
+ <triggerId>
+ <source>0</source>
+ <identifier>0</identifier>
+ </triggerId>
+ </activity>
+
+ <!-- SAP 1 -->
+ <activity>
+ <temporaryIdentifier>
+ <source>0</source>
+ <identifier>301</identifier>
+ <description>0</description>
+ </temporaryIdentifier>
+ <measurement xsi:type="base:BeamMeasurement">
+ <name>Calibrator</name>
+ <description>Calibrator</description>
+ <ra>123.400291667</ra>
+ <dec>48.2173833333</dec>
+ <equinox>J2000</equinox>
+ <subbandsSpecification>
+ <subbands>160..339</subbands>
+ </subbandsSpecification>
+ <measurementType>Calibration</measurementType>
+ </measurement>
+
+ <status>approved</status>
+ <qualityOfService>LATENCY</qualityOfService>
+ <priority>10</priority>
+ <triggerId>
+ <source>0</source>
+ <identifier>0</identifier>
+ </triggerId>
+ </activity>
+
+ <!-- Calibrator Averaging Pipeline -->
+ <activity>
+ <temporaryIdentifier>
+ <source>0</source>
+ <identifier>201</identifier>
+ <description>0</description>
+ </temporaryIdentifier>
+ <pipeline xsi:type="base:AveragingPipeline">
+ <name>Calibrator/1/CPT</name>
+ <description>Calibrator/1/CPT (Preprocessing)</description>
+ <processingCluster>
+ <name>CEP4</name>
+ <partition>cpu</partition>
+ <numberOfTasks>24</numberOfTasks>
+ <minRAMPerTask unit="byte">1000000000</minRAMPerTask>
+ <minScratchPerTask unit="byte">100000000</minScratchPerTask>
+ <maxDurationPerTask>PT600S</maxDurationPerTask>
+ <numberOfCoresPerTask>20</numberOfCoresPerTask>
+ <runSimultaneous>true</runSimultaneous>
+ </processingCluster>
+ <defaultTemplate>Preprocessing Pipeline</defaultTemplate>
+ <demixingParameters>
+ <averagingFreqStep>16</averagingFreqStep>
+ <averagingTimeStep>1</averagingTimeStep>
+ <demixFreqStep>16</demixFreqStep>
+ <demixTimeStep>5</demixTimeStep>
+ <demixAlways />
+ <demixIfNeeded />
+ <ignoreTarget>false</ignoreTarget>
+ </demixingParameters>
+ <flaggingStrategy>LBAdefault</flaggingStrategy>
+ </pipeline>
+ <status>approved</status>
+ <qualityOfService>LATENCY</qualityOfService>
+ <priority>10</priority>
+ <triggerId>
+ <source>0</source>
+ <identifier>0</identifier>
+ </triggerId>
+ </activity>
+
+ <!-- Target Averaging Pipeline -->
+ <activity>
+ <temporaryIdentifier>
+ <source>0</source>
+ <identifier>202</identifier>
+ <description>0</description>
+ </temporaryIdentifier>
+ <pipeline xsi:type="base:AveragingPipeline">
+ <name>Calibrator/1/CPT</name>
+ <description>Calibrator/1/CPT (Preprocessing)</description>
+ <processingCluster>
+ <name>CEP4</name>
+ <partition>cpu</partition>
+ <numberOfTasks>24</numberOfTasks>
+ <minRAMPerTask unit="byte">1000000000</minRAMPerTask>
+ <minScratchPerTask unit="byte">100000000</minScratchPerTask>
+ <maxDurationPerTask>PT600S</maxDurationPerTask>
+ <numberOfCoresPerTask>20</numberOfCoresPerTask>
+ <runSimultaneous>true</runSimultaneous>
+ </processingCluster>
+ <defaultTemplate>Preprocessing Pipeline</defaultTemplate>
+ <demixingParameters>
+ <averagingFreqStep>16</averagingFreqStep>
+ <averagingTimeStep>1</averagingTimeStep>
+ <demixFreqStep>16</demixFreqStep>
+ <demixTimeStep>5</demixTimeStep>
+ <demixAlways />
+ <demixIfNeeded />
+ <ignoreTarget>false</ignoreTarget>
+ </demixingParameters>
+ <flaggingStrategy>LBAdefault</flaggingStrategy>
+ </pipeline>
+ <status>approved</status>
+ <qualityOfService>LATENCY</qualityOfService>
+ <priority>10</priority>
+ <triggerId>
+ <source>0</source>
+ <identifier>0</identifier>
+ </triggerId>
+ </activity>
+
+ <!-- SAP 0 data products -->
+ <entity>
+ <temporaryIdentifier>
+ <source>0</source>
+ <identifier>400</identifier>
+ </temporaryIdentifier>
+ <dataproductType>UVDataProduct</dataproductType>
+ <storageCluster>
+
+ <name>CEP4</name>
+ <partition>/data/projects/</partition>
+ </storageCluster>
+ </entity>
+
+ <!-- SAP 1 data products -->
+ <entity>
+ <temporaryIdentifier>
+ <source>0</source>
+ <identifier>401</identifier>
+ </temporaryIdentifier>
+ <dataproductType>UVDataProduct</dataproductType>
+ <storageCluster>
+ <name>CEP4</name>
+ <partition>/data/projects/</partition>
+ </storageCluster>
+ </entity>
+
+ <!-- Calibrator Pipeline dataproducts -->
+ <entity>
+ <temporaryIdentifier>
+ <source>0</source>
+ <identifier>402</identifier>
+ </temporaryIdentifier>
+ <dataproductType>UVDataProduct</dataproductType>
+ <storageCluster>
+ <name>CEP4</name>
+ <partition>/data/projects/</partition>
+ </storageCluster>
+ </entity>
+
+ <!-- Target Pipeline dataproducts -->
+ <entity>
+ <temporaryIdentifier>
+ <source>0</source>
+ <identifier>403</identifier>
+ </temporaryIdentifier>
+ <dataproductType>UVDataProduct</dataproductType>
+ <storageCluster>
+ <name>CEP4</name>
+ <partition>/data/projects/</partition>
+ </storageCluster>
+ </entity>
+
+ <!-- folder 101 is child of folder 100 -->
+ <relation xsi:type="spec:ChildRelation">
+ <parent>
+ <source>0</source>
+ <identifier>100</identifier>
+ </parent>
+ <child>
+ <source>0</source>
+ <identifier>101</identifier>
+ </child>
+ <type>folder-folder</type>
+ </relation>
+
+ <!-- observation 200 is child of folder 101 -->
+ <relation xsi:type="spec:ChildRelation">
+ <parent>
+ <source>0</source>
+ <identifier>101</identifier>
+ </parent>
+ <child>
+ <source>0</source>
+ <identifier>200</identifier>
+ </child>
+ <type>folder-activity</type>
+ </relation>
+
+ <!-- measurements 300 is a child of observation 200 -->
+ <relation xsi:type="spec:ChildRelation">
+ <parent>
+ <source>0</source>
+ <identifier>200</identifier>
+ </parent>
+ <child>
+ <source>0</source>
+ <identifier>300</identifier>
+ </child>
+ <type>observation-measurement</type>
+ </relation>
+
+ <!-- measurement 301 is a child of observation 200 -->
+ <relation xsi:type="spec:ChildRelation">
+ <parent>
+ <source>0</source>
+ <identifier>200</identifier>
+ <description>0</description>
+ </parent>
+ <child>
+ <source>0</source>
+ <identifier>301</identifier>
+ <description>0</description>
+ </child>
+ <type>observation-measurement</type>
+ </relation>
+
+ <!-- dataproducts 400 are output of measurement 300 -->
+ <relation xsi:type="spec:ActivityEntityRelation">
+ <entity>
+ <source>0</source>
+ <identifier>400</identifier>
+ </entity>
+ <activity>
+ <source>0</source>
+ <identifier>300</identifier>
+ </activity>
+ <type>producer</type>
+ </relation>
+
+ <!-- dataproducts 401 are output of measurement 301 -->
+ <relation xsi:type="spec:ActivityEntityRelation">
+ <entity>
+ <source>0</source>
+ <identifier>401</identifier>
+ </entity>
+ <activity>
+ <source>0</source>
+ <identifier>301</identifier>
+ </activity>
+ <type>producer</type>
+ </relation>
+
+
+ <!-- SAP 1 is the calibrator for SAP 0 -->
+ <relation xsi:type="spec:TwinRelation">
+ <first>
+ <source>0</source>
+ <identifier>301</identifier>
+ </first>
+ <second>
+ <source>0</source>
+ <identifier>300</identifier>
+ </second>
+ <type>calibrator-target</type>
+ </relation>
+
+
+ <!-- dataproducts 401 are input for pipeline 201 -->
+ <relation xsi:type="spec:ActivityEntityRelation">
+ <entity>
+ <source>0</source>
+ <identifier>401</identifier>
+ </entity>
+ <activity>
+ <source>0</source>
+ <identifier>201</identifier>
+ </activity>
+ <type>user</type>
+ </relation>
+
+ <!-- dataproducts 402 are output of pipeline 201 -->
+ <relation xsi:type="spec:ActivityEntityRelation">
+ <entity>
+ <source>0</source>
+ <identifier>402</identifier>
+ </entity>
+ <activity>
+ <source>0</source>
+ <identifier>201</identifier>
+ </activity>
+ <type>producer</type>
+ </relation>
+
+ <!-- pipeline 201 is child of folder 101 -->
+ <relation xsi:type="spec:ChildRelation">
+ <parent>
+ <source>0</source>
+ <identifier>101</identifier>
+ </parent>
+ <child>
+ <source>0</source>
+ <identifier>201</identifier>
+ </child>
+ <type>folder-activity</type>
+ </relation>
+
+ <!-- dataproducts 400 are input for pipeline 202 -->
+ <relation xsi:type="spec:ActivityEntityRelation">
+ <entity>
+ <source>0</source>
+ <identifier>400</identifier>
+ </entity>
+ <activity>
+ <source>0</source>
+ <identifier>202</identifier>
+ </activity>
+ <type>user</type>
+ </relation>
+
+ <!-- pipeline 202 is child of folder 101 -->
+ <relation xsi:type="spec:ChildRelation">
+ <parent>
+ <source>0</source>
+ <identifier>101</identifier>
+ </parent>
+ <child>
+ <source>0</source>
+ <identifier>202</identifier>
+ </child>
+ <type>folder-activity</type>
+ </relation>
+
+ <!-- dataproducts 403 are output of pipeline 202 -->
+ <relation xsi:type="spec:ActivityEntityRelation">
+ <entity>
+ <source>0</source>
+ <identifier>403</identifier>
+ </entity>
+ <activity>
+ <source>0</source>
+ <identifier>202</identifier>
+ </activity>
+ <type>producer</type>
+ </relation>
+ </specification>
+ <generatorName>Jan David Mol</generatorName>
+ <generatorVersion>0.0</generatorVersion>
+</trigger:trigger>
+"""
+
+ message_content = {
+ "trigger_id": trigger_id,
+ "project": project_name,
+ "metadata": xml
+ }
+
+ def setUp(self):
+ self.momqueryrpc_mock = mock.MagicMock()
+ self.momqueryrpc_mock.getProject.return_value = {"name": self.project_name}
+ self.momqueryrpc_mock.getProjects.return_value = [{"name": self.project_name, "mom2id": self.project_mom_id}]
+ self.momqueryrpc_mock.get_project_details.return_value = {
+ "author_email": "author@example.com", "pi_email": "pi@example.com"
+ }
+
+ email_patcher = mock.patch('lofar.sas.TriggerEmailService.TriggerEmailService.email')
+ self.addCleanup(email_patcher.stop)
+ self.email_mock = email_patcher.start()
+
+ self.message = mock.MagicMock()
+ self.message.content = self.message_content
+
+ @mock.patch('lofar.messaging.messagebus.AbstractBusListener.start_listening')
+ def test_start_listening_should_open_momquery_rpc(self, super_mock):
+ listener = TriggerNotificationListener(self.momqueryrpc_mock)
+
+ listener.start_listening()
+
+ self.momqueryrpc_mock.open.assert_called()
+ super_mock.assert_called()
+
+ @mock.patch('lofar.messaging.messagebus.AbstractBusListener.stop_listening')
+ def test_stop_listening_should_close_momquery_rpc(self, super_mock):
+ listener = TriggerNotificationListener(self.momqueryrpc_mock)
+
+ listener.stop_listening()
+
+ self.momqueryrpc_mock.close.assert_called()
+ super_mock.assert_called()
+
+ def test_handleMessage_should_email(self):
+ listener = TriggerNotificationListener(self.momqueryrpc_mock)
+
+ listener._handleMessage(self.message)
+
+ self.email_mock.assert_called()
+
+ def test_handleMessage_should_set_correct_subject(self):
+ listener = TriggerNotificationListener(self.momqueryrpc_mock)
+
+ listener._handleMessage(self.message)
+
+ self.assertIn(str(self.trigger_id), self.email_mock.call_args[0][1])
+ self.assertIn(self.project_name, self.email_mock.call_args[0][1])
+
+ def test_handleMessage_should_set_correct_body(self):
+ listener = TriggerNotificationListener(self.momqueryrpc_mock)
+
+ listener._handleMessage(self.message)
+
+ self.assertIn(str(self.trigger_id), self.email_mock.call_args[0][2])
+ self.assertIn(self.project_name, self.email_mock.call_args[0][2])
+ self.assertIn(str(self.start_time), self.email_mock.call_args[0][2])
+ self.assertIn(str(self.stop_time), self.email_mock.call_args[0][2])
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/SAS/XML_generator/src/xmlgen.py b/SAS/XML_generator/src/xmlgen.py
index 0d229cb357287f5462dc40041689cc6c869e7ac0..eed2804ebda128b4222d5311273a61bed9f82608 100755
--- a/SAS/XML_generator/src/xmlgen.py
+++ b/SAS/XML_generator/src/xmlgen.py
@@ -2,7 +2,7 @@
# XML generator
# xmlgen.py
#
-# Copyright (C) 2016
+# Copyright (C) 2016, 2017
# ASTRON (Netherlands Institute for Radio Astronomy)
# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
#
@@ -25,11 +25,8 @@
# Revision : $Revision: 34492 $
# Last change by : $Author: renting $
# Change date : $Date: 2016-05-18 11:47:57 +0200 (wo, 18 mei 2016) $
-# First creation : unknown
-# URL : $URL: https://svn.astron.nl/ROD/trunk/LOFAR_Scheduler/DataHandler.cpp $
-
-VERSION = "2.20.0"
+VERSION = "2.21.0"
import sys, getopt, time
from xml.sax.saxutils import escape as XMLescape
@@ -760,13 +757,13 @@ def writeSkyImageOutputDataproduct(ofile, topology, storageCluster):
</lofar:pipeline>
</item>""" % (topology, topology, stor_cluster)
-def writeFolderStart(ofile, packageName, packageDescription, processing):
+def writeFolderStart(ofile, folderNr, packageName, packageDescription, processing):
print >>ofile, r""" <item index="0">
<lofar:folder topology_parent="true">
- <topology>0</topology>
+ <topology>%s</topology>
<name>%s</name>
<description>%s (%s)</description>
- <children>""" % (packageName, packageDescription, processing)
+ <children>""" % (folderNr, packageName, packageDescription, processing)
def writeFolderEnd(ofile):
print >> ofile, r"""</children>
@@ -1706,6 +1703,7 @@ def checkSettings(settings, blockNr):
elif settings["imagingPipeline"] == 'MSSS':
settings["imaging_pipe_type"] = 'ImagingPipelineMSSSType'
settings["imaging_pipe_default_template"] = "MSSS Imaging Pipeline"
+ settings["imaging_pipe_duration"] = 0 # img pipeline duration placeholder, MoM rejects <duration></duration>
# determine nrImages
settings["nrImages"] = determineNrImages(settings["targetBeams"], settings["nrSubbandsPerImage"], "nrSubbandsPerImage")
else:
@@ -2244,7 +2242,7 @@ def writeBlock(ofile, settings, projectName, blockNr, status, nr_nodes):
#There's a lot of stuff in settings that's only relevant to the imaging pipelines
#otherSettings = { key: settings[key] for key not in imagingPipelineKeys }
- writeFolderStart(ofile, settings["packageName"], settings["packageDescription"], settings["processing"])
+ writeFolderStart(ofile, blockNr-1, settings["packageName"], settings["packageDescription"], settings["processing"])
if settings["set_starttime"]:
settings["startTimeObs"] = settings["startTime"]
diff --git a/SAS/XML_generator/test/test_regression.in_data/txt/old_Long_Baseline_test.txt b/SAS/XML_generator/test/test_regression.in_data/txt/old_Long_Baseline_test.txt
index eebd87a3af0b89e9116f72f6d028ce3fa9ed533d..7b3ff3bbe6e3335a0d3bd9ac027318c1a04a1ff2 100644
--- a/SAS/XML_generator/test/test_regression.in_data/txt/old_Long_Baseline_test.txt
+++ b/SAS/XML_generator/test/test_regression.in_data/txt/old_Long_Baseline_test.txt
@@ -1,5 +1,5 @@
-projectName=test_lofar
+projectName=test-lofar
mainFolderName=
mainFolderDescription=
diff --git a/SAS/XML_generator/test/test_regression.in_data/txt/old_input.txt b/SAS/XML_generator/test/test_regression.in_data/txt/old_input.txt
index 56ccd64a0a3b2ef10291aba384fe979807798744..fa56a8fc9bd1eb26f57f484903b29c29a6ffacb2 100644
--- a/SAS/XML_generator/test/test_regression.in_data/txt/old_input.txt
+++ b/SAS/XML_generator/test/test_regression.in_data/txt/old_input.txt
@@ -1,5 +1,5 @@
-projectName=test_lofar
+projectName=test-lofar
mainFolderName=
mainFolderDescription=MSSS HBA session
diff --git a/SAS/XML_generator/test/test_regression.in_data/txt/old_pulsar_pipe_test.txt b/SAS/XML_generator/test/test_regression.in_data/txt/old_pulsar_pipe_test.txt
index 5e4127443792a89a380843a6b3be0c43ebb6390f..2b06b1882d21c152cb29e67fabe8718fad50e973 100644
--- a/SAS/XML_generator/test/test_regression.in_data/txt/old_pulsar_pipe_test.txt
+++ b/SAS/XML_generator/test/test_regression.in_data/txt/old_pulsar_pipe_test.txt
@@ -1,5 +1,5 @@
-projectName=test_lofar
+projectName=test-lofar
mainFolderName=
mainFolderDescription=
diff --git a/SAS/XML_generator/test/test_regression.in_data/txt/test_input.txt b/SAS/XML_generator/test/test_regression.in_data/txt/test_input.txt
index 4027dc0e2f21d084b9f13a6b8133c5d6a9c544e6..9353c3e8801f73fdd0a1430192bf0381b361b604 100644
--- a/SAS/XML_generator/test/test_regression.in_data/txt/test_input.txt
+++ b/SAS/XML_generator/test/test_regression.in_data/txt/test_input.txt
@@ -1,4 +1,4 @@
-projectName=test_lofar # be careful! Don't make typos here. Or otherwise a non-valid project will be created in mom
+projectName=test-lofar # be careful! Don't make typos here. Or otherwise a non-valid project will be created in mom
mainFolderName= # the name of the folder that will contain all blocks, if left unspecified no main folder will be created
mainFolderDescription=
@@ -83,4 +83,4 @@ targetBeams=
#0.0417300951946;0.00558069028325;Sun;54..297;244;;;T;8100
#Demix=16;1;16;1
-BLOCK
\ No newline at end of file
+BLOCK
diff --git a/SAS/XML_generator/test/test_regression.in_data/txt/test_input_cep4.txt b/SAS/XML_generator/test/test_regression.in_data/txt/test_input_cep4.txt
index 14c4ed71de96de54af8637d8a3fda63e59139237..b656642e4b11fd7d1437602e1bebf5e7cc31e68c 100644
--- a/SAS/XML_generator/test/test_regression.in_data/txt/test_input_cep4.txt
+++ b/SAS/XML_generator/test/test_regression.in_data/txt/test_input_cep4.txt
@@ -1,4 +1,4 @@
-projectName=test_lofar # be careful! Don't make typos here. Or otherwise a non-valid project will be created in mom
+projectName=test-lofar # be careful! Don't make typos here. Or otherwise a non-valid project will be created in mom
mainFolderName= # the name of the folder that will contain all blocks, if left unspecified no main folder will be created
mainFolderDescription=
diff --git a/SAS/XML_generator/test/test_regression.in_data/txt/test_input_commensal_obs_DRAGNET.txt b/SAS/XML_generator/test/test_regression.in_data/txt/test_input_commensal_obs_DRAGNET.txt
new file mode 100644
index 0000000000000000000000000000000000000000..58e74b2404088819b9fe17b3b9ba922e8e07626f
--- /dev/null
+++ b/SAS/XML_generator/test/test_regression.in_data/txt/test_input_commensal_obs_DRAGNET.txt
@@ -0,0 +1,78 @@
+projectName=test-lofar # be careful! Don't make typos here. Or otherwise a non-valid project will be created in mom
+mainFolderName=xmlgen_commensal_test # the name of the folder that will contain all blocks, if left unspecified no main folder will be created
+mainFolderDescription=Commensal Observation DRAGNET (XMLgenerator test)
+
+BLOCK
+
+cluster=DRAGNET
+split_targets = F # true:create a separate target observation for every target (beam) line or false:combine them in a multi-beam observation
+calibration = none # internal / external / none
+processing=none # can be one of Calibration, Preprocessing, Imaging, Pulsar, LongBaseline, none
+imagingPipeline=none # can be one of MSSS, standard, none
+repeat=1 # the number of repeats of this block
+
+packageName=commensalobs-DRAGNET # the name that will be given to the package folder that contains this block's observation and pipelines
+packageDescription=Commensal Obs package description
+packageTag= # optional tag that will be prepended before every observation and pipeline name/description (Max 8 characters).
+
+antennaMode=HBA Dual
+clock=200 MHz
+instrumentFilter=110-190 MHz
+numberOfBitsPerSample=8
+integrationTime=4.0
+channelsPerSubband=16
+stationList=core # comma-separated list of station names and/or the following aliasses: core, superterp, remote, international, all, nl
+tbbPiggybackAllowed=T
+aartfaacPiggybackAllowed=T
+
+###### Which data types should be produced: ######
+correlatedData=T
+coherentStokesData=T
+incoherentStokesData=T
+flysEye=F
+coherentDedisperseChannels=False
+
+###### Coherent Stokes parameters ######
+subbandsPerFileCS=20
+numberCollapsedChannelsCS=16
+stokesDownsamplingStepsCS=6
+whichCS=XXYY
+
+###### Incoherent Stokes parameters ######
+subbandsPerFileIS=40
+numberCollapsedChannelsIS=1
+stokesDownsamplingStepsIS=12
+whichIS=I
+
+#calibratorDuration_s=120 # duration of calibration observations in seconds
+targetDuration_s=600 # duration of target observations in seconds
+
+###### Globals are used when a target/calibration line does not have its own specific parameter specified for that property ######
+#Global_TAB=
+#Global_TABrings=2;0.1 # nr of tabrings and tabring size
+#Global_Pulsar=B0329+54;T;T;DSPSR EXTRA OPTIONS;PREPDATA;5.1;-2;F;F;F;F;F;2BF2FITS;4;101;RFI FIND EXTRA OPTIONS;PREPFOLD EXTRA;PREPSUBBAND Extra; 0.6;T;T
+#Global_Subbands=98..297;200
+
+# startTimeUTC, the start time of the first observation. format: yyyy-MM-dd hh:mm:ss
+# un-comment the startTimeUTC to have the observation start times generated
+startTimeUTC=2017-05-09 03:00:00
+# timeStep's in seconds
+timeStep1=0
+#timeStep2=60
+
+#calibratorBeam=
+#19:59:28.35;+40:44:02.0;3C295;;;;;T;60
+
+TAB:
+c;05:34:51.94;+22:00:52.2 # a coherent tab with absolute coordinates 05:34:51.94, +22:00:52.2
+i;05:34:51.94;+22:00:52.2
+c;05:34:52.02;+22:00:52.5
+i;05:34:52.02;+22:00:52.5
+
+# target beams and target pipelines
+# ra ;dec; targetname; subbandList; nrSubbands; nrTABrings; TABringSize; create_pipeline [;pipeline duration seconds]
+# optionally followed by BBS and/or demixing settings
+targetBeams=
+05:34:51.94;+22:00:52.2;Crab pointing 1;98..297;200;2;0.1;F;600
+05:34:52.02;+22:00:52.5;Crab pointing 2;297..396;100;2;0.1;F;600
+
diff --git a/SAS/XML_generator/test/test_regression.in_data/txt/test_input_long_baseline_pipeline.txt b/SAS/XML_generator/test/test_regression.in_data/txt/test_input_long_baseline_pipeline.txt
index 8106327a4bc077649ebb129e1e6bc55836db52f0..4e9f2981dfb0f59379063694dbe9d5fefd96e5f3 100644
--- a/SAS/XML_generator/test/test_regression.in_data/txt/test_input_long_baseline_pipeline.txt
+++ b/SAS/XML_generator/test/test_regression.in_data/txt/test_input_long_baseline_pipeline.txt
@@ -42,8 +42,8 @@ whichCS=IQUV
#stokesDownsamplingStepsIS=12
#whichIS=IQUV
-subbandsPerSubbandGroup=8 # number of subbands per subbandGroup for Long Baseline Pipeline (Concat)
-subbandGroupsPerMS=8 # number of subbandgroups per MeasurementSet for Long Baseline Pipeline (Concat)
+subbandsPerSubbandGroup=4 # number of subbands per subbandGroup for Long Baseline Pipeline (Concat)
+subbandGroupsPerMS=6 # number of subbandgroups per MeasurementSet for Long Baseline Pipeline (Concat)
flaggingStrategy=HBAdefault # flagging strategy used for AOflagger
calibratorDuration_s=120 # duration of calibration observations in seconds
targetDuration_s=600 # duration of target observations in seconds
diff --git a/SAS/XML_generator/test/test_regression.in_data/xml/20150731_G46_run1_HBA.xml b/SAS/XML_generator/test/test_regression.in_data/xml/20150731_G46_run1_HBA.xml
index 42b7547aee6a1db6317770a225229635e6442fd0..e22f8a789c25004e2e7c6811ca7b465c9595bb53 100644
--- a/SAS/XML_generator/test/test_regression.in_data/xml/20150731_G46_run1_HBA.xml
+++ b/SAS/XML_generator/test/test_regression.in_data/xml/20150731_G46_run1_HBA.xml
@@ -275,7 +275,7 @@
</item>
<item index="0">
<lofar:folder topology_parent="true">
- <topology>0</topology>
+ <topology>1</topology>
<name>20150731_G46_run1_HBA</name>
<description>20150731_G46_run1_HBA (Preprocessing)</description>
<children>
@@ -635,7 +635,7 @@
</item>
<item index="0">
<lofar:folder topology_parent="true">
- <topology>0</topology>
+ <topology>2</topology>
<name>3C380</name>
<description>3C380 (Preprocessing)</description>
<children>
diff --git a/SAS/XML_generator/test/test_regression.in_data/xml/20150810_P247P244.xml b/SAS/XML_generator/test/test_regression.in_data/xml/20150810_P247P244.xml
index 36927a1a3908c0444588c071468b1605ae19cd3e..d20df7175bf39ca73ed1eaf8c602f64890a887da 100644
--- a/SAS/XML_generator/test/test_regression.in_data/xml/20150810_P247P244.xml
+++ b/SAS/XML_generator/test/test_regression.in_data/xml/20150810_P247P244.xml
@@ -177,7 +177,7 @@
</item>
<item index="0">
<lofar:folder topology_parent="true">
- <topology>0</topology>
+ <topology>1</topology>
<name>20150810P247+45P244+48</name>
<description>HBA Dual Inner, 110-190MHz, 8bits, 96MHz@144MHz, 1s, 64ch/sb (Preprocessing)</description>
<children>
@@ -439,7 +439,7 @@
</item>
<item index="0">
<lofar:folder topology_parent="true">
- <topology>0</topology>
+ <topology>2</topology>
<name>20150811_3C295</name>
<description>HBA Dual Inner, 110-190MHz, 8bits, 96MHz@144MHz, 1s, 64ch/sb (Preprocessing)</description>
<children>
diff --git a/SAS/XML_generator/test/test_regression.in_data/xml/2MASS_1.xml b/SAS/XML_generator/test/test_regression.in_data/xml/2MASS_1.xml
index 96c89097f95765ce834c5d9912a3598732e09c92..ea863903712119cb75d6929c773b659bbe0095b2 100644
--- a/SAS/XML_generator/test/test_regression.in_data/xml/2MASS_1.xml
+++ b/SAS/XML_generator/test/test_regression.in_data/xml/2MASS_1.xml
@@ -332,7 +332,7 @@
</item>
<item index="0">
<lofar:folder topology_parent="true">
- <topology>0</topology>
+ <topology>1</topology>
<name>3C295_cal2</name>
<description>HBA Dual, 110-190MHz,8bits, 48MHz@144MHz,1s,64ch/sb (Preprocessing)</description>
<children>
diff --git a/SAS/XML_generator/test/test_regression.in_data/xml/LC4_022_3C58_HBA_parameters.xml b/SAS/XML_generator/test/test_regression.in_data/xml/LC4_022_3C58_HBA_parameters.xml
index 7cbc96cf67509787e1ec77ce1a73f3a90c6cae1a..15492c3866d1077feb0c79d1b19d6d22322c3798 100644
--- a/SAS/XML_generator/test/test_regression.in_data/xml/LC4_022_3C58_HBA_parameters.xml
+++ b/SAS/XML_generator/test/test_regression.in_data/xml/LC4_022_3C58_HBA_parameters.xml
@@ -384,7 +384,7 @@
</processingCluster>
<imagingPipelineAttributes>
<defaultTemplate>Imaging Pipeline HBA</defaultTemplate>
- <duration></duration>
+ <duration>0</duration>
<nrOfOutputSkyImage>12</nrOfOutputSkyImage>
<imagingParameters>
<nrSlicesPerImage>1</nrSlicesPerImage>
@@ -428,7 +428,7 @@
</item>
<item index="0">
<lofar:folder topology_parent="true">
- <topology>0</topology>
+ <topology>1</topology>
<name>3C196</name>
<description>3C196 Bookend Scan (Preprocessing)</description>
<children>
diff --git a/SAS/XML_generator/test/test_regression.in_data/xml/Ticket_6923.xml b/SAS/XML_generator/test/test_regression.in_data/xml/Ticket_6923.xml
index 8c39674edee3f173b1e06004aa0580ac0bf99a2f..b9dbab3bb3a8e9300bbd618111260eb165ae0e00 100644
--- a/SAS/XML_generator/test/test_regression.in_data/xml/Ticket_6923.xml
+++ b/SAS/XML_generator/test/test_regression.in_data/xml/Ticket_6923.xml
@@ -281,7 +281,7 @@
</item>
<item index="0">
<lofar:folder topology_parent="true">
- <topology>0</topology>
+ <topology>1</topology>
<name>HBA_150-190</name>
<description>HBA 150-190 (none)</description>
<children>
@@ -549,7 +549,7 @@
</item>
<item index="0">
<lofar:folder topology_parent="true">
- <topology>0</topology>
+ <topology>2</topology>
<name>HBA_210-250 A</name>
<description>HBA 210-250 part A (none)</description>
<children>
@@ -817,7 +817,7 @@
</item>
<item index="0">
<lofar:folder topology_parent="true">
- <topology>0</topology>
+ <topology>3</topology>
<name>LBA_10-50</name>
<description>Test LBA 10-50 (none)</description>
<children>
@@ -1085,7 +1085,7 @@
</item>
<item index="0">
<lofar:folder topology_parent="true">
- <topology>0</topology>
+ <topology>4</topology>
<name>LBA_50-90</name>
<description>Test LBA 50-90 (none)</description>
<children>
diff --git a/SAS/XML_generator/test/test_regression.in_data/xml/test_input.xml b/SAS/XML_generator/test/test_regression.in_data/xml/test_input.xml
index 7ef0ed075ff688f29caa7ea29e725cf90a2cf7df..bb11425814a430b394bb21fa522cb34777915071 100644
--- a/SAS/XML_generator/test/test_regression.in_data/xml/test_input.xml
+++ b/SAS/XML_generator/test/test_regression.in_data/xml/test_input.xml
@@ -4,7 +4,7 @@
<template version="2.17.0" author="Alwin de Jong,Adriaan Renting" changedBy="Adriaan Renting">
<description>XML Template generator version 2.17.0</description>
</template>
- <name>test_lofar</name>
+ <name>test-lofar</name>
<children>
<item index="0">
<lofar:folder topology_parent="true">
@@ -25,7 +25,7 @@
<observationId>
</observationId>
<name>Crab test/1/TO</name>
- <projectName>test_lofar</projectName>
+ <projectName>test-lofar</projectName>
<instrument>Beam Observation</instrument>
<defaultTemplate>BeamObservation</defaultTemplate>
<tbbPiggybackAllowed>true</tbbPiggybackAllowed>
@@ -203,7 +203,7 @@
<observationId>
</observationId>
<name>Crab test/2/TO</name>
- <projectName>test_lofar</projectName>
+ <projectName>test-lofar</projectName>
<instrument>Beam Observation</instrument>
<defaultTemplate>BeamObservation</defaultTemplate>
<tbbPiggybackAllowed>true</tbbPiggybackAllowed>
diff --git a/SAS/XML_generator/test/test_regression.in_data/xml/test_input_cep4.xml b/SAS/XML_generator/test/test_regression.in_data/xml/test_input_cep4.xml
index 7ef0ed075ff688f29caa7ea29e725cf90a2cf7df..bb11425814a430b394bb21fa522cb34777915071 100644
--- a/SAS/XML_generator/test/test_regression.in_data/xml/test_input_cep4.xml
+++ b/SAS/XML_generator/test/test_regression.in_data/xml/test_input_cep4.xml
@@ -4,7 +4,7 @@
<template version="2.17.0" author="Alwin de Jong,Adriaan Renting" changedBy="Adriaan Renting">
<description>XML Template generator version 2.17.0</description>
</template>
- <name>test_lofar</name>
+ <name>test-lofar</name>
<children>
<item index="0">
<lofar:folder topology_parent="true">
@@ -25,7 +25,7 @@
<observationId>
</observationId>
<name>Crab test/1/TO</name>
- <projectName>test_lofar</projectName>
+ <projectName>test-lofar</projectName>
<instrument>Beam Observation</instrument>
<defaultTemplate>BeamObservation</defaultTemplate>
<tbbPiggybackAllowed>true</tbbPiggybackAllowed>
@@ -203,7 +203,7 @@
<observationId>
</observationId>
<name>Crab test/2/TO</name>
- <projectName>test_lofar</projectName>
+ <projectName>test-lofar</projectName>
<instrument>Beam Observation</instrument>
<defaultTemplate>BeamObservation</defaultTemplate>
<tbbPiggybackAllowed>true</tbbPiggybackAllowed>
diff --git a/SAS/XML_generator/test/test_regression.in_data/xml/test_input_commensal_obs_DRAGNET.xml b/SAS/XML_generator/test/test_regression.in_data/xml/test_input_commensal_obs_DRAGNET.xml
new file mode 100644
index 0000000000000000000000000000000000000000..91a039e2ac03eef0d7fc553f47d94fed311e4c2e
--- /dev/null
+++ b/SAS/XML_generator/test/test_regression.in_data/xml/test_input_commensal_obs_DRAGNET.xml
@@ -0,0 +1,194 @@
+<?xml version="1.0" encoding="UTF-8"?>
+ <lofar:project xmlns:lofar="http://www.astron.nl/MoM2-Lofar" xmlns:mom2="http://www.astron.nl/MoM2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.astron.nl/MoM2-Lofar http://lofar.astron.nl:8080/mom3/schemas/LofarMoM2.xsd http://www.astron.nl/MoM2 http://lofar.astron.nl:8080/mom3/schemas/MoM2.xsd ">
+ <version>2.21.0</version>
+ <template version="2.21.0" author="Alwin de Jong,Adriaan Renting" changedBy="Adriaan Renting">
+ <description>XML Template generator version 2.21.0</description>
+ </template>
+ <name>test-lofar</name>
+ <children>
+ <item index="0">
+ <lofar:folder topology_parent="false">
+ <name>xmlgen_commensal_test</name>
+ <description>Commensal Observation DRAGNET (XMLgenerator test)</description>
+ <children>
+ <item index="0">
+ <lofar:folder topology_parent="true">
+ <topology>0</topology>
+ <name>commensalobs-DRAGNET</name>
+ <description>Commensal Obs package description (none)</description>
+ <children>
+ <item index="0">
+ <lofar:observation>
+ <name>Crab pointing 1/1/TO</name>
+ <description>Crab pointing 1/1/TO (Target Observation)</description>
+ <topology>B0.1.T</topology>
+ <predecessor_topology></predecessor_topology>
+ <currentStatus>
+ <mom2:openedStatus/>
+ </currentStatus>
+ <lofar:observationAttributes>
+ <observationId>
+ </observationId>
+ <name>Crab pointing 1/1/TO</name>
+ <projectName>test-lofar</projectName>
+ <instrument>Beam Observation</instrument>
+ <defaultTemplate>BeamObservation</defaultTemplate>
+ <tbbPiggybackAllowed>true</tbbPiggybackAllowed>
+ <aartfaacPiggybackAllowed>true</aartfaacPiggybackAllowed>
+ <userSpecification>
+ <correlatedData>true</correlatedData>
+ <coherentStokesData>true</coherentStokesData>
+ <incoherentStokesData>true</incoherentStokesData>
+ <antenna>HBA Dual</antenna>
+ <clock mode="200 MHz"/>
+ <instrumentFilter>110-190 MHz</instrumentFilter>
+ <integrationInterval>4.0</integrationInterval>
+ <channelsPerSubband>16</channelsPerSubband>
+ <coherentDedisperseChannels>false</coherentDedisperseChannels>
+ <tiedArrayBeams>
+ <flyseye>false</flyseye>
+ </tiedArrayBeams>
+ <stokes>
+ <integrateChannels>false</integrateChannels>
+ <subbandsPerFileCS>20</subbandsPerFileCS>
+ <numberCollapsedChannelsCS>16</numberCollapsedChannelsCS>
+ <stokesDownsamplingStepsCS>6</stokesDownsamplingStepsCS>
+ <whichCS>XXYY</whichCS>
+ <subbandsPerFileIS>40</subbandsPerFileIS>
+ <numberCollapsedChannelsIS>1</numberCollapsedChannelsIS>
+ <stokesDownsamplingStepsIS>12</stokesDownsamplingStepsIS>
+ <whichIS>I</whichIS>
+ </stokes>
+ <stationSet>Custom</stationSet>
+ <stations>CS001,CS002,CS003,CS004,CS005,CS006,CS007,CS011,CS013,CS017,CS021,CS024,CS026,CS028,CS030,CS031,CS032,CS101,CS103,CS201,CS301,CS302,CS401,CS501</stations>
+ <timeFrame>UT</timeFrame>
+ <startTime>2017-05-09T03:00:00</startTime>
+ <endTime>2017-05-09T03:10:00</endTime>
+ <duration>600</duration>
+ <bypassPff>false</bypassPff>
+ <enableSuperterp>false</enableSuperterp>
+ <numberOfBitsPerSample>8</numberOfBitsPerSample>
+ </userSpecification>
+ </lofar:observationAttributes>
+ <children>
+<item index="0">
+ <lofar:measurement xsi:type="lofar:BFMeasurementType">
+ <name>Crab pointing 1</name>
+ <description>Crab pointing 1</description>
+ <topology>B0.1.T.SAP000</topology>
+ <currentStatus>
+ <mom2:openedStatus/>
+ </currentStatus>
+ <lofar:bfMeasurementAttributes>
+ <measurementType>Target</measurementType>
+ <specification>
+ <targetName>Crab pointing 1</targetName>
+ <ra>83.7164166667</ra>
+ <dec>22.0145</dec>
+ <equinox>J2000</equinox>
+ <duration>0</duration>
+ <subbandsSpecification>
+ <subbands>98..297</subbands>
+ </subbandsSpecification>
+ <tiedArrayBeams>
+ <flyseye>false</flyseye>
+ <nrTabRings>2</nrTabRings>
+ <tabRingSize>0.1</tabRingSize>
+ <tiedArrayBeamList>
+
+ </tiedArrayBeamList>
+ </tiedArrayBeams>
+ </specification>
+ </lofar:bfMeasurementAttributes>
+ <resultDataProducts>
+ <item>
+ <lofar:uvDataProduct>
+ <name>B0.1.T.SAP000.uv.dps</name>
+ <topology>B0.1.T.SAP000.uv.dps</topology>
+ <status>no_data</status>
+ <storageCluster>
+ <name>DRAGNET</name>
+ <partition></partition>
+ </storageCluster>
+ </lofar:uvDataProduct>
+ </item>
+ <item>
+ <lofar:bfDataProduct>
+ <name>B0.1.T.SAP000.csis</name>
+ <topology>B0.1.T.SAP000.csis</topology>
+ <status>no_data</status>
+ <storageCluster>
+ <name>DRAGNET</name>
+ <partition></partition>
+ </storageCluster>
+ </lofar:bfDataProduct>
+ </item>
+ </resultDataProducts>
+ </lofar:measurement>
+ </item>
+<item index="0">
+ <lofar:measurement xsi:type="lofar:BFMeasurementType">
+ <name>Crab pointing 2</name>
+ <description>Crab pointing 2</description>
+ <topology>B0.1.T.SAP001</topology>
+ <currentStatus>
+ <mom2:openedStatus/>
+ </currentStatus>
+ <lofar:bfMeasurementAttributes>
+ <measurementType>Target</measurementType>
+ <specification>
+ <targetName>Crab pointing 2</targetName>
+ <ra>83.71675</ra>
+ <dec>22.0145833333</dec>
+ <equinox>J2000</equinox>
+ <duration>0</duration>
+ <subbandsSpecification>
+ <subbands>297..396</subbands>
+ </subbandsSpecification>
+ <tiedArrayBeams>
+ <flyseye>false</flyseye>
+ <nrTabRings>2</nrTabRings>
+ <tabRingSize>0.1</tabRingSize>
+ <tiedArrayBeamList>
+
+ </tiedArrayBeamList>
+ </tiedArrayBeams>
+ </specification>
+ </lofar:bfMeasurementAttributes>
+ <resultDataProducts>
+ <item>
+ <lofar:uvDataProduct>
+ <name>B0.1.T.SAP001.uv.dps</name>
+ <topology>B0.1.T.SAP001.uv.dps</topology>
+ <status>no_data</status>
+ <storageCluster>
+ <name>DRAGNET</name>
+ <partition></partition>
+ </storageCluster>
+ </lofar:uvDataProduct>
+ </item>
+ <item>
+ <lofar:bfDataProduct>
+ <name>B0.1.T.SAP001.csis</name>
+ <topology>B0.1.T.SAP001.csis</topology>
+ <status>no_data</status>
+ <storageCluster>
+ <name>DRAGNET</name>
+ <partition></partition>
+ </storageCluster>
+ </lofar:bfDataProduct>
+ </item>
+ </resultDataProducts>
+ </lofar:measurement>
+ </item>
+</children>
+ </lofar:observation>
+ </item>
+</children>
+ </lofar:folder>
+ </item>
+</children>
+ </lofar:folder>
+ </item>
+ </children>
+ </lofar:project>
diff --git a/SAS/XML_generator/test/test_regression.in_data/xml/test_input_long_baseline_pipeline.xml b/SAS/XML_generator/test/test_regression.in_data/xml/test_input_long_baseline_pipeline.xml
index 1a3a088592ac9ecbe41ccbac5d71295cb6a48d71..dc4f371d0c49f1d1c7586de32b8d7230fa458a0f 100644
--- a/SAS/XML_generator/test/test_regression.in_data/xml/test_input_long_baseline_pipeline.xml
+++ b/SAS/XML_generator/test/test_regression.in_data/xml/test_input_long_baseline_pipeline.xml
@@ -594,8 +594,8 @@
<pipelineAttributes>
<defaultTemplate>Long-Baseline Pipeline</defaultTemplate>
<duration>600</duration>
- <subbandsPerSubbandGroup>8</subbandsPerSubbandGroup>
- <subbandGroupsPerMS>8</subbandGroupsPerMS>
+ <subbandsPerSubbandGroup>4</subbandsPerSubbandGroup>
+ <subbandGroupsPerMS>6</subbandGroupsPerMS>
</pipelineAttributes>
<usedDataProducts>
<item>