diff --git a/SAS/ResourceAssignment/ResourceAssigner/lib/assignment.py b/SAS/ResourceAssignment/ResourceAssigner/lib/assignment.py index 64caf997b00f6f17c305ce797662babdadb56085..a7df1b6dac0f60a7815d1471fb84709f5f0ab562 100755 --- a/SAS/ResourceAssignment/ResourceAssigner/lib/assignment.py +++ b/SAS/ResourceAssignment/ResourceAssigner/lib/assignment.py @@ -495,8 +495,7 @@ class ResourceAssigner(): # Compile list of resource group IDs whose resources we care about, # then filter db resource list with it. -# TODO: tmp: deal with [0] only - needed_root_resource_group_name = needed_res['root_resource_groups'][0] + needed_root_resource_group_name = needed_res['root_resource_group'] target_resgids = self._get_subtree_resource_group_ids(db_rgp2rgp, needed_root_resource_group_name) db_resource_list.extend([r for r in db_needed_resource_list if db_r2rgp[r['id']]['parent_group_ids'][0] in target_resgids]) diff --git a/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/observation.py b/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/observation.py index 226626131e3886944f7c1053e28818787f353685..5161b7bd980fb21820eda83a9f65ee5e72b38271 100644 --- a/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/observation.py +++ b/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/observation.py @@ -65,7 +65,7 @@ class ObservationResourceEstimator(BaseResourceEstimator): The following return value example is for an obs duration of 240.0 s and 3 data product types for 2 clusters. Here we have UV data for CEP4 (100 files), and we have for DRAGNET 2 IS (IQUV) and 7 CS TABs (also IQUV), each split across 5 parts (i.e. 7*4*5 + 2*4*5 = 180 files). - If estimate sizes and root_resource_groups are the same, estimates can be combined into 1 estimate (partly done here). + If estimate sizes and root_resource_group are the same, estimates can be combined into 1 estimate (partly done here). (Not shown here, but note that for CS complex voltage data (XXYY), we must produce 1 estimate per 4 files (XXYY), such that the XXYY files for a part end up on the same storage. Such a beam can still be split across >1 parts.) @@ -77,7 +77,7 @@ class ObservationResourceEstimator(BaseResourceEstimator): 'errors': [], 'estimates': [{ 'resource_types': {'bandwidth': 536870912, 'storage': 128849018880}, - 'count': 1, 'root_resource_groups': ['CEP4'], # count is per root resource group! + 'count': 1, 'root_resource_group': 'CEP4', 'output_files': { 'uv': {'nr_of_uv_files': 100, 'uv_file_size': 1073741824, 'identifications': [...]}, 'saps': [{'sap_nr': 0, 'properties': {'nr_of_uv_files': 80, 'start_sb_nr': 0}}, @@ -86,7 +86,7 @@ class ObservationResourceEstimator(BaseResourceEstimator): ] } }, {'resources': {'bandwidth': 8947849, 'storage': 2147483648}, - 'count': 180, 'root_resource_groups': ['DRAGNET'], # count is per root resource group! + 'count': 180, 'root_resource_group': 'DRAGNET', 'output_files': { 'cs': {'nr_of_cs_files': 140, 'cs_file_size': 2147483648, 'nr_of_cs_stokes': 4, 'identifications': [...]}, 'is': {'nr_of_is_files': 40, 'is_file_size': 2147483648, 'nr_of_is_stokes': 4, 'identifications': [...]}, @@ -199,7 +199,7 @@ class ObservationResourceEstimator(BaseResourceEstimator): total_files = 1 estimate = {'resource_types': {'bandwidth': bandwidth, 'storage': file_size}, - 'count': total_files, 'root_resource_groups': [root_resource_group], + 'count': total_files, 'root_resource_group': root_resource_group, 'output_files': output_files} logger.debug("Correlated data estimate: {}".format(estimate)) return estimate @@ -289,7 +289,7 @@ class ObservationResourceEstimator(BaseResourceEstimator): total_files = 1 estimate = {'resource_types': {'bandwidth': bandwidth, 'storage': file_size}, - 'count': total_files, 'root_resource_groups': [root_resource_group], + 'count': total_files, 'root_resource_group': root_resource_group, 'output_files': output_files} estimate['output_files']['cs']['nr_of_cs_stokes'] = nr_coherent @@ -367,7 +367,7 @@ class ObservationResourceEstimator(BaseResourceEstimator): total_files = 1 estimate = {'resource_types': {'bandwidth': bandwidth, 'storage': file_size}, - 'count': total_files, 'root_resource_groups': [root_resource_group], + 'count': total_files, 'root_resource_group': root_resource_group, 'output_files': output_files} estimate['output_files']['is']['nr_of_is_stokes'] = nr_incoherent @@ -375,17 +375,17 @@ class ObservationResourceEstimator(BaseResourceEstimator): return estimate def _merge_estimates(self, estimates): - """ Estimates can only be merged if same root_resource_groups and bandwidth and storage, - or if the root_resource_groups have a (single) global filesystem. + """ Estimates can only be merged if same root_resource_group and bandwidth and storage, + or if the root_resource_group has a (single) global filesystem. NOTE: assumed good enough to only merge conseq pairs, not all pairs. """ i = 1 while i < len(estimates): # careful iterating while modifying - if estimates[i-1]['root_resource_groups'] == estimates[i]['root_resource_groups'] and \ + if estimates[i-1]['root_resource_group'] == estimates[i]['root_resource_group'] and \ ((estimates[i-1]['resource_types']['bandwidth'] == estimates[i]['resource_types']['bandwidth'] and \ estimates[i-1]['resource_types']['storage'] == estimates[i]['resource_types']['storage']) or \ - all(self._hasGlobalStorage(rg) for rg in root_resource_groups) + self._hasGlobalStorage(estimates[i]['root_resource_group']) ): # Mergeable. Add uv, cs, is from estimates[i] into estimates[i-1] if 'uv' in estimates[i]['output_files']: @@ -414,7 +414,7 @@ class ObservationResourceEstimator(BaseResourceEstimator): else: j += 1 - if all(self._hasGlobalStorage(rg) for rg in estimates[i]['root_resource_groups']): + if self._hasGlobalStorage(estimates[i]['root_resource_group']): # for global fs, collapse regardless estimates[i-1]['resource_types']['bandwidth'] *= estimates[i-1]['count'] # *= 1, but to be robust and clear estimates[i-1]['resource_types']['bandwidth'] += estimates[i]['resource_types']['bandwidth'] * estimates[i]['count'] @@ -422,7 +422,7 @@ class ObservationResourceEstimator(BaseResourceEstimator): estimates[i-1]['resource_types']['storage'] += estimates[i]['resource_types']['storage'] * estimates[i]['count'] estimates[i-1]['count'] = 1 # already 1, but to be robust and clear else: - # root_resource_groups and values of bandwidth and storage are equal for both estimates + # root_resource_group and values of bandwidth and storage are equal for both estimates estimates[i-1]['count'] += estimates[i]['count'] logger.info('Merged observation resource estimate {} into {}'.format(i, i-1))