Task #9939: Resource estimate fmt: pass single root_resource_group instead of...

Task #9939: Resource estimate fmt: pass single root_resource_group instead of list: enough for DRAGNET work.

SAS/ResourceAssignment/ResourceAssigner/lib/assignment.py

@@ -495,8 +495,7 @@ class ResourceAssigner():
 
             # Compile list of resource group IDs whose resources we care about,
             # then filter db resource list with it.
-# TODO: tmp: deal with [0] only
-            needed_root_resource_group_name = needed_res['root_resource_groups'][0]
+            needed_root_resource_group_name = needed_res['root_resource_group']
             target_resgids = self._get_subtree_resource_group_ids(db_rgp2rgp, needed_root_resource_group_name)
             db_resource_list.extend([r for r in db_needed_resource_list if db_r2rgp[r['id']]['parent_group_ids'][0] in target_resgids])
 

SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/observation.py

@@ -65,7 +65,7 @@ class ObservationResourceEstimator(BaseResourceEstimator):
             The following return value example is for an obs duration of 240.0 s and 3 data product types for 2 clusters.
             Here we have UV data for CEP4 (100 files), and we have for DRAGNET 2 IS (IQUV) and 7 CS TABs (also IQUV),
             each split across 5 parts (i.e. 7*4*5 + 2*4*5 = 180 files).
-            If estimate sizes and root_resource_groups are the same, estimates can be combined into 1 estimate (partly done here).
+            If estimate sizes and root_resource_group are the same, estimates can be combined into 1 estimate (partly done here).
             (Not shown here, but note that for CS complex voltage data (XXYY), we must produce 1 estimate per 4 files (XXYY),
              such that the XXYY files for a part end up on the same storage. Such a beam can still be split across >1 parts.)
 
@@ -77,7 +77,7 @@ class ObservationResourceEstimator(BaseResourceEstimator):
             'errors': [],
             'estimates': [{
                 'resource_types': {'bandwidth': 536870912, 'storage': 128849018880},
-                'count': 1, 'root_resource_groups': ['CEP4'],  # count is per root resource group!
+                'count': 1, 'root_resource_group': 'CEP4',
                 'output_files': {
                     'uv': {'nr_of_uv_files': 100, 'uv_file_size': 1073741824, 'identifications': [...]},
                     'saps': [{'sap_nr': 0, 'properties': {'nr_of_uv_files': 80, 'start_sb_nr': 0}},
@@ -86,7 +86,7 @@ class ObservationResourceEstimator(BaseResourceEstimator):
                     ]
                 }
             }, {'resources': {'bandwidth': 8947849, 'storage': 2147483648},
-                'count': 180, 'root_resource_groups': ['DRAGNET'],  # count is per root resource group!
+                'count': 180, 'root_resource_group': 'DRAGNET',
                 'output_files': {
                     'cs': {'nr_of_cs_files': 140, 'cs_file_size': 2147483648, 'nr_of_cs_stokes': 4, 'identifications': [...]},
                     'is': {'nr_of_is_files': 40,  'is_file_size': 2147483648, 'nr_of_is_stokes': 4, 'identifications': [...]},
@@ -199,7 +199,7 @@ class ObservationResourceEstimator(BaseResourceEstimator):
             total_files = 1
 
         estimate = {'resource_types': {'bandwidth': bandwidth, 'storage': file_size},
-                    'count': total_files, 'root_resource_groups': [root_resource_group],
+                    'count': total_files, 'root_resource_group': root_resource_group,
                     'output_files': output_files}
         logger.debug("Correlated data estimate: {}".format(estimate))
         return estimate
@@ -289,7 +289,7 @@ class ObservationResourceEstimator(BaseResourceEstimator):
             total_files = 1
 
         estimate = {'resource_types': {'bandwidth': bandwidth, 'storage': file_size},
-                    'count': total_files, 'root_resource_groups': [root_resource_group],
+                    'count': total_files, 'root_resource_group': root_resource_group,
                     'output_files': output_files}
         estimate['output_files']['cs']['nr_of_cs_stokes'] = nr_coherent
 
@@ -367,7 +367,7 @@ class ObservationResourceEstimator(BaseResourceEstimator):
             total_files = 1
 
         estimate = {'resource_types': {'bandwidth': bandwidth, 'storage': file_size},
-                    'count': total_files, 'root_resource_groups': [root_resource_group],
+                    'count': total_files, 'root_resource_group': root_resource_group,
                     'output_files': output_files}
         estimate['output_files']['is']['nr_of_is_stokes'] = nr_incoherent
 
@@ -375,17 +375,17 @@ class ObservationResourceEstimator(BaseResourceEstimator):
         return estimate
 
     def _merge_estimates(self, estimates):
-        """ Estimates can only be merged if same root_resource_groups and bandwidth and storage,
-            or if the root_resource_groups have a (single) global filesystem.
+        """ Estimates can only be merged if same root_resource_group and bandwidth and storage,
+            or if the root_resource_group has a (single) global filesystem.
 
             NOTE: assumed good enough to only merge conseq pairs, not all pairs.
         """
         i = 1
         while i < len(estimates):  # careful iterating while modifying
-            if estimates[i-1]['root_resource_groups'] == estimates[i]['root_resource_groups'] and \
+            if estimates[i-1]['root_resource_group'] == estimates[i]['root_resource_group'] and \
                ((estimates[i-1]['resource_types']['bandwidth'] == estimates[i]['resource_types']['bandwidth'] and \
                  estimates[i-1]['resource_types']['storage']   == estimates[i]['resource_types']['storage']) or \
-                all(self._hasGlobalStorage(rg) for rg in root_resource_groups)
+                self._hasGlobalStorage(estimates[i]['root_resource_group'])
                ):
                 # Mergeable. Add uv, cs, is from estimates[i] into estimates[i-1]
                 if 'uv' in estimates[i]['output_files']:
@@ -414,7 +414,7 @@ class ObservationResourceEstimator(BaseResourceEstimator):
                     else:
                         j += 1
 
-                if all(self._hasGlobalStorage(rg) for rg in estimates[i]['root_resource_groups']):
+                if self._hasGlobalStorage(estimates[i]['root_resource_group']):
                     # for global fs, collapse regardless
                     estimates[i-1]['resource_types']['bandwidth'] *= estimates[i-1]['count']  # *= 1, but to be robust and clear
                     estimates[i-1]['resource_types']['bandwidth'] += estimates[i]['resource_types']['bandwidth'] * estimates[i]['count']
@@ -422,7 +422,7 @@ class ObservationResourceEstimator(BaseResourceEstimator):
                     estimates[i-1]['resource_types']['storage'] += estimates[i]['resource_types']['storage'] * estimates[i]['count']
                     estimates[i-1]['count'] = 1  # already 1, but to be robust and clear
                 else:
-                    # root_resource_groups and values of bandwidth and storage are equal for both estimates
+                    # root_resource_group and values of bandwidth and storage are equal for both estimates
                     estimates[i-1]['count'] += estimates[i]['count']
  
                 logger.info('Merged observation resource estimate {} into {}'.format(i, i-1))