diff --git a/SAS/ResourceAssignment/RAtoOTDBTaskSpecificationPropagator/lib/translator.py b/SAS/ResourceAssignment/RAtoOTDBTaskSpecificationPropagator/lib/translator.py
index e1d1e2cd4e532f1e82934a08c66b045c529502bf..8cbd4ac388fcfd5a45287ed45bcee1de577d6505 100755
--- a/SAS/ResourceAssignment/RAtoOTDBTaskSpecificationPropagator/lib/translator.py
+++ b/SAS/ResourceAssignment/RAtoOTDBTaskSpecificationPropagator/lib/translator.py
@@ -95,9 +95,11 @@ class RAtoOTDBTranslator():
result = {}
nr_stokes = storage_properties['nr_of_cs_stokes']
for sap in storage_properties["saps"]: ##We might need to sort saps?
- if "nr_of_cs_files" in sap['properties']:
+ if 'nr_of_cs_files' in sap['properties']:
nr_files = sap['properties']['nr_of_cs_files']
nr_tabs = sap['properties']['nr_of_tabs']
+ if 'is_tab_nr' in sap['properties']:
+ nr_tabs -= 1
nr_parts = int(ceil(nr_files/float(nr_tabs * nr_stokes)))
for tab in xrange(nr_tabs):
for stokes in xrange(nr_stokes):
@@ -118,15 +120,14 @@ class RAtoOTDBTranslator():
nr_stokes = storage_properties['nr_of_is_stokes']
for sap in storage_properties["saps"]: ##We might need to sort saps?
if "nr_of_is_files" in sap['properties']:
- nr_files = sap['properties']['nr_of_is_files']
- nr_tabs = sap['properties']['nr_of_tabs']
- nr_parts = int(ceil(nr_files/float(nr_tabs * nr_stokes)))
- for tab in xrange(nr_tabs):
- for stokes in xrange(nr_stokes):
- for part in xrange(nr_parts):
- locations.append(self.locationPath(project_name, otdb_id) + '/is')
- filenames.append("L%d_SAP%03d_B%03d_S%d_P%03d_bf.h5" % (otdb_id, sap['sap_nr'], tab, stokes, part))
- skip.append("0")
+ nr_files = sap['properties']['nr_of_is_files']
+ is_tab_nr = sap['properties']['is_tab_nr']
+ nr_parts = int(ceil(nr_files/float(nr_tabs * nr_stokes)))
+ for stokes in xrange(nr_stokes):
+ for part in xrange(nr_parts):
+ locations.append(self.locationPath(project_name, otdb_id) + '/is')
+ filenames.append("L%d_SAP%03d_B%03d_S%d_P%03d_bf.h5" % (otdb_id, sap['sap_nr'], is_tab_nr, stokes, part))
+ skip.append("0")
result[PREFIX + 'DataProducts.%s_IncoherentStokes.locations' % (io_type)] = '[' + to_csv_string(locations) + ']'
result[PREFIX + 'DataProducts.%s_IncoherentStokes.filenames' % (io_type)] = '[' + to_csv_string(filenames) + ']'
result[PREFIX + 'DataProducts.%s_IncoherentStokes.skip' % (io_type)] = '[' + to_csv_string(skip) + ']'
diff --git a/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/calibration_pipeline.py b/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/calibration_pipeline.py
index 5d7cb3a07ee81923c567dc8c6193b36adf8b18da..524a0356c254daa6c5cb32bea6866b9dd1386dc3 100644
--- a/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/calibration_pipeline.py
+++ b/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/calibration_pipeline.py
@@ -115,7 +115,15 @@ class CalibrationPipelineResourceEstimator(BaseResourceEstimator):
#total_data_size = result['storage']['output_files']['uv']['nr_of_uv_files'] * result['storage']['output_files']['uv']['uv_file_size'] + \
# result['storage']['output_files']['im']['nr_of_im_files'] * result['storage']['output_files']['im']['im_file_size'] # bytes
total_bandwidth = int(ceil((total_data_size * 8) / duration)) # bits/second
- result['storage']['total_size'] = total_data_size
- result['bandwidth']['total_size'] = total_bandwidth
+ if total_data_size and output_files:
+ result['storage']['total_size'] = total_data_size
+ result['bandwidth']['total_size'] = total_bandwidth
+ else:
+ if not total_data_size:
+ result['errors'].append('Total data size is zero!')
+ logger.warning('ERROR: A datasize of zero was calculated!')
+ if not output_files:
+ result['errors'].append('No output files!')
+ logger.warning('ERROR: No output files were calculated!')
return result
diff --git a/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/image_pipeline.py b/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/image_pipeline.py
index 5314e3b81ecd5843da060da7a4c0a4855255c626..3050fd401346e40254b806c93dd4a10bc3fde36a 100644
--- a/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/image_pipeline.py
+++ b/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/image_pipeline.py
@@ -98,7 +98,15 @@ class ImagePipelineResourceEstimator(BaseResourceEstimator):
# count total data size
total_data_size = result['storage']['output_files']['img']['nr_of_img_files'] * result['storage']['output_files']['img']['img_file_size'] # bytes
total_bandwidth = int(ceil((total_data_size * 8) / duration)) # bits/second
- result['storage']['total_size'] = total_data_size
- result['bandwidth']['total_size'] = total_bandwidth
+ if total_data_size and output_files:
+ result['storage']['total_size'] = total_data_size
+ result['bandwidth']['total_size'] = total_bandwidth
+ else:
+ if not total_data_size:
+ result['errors'].append('Total data size is zero!')
+ logger.warning('ERROR: A datasize of zero was calculated!')
+ if not output_files:
+ result['errors'].append('No output files!')
+ logger.warning('ERROR: No output files were calculated!')
return result
diff --git a/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/longbaseline_pipeline.py b/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/longbaseline_pipeline.py
index 6014e46e9fc315cc5e52171b09d1924a67e45f66..2be090a2cabe54c61b2be7967397f3a8e4dbf579 100644
--- a/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/longbaseline_pipeline.py
+++ b/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/longbaseline_pipeline.py
@@ -99,6 +99,14 @@ class LongBaselinePipelineResourceEstimator(BaseResourceEstimator):
# count total data size
total_data_size = result['storage']['output_files']['uv']['nr_of_uv_files'] * result['storage']['output_files']['uv']['uv_file_size'] # bytes
total_bandwidth = int(ceil((total_data_size * 8) / duration)) # bits/second
- result['storage']['total_size'] = total_data_size
- result['bandwidth']['total_size'] = total_bandwidth
+ if total_data_size and output_files:
+ result['storage']['total_size'] = total_data_size
+ result['bandwidth']['total_size'] = total_bandwidth
+ else:
+ if not total_data_size:
+ result['errors'].append('Total data size is zero!')
+ logger.warning('ERROR: A datasize of zero was calculated!')
+ if not output_files:
+ result['errors'].append('No output files!')
+ logger.warning('ERROR: No output files were calculated!')
return result
diff --git a/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/observation.py b/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/observation.py
index cdcca21fce53e991da408a26f26aef6f4edfb990..a893e1bac20a9e8610c8536ee002c5541a21bfbc 100644
--- a/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/observation.py
+++ b/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/observation.py
@@ -95,6 +95,10 @@ class ObservationResourceEstimator(BaseResourceEstimator):
sap['properties'].update(coherentstokes_saps[sap_nr])
if sap_nr in incoherentstokes_saps:
sap['properties'].update(incoherentstokes_saps[sap_nr])
+ if 'nr_of_tabs' in sap['properties']: # These are coherent TABs
+ sap['properties']['nr_of_tabs'] = sap['properties']['nr_of_tabs'] + 1
+ else:
+ sap['properties']['nr_of_tabs'] = 1 # Only an incoherent TAB for this SAP
output_files['saps'].append(sap)
@@ -187,7 +191,7 @@ class ObservationResourceEstimator(BaseResourceEstimator):
logger.info("checking TAB {}".format(tab_nr))
if parset.getBool("Observation.Beam[%d].TiedArrayBeam[%d].coherent" % (sap_nr, tab_nr)):
logger.info("adding coherentstokes size")
- nr_stokes = nr_coherent #TODO what does min mean here?
+ nr_stokes = nr_coherent # TODO, there used to be a function with min() here?
total_nr_tabs += 1
total_nr_stokes += nr_stokes
nr_files += int(nr_stokes * ceil(nr_subbands / float(subbands_per_file)))
@@ -238,7 +242,7 @@ class ObservationResourceEstimator(BaseResourceEstimator):
channels_per_subband = parset.getInt(COBALT + 'Correlator.nrChannelsPerSubband', 64) #TODO should these have defaults?
incoherent_channels_per_subband = parset.getInt(COBALT + 'BeamFormer.IncoherentStokes.nrChannelsPerSubband', 0)
- nr_incoherent = 4 if incoherent_type in ('IQUV',) else 1
+ nr_incoherent = 4 if incoherent_type in ('IQUV',) else 1 # Should this also include XXYY ?
total_nr_stokes = 0
total_files = 0
@@ -251,17 +255,22 @@ class ObservationResourceEstimator(BaseResourceEstimator):
nr_subbands = len(subbandList)
max_nr_subbands = max(nr_subbands, max_nr_subbands)
nr_files = 0
+ is_tab_nr = -1
total_nr_tabs = parset.getInt('Observation.Beam[%d].nrTiedArrayBeams' % sap_nr)
for tab_nr in xrange(total_nr_tabs):
logger.info("checking TAB {}".format(tab_nr))
if not parset.getBool("Observation.Beam[%d].TiedArrayBeam[%d].coherent" % (sap_nr, tab_nr)): #not coherent is incoherent
logger.info("Found incoherent stokes TAB: %i" % tab_nr)
- is_tab_nr = tab_nr
- total_nr_stokes += nr_incoherent
- nr_files += int(nr_incoherent * ceil(nr_subbands / float(subbands_per_file)))
+ if is_tab_nr >= 0:
+ logger.warning("TAB nr %i can't be incoherent as %i already is!" % (tab_nr, is_tab_nr))
+ # TODO We need to generate an error here, or preferably check before we get here
+ else:
+ is_tab_nr = tab_nr
+ total_nr_stokes += nr_incoherent
+ nr_files += int(nr_incoherent * ceil(nr_subbands / float(subbands_per_file)))
if nr_files:
- sap_files[sap_nr] = {'nr_of_is_files': nr_files, 'nr_of_tabs': total_nr_tabs, 'is_tab_nr': is_tab_nr}
+ sap_files[sap_nr] = {'nr_of_is_files': nr_files, 'is_tab_nr': is_tab_nr}
total_files += nr_files
if incoherent_channels_per_subband > 0:
diff --git a/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/pulsar_pipeline.py b/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/pulsar_pipeline.py
index a024a9db3c34167516b8fa9bacbecb787d2b5846..7dbe0a2b79085be1801d75a2075d8d233ebe1c1a 100644
--- a/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/pulsar_pipeline.py
+++ b/SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/pulsar_pipeline.py
@@ -99,6 +99,14 @@ class PulsarPipelineResourceEstimator(BaseResourceEstimator):
# count total data size
total_data_size = result['storage']['output_files']['pulp']['nr_of_pulp_files'] * result['storage']['output_files']['pulp']['pulp_file_size'] # bytes
total_bandwidth = int(ceil((total_data_size * 8) / duration)) # bits/second
- result['storage']['total_size'] = total_data_size
- result['bandwidth']['total_size'] = total_bandwidth
+ if total_data_size and output_files:
+ result['storage']['total_size'] = total_data_size
+ result['bandwidth']['total_size'] = total_bandwidth
+ else:
+ if not total_data_size:
+ result['errors'].append('Total data size is zero!')
+ logger.warning('ERROR: A datasize of zero was calculated!')
+ if not output_files:
+ result['errors'].append('No output files!')
+ logger.warning('ERROR: No output files were calculated!')
return result