Task #9667: Fixes for correct number of output files when using beamformed with tab rings

SAS/ResourceAssignment/RAtoOTDBTaskSpecificationPropagator/lib/translator.py

@@ -95,9 +95,11 @@ class RAtoOTDBTranslator():
         result = {}
         nr_stokes = storage_properties['nr_of_cs_stokes']
         for sap in storage_properties["saps"]: ##We might need to sort saps?
-            if "nr_of_cs_files" in sap['properties']:
+            if 'nr_of_cs_files' in sap['properties']:
                 nr_files = sap['properties']['nr_of_cs_files']
                 nr_tabs  = sap['properties']['nr_of_tabs']
+                if 'is_tab_nr' in sap['properties']:
+                    nr_tabs -= 1
                 nr_parts = int(ceil(nr_files/float(nr_tabs * nr_stokes)))
                 for tab in xrange(nr_tabs):
                     for stokes in xrange(nr_stokes):
@@ -119,13 +121,12 @@ class RAtoOTDBTranslator():
         for sap in storage_properties["saps"]: ##We might need to sort saps?
             if "nr_of_is_files" in sap['properties']:
                 nr_files  = sap['properties']['nr_of_is_files']
-                nr_tabs  = sap['properties']['nr_of_tabs']
+                is_tab_nr = sap['properties']['is_tab_nr']
                 nr_parts  = int(ceil(nr_files/float(nr_tabs * nr_stokes)))
-                for tab in xrange(nr_tabs):
                 for stokes in xrange(nr_stokes):
                     for part in xrange(nr_parts):
                         locations.append(self.locationPath(project_name, otdb_id) + '/is')
-                            filenames.append("L%d_SAP%03d_B%03d_S%d_P%03d_bf.h5" % (otdb_id, sap['sap_nr'], tab, stokes, part))
+                        filenames.append("L%d_SAP%03d_B%03d_S%d_P%03d_bf.h5" % (otdb_id, sap['sap_nr'], is_tab_nr, stokes, part))
                         skip.append("0")
         result[PREFIX + 'DataProducts.%s_IncoherentStokes.locations' % (io_type)] = '[' + to_csv_string(locations) + ']'
         result[PREFIX + 'DataProducts.%s_IncoherentStokes.filenames' % (io_type)] = '[' + to_csv_string(filenames) + ']'

SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/calibration_pipeline.py

@@ -115,7 +115,15 @@ class CalibrationPipelineResourceEstimator(BaseResourceEstimator):
         #total_data_size = result['storage']['output_files']['uv']['nr_of_uv_files'] * result['storage']['output_files']['uv']['uv_file_size'] + \
         #                  result['storage']['output_files']['im']['nr_of_im_files'] * result['storage']['output_files']['im']['im_file_size']  # bytes
         total_bandwidth = int(ceil((total_data_size * 8) / duration))  # bits/second
+        if total_data_size and output_files:
             result['storage']['total_size'] = total_data_size
             result['bandwidth']['total_size'] = total_bandwidth
+        else:
+            if not total_data_size:
+                result['errors'].append('Total data size is zero!')
+                logger.warning('ERROR: A datasize of zero was calculated!')
+            if not output_files:
+                result['errors'].append('No output files!')
+                logger.warning('ERROR: No output files were calculated!')
         return result
 

SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/image_pipeline.py

@@ -98,7 +98,15 @@ class ImagePipelineResourceEstimator(BaseResourceEstimator):
         # count total data size
         total_data_size = result['storage']['output_files']['img']['nr_of_img_files'] * result['storage']['output_files']['img']['img_file_size'] # bytes
         total_bandwidth = int(ceil((total_data_size * 8) / duration))  # bits/second
+        if total_data_size and output_files:
             result['storage']['total_size'] = total_data_size
             result['bandwidth']['total_size'] = total_bandwidth
+        else:
+            if not total_data_size:
+                result['errors'].append('Total data size is zero!')
+                logger.warning('ERROR: A datasize of zero was calculated!')
+            if not output_files:
+                result['errors'].append('No output files!')
+                logger.warning('ERROR: No output files were calculated!')
         return result
 

SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/longbaseline_pipeline.py

@@ -99,6 +99,14 @@ class LongBaselinePipelineResourceEstimator(BaseResourceEstimator):
         # count total data size
         total_data_size = result['storage']['output_files']['uv']['nr_of_uv_files'] * result['storage']['output_files']['uv']['uv_file_size'] # bytes
         total_bandwidth = int(ceil((total_data_size * 8) / duration))  # bits/second
+        if total_data_size and output_files:
             result['storage']['total_size'] = total_data_size
             result['bandwidth']['total_size'] = total_bandwidth
+        else:
+            if not total_data_size:
+                result['errors'].append('Total data size is zero!')
+                logger.warning('ERROR: A datasize of zero was calculated!')
+            if not output_files:
+                result['errors'].append('No output files!')
+                logger.warning('ERROR: No output files were calculated!')
         return result

SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/observation.py

@@ -95,6 +95,10 @@ class ObservationResourceEstimator(BaseResourceEstimator):
                 sap['properties'].update(coherentstokes_saps[sap_nr])
             if sap_nr in incoherentstokes_saps:
                 sap['properties'].update(incoherentstokes_saps[sap_nr])
+                if 'nr_of_tabs' in sap['properties']: # These are coherent TABs
+                    sap['properties']['nr_of_tabs'] = sap['properties']['nr_of_tabs'] + 1
+                else:
+                    sap['properties']['nr_of_tabs'] = 1 # Only an incoherent TAB for this SAP
             output_files['saps'].append(sap)
 
 
@@ -187,7 +191,7 @@ class ObservationResourceEstimator(BaseResourceEstimator):
                 logger.info("checking TAB {}".format(tab_nr))
                 if parset.getBool("Observation.Beam[%d].TiedArrayBeam[%d].coherent" % (sap_nr, tab_nr)):
                     logger.info("adding coherentstokes size")
-                    nr_stokes = nr_coherent #TODO what does min mean here?
+                    nr_stokes = nr_coherent # TODO, there used to be a function with min() here?
                     total_nr_tabs += 1
                     total_nr_stokes += nr_stokes
                     nr_files += int(nr_stokes * ceil(nr_subbands / float(subbands_per_file)))
@@ -238,7 +242,7 @@ class ObservationResourceEstimator(BaseResourceEstimator):
         channels_per_subband = parset.getInt(COBALT + 'Correlator.nrChannelsPerSubband', 64) #TODO should these have defaults?
         incoherent_channels_per_subband = parset.getInt(COBALT + 'BeamFormer.IncoherentStokes.nrChannelsPerSubband', 0)
 
-        nr_incoherent = 4 if incoherent_type in ('IQUV',) else 1
+        nr_incoherent = 4 if incoherent_type in ('IQUV',) else 1 # Should this also include XXYY ?
 
         total_nr_stokes = 0
         total_files     = 0
@@ -251,17 +255,22 @@ class ObservationResourceEstimator(BaseResourceEstimator):
             nr_subbands = len(subbandList)
             max_nr_subbands = max(nr_subbands, max_nr_subbands)
             nr_files = 0
+            is_tab_nr = -1
             total_nr_tabs = parset.getInt('Observation.Beam[%d].nrTiedArrayBeams' % sap_nr)
             for tab_nr in xrange(total_nr_tabs):
                 logger.info("checking TAB {}".format(tab_nr))
                 if not parset.getBool("Observation.Beam[%d].TiedArrayBeam[%d].coherent" % (sap_nr, tab_nr)): #not coherent is incoherent
                     logger.info("Found incoherent stokes TAB: %i" % tab_nr)
+                    if is_tab_nr >= 0:
+                        logger.warning("TAB nr %i can't be incoherent as %i already is!" % (tab_nr, is_tab_nr))
+                        # TODO We need to generate an error here, or preferably check before we get here
+                    else:
                         is_tab_nr = tab_nr
                         total_nr_stokes += nr_incoherent
                         nr_files += int(nr_incoherent * ceil(nr_subbands / float(subbands_per_file)))
             
             if nr_files:
-                sap_files[sap_nr] = {'nr_of_is_files': nr_files, 'nr_of_tabs': total_nr_tabs, 'is_tab_nr': is_tab_nr}
+                sap_files[sap_nr] = {'nr_of_is_files': nr_files, 'is_tab_nr': is_tab_nr}
                 total_files += nr_files
 
         if incoherent_channels_per_subband > 0:

SAS/ResourceAssignment/ResourceAssignmentEstimator/resource_estimators/pulsar_pipeline.py

@@ -99,6 +99,14 @@ class PulsarPipelineResourceEstimator(BaseResourceEstimator):
         # count total data size
         total_data_size = result['storage']['output_files']['pulp']['nr_of_pulp_files'] * result['storage']['output_files']['pulp']['pulp_file_size'] # bytes
         total_bandwidth = int(ceil((total_data_size * 8) / duration))  # bits/second
+        if total_data_size and output_files:
             result['storage']['total_size'] = total_data_size
             result['bandwidth']['total_size'] = total_bandwidth
+        else:
+            if not total_data_size:
+                result['errors'].append('Total data size is zero!')
+                logger.warning('ERROR: A datasize of zero was calculated!')
+            if not output_files:
+                result['errors'].append('No output files!')
+                logger.warning('ERROR: No output files were calculated!')
         return result