diff --git a/SAS/XML_generator/src/xmlgen.py b/SAS/XML_generator/src/xmlgen.py
index f610f7f5ea452cda0f023761dfe20f34a9263089..a982b715e6fce66c9741a58bbe80e53c4553533d 100755
--- a/SAS/XML_generator/src/xmlgen.py
+++ b/SAS/XML_generator/src/xmlgen.py
@@ -300,7 +300,7 @@ def processingCluster(cluster, number_of_tasks, number_of_cores_per_task):
<partition>cpu</partition>
<numberOfTasks>%i</numberOfTasks>
<minRAMPerTask unit="byte">1000000000</minRAMPerTask>
- <minScratchPerTask unit="byte">100000000</minScratchPerTask>
+ <minScratchPerTask unit="byte">100000000</minScratchPerTask>
<maxDurationPerTask>P7DT0S</maxDurationPerTask>
<numberOfCoresPerTask>%i</numberOfCoresPerTask>
<runSimultaneous>true</runSimultaneous>
@@ -427,7 +427,7 @@ def writeXMLBeam(ofile, name, description, topo, beamtype, target, ra, dec, subb
</lofar:bfMeasurementAttributes>
<resultDataProducts>
%s
- </resultDataProducts>
+ </resultDataProducts>
</lofar:measurement>
</item>""" % (
name, description, topo, status, beamtype, target, ra, dec, subbands, writeBoolean(flyseye),
@@ -547,8 +547,8 @@ def writeXMLTargetPipeline(ofile, topo, pred_topo, name, descr, defaulttemplate,
<status>no_data</status>
%s
</lofar:uvDataProduct>
- </item>
- </resultDataProducts>
+ </item>
+ </resultDataProducts>
</lofar:pipeline>
</item>""" % (uvintopo, uvinname, instrintopo, instrinname, uvoutname, uvouttopo, stor_cluster)
@@ -1975,73 +1975,18 @@ def determineBfDataExtension(coherentStokesData, incoherentStokesData):
def writeRepeat(ofile, projectName, blockTopo, repeatNr, settings, imaging_pipe_inputs,
imaging_pipe_predecessors, status, nr_tasks, nr_cores_per_task, miscParameters):
- # Try to read all the variables that are required in the following.
- # (This is to get rid of the crazy hack that declared all keys in settings as a local variable.)
- #
- # Dependencies are an issue. I resolved the once that popped up with the regression tests, but there might be more
- # There is the checkSettings already, so we could also just read these value from the settings in the blocks where
- # they are actually needed. But it's all a bit entangled and this will help to clear things up.
+ # This is to get rid of the crazy hack that declared all keys in settings as a local variable. I first tried to
+ # read all the variables that are required in the following and raise an error if something is missing.
+ # Dependencies are an issue, though, and resolving the ones that pop up with the regression tests might not be
+ # sufficient. After consulting Adriaan, we came to the conclusion that it is best to go the nasty way of just
+ # accessing things on demand in the code paths where they are actually required, to have no impact on behavior.
+
+ # These items from settings are used on top level, so they must be there:
try:
processing = settings['processing']
- calibration_mode = settings['calibration_mode']
coherentStokesData = settings['coherentStokesData']
incoherentStokesData = settings['incoherentStokesData']
nr_beams = settings['nr_beams']
- create_calibrator_observations = settings['create_calibrator_observations']
- writePackageTag = settings['writePackageTag']
- packageTag = settings['packageTag']
- do_imaging = settings['do_imaging']
- set_starttime = settings['set_starttime']
- startTimeObs = settings['startTimeObs']
- tbbPiggybackAllowed = settings['tbbPiggybackAllowed']
- aartfaacPiggybackAllowed = settings['aartfaacPiggybackAllowed']
- correlatedData = settings['correlatedData']
- antennaMode = settings['antennaMode']
- clock = settings['clock']
- instrumentFilter = settings['instrumentFilter']
- integrationTime = settings['integrationTime']
- channelsPerSubband = settings['channelsPerSubband']
- coherentDedisperseChannels = settings['coherentDedisperseChannels']
- subbandsPerFileCS = settings['subbandsPerFileCS']
- numberCollapsedChannelsCS = settings['numberCollapsedChannelsCS']
- stokesDownsamplingStepsCS = settings['stokesDownsamplingStepsCS']
- whichCS = settings['whichCS']
- subbandsPerFileIS = settings['subbandsPerFileIS']
- numberCollapsedChannelsIS = settings['numberCollapsedChannelsIS']
- stokesDownsamplingStepsIS = settings['stokesDownsamplingStepsIS']
- whichIS = settings['whichIS']
- stationList = settings['stationList']
- numberOfBitsPerSample = settings['numberOfBitsPerSample']
- flysEye = settings['flysEye']
- cluster = settings['cluster']
- timeStep1 = settings['timeStep1']
- targetDuration_s = settings['targetDuration_s']
- flaggingStrategy = settings['flaggingStrategy']
- split_targets = settings['split_targets']
- targetBeams = settings['targetBeams']
- targetTAB = settings['targetTAB']
- create_target_cal_beam = settings['create_target_cal_beam']
- targetDemix = settings['targetDemix']
- targetBBS = settings['targetBBS']
-
- if create_calibrator_observations or create_target_cal_beam:
- calibratorBeam = settings['calibratorBeam']
- calibratorTAB = settings['calibratorTAB']
- calibratorBBS = settings['calibratorBBS']
- calibratorDemix = settings['calibratorDemix']
-
- if create_calibrator_observations:
- calibratorDuration_s = settings['calibratorDuration_s']
- create_calibrator_pipeline = settings['create_calibrator_pipeline']
- timeStep2 = settings['timeStep2']
-
- if processing == 'Pulsar':
- targetPulsar = settings['targetPulsar']
-
- if processing == 'LongBaseline':
- subbandsPerSubbandGroup = settings['subbandsPerSubbandGroup']
- subbandGroupsPerMS = settings['subbandGroupsPerMS']
-
except KeyError as ex:
print ex
raise GenException("Could not read required setting! (%s)" % ex.message)
@@ -2069,14 +2014,14 @@ def writeRepeat(ofile, projectName, blockTopo, repeatNr, settings, imaging_pipe_
cal_pipe_target_topology = repeatTopo + '.CPT' # 1.CPT
if processing == 'Imaging':
- if calibration_mode == "internal":
+ if settings['calibration_mode'] == "internal":
cal_obs_pipe_default_template = "Calibrator Pipeline (export)"
cal_tar_pipe_default_template = "Calibrator Pipeline (no export)"
cal_pipe_calibrator_description = "Cal Pipe Calibrator"
cal_pipe_target_description = "Cal Pipe Target"
tar_pipe_predecessor = tar_obs_topology + ',' + cal_pipe_target_topology # 1.T,1.CPT
tar_pipe_input_INST_topo = cal_pipe_target_topology + '.inst.dps' # 1.P1.dps
- elif calibration_mode == "external":
+ elif settings['calibration_mode'] == "external":
cal_obs_pipe_default_template = "Calibrator Pipeline (export)"
cal_tar_pipe_default_template = "Calibrator Pipeline (no export)"
cal_pipe_calibrator_description = "Cal Pipe Calibrator"
@@ -2101,14 +2046,14 @@ def writeRepeat(ofile, projectName, blockTopo, repeatNr, settings, imaging_pipe_
# pulsar_pipe_predecessor = tar_obs_topology
pulsar_pipe_default_template = "Pulsar Pipeline"
elif processing == 'LongBaseline':
- if calibration_mode == "internal": # internal calibration (previously Calbeam)
+ if settings['calibration_mode'] == "internal": # internal calibration (previously Calbeam)
cal_obs_pipe_default_template = "Calibrator Pipeline (export)"
cal_tar_pipe_default_template = "Calibrator Pipeline (no export)"
cal_pipe_calibrator_description = "Cal Pipe Calibrator"
cal_pipe_target_description = "Cal Pipe Target"
tar_pipe_predecessor = tar_obs_topology + ',' + cal_pipe_target_topology # 1.T,1.CPT
tar_pipe_input_INST_topo = cal_pipe_target_topology + '.inst.dps' # 1.P1.dps
- elif calibration_mode == "external": # external calibration (previously calObs)
+ elif settings['calibration_mode'] == "external": # external calibration (previously calObs)
cal_obs_pipe_default_template = "Calibrator Pipeline (export)"
cal_tar_pipe_default_template = "Calibrator Pipeline (no export)"
cal_pipe_calibrator_description = "Cal Pipe Calibrator"
@@ -2120,11 +2065,11 @@ def writeRepeat(ofile, projectName, blockTopo, repeatNr, settings, imaging_pipe_
for beamNr in range(0, nr_beams):
beam_nr_str = str(beamNr)
- if create_calibrator_observations:
- if writePackageTag:
- cal_obs_name = packageTag + "/" + calibratorBeam[2] + "/" + str(repeatNr) + "/CO"
+ if settings['create_calibrator_observations']:
+ if settings['writePackageTag']:
+ cal_obs_name = settings['packageTag'] + "/" + settings['calibratorBeam'][2] + "/" + str(repeatNr) + "/CO"
else:
- cal_obs_name = calibratorBeam[2] + "/" + str(repeatNr) + "/CO"
+ cal_obs_name = settings['calibratorBeam'][2] + "/" + str(repeatNr) + "/CO"
# TODO: for multiObs this is not ok. The SAP numbers should start from scratch again with every new target observation
# and there should be a .beamnr added before the .SAP in the topology
@@ -2147,7 +2092,7 @@ def writeRepeat(ofile, projectName, blockTopo, repeatNr, settings, imaging_pipe_
LB_pipeline_input_uv_topologies.append(LB_preproc_pipe_output_MS_topologies[beamNr])
LB_pipeline_output_uv_topologies.append(LB_pipeline_topologies[beamNr] + ".uv.dps")
- if do_imaging:
+ if settings['do_imaging']:
imaging_pipe_inputs[beamNr].append(tar_pipe_topologies[beamNr] + ".uv.dps")
imaging_pipe_predecessors[beamNr].append(tar_pipe_topologies[beamNr])
@@ -2160,11 +2105,11 @@ def writeRepeat(ofile, projectName, blockTopo, repeatNr, settings, imaging_pipe_
tar_obs_uv_data_topologies.append(tarObsCalBeamDataTopoStr + ".uv.dps")
tar_obs_predecessor = ''
- if create_calibrator_observations:
+ if settings['create_calibrator_observations']:
tar_obs_predecessor = cal_obs_topology # 1.C
- if set_starttime:
- startTimeStr = startTimeObs.strftime('%Y-%m-%dT%H:%M:%S')
- endTimeStr = (startTimeObs + timedelta(seconds=calibratorDuration_s)).strftime('%Y-%m-%dT%H:%M:%S')
+ if settings['set_starttime']:
+ startTimeStr = settings['startTimeObs'].strftime('%Y-%m-%dT%H:%M:%S')
+ endTimeStr = (settings['startTimeObs'] + timedelta(seconds=settings['calibratorDuration_s'])).strftime('%Y-%m-%dT%H:%M:%S')
else:
startTimeStr = ''
endTimeStr = ''
@@ -2176,33 +2121,34 @@ def writeRepeat(ofile, projectName, blockTopo, repeatNr, settings, imaging_pipe_
'',
cal_obs_name,
projectName,
- tbbPiggybackAllowed,
- aartfaacPiggybackAllowed,
- correlatedData,
+ settings['tbbPiggybackAllowed'],
+ settings['aartfaacPiggybackAllowed'],
+ settings['correlatedData'],
coherentStokesData,
incoherentStokesData,
- antennaMode,
- clock,
- instrumentFilter,
- integrationTime,
- channelsPerSubband,
- coherentDedisperseChannels,
- flysEye,
- subbandsPerFileCS,
- numberCollapsedChannelsCS,
- stokesDownsamplingStepsCS,
- whichCS,
- subbandsPerFileIS,
- numberCollapsedChannelsIS,
- stokesDownsamplingStepsIS,
- whichIS,
- stationList,
+ settings['antennaMode'],
+ settings['clock'],
+ settings['instrumentFilter'],
+ settings['integrationTime'],
+ settings['channelsPerSubband'],
+ settings['coherentDedisperseChannels'],
+ settings['flysEye'],
+ settings['subbandsPerFileCS'],
+ settings['numberCollapsedChannelsCS'],
+ settings['stokesDownsamplingStepsCS'],
+ settings['whichCS'],
+ settings['subbandsPerFileIS'],
+ settings['numberCollapsedChannelsIS'],
+ settings['stokesDownsamplingStepsIS'],
+ settings['whichIS'],
+ settings['stationList'],
startTimeStr,
endTimeStr,
- calibratorDuration_s,
- numberOfBitsPerSample,
+ settings['calibratorDuration_s'],
+ settings['numberOfBitsPerSample'],
status)
+ calibratorBeam = settings['calibratorBeam']
writeXMLBeam(ofile,
calibratorBeam[2],
calibratorBeam[2],
@@ -2212,36 +2158,37 @@ def writeRepeat(ofile, projectName, blockTopo, repeatNr, settings, imaging_pipe_
calibratorBeam[0],
calibratorBeam[1],
calibratorBeam[3],
- flysEye,
+ settings['flysEye'],
str(calibratorBeam[5]),
str(calibratorBeam[6]),
- writeTABXML(calibratorTAB),
+ writeTABXML(settings['calibratorTAB']),
writeDataProducts(cal_obs_beam0_topology,
- correlatedData,
+ settings['correlatedData'],
coherentStokesData,
incoherentStokesData,
- cluster),
+ settings['cluster']),
status)
writeXMLObsEnd(ofile)
# target start and end time:
- if set_starttime:
- if create_calibrator_observations:
- startTimeObs = startTimeObs + timedelta(seconds=timeStep1 + calibratorDuration_s)
- startTimeStr = startTimeObs.strftime('%Y-%m-%dT%H:%M:%S')
- endTimeStr = (startTimeObs + timedelta(seconds=targetDuration_s)).strftime('%Y-%m-%dT%H:%M:%S')
+ if settings['set_starttime']:
+ if settings['create_calibrator_observations']:
+ settings['startTimeObs'] = settings['startTimeObs'] + timedelta(seconds=settings['timeStep1'] + settings['calibratorDuration_s'])
+ startTimeStr = settings['startTimeObs'].strftime('%Y-%m-%dT%H:%M:%S')
+ endTimeStr = (settings['startTimeObs'] + timedelta(seconds=settings['targetDuration_s'])).strftime('%Y-%m-%dT%H:%M:%S')
else:
startTimeStr = ''
endTimeStr = ''
- if create_calibrator_observations and create_calibrator_pipeline:
+ if settings['create_calibrator_observations'] and settings['create_calibrator_pipeline']:
- if writePackageTag:
- cal_pipe_name = packageTag + "/" + calibratorBeam[2] + "/" + str(repeatNr) + "/CPC"
+ if settings['writePackageTag']:
+ cal_pipe_name = settings['packageTag'] + "/" + settings['calibratorBeam'][2] + "/" + str(repeatNr) + "/CPC"
else:
- cal_pipe_name = calibratorBeam[2] + "/" + str(repeatNr) + "/CPC"
+ cal_pipe_name = settings['calibratorBeam'][2] + "/" + str(repeatNr) + "/CPC"
if processing == 'Imaging' or processing == 'LongBaseline':
+ calibratorBBS = settings['calibratorBBS']
if not calibratorBBS:
raise GenException("BBS SkyModel is not specified for pipeline coupled to calibrator beam")
@@ -2251,25 +2198,25 @@ def writeRepeat(ofile, projectName, blockTopo, repeatNr, settings, imaging_pipe_
cal_pipe_name,
cal_pipe_calibrator_description,
cal_obs_pipe_default_template,
- flaggingStrategy,
- calibratorBeam[8],
+ settings['flaggingStrategy'],
+ settings['calibratorBeam'][8],
calibratorBBS[0][0],
- calibratorDemix[0],
+ settings['calibratorDemix'][0],
calibratorBBS[0][1:],
cal_obs_beam0_topology + '.uv.dps',
cal_pipe_calibrator_topology + '.inst.dps',
cal_pipe_calibrator_topology + '.inst.dps',
cal_pipe_calibrator_topology + '.uv.dps',
- cluster,
+ settings['cluster'],
status,
nr_tasks,
nr_cores_per_task,
miscParameters)
elif processing == 'Preprocessing':
+ calibratorDemix = settings['calibratorDemix']
for i in range(0, len(calibratorDemix)):
- if len(
- calibratorDemix) > 1: # TODO a cludge right now, but want to refactor how to call the writeXML soon
+ if len(calibratorDemix) > 1: # TODO a cludge right now, but want to refactor how to call the writeXML soon
cal_pipe_calibrator_topology_tmp = cal_pipe_calibrator_topology + ".%i" % i
cal_pipe_name_tmp = cal_pipe_name + ".%i" % i
else:
@@ -2282,12 +2229,12 @@ def writeRepeat(ofile, projectName, blockTopo, repeatNr, settings, imaging_pipe_
cal_pipe_name_tmp,
cal_pipe_calibrator_description,
cal_obs_pipe_default_template,
- flaggingStrategy,
- calibratorBeam[8],
+ settings['flaggingStrategy'],
+ settings['calibratorBeam'][8],
calibratorDemix[i],
cal_obs_beam0_topology + '.uv.dps',
cal_pipe_calibrator_topology_tmp + '.uv.dps',
- cluster,
+ settings['cluster'],
status,
nr_tasks,
nr_cores_per_task,
@@ -2295,6 +2242,7 @@ def writeRepeat(ofile, projectName, blockTopo, repeatNr, settings, imaging_pipe_
elif processing == 'Calibration':
+ calibratorBBS = settings['calibratorBBS']
if not calibratorBBS:
raise GenException("BBS SkyModel is not specified for pipeline coupled to calibrator beam")
@@ -2305,26 +2253,26 @@ def writeRepeat(ofile, projectName, blockTopo, repeatNr, settings, imaging_pipe_
cal_pipe_name,
cal_pipe_calibrator_description,
cal_obs_pipe_default_template,
- flaggingStrategy,
- calibratorBeam[8],
+ settings['flaggingStrategy'],
+ settings['calibratorBeam'][8],
calibratorBBS[0][0],
- calibratorDemix[0],
+ settings['calibratorDemix'][0],
['', '', '', '', '', '', ''],
cal_obs_beam0_topology + '.uv.dps',
cal_pipe_calibrator_topology + '.inst.dps',
cal_pipe_calibrator_topology + '.inst.dps',
cal_pipe_calibrator_topology + '.uv.dps',
- cluster,
+ settings['cluster'],
status,
nr_tasks,
nr_cores_per_task,
miscParameters)
- if not split_targets:
- if writePackageTag:
- tar_obs_name = packageTag + "/" + targetBeams[0][2] + "/" + str(repeatNr) + "/TO"
+ if not settings['split_targets']:
+ if settings['writePackageTag']:
+ tar_obs_name = settings['packageTag'] + "/" + settings['targetBeams'][0][2] + "/" + str(repeatNr) + "/TO"
else:
- tar_obs_name = targetBeams[0][2] + "/" + str(repeatNr) + "/TO"
+ tar_obs_name = settings['targetBeams'][0][2] + "/" + str(repeatNr) + "/TO"
writeXMLObs(ofile,
tar_obs_name,
@@ -2333,40 +2281,41 @@ def writeRepeat(ofile, projectName, blockTopo, repeatNr, settings, imaging_pipe_
tar_obs_predecessor,
tar_obs_name,
projectName,
- tbbPiggybackAllowed,
- aartfaacPiggybackAllowed,
- correlatedData,
+ settings['tbbPiggybackAllowed'],
+ settings['aartfaacPiggybackAllowed'],
+ settings['correlatedData'],
coherentStokesData,
incoherentStokesData,
- antennaMode,
- clock,
- instrumentFilter,
- integrationTime,
- channelsPerSubband,
- coherentDedisperseChannels,
- flysEye,
- subbandsPerFileCS,
- numberCollapsedChannelsCS,
- stokesDownsamplingStepsCS,
- whichCS,
- subbandsPerFileIS,
- numberCollapsedChannelsIS,
- stokesDownsamplingStepsIS,
- whichIS,
- stationList,
+ settings['antennaMode'],
+ settings['clock'],
+ settings['instrumentFilter'],
+ settings['integrationTime'],
+ settings['channelsPerSubband'],
+ settings['coherentDedisperseChannels'],
+ settings['flysEye'],
+ settings['subbandsPerFileCS'],
+ settings['numberCollapsedChannelsCS'],
+ settings['stokesDownsamplingStepsCS'],
+ settings['whichCS'],
+ settings['subbandsPerFileIS'],
+ settings['numberCollapsedChannelsIS'],
+ settings['stokesDownsamplingStepsIS'],
+ settings['whichIS'],
+ settings['stationList'],
startTimeStr,
endTimeStr,
- targetDuration_s,
- numberOfBitsPerSample,
+ settings['targetDuration_s'],
+ settings['numberOfBitsPerSample'],
status)
- if set_starttime:
- if create_calibrator_observations:
- startTimeObs = startTimeObs + timedelta(seconds=timeStep2 + targetDuration_s)
+ if settings['set_starttime']:
+ if settings['create_calibrator_observations']:
+ settings['startTimeObs'] = settings['startTimeObs'] + timedelta(seconds=settings['timeStep2'] + settings['targetDuration_s'])
else:
- startTimeObs = startTimeObs + timedelta(seconds=timeStep1 + targetDuration_s)
+ settings['startTimeObs'] = settings['startTimeObs'] + timedelta(seconds=settings['timeStep1'] + settings['targetDuration_s'])
for beamNr in range(0, nr_beams):
+ targetBeams = settings['targetBeams']
writeXMLBeam(ofile,
targetBeams[beamNr][2],
targetBeams[beamNr][2],
@@ -2376,44 +2325,46 @@ def writeRepeat(ofile, projectName, blockTopo, repeatNr, settings, imaging_pipe_
targetBeams[beamNr][0],
targetBeams[beamNr][1],
targetBeams[beamNr][3],
- flysEye,
+ settings['flysEye'],
targetBeams[beamNr][5],
targetBeams[beamNr][6],
- writeTABXML(targetTAB[beamNr]),
+ writeTABXML(settings['targetTAB'][beamNr]),
writeDataProducts(tar_obs_beam_topologies[beamNr],
- correlatedData,
+ settings['correlatedData'],
coherentStokesData,
incoherentStokesData,
- cluster),
+ settings['cluster']),
status)
# create the extra polarization beam?
if "create_extra_ncp_beam" in settings and settings["create_extra_ncp_beam"]:
polBeamTopo = tar_obs_topology + ".SAP" + str(beamNr + 1).rjust(3, '0') # FIXME beamNr referenced outside loop?!
+ targetBeams = settings['targetBeams']
writeXMLBeam(ofile,
targetBeams[0][2],
targetBeams[0][2],
targetBeams[0][2],
'Target',
targetBeams[0][0],
- flysEye,
+ settings['flysEye'],
targetBeams[0][5],
targetBeams[0][6],
- writeTABXML(targetTAB[0]),
+ writeTABXML(settings['targetTAB'][0]),
writeDataProducts(polBeamTopo,
- correlatedData,
+ settings['correlatedData'],
coherentStokesData,
incoherentStokesData,
- cluster),
- status)
+ settings['cluster']),
+ status) # FIXME missing parameters
# create a calibrator beam in the target observation?
- if create_target_cal_beam:
+ if settings['create_target_cal_beam']:
if "create_extra_ncp_beam" in settings and settings["create_extra_ncp_beam"]:
calBeamTopo = tar_obs_topology + ".SAP" + str(beamNr + 2).rjust(3, '0')
else:
calBeamTopo = tar_obs_topology + ".SAP" + str(beamNr + 1).rjust(3, '0')
+ calibratorBeam = settings['calibratorBeam']
writeXMLBeam(ofile,
calibratorBeam[2],
calibratorBeam[2],
@@ -2423,27 +2374,28 @@ def writeRepeat(ofile, projectName, blockTopo, repeatNr, settings, imaging_pipe_
calibratorBeam[0],
calibratorBeam[1],
calibratorBeam[3],
- flysEye,
+ settings['flysEye'],
calibratorBeam[5],
calibratorBeam[6],
- writeTABXML(calibratorTAB),
+ writeTABXML(settings['calibratorTAB']),
writeDataProducts(tar_obs_beam_topologies[nr_beams],
- correlatedData,
+ settings['correlatedData'],
coherentStokesData,
incoherentStokesData,
- cluster),
+ settings['cluster']),
status)
writeXMLObsEnd(ofile)
- if writePackageTag:
- cal_pipe_target_name = packageTag + "/" + calibratorBeam[2] + "/" + str(repeatNr) + "/CPT"
+ if settings['writePackageTag']:
+ cal_pipe_target_name = settings['packageTag'] + "/" + calibratorBeam[2] + "/" + str(repeatNr) + "/CPT"
else:
cal_pipe_target_name = calibratorBeam[2] + "/" + str(repeatNr) + "/CPT"
create_pipeline = calibratorBeam[7]
if create_pipeline:
if processing == 'Imaging' or processing == 'LongBaseline':
+ calibratorBBS = settings['calibratorBBS']
if not calibratorBBS:
raise GenException("BBS SkyModel is not specified for pipeline coupled to calibration beam")
@@ -2453,22 +2405,23 @@ def writeRepeat(ofile, projectName, blockTopo, repeatNr, settings, imaging_pipe_
cal_pipe_target_name,
cal_pipe_target_description,
cal_tar_pipe_default_template,
- flaggingStrategy,
+ settings['flaggingStrategy'],
calibratorBeam[8],
calibratorBBS[0][0],
- calibratorDemix[0],
+ settings['calibratorDemix'][0],
calibratorBBS[0][1:],
tar_obs_uv_data_topologies[nr_beams],
cal_pipe_target_topology + '.inst.dps',
cal_pipe_target_topology + '.inst.dps',
cal_pipe_target_topology + '.uv.dps',
- cluster,
+ settings['cluster'],
status,
nr_tasks,
nr_cores_per_task,
miscParameters)
elif processing == 'Preprocessing':
+ calibratorDemix = settings['calibratorDemix']
for i in range(0, len(calibratorDemix)):
if len(calibratorDemix) > 1: # TODO a cludge right now, but want to refactor how to call the writeXML soon
cal_pipe_target_topology_tmp = cal_pipe_target_topology + ".%i" % i
@@ -2483,12 +2436,12 @@ def writeRepeat(ofile, projectName, blockTopo, repeatNr, settings, imaging_pipe_
cal_pipe_target_name_tmp,
cal_pipe_target_description, # FIXME: Might be undefined
cal_tar_pipe_default_template, # FIXME: Might be undefined
- flaggingStrategy,
+ settings['flaggingStrategy'],
calibratorBeam[8],
calibratorDemix[i],
tar_obs_uv_data_topologies[nr_beams],
cal_pipe_target_topology_tmp + '.uv.dps',
- cluster,
+ settings['cluster'],
status,
nr_tasks,
nr_cores_per_task,
@@ -2496,6 +2449,7 @@ def writeRepeat(ofile, projectName, blockTopo, repeatNr, settings, imaging_pipe_
elif processing == 'Calibration':
+ calibratorBBS = settings['calibratorBBS']
if not calibratorBBS:
raise GenException("BBS SkyModel is not specified for pipeline coupled to calibration beam")
@@ -2505,16 +2459,16 @@ def writeRepeat(ofile, projectName, blockTopo, repeatNr, settings, imaging_pipe_
cal_pipe_target_name,
cal_pipe_target_description, # FIXME: Might be undefined
cal_tar_pipe_default_template, # FIXME: Might be undefined
- flaggingStrategy,
+ settings['flaggingStrategy'],
calibratorBeam[8],
calibratorBBS[0][0],
- calibratorDemix[0],
+ settings['calibratorDemix'][0],
calibratorBBS[0][1:],
tar_obs_uv_data_topologies[nr_beams],
cal_pipe_target_topology + '.inst.dps',
cal_pipe_target_topology + '.inst.dps',
cal_pipe_target_topology + '.uv.dps',
- cluster,
+ settings['cluster'],
status,
nr_tasks,
nr_cores_per_task,
@@ -2524,10 +2478,10 @@ def writeRepeat(ofile, projectName, blockTopo, repeatNr, settings, imaging_pipe_
else: # split target sources into separate observations
for beamNr in range(0, nr_beams):
- if writePackageTag:
- tar_obs_name = packageTag + "/" + targetBeams[beamNr][2] + "/" + str(repeatNr) + "/TO"
+ if settings['writePackageTag']:
+ tar_obs_name = settings['packageTag'] + "/" + settings['targetBeams'][beamNr][2] + "/" + str(repeatNr) + "/TO"
else:
- tar_obs_name = targetBeams[beamNr][2] + "/" + str(repeatNr) + "/TO"
+ tar_obs_name = settings['targetBeams'][beamNr][2] + "/" + str(repeatNr) + "/TO"
tar_obs_topology_MultiObs = tar_obs_topology + '.' + str(beamNr)
writeXMLObs(ofile,
@@ -2537,33 +2491,34 @@ def writeRepeat(ofile, projectName, blockTopo, repeatNr, settings, imaging_pipe_
'',
tar_obs_name,
projectName,
- tbbPiggybackAllowed,
- aartfaacPiggybackAllowed,
- correlatedData,
+ settings['tbbPiggybackAllowed'],
+ settings['aartfaacPiggybackAllowed'],
+ settings['correlatedData'],
coherentStokesData,
incoherentStokesData,
- antennaMode,
- clock,
- instrumentFilter,
- integrationTime,
- channelsPerSubband,
- coherentDedisperseChannels,
- flysEye,
- subbandsPerFileCS,
- numberCollapsedChannelsCS,
- stokesDownsamplingStepsCS,
- whichCS,
- subbandsPerFileIS,
- numberCollapsedChannelsIS,
- stokesDownsamplingStepsIS,
- whichIS,
- stationList,
+ settings['antennaMode'],
+ settings['clock'],
+ settings['instrumentFilter'],
+ settings['integrationTime'],
+ settings['channelsPerSubband'],
+ settings['coherentDedisperseChannels'],
+ settings['flysEye'],
+ settings['subbandsPerFileCS'],
+ settings['numberCollapsedChannelsCS'],
+ settings['stokesDownsamplingStepsCS'],
+ settings['whichCS'],
+ settings['subbandsPerFileIS'],
+ settings['numberCollapsedChannelsIS'],
+ settings['stokesDownsamplingStepsIS'],
+ settings['whichIS'],
+ settings['stationList'],
startTimeStr,
endTimeStr,
- targetDuration_s,
- numberOfBitsPerSample,
+ settings['targetDuration_s'],
+ settings['numberOfBitsPerSample'],
status)
+ targetBeams = settings['targetBeams']
writeXMLBeam(ofile,
targetBeams[beamNr][2],
targetBeams[beamNr][2],
@@ -2573,25 +2528,26 @@ def writeRepeat(ofile, projectName, blockTopo, repeatNr, settings, imaging_pipe_
targetBeams[beamNr][0],
targetBeams[beamNr][1],
targetBeams[beamNr][3],
- flysEye,
+ settings['flysEye'],
targetBeams[beamNr][5],
targetBeams[beamNr][6],
- writeTABXML(targetTAB[beamNr]),
+ writeTABXML(settings['targetTAB'][beamNr]),
writeDataProducts(tar_obs_beam_topologies[beamNr],
- correlatedData,
+ settings['correlatedData'],
coherentStokesData,
incoherentStokesData,
- cluster),
+ settings['cluster']),
status)
writeXMLObsEnd(ofile)
- if set_starttime:
- startTimeObs = startTimeObs + timedelta(seconds=timeStep1 + targetDuration_s)
+ if settings['set_starttime']:
+ settings['startTimeObs'] = settings['startTimeObs'] + timedelta(seconds=settings['timeStep1'] + settings['targetDuration_s'])
# Target PIPELINES generation from here on
for beamNr in range(0, nr_beams):
+ targetBeams = settings['targetBeams']
create_pipeline = targetBeams[beamNr][7]
if create_pipeline:
tar_pipe_ID = "/TP"
@@ -2612,8 +2568,8 @@ def writeRepeat(ofile, projectName, blockTopo, repeatNr, settings, imaging_pipe_
tar_pipe_default_template = "Calibration Pipeline Target"
tar_pipe_description = "Target Pipeline"
- if writePackageTag:
- tar_pipe_name = packageTag + "/" + targetBeams[beamNr][2] + "/" + str(repeatNr) + "." + \
+ if settings['writePackageTag']:
+ tar_pipe_name = settings['packageTag'] + "/" + targetBeams[beamNr][2] + "/" + str(repeatNr) + "." + \
str(beamNr) + tar_pipe_ID
else:
tar_pipe_name = targetBeams[beamNr][2] + "/" + str(repeatNr) + "." + str(beamNr) + tar_pipe_ID
@@ -2625,23 +2581,24 @@ def writeRepeat(ofile, projectName, blockTopo, repeatNr, settings, imaging_pipe_
tar_pipe_name,
tar_pipe_description,
tar_pipe_default_template,
- flaggingStrategy,
+ settings['flaggingStrategy'],
targetBeams[beamNr][8],
- targetDemix[beamNr][0],
- targetBBS[beamNr][0][1:],
+ settings['targetDemix'][beamNr][0],
+ settings['targetBBS'][beamNr][0][1:],
tar_obs_uv_data_topologies[beamNr],
tar_obs_uv_data_topologies[beamNr],
tar_pipe_input_INST_topo,
tar_pipe_input_INST_topo,
tar_pipe_topologies[beamNr] + ".uv.dps",
tar_pipe_topologies[beamNr] + ".uv.dps",
- cluster,
+ settings['cluster'],
status,
nr_tasks,
nr_cores_per_task,
miscParameters)
elif processing == 'Preprocessing':
+ targetDemix = settings['targetDemix']
for i in range(0, len(targetDemix[beamNr])):
if len(targetDemix[
beamNr]) > 1: # TODO a cludge right now, but want to refactor how to call the writeXML soon
@@ -2658,18 +2615,19 @@ def writeRepeat(ofile, projectName, blockTopo, repeatNr, settings, imaging_pipe_
tar_pipe_name_tmp,
tar_pipe_description,
tar_pipe_default_template,
- flaggingStrategy,
+ settings['flaggingStrategy'],
targetBeams[beamNr][8],
targetDemix[beamNr][i],
tar_obs_uv_data_topologies[beamNr],
tar_pipe_topology_output_MS_tmp,
- cluster,
+ settings['cluster'],
status,
nr_tasks,
nr_cores_per_task,
miscParameters)
elif processing == 'Calibration': # TODO currently doesn't work according to Alwin's wiki, why?
+ targetBBS = settings['targetBBS']
if targetBBS[beamNr][0][0] == '':
raise GenException(
"BBS SkyModel is not specified for pipeline coupled to target beam " + str(beamNr))
@@ -2680,16 +2638,16 @@ def writeRepeat(ofile, projectName, blockTopo, repeatNr, settings, imaging_pipe_
tar_pipe_name,
tar_pipe_description,
tar_pipe_default_template,
- flaggingStrategy,
+ settings['flaggingStrategy'],
targetBeams[beamNr][8],
targetBBS[beamNr][0][0],
- targetDemix[beamNr][0],
+ settings['targetDemix'][beamNr][0],
targetBBS[beamNr][0][1:],
tar_obs_uv_data_topologies[beamNr],
tar_pipe_topologies[beamNr] + ".inst.dps",
tar_pipe_topologies[beamNr] + ".inst.dps",
tar_pipe_topologies[beamNr] + ".uv.dps",
- cluster,
+ settings['cluster'],
status,
nr_tasks,
nr_cores_per_task,
@@ -2698,7 +2656,7 @@ def writeRepeat(ofile, projectName, blockTopo, repeatNr, settings, imaging_pipe_
elif processing == 'Pulsar':
# tar_obs_topology_MultiObs = tar_obs_topology + '.' + str(beamNr)
tar_pipe_predecessor = tar_obs_topology
-
+ targetPulsar = settings['targetPulsar']
writeXMLPulsarPipe(ofile,
tar_pipe_topologies[beamNr],
tar_obs_topology,
@@ -2708,7 +2666,7 @@ def writeRepeat(ofile, projectName, blockTopo, repeatNr, settings, imaging_pipe_
targetBeams[beamNr][8],
tar_obs_bf_data_topologies[beamNr],
tar_pipe_topologies[beamNr] + ".pu.dps",
- cluster,
+ settings['cluster'],
status,
nr_tasks,
nr_cores_per_task,
@@ -2745,11 +2703,12 @@ def writeRepeat(ofile, projectName, blockTopo, repeatNr, settings, imaging_pipe_
LB_pipeline_description = "Long-Baseline Concat"
for beamNr in range(0, nr_beams):
+ targetBeams = settings['targetBeams']
if targetBeams[beamNr][7]: # create pipelines for this beam
- if writePackageTag:
- LB_preproc_pipe_name = packageTag + "/" + targetBeams[beamNr][2] + "/" + str(repeatNr) + "." + str(
+ if settings['writePackageTag']:
+ LB_preproc_pipe_name = settings['packageTag'] + "/" + targetBeams[beamNr][2] + "/" + str(repeatNr) + "." + str(
beamNr) + "/PP"
- LB_pipeline_name = packageTag + "/" + targetBeams[beamNr][2] + "/" + str(repeatNr) + "." + str(
+ LB_pipeline_name = settings['packageTag'] + "/" + targetBeams[beamNr][2] + "/" + str(repeatNr) + "." + str(
beamNr) + "/LBP"
else:
LB_preproc_pipe_name = targetBeams[beamNr][2] + "/" + str(repeatNr) + "." + str(beamNr) + "/PP"
@@ -2761,12 +2720,12 @@ def writeRepeat(ofile, projectName, blockTopo, repeatNr, settings, imaging_pipe_
LB_preproc_pipe_name,
LB_preproc_pipe_description,
LB_preproc_pipe_template,
- flaggingStrategy,
+ settings['flaggingStrategy'],
targetBeams[beamNr][8],
- targetDemix[beamNr][0],
+ settings['targetDemix'][beamNr][0],
tar_pipe_topologies[beamNr] + ".uv.dps",
LB_preproc_pipe_output_MS_topologies[beamNr],
- cluster,
+ settings['cluster'],
status,
nr_tasks,
nr_cores_per_task,
@@ -2780,16 +2739,16 @@ def writeRepeat(ofile, projectName, blockTopo, repeatNr, settings, imaging_pipe_
LB_pipeline_description,
LB_pipeline_default_template,
targetBeams[beamNr][8],
- subbandsPerSubbandGroup,
- subbandGroupsPerMS,
+ settings['subbandsPerSubbandGroup'],
+ settings['subbandGroupsPerMS'],
LB_pipeline_input_uv_topologies[beamNr],
LB_pipeline_output_uv_topologies[beamNr],
- cluster,
+ settings['cluster'],
status,
nr_tasks,
nr_cores_per_task)
- return imaging_pipe_inputs, imaging_pipe_predecessors, startTimeObs
+ return imaging_pipe_inputs, imaging_pipe_predecessors, settings['startTimeObs']
def writeBlock(ofile, settings, projectName, blockNr, status):