Commit 887ef703 authored by Vlad Kondratiev's avatar Vlad Kondratiev

another fix related to using global FS on CEP4

parent 6d9cb38e
......@@ -66,23 +66,33 @@ class PipeUnitPart(PipeUnit):
# make a soft links in the current dir (in order for processing to be consistent with the case when data are in many nodes)
self.log.info("Making links to input files in the current directory...")
if (self.tab.is_coherent and obs.stokesCS == "IQUV") or (self.tab.is_coherent == False and obs.stokesIS == "IQUV"):
# in this case we get files only for needed Stokes
infiles=[ff for ff in self.tab.rawfiles[cep2.current_node] if re.search("_S%d_P%03d_" % (self.stokes_index, self.part), ff)]
else:
infiles=[ff for ff in self.tab.rawfiles[cep2.current_node] if re.search("_P%03d_" % self.part, ff)]
if not cmdline.opts.is_globalfs:
if (self.tab.is_coherent and obs.stokesCS == "IQUV") or (self.tab.is_coherent == False and obs.stokesIS == "IQUV"):
# in this case we get files only for needed Stokes
infiles=[ff for ff in self.tab.rawfiles[cep2.current_node] if re.search("_S%d_P%03d_" % (self.stokes_index, self.part), ff)]
else:
infiles=[ff for ff in self.tab.rawfiles[cep2.current_node] if re.search("_P%03d_" % self.part, ff)]
else: # global FS
input_files=[]
for val in self.tab.rawfiles.values(): input_files.extend(val)
if (self.tab.is_coherent and obs.stokesCS == "IQUV") or (self.tab.is_coherent == False and obs.stokesIS == "IQUV"):
# in this case we get files only for needed Stokes
infiles=[ff for ff in input_files if re.search("_S%d_P%03d_" % (self.stokes_index, self.part), ff)]
else:
infiles=[ff for ff in input_files if re.search("_P%03d_" % self.part, ff)]
for f in infiles:
# links to the *.raw files
cmd="ln -sf %s ." % (f)
try:
self.execute(cmd, workdir=self.curdir)
except: pass
try:
self.execute(cmd, workdir=self.curdir)
except: pass
# copy *.h5 files
cmd="cp -f %s.h5 ." % (f.split(".raw")[0])
try:
self.execute(cmd, workdir=self.curdir)
except: pass
try:
self.execute(cmd, workdir=self.curdir)
except: pass
# second, we need to rsync other files for this part from other locus nodes
# forming the list of dependent locus nodes that have the rest of the data for this part
......@@ -309,14 +319,6 @@ class PipeUnitPart(PipeUnit):
else:
infiles=[ff for ff in self.tab.rawfiles[cep2.current_node] if re.search("_P%03d_" % self.part, ff)]
for f in infiles:
# forming the input file string
input_file=f # should be only one file for a part
# copy *.h5 files
cmd="cp -f %s.h5 ." % (f.split(".raw")[0])
try:
self.execute(cmd, workdir=self.curdir)
except: pass
else: # global FS
input_files=[]
for val in self.tab.rawfiles.values(): input_files.extend(val)
......@@ -326,14 +328,14 @@ class PipeUnitPart(PipeUnit):
else:
infiles=[ff for ff in input_files if re.search("_P%03d_" % self.part, ff)]
for f in infiles:
# forming the input file string
input_file=f # should be only one file for a part
# copy *.h5 files
cmd="cp -f %s.h5 ." % (f.split(".raw")[0])
try:
self.execute(cmd, workdir=self.curdir)
except: pass
for f in infiles:
# forming the input file string
input_file=f # should be only one file for a part
# copy *.h5 files
cmd="cp -f %s.h5 ." % (f.split(".raw")[0])
try:
self.execute(cmd, workdir=self.curdir)
except: pass
self.log.info("Input data: %s" % (input_file))
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment