From 785c71a1d70cbab4643f4282c2e465131cb4aca3 Mon Sep 17 00:00:00 2001
From: Jan David Mol <mol@astron.nl>
Date: Thu, 19 Feb 2015 11:17:39 +0000
Subject: [PATCH] Task #7336: Give all temporary directories a suffix of the
python file that created them, and clean up dangling temporary directories
from tests
---
CEP/Pipeline/recipes/sip/master/deprecated/bbs.py | 4 ++--
CEP/Pipeline/recipes/sip/master/new_bbs.py | 2 +-
CEP/Pipeline/recipes/sip/master/setupparmdb.py | 3 ++-
CEP/Pipeline/recipes/sip/nodes/bbs_reducer.py | 2 +-
CEP/Pipeline/recipes/sip/nodes/deprecated/bbs.py | 2 +-
CEP/Pipeline/recipes/sip/nodes/deprecated/cimager.py | 2 +-
CEP/Pipeline/recipes/sip/nodes/deprecated/demixing.py | 2 +-
CEP/Pipeline/recipes/sip/nodes/dppp.py | 2 +-
CEP/Pipeline/recipes/sip/nodes/gainoutliercorrection.py | 2 +-
CEP/Pipeline/recipes/sip/nodes/imager_finalize.py | 2 +-
CEP/Pipeline/recipes/sip/nodes/new_bbs.py | 2 +-
CEP/Pipeline/recipes/sip/nodes/rficonsole.py | 2 +-
CEP/Pipeline/recipes/sip/nodes/setupsourcedb.py | 2 +-
CEP/Pipeline/test/recipes/helpers/WritableParmDB_test.py | 2 +-
CEP/Pipeline/test/recipes/master/copier_test.py | 8 ++++----
CEP/Pipeline/test/recipes/master/imager_bbs_test.py | 4 ++--
.../test/recipes/master/imager_create_dbs_test.py | 4 ++--
CEP/Pipeline/test/recipes/master/imager_prepare_test.py | 4 ++--
CEP/Pipeline/test/recipes/nodes/copier_test.py | 4 ++--
.../test/recipes/nodes/gainoutliercorrection_test.py | 2 +-
CEP/Pipeline/test/recipes/nodes/imager_bbs_test.py | 3 ++-
CEP/Pipeline/test/recipes/nodes/imager_create_dbs_test.py | 3 ++-
CEP/Pipeline/test/recipes/nodes/imager_prepare_test.py | 7 +++++--
CEP/Pipeline/test/support/data_map_test.py | 6 +++---
24 files changed, 41 insertions(+), 35 deletions(-)
diff --git a/CEP/Pipeline/recipes/sip/master/deprecated/bbs.py b/CEP/Pipeline/recipes/sip/master/deprecated/bbs.py
index d1f4a665e4a..491d7c0860d 100644
--- a/CEP/Pipeline/recipes/sip/master/deprecated/bbs.py
+++ b/CEP/Pipeline/recipes/sip/master/deprecated/bbs.py
@@ -196,7 +196,7 @@ class bbs(BaseRecipe):
# already available on disk.
# ------------------------------------------------------------------
self.logger.debug("Building VDS file describing data for BBS run")
- vds_dir = tempfile.mkdtemp()
+ vds_dir = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
vds_file = os.path.join(vds_dir, "bbs.gvds")
combineproc = utilities.spawn_process(
[
@@ -344,7 +344,7 @@ class bbs(BaseRecipe):
"""
env = utilities.read_initscript(self.logger, self.inputs['initscript'])
self.logger.info("Running BBS GlobalControl")
- working_dir = tempfile.mkdtemp()
+ working_dir = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
with CatchLog4CPlus(
working_dir,
self.logger.name + ".GlobalControl",
diff --git a/CEP/Pipeline/recipes/sip/master/new_bbs.py b/CEP/Pipeline/recipes/sip/master/new_bbs.py
index 02b99cb8076..acdaa5f073b 100644
--- a/CEP/Pipeline/recipes/sip/master/new_bbs.py
+++ b/CEP/Pipeline/recipes/sip/master/new_bbs.py
@@ -355,7 +355,7 @@ class new_bbs(BaseRecipe):
code.
"""
self.logger.info("Running BBS GlobalControl")
- working_dir = tempfile.mkdtemp()
+ working_dir = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
with CatchLog4CPlus(
working_dir,
self.logger.name + ".GlobalControl",
diff --git a/CEP/Pipeline/recipes/sip/master/setupparmdb.py b/CEP/Pipeline/recipes/sip/master/setupparmdb.py
index e887d33b1dd..ad54b38277b 100644
--- a/CEP/Pipeline/recipes/sip/master/setupparmdb.py
+++ b/CEP/Pipeline/recipes/sip/master/setupparmdb.py
@@ -90,7 +90,8 @@ class setupparmdb(BaseRecipe, RemoteCommandRecipeMixIn):
# generate a temp dir
pdbdir = tempfile.mkdtemp(
- dir=self.config.get("layout", "job_directory")
+ dir=self.config.get("layout", "job_directory"),
+ suffix=".%s" % (os.path.basename(__file__),)
)
pdbfile = os.path.join(pdbdir, self.inputs['suffix'])
diff --git a/CEP/Pipeline/recipes/sip/nodes/bbs_reducer.py b/CEP/Pipeline/recipes/sip/nodes/bbs_reducer.py
index d327ae04c4d..64e3bb9054f 100644
--- a/CEP/Pipeline/recipes/sip/nodes/bbs_reducer.py
+++ b/CEP/Pipeline/recipes/sip/nodes/bbs_reducer.py
@@ -46,7 +46,7 @@ class bbs_reducer(LOFARnodeTCP):
return 1
# Run bbs-reducer. Catch log output from bbs-reducer and stdout.
- scratch_dir = mkdtemp()
+ scratch_dir = mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
try:
cmd = [executable,
"--parmdb=%s" % parmdb,
diff --git a/CEP/Pipeline/recipes/sip/nodes/deprecated/bbs.py b/CEP/Pipeline/recipes/sip/nodes/deprecated/bbs.py
index 6fbaca14092..8ab25875631 100644
--- a/CEP/Pipeline/recipes/sip/nodes/deprecated/bbs.py
+++ b/CEP/Pipeline/recipes/sip/nodes/deprecated/bbs.py
@@ -66,7 +66,7 @@ class bbs(LOFARnodeTCP):
# Run the kernel
# Catch & log output from the kernel logger and stdout
# ------------------------------------------------------------------
- working_dir = mkdtemp()
+ working_dir = mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
env = read_initscript(self.logger, initscript)
try:
cmd = [executable, parset_filename, "0"]
diff --git a/CEP/Pipeline/recipes/sip/nodes/deprecated/cimager.py b/CEP/Pipeline/recipes/sip/nodes/deprecated/cimager.py
index 4b1d715c61f..278d47336c6 100644
--- a/CEP/Pipeline/recipes/sip/nodes/deprecated/cimager.py
+++ b/CEP/Pipeline/recipes/sip/nodes/deprecated/cimager.py
@@ -47,7 +47,7 @@ class cimager(LOFARnodeTCP):
self.logger.info("Image already exists: aborting.")
return 0
try:
- working_dir = mkdtemp()
+ working_dir = mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
# If a time range has been specified, copy that section of the
# input MS and only image that.
diff --git a/CEP/Pipeline/recipes/sip/nodes/deprecated/demixing.py b/CEP/Pipeline/recipes/sip/nodes/deprecated/demixing.py
index c72a5cf9e8f..cbd49e5a281 100644
--- a/CEP/Pipeline/recipes/sip/nodes/deprecated/demixing.py
+++ b/CEP/Pipeline/recipes/sip/nodes/deprecated/demixing.py
@@ -54,7 +54,7 @@ class demixing(LOFARnodeTCP):
def _execute(self, cmd):
try:
- temp_dir = tempfile.mkdtemp()
+ temp_dir = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
with CatchLog4CPlus(temp_dir,
self.logger.name,
os.path.basename(cmd[0])
diff --git a/CEP/Pipeline/recipes/sip/nodes/dppp.py b/CEP/Pipeline/recipes/sip/nodes/dppp.py
index de369323e88..f77761729c5 100644
--- a/CEP/Pipeline/recipes/sip/nodes/dppp.py
+++ b/CEP/Pipeline/recipes/sip/nodes/dppp.py
@@ -142,7 +142,7 @@ class dppp(LOFARnodeTCP):
temp_parset_filename
)
try:
- working_dir = tempfile.mkdtemp()
+ working_dir = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
# ****************************************************************
# 6. Run ndppp
cmd = [executable, temp_parset_filename, '1']
diff --git a/CEP/Pipeline/recipes/sip/nodes/gainoutliercorrection.py b/CEP/Pipeline/recipes/sip/nodes/gainoutliercorrection.py
index 03ee5128f59..db263daa859 100644
--- a/CEP/Pipeline/recipes/sip/nodes/gainoutliercorrection.py
+++ b/CEP/Pipeline/recipes/sip/nodes/gainoutliercorrection.py
@@ -77,7 +77,7 @@ class gainoutliercorrection(LOFARnodeTCP):
self.logger.info(
"Using the gainoutlier correction based on parmexportcal")
try:
- temp_dir = tempfile.mkdtemp()
+ temp_dir = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
with CatchLog4CPlus(
temp_dir,
self.logger.name + '.' + os.path.basename(infile),
diff --git a/CEP/Pipeline/recipes/sip/nodes/imager_finalize.py b/CEP/Pipeline/recipes/sip/nodes/imager_finalize.py
index 81cc87d795a..a06849c3dbd 100644
--- a/CEP/Pipeline/recipes/sip/nodes/imager_finalize.py
+++ b/CEP/Pipeline/recipes/sip/nodes/imager_finalize.py
@@ -116,7 +116,7 @@ class imager_finalize(LOFARnodeTCP):
os.unlink(fits_output)
try:
- temp_dir = tempfile.mkdtemp()
+ temp_dir = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
with CatchLog4CPlus(temp_dir,
self.logger.name + '.' + os.path.basename(awimager_output),
"image2fits") as logger:
diff --git a/CEP/Pipeline/recipes/sip/nodes/new_bbs.py b/CEP/Pipeline/recipes/sip/nodes/new_bbs.py
index a0ee543182e..0123140135f 100644
--- a/CEP/Pipeline/recipes/sip/nodes/new_bbs.py
+++ b/CEP/Pipeline/recipes/sip/nodes/new_bbs.py
@@ -75,7 +75,7 @@ class new_bbs(LOFARnodeTCP):
# Run the kernel
# Catch & log output from the kernel logger and stdout
# ------------------------------------------------------------------
- working_dir = mkdtemp()
+ working_dir = mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
try:
self.logger.info("******** {0}".format(open(parset_file).read()))
cmd = [executable, parset_file, "0"]
diff --git a/CEP/Pipeline/recipes/sip/nodes/rficonsole.py b/CEP/Pipeline/recipes/sip/nodes/rficonsole.py
index d9869abff65..ceecf728f59 100644
--- a/CEP/Pipeline/recipes/sip/nodes/rficonsole.py
+++ b/CEP/Pipeline/recipes/sip/nodes/rficonsole.py
@@ -27,7 +27,7 @@ class rficonsole(LOFARnodeTCP):
if not os.access(executable, os.X_OK):
raise ExecutableMissing(executable)
- working_dir = tempfile.mkdtemp(dir=wd)
+ working_dir = tempfile.mkdtemp(dir=wd,suffix=".%s" % (os.path.basename(__file__),))
cmd = [executable, "-j", str(nthreads)]
if strategy:
if os.path.exists(strategy):
diff --git a/CEP/Pipeline/recipes/sip/nodes/setupsourcedb.py b/CEP/Pipeline/recipes/sip/nodes/setupsourcedb.py
index ee35999e2d9..af1212d13c4 100644
--- a/CEP/Pipeline/recipes/sip/nodes/setupsourcedb.py
+++ b/CEP/Pipeline/recipes/sip/nodes/setupsourcedb.py
@@ -49,7 +49,7 @@ class setupsourcedb(LOFARnodeTCP):
shutil.rmtree(skydb, ignore_errors=True)
self.logger.info("Creating skymodel: %s" % (skydb))
- scratch_dir = tempfile.mkdtemp()
+ scratch_dir = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
try:
cmd = [executable,
"in=%s" % catalogue,
diff --git a/CEP/Pipeline/test/recipes/helpers/WritableParmDB_test.py b/CEP/Pipeline/test/recipes/helpers/WritableParmDB_test.py
index adc4651d849..6da1b92ed44 100644
--- a/CEP/Pipeline/test/recipes/helpers/WritableParmDB_test.py
+++ b/CEP/Pipeline/test/recipes/helpers/WritableParmDB_test.py
@@ -28,7 +28,7 @@ class writableParmdbTest(unittest.TestCase):
super(writableParmdbTest, self).__init__(arg)
def setUp(self):
- self.tempDir = tempfile.mkdtemp()
+ self.tempDir = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
self.tempParmDB = os.path.join(self.tempDir, "parmDB")
self.writable_parmdb = writableParmdbWrapper(self.tempParmDB)
diff --git a/CEP/Pipeline/test/recipes/master/copier_test.py b/CEP/Pipeline/test/recipes/master/copier_test.py
index 518445c1a1e..b94b35ce06c 100644
--- a/CEP/Pipeline/test/recipes/master/copier_test.py
+++ b/CEP/Pipeline/test/recipes/master/copier_test.py
@@ -33,10 +33,10 @@ class copierTest(unittest.TestCase):
def setUp(self):
self.imager_create_dbs = copierWrapper()
- self.test_path = temp_path = tempfile.mkdtemp()
+ self.test_path = temp_path = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
def tearDown(self):
- #shutil.rmtree(self.test_path)
+ shutil.rmtree(self.test_path)
pass
def test_validate_mapfiles_norename(self):
@@ -120,10 +120,10 @@ class MasterNodeInterfaceTest(unittest.TestCase):
super(MasterNodeInterfaceTest, self).__init__(arg)
def setUp(self):
- self.test_path = temp_path = tempfile.mkdtemp()
+ self.test_path = temp_path = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
def tearDown(self):
- #shutil.rmtree(self.test_path)
+ shutil.rmtree(self.test_path)
pass
def test__init__raise_exception(self):
diff --git a/CEP/Pipeline/test/recipes/master/imager_bbs_test.py b/CEP/Pipeline/test/recipes/master/imager_bbs_test.py
index 5bc5e2edea0..d85e0d7c74b 100644
--- a/CEP/Pipeline/test/recipes/master/imager_bbs_test.py
+++ b/CEP/Pipeline/test/recipes/master/imager_bbs_test.py
@@ -35,10 +35,10 @@ class imager_bbsTest(unittest.TestCase):
super(imager_bbsTest, self).__init__(arg)
def setUp(self):
- self.test_path = temp_path = tempfile.mkdtemp()
+ self.test_path = temp_path = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
def tearDown(self):
- #shutil.rmtree(self.test_path)
+ shutil.rmtree(self.test_path)
pass
def test_constructor(self):
diff --git a/CEP/Pipeline/test/recipes/master/imager_create_dbs_test.py b/CEP/Pipeline/test/recipes/master/imager_create_dbs_test.py
index af56368efd0..ca5b7591241 100644
--- a/CEP/Pipeline/test/recipes/master/imager_create_dbs_test.py
+++ b/CEP/Pipeline/test/recipes/master/imager_create_dbs_test.py
@@ -35,10 +35,10 @@ class imager_create_dbsTest(unittest.TestCase):
super(imager_create_dbsTest, self).__init__(arg)
def setUp(self):
- self.test_path = temp_path = tempfile.mkdtemp()
+ self.test_path = temp_path = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
def tearDown(self):
- #shutil.rmtree(self.test_path)
+ shutil.rmtree(self.test_path)
pass
def test_validate_input_data(self):
diff --git a/CEP/Pipeline/test/recipes/master/imager_prepare_test.py b/CEP/Pipeline/test/recipes/master/imager_prepare_test.py
index a4504406a2d..5eb2c34067c 100644
--- a/CEP/Pipeline/test/recipes/master/imager_prepare_test.py
+++ b/CEP/Pipeline/test/recipes/master/imager_prepare_test.py
@@ -33,10 +33,10 @@ class imager_prepareTest(unittest.TestCase):
super(imager_prepareTest, self).__init__(arg)
def setUp(self):
- self.test_path = temp_path = tempfile.mkdtemp()
+ self.test_path = temp_path = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
def tearDown(self):
- #shutil.rmtree(self.test_path)
+ shutil.rmtree(self.test_path)
pass
def test_create_input_map_for_sbgroup_single_ms(self):
diff --git a/CEP/Pipeline/test/recipes/nodes/copier_test.py b/CEP/Pipeline/test/recipes/nodes/copier_test.py
index d72935df958..fe67f3444fb 100644
--- a/CEP/Pipeline/test/recipes/nodes/copier_test.py
+++ b/CEP/Pipeline/test/recipes/nodes/copier_test.py
@@ -38,7 +38,7 @@ class copierTest(unittest.TestCase):
def tearDown(self):
pass
- #shutil.rmtree(self.test_path)
+ shutil.rmtree(self.test_path)
# # refactor of copier means this test is not needed anymore. Comment out for
# # now
@@ -49,7 +49,7 @@ class copierTest(unittest.TestCase):
# rsync return value
#
# """
-# temp_dir = tempfile.mkdtemp()
+# temp_dir = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
# path_to_unowned_dir = "/home/klijntest/testdir" #
# file_to_copy = open(os.path.join(temp_dir, "test.txt"), 'w')
# file_to_copy.close()
diff --git a/CEP/Pipeline/test/recipes/nodes/gainoutliercorrection_test.py b/CEP/Pipeline/test/recipes/nodes/gainoutliercorrection_test.py
index 60b446fd509..014fce027a8 100644
--- a/CEP/Pipeline/test/recipes/nodes/gainoutliercorrection_test.py
+++ b/CEP/Pipeline/test/recipes/nodes/gainoutliercorrection_test.py
@@ -30,7 +30,7 @@ class GainOutlierDetectionTest(unittest.TestCase):
super(GainOutlierDetectionTest, self).__init__(arg)
def setUp(self):
- self.tempDir = tempfile.mkdtemp()
+ self.tempDir = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
def tearDown(self):
diff --git a/CEP/Pipeline/test/recipes/nodes/imager_bbs_test.py b/CEP/Pipeline/test/recipes/nodes/imager_bbs_test.py
index 9a17e1faf8b..cb67c88998a 100644
--- a/CEP/Pipeline/test/recipes/nodes/imager_bbs_test.py
+++ b/CEP/Pipeline/test/recipes/nodes/imager_bbs_test.py
@@ -33,9 +33,10 @@ class imager_bbsTest(unittest.TestCase):
super(imager_bbsTest, self).__init__(arg)
def setUp(self):
- self.test_path = tempfile.mkdtemp()
+ self.test_path = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
def tearDown(self):
+ shutil.rmtree(self.test_path)
pass
def test_constructor(self):
diff --git a/CEP/Pipeline/test/recipes/nodes/imager_create_dbs_test.py b/CEP/Pipeline/test/recipes/nodes/imager_create_dbs_test.py
index bce1838dae1..004cd81676e 100644
--- a/CEP/Pipeline/test/recipes/nodes/imager_create_dbs_test.py
+++ b/CEP/Pipeline/test/recipes/nodes/imager_create_dbs_test.py
@@ -46,9 +46,10 @@ class ImagerCreateDBsTest(unittest.TestCase):
def setUp(self):
self.imager_create_dbs = ImagerCreateDBsTestWrapper()
- self.test_path = tempfile.mkdtemp()
+ self.test_path = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
def tearDown(self):
+ shutil.rmtree(self.test_path)
pass
# New version of gsm: Quick fix to allow tests to succeed
diff --git a/CEP/Pipeline/test/recipes/nodes/imager_prepare_test.py b/CEP/Pipeline/test/recipes/nodes/imager_prepare_test.py
index 85cc51057fc..35e61a823ac 100644
--- a/CEP/Pipeline/test/recipes/nodes/imager_prepare_test.py
+++ b/CEP/Pipeline/test/recipes/nodes/imager_prepare_test.py
@@ -40,9 +40,10 @@ class ImagerPrepareTest(unittest.TestCase):
def setUp(self):
self.ImagerPrepareTestWrapper = ImagerPrepareTestWrapper()
- self.test_path = tempfile.mkdtemp()
+ self.test_path = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
def tearDown(self):
+ shutil.rmtree(self.test_path)
pass
def test__copy_input_files_multi_file(self):
@@ -97,7 +98,7 @@ class ImagerPrepareTest(unittest.TestCase):
"""
working_dir = ""
- time_slice_dir_path = tempfile.mkdtemp()
+ time_slice_dir_path = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
slices_per_image = 2
input_map = [("lce072", "test_file_path1"),
("lce072", "test_file_path2"),
@@ -145,6 +146,8 @@ class ImagerPrepareTest(unittest.TestCase):
"time_slice_1.dppp.ms.ndppp.par")).read())
self.assertTrue(parset_2_output == parset_2_content_expected,
"\n{0} != \n{1}".format(parset_2_output, parset_2_content_expected))
+
+ shutil.rmtree(time_slice_dir_path)
if __name__ == "__main__":
diff --git a/CEP/Pipeline/test/support/data_map_test.py b/CEP/Pipeline/test/support/data_map_test.py
index 17ad4c607eb..5321e766e94 100644
--- a/CEP/Pipeline/test/support/data_map_test.py
+++ b/CEP/Pipeline/test/support/data_map_test.py
@@ -33,7 +33,7 @@ class DataMapTest(unittest.TestCase):
"""
Create scratch directory and create required input files in there.
"""
- self.tmpdir = tempfile.mkdtemp()
+ self.tmpdir = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
self.old_style_map_file = self._create_old_style_map_file()
self.new_style_map_file = self._create_new_style_map_file()
self.syntax_error_map_file = self._create_syntax_error_map_file()
@@ -155,7 +155,7 @@ class MultiDataMapTest(unittest.TestCase):
"""
Create scratch directory and create required input files in there.
"""
- self.tmpdir = tempfile.mkdtemp()
+ self.tmpdir = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
self.old_style_map_file = self._create_old_style_map_file()
self.new_style_map_file = self._create_new_style_map_file()
self.syntax_error_map_file = self._create_syntax_error_map_file()
@@ -290,7 +290,7 @@ class HelperFunctionDataMapTest(unittest.TestCase):
"""
Create scratch directory and create required input files in there.
"""
- self.tmpdir = tempfile.mkdtemp()
+ self.tmpdir = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
self.old_style_map_file = self._create_old_style_map_file()
self.new_style_map_file = self._create_new_style_map_file()
self.syntax_error_map_file = self._create_syntax_error_map_file()
--
GitLab