From f5d56aaee23b6ef1d093bc45e7d55a4106644b4d Mon Sep 17 00:00:00 2001
From: Stefan Froehlich <s.froehlich@fz-juelich.de>
Date: Tue, 12 Jan 2016 12:54:04 +0000
Subject: [PATCH] Task #8746 Adding a error_tolerance option to the
executable_args master script. This controls if the step exits at the first
error or if a pipeline should continue with the successfull processed steps.
Usage in the parset on a step by step basis:
stepname.control.error_tolerance=<Bool>
---
.../recipes/sip/master/executable_args.py | 22 ++++++++++++++++++-
1 file changed, 21 insertions(+), 1 deletion(-)
diff --git a/CEP/Pipeline/recipes/sip/master/executable_args.py b/CEP/Pipeline/recipes/sip/master/executable_args.py
index 1b1e55b919e..4c07727745f 100644
--- a/CEP/Pipeline/recipes/sip/master/executable_args.py
+++ b/CEP/Pipeline/recipes/sip/master/executable_args.py
@@ -171,6 +171,12 @@ class executable_args(BaseRecipe, RemoteCommandRecipeMixIn):
'--environment',
help="Update environment variables for this step.",
optional=True
+ ),
+ 'error_tolerance': ingredient.BoolField(
+ '--error_tolerance',
+ help="Controls if the program exits on the first error or continues with succeeded MS.",
+ default=True,
+ optional=True
)
}
@@ -313,7 +319,18 @@ class executable_args(BaseRecipe, RemoteCommandRecipeMixIn):
# ********************************************************************
# Call the node side of the recipe
# Create and schedule the compute jobs
- command = "python %s" % (self.__file__.replace('master', 'nodes')).replace('executable_args', self.inputs['nodescript'])
+ #command = "python %s" % (self.__file__.replace('master', 'nodes')).replace('executable_args', self.inputs['nodescript'])
+ recipe_dir_str = str(self.config.get('DEFAULT', 'recipe_directories'))
+ recipe_directories = recipe_dir_str.rstrip(']').lstrip('[').split(',')
+ pylist = os.getenv('PYTHONPATH').split(':')
+ command = None
+ for pl in pylist:
+ if os.path.isfile(os.path.join(pl,'lofarpipe/recipes/nodes/'+self.inputs['nodescript']+'.py')):
+ command = "python %s" % os.path.join(pl,'lofarpipe/recipes/nodes/'+self.inputs['nodescript']+'.py')
+ for pl in recipe_directories:
+ if os.path.isfile(os.path.join(pl,'nodes/'+self.inputs['nodescript']+'.py')):
+ command = "python %s" % os.path.join(pl,'nodes/'+self.inputs['nodescript']+'.py')
+
inputmapfiles[0].iterator = outputmapfiles[0].iterator = DataMap.SkipIterator
jobs = []
for i, (outp, inp,) in enumerate(zip(
@@ -362,6 +379,9 @@ class executable_args(BaseRecipe, RemoteCommandRecipeMixIn):
for job, outp in zip(jobs, outputmapfiles[0]):
if job.results['returncode'] != 0:
outp.skip = True
+ if not self.inputs['error_tolerance']:
+ self.logger.error("A job has failed and error_tolerance is not set. Bailing out!")
+ return 1
for k, v in job.results.items():
if not k in jobresultdict:
jobresultdict[k] = []
--
GitLab