diff --git a/MAC/Services/src/PipelineControl.py b/MAC/Services/src/PipelineControl.py index 40ab25b34425d4aee1c5ae5d69810c40930b1c27..1a751d8fc991ebd9a5f7df03feccf113b023a375 100755 --- a/MAC/Services/src/PipelineControl.py +++ b/MAC/Services/src/PipelineControl.py @@ -314,6 +314,9 @@ class PipelineControl(OTDBBusListener): # Maximum run time for job (31 days) "--time=31-0", + + # Lower priority to drop below inspection plots + "--nice=1000", "--partition=%s" % parset.processingPartition(), "--nodes=%s" % parset.processingNumberOfTasks(), @@ -344,11 +347,11 @@ class PipelineControl(OTDBBusListener): "{setStatus_active}\n" # pull docker image from repository on all nodes "srun --nodelist=$SLURM_NODELIST --cpus-per-task=1 --job-name=docker-pull" - " --no-kill" + " --kill-on-bad-exit=0 --wait=0" " docker pull {repository}/{image}\n" # put a local tag on the pulled image "srun --nodelist=$SLURM_NODELIST --cpus-per-task=1 --job-name=docker-tag" - " --no-kill" + " --kill-on-bad-exit=0 --wait=0" " docker tag -f {repository}/{image} {image}\n" # call runPipeline.sh in the image on this node "docker run --rm" @@ -361,7 +364,7 @@ class PipelineControl(OTDBBusListener): " -e SLURM_JOB_ID=$SLURM_JOB_ID" " -v /data:/data" " {image}" - " runPipeline.sh -o {obsid} -c /opt/lofar/share/pipeline/pipeline.cfg.{cluster}\n" + " runPipeline.sh -o {obsid} -c /opt/lofar/share/pipeline/pipeline.cfg.{cluster} || exit $?\n" # notify that we're tearing down "{setStatus_completing}\n"