Task #9192: Run pipelines with lower priority (than inspection plots etc),...

Task #9192: Run pipelines with lower priority (than inspection plots etc), prevent cancellation of docker pull/tag jobs, propagate runPipeline exit code

MAC/Services/src/PipelineControl.py

@@ -314,6 +314,9 @@ class PipelineControl(OTDBBusListener):
 
                      # Maximum run time for job (31 days)
                      "--time=31-0",
+
+                     # Lower priority to drop below inspection plots
+                     "--nice=1000",
                    
                      "--partition=%s" % parset.processingPartition(),
                      "--nodes=%s" % parset.processingNumberOfTasks(),
@@ -344,11 +347,11 @@ class PipelineControl(OTDBBusListener):
       "{setStatus_active}\n"
       # pull docker image from repository on all nodes
       "srun --nodelist=$SLURM_NODELIST --cpus-per-task=1 --job-name=docker-pull"
-        " --no-kill"
+        " --kill-on-bad-exit=0 --wait=0"
         " docker pull {repository}/{image}\n"
       # put a local tag on the pulled image
       "srun --nodelist=$SLURM_NODELIST --cpus-per-task=1 --job-name=docker-tag"
-        " --no-kill"
+        " --kill-on-bad-exit=0 --wait=0"
         " docker tag -f {repository}/{image} {image}\n"
       # call runPipeline.sh in the image on this node
       "docker run --rm"
@@ -361,7 +364,7 @@ class PipelineControl(OTDBBusListener):
         " -e SLURM_JOB_ID=$SLURM_JOB_ID"
         " -v /data:/data"
         " {image}"
-        " runPipeline.sh -o {obsid} -c /opt/lofar/share/pipeline/pipeline.cfg.{cluster}\n"
+        " runPipeline.sh -o {obsid} -c /opt/lofar/share/pipeline/pipeline.cfg.{cluster} || exit $?\n"
 
         # notify that we're tearing down
         "{setStatus_completing}\n"