Task #9682: Let SLURM jobs result in FAILED if pipeline fails, but do not run...

Task #9682: Let SLURM jobs result in FAILED if pipeline fails, but do not run abort-trigger if pipeline could set the ABORTED status itself

MAC/Services/src/PipelineControl.py

@@ -512,7 +512,9 @@ runcmd docker run --rm --net=host \
       {image} \
   runPipeline.sh -o {obsid} -c /opt/lofar/share/pipeline/pipeline.cfg.{cluster}
 
-if [ $? -eq 0 ]; then
+RESULT=$?
+
+if [ $RESULT -eq 0 ]; then
     # notify that we're tearing down
     runcmd {setStatus_completing}
 
@@ -526,15 +528,18 @@ if [ $? -eq 0 ]; then
     wget -O - -q "http://ganglia.control.lofar/ganglia/api/events.php?action=add&start_time=now&summary=Pipeline {obsid} FINISHED&host_regex="
 else
     # notify system that we've aborted
-    {setStatus_aborted}
+    runcmd {setStatus_aborted}
 
     # notify ganglia
     wget -O - -q "http://ganglia.control.lofar/ganglia/api/events.php?action=add&start_time=now&summary=Pipeline {obsid} ABORTED&host_regex="
 fi
 
-# if we've reached this point, the process (not the pipeline) ran ok.
-# we do not want to trigger the $OBSID-aborted job.
-exit 0
+# remove the abort-trigger job
+scancel --jobname={obsid}-abort-trigger
+
+# report status back to SLURM
+echo "Pipeline exited with status $RESULT"
+exit $RESULT
 """.format(
         lofarenv = os.environ.get("LOFARENV", ""),
         obsid = otdbId,