From 2b14f238f33d42cb342cf0a036f2fdfadb85163c Mon Sep 17 00:00:00 2001
From: Jan David Mol <mol@astron.nl>
Date: Tue, 15 Mar 2016 10:26:21 +0000
Subject: [PATCH] Task #8437: Proper command-line parsing in runPipeline.sh and
 pipelineAborted.sh, and allow parset to be provided manually

---
 .../recipes/sip/bin/pipelineAborted.sh        | 38 +++++++++--
 CEP/Pipeline/recipes/sip/bin/runPipeline.sh   | 66 +++++++++++++++----
 MAC/Services/src/PipelineControl.py           |  8 ++-
 3 files changed, 92 insertions(+), 20 deletions(-)

diff --git a/CEP/Pipeline/recipes/sip/bin/pipelineAborted.sh b/CEP/Pipeline/recipes/sip/bin/pipelineAborted.sh
index f4a2bc76f2d..68031f76983 100755
--- a/CEP/Pipeline/recipes/sip/bin/pipelineAborted.sh
+++ b/CEP/Pipeline/recipes/sip/bin/pipelineAborted.sh
@@ -8,18 +8,44 @@
 #
 # Syntax:
 #
-#   runPipeline.sh <obsid> || pipelineAborted.sh <obsid>
+#   runPipeline.sh -o <obsid> || pipelineAborted.sh -o <obsid>
 
-OBSID=$1
+# ======= Defaults
 
-if [ -z "$OBSID" ]; then
-  echo "Usage: $0 <obsid>"
-  exit 1
-fi
+# Obs ID
+OBSID=
 
 # Queue on which to post status changes
 SETSTATUS_BUS=lofar.otdb.setStatus
 
+# ======= Parse command-line parameters
+
+function usage() {
+  echo "$0 -o OBSID [options]"
+  echo ""
+  echo "  -o OBSID           Task identifier"
+  echo "  -b busname         Bus name to post status changes on (default: $SETSTATUS_BUS)"
+  exit 1
+}
+
+while getopts "o:c:p:b:" opt; do
+  case $opt in
+    h)  usage
+        ;;
+    o)  OBSID="$OPTARG"
+        ;;
+    b)  SETSTATUS_BUS="$OPTARG"
+        ;;
+    \?) error "Invalid option: -$OPTARG"
+        ;;
+    :)  error "Option requires an argument: -$OPTARG"
+        ;;
+  esac
+done
+[ -z "$OBSID" ] && usage
+
+# ======= Run
+
 # Mark as aborted
 setStatus.py -o $OBSID -s aborted -b $SETSTATUS_BUS || true
 
diff --git a/CEP/Pipeline/recipes/sip/bin/runPipeline.sh b/CEP/Pipeline/recipes/sip/bin/runPipeline.sh
index 2fdfe4fabdc..74e28c521df 100755
--- a/CEP/Pipeline/recipes/sip/bin/runPipeline.sh
+++ b/CEP/Pipeline/recipes/sip/bin/runPipeline.sh
@@ -12,31 +12,72 @@
 #
 # Syntax:
 #
-#   runPipeline.sh <obsid> || pipelineAborted.sh <obsid>
+#   runPipeline.sh -o <obsid> || pipelineAborted.sh -o <obsid>
 
-OBSID=$1
-shift
+# ======= Defaults
 
-if [ -z "$OBSID" ]; then
-  echo "Usage: $0 <obsid> <pipeline parameters>"
-  exit 1
-fi
+# Obs ID
+OBSID=
+
+# Parset (will be requested if not set)
+PARSET=
+
+# Location of pipeline-framework configuration file
+PIPELINE_CONFIG=$LOFARROOT/share/pipeline/pipeline.cfg
 
 # Queue on which to post status changes
 SETSTATUS_BUS=lofar.otdb.setStatus
 
+# ======= Parse command-line parameters
+
+function usage() {
+  echo "$0 -o OBSID [options]"
+  echo ""
+  echo "  -o OBSID           Task identifier"
+  echo "  -c pipeline.cfg    Override pipeline configuration file (default: $PIPELINE_CONFIG)"
+  echo "  -p pipeline.parset Provide parset (default: request through QPID)"
+  echo "  -b busname         Bus name to post status changes on (default: $SETSTATUS_BUS)"
+  exit 1
+}
+
+while getopts "o:c:p:b:" opt; do
+  case $opt in
+    h)  usage
+        ;;
+    o)  OBSID="$OPTARG"
+        ;;
+    c)  PIPELINE_CONFIG="$OPTARG"
+        ;;
+    p)  PARSET="$OPTARG"
+        ;;
+    b)  SETSTATUS_BUS="$OPTARG"
+        ;;
+    \?) error "Invalid option: -$OPTARG"
+        ;;
+    :)  error "Option requires an argument: -$OPTARG"
+        ;;
+  esac
+done
+[ -z "$OBSID" ] && usage
+
+# ======= Init
+
 # Mark as started
 setStatus.py -o $OBSID -s active -b $SETSTATUS_BUS || true
 
-# Fetch parset
-PARSET=${LOFARROOT}/var/run/Observation${OBSID}.parset
-getParset.py -o $OBSID >$PARSET
+if [ -z "$PARSET" ]; then
+  # Fetch parset
+  PARSET=${LOFARROOT}/var/run/Observation${OBSID}.parset
+  getParset.py -o $OBSID >$PARSET
+fi
+
+# ======= Run
 
 # Fetch parameters from parset
 PROGRAM_NAME=$(getparsetvalue $PARSET "ObsSW.Observation.ObservationControl.PythonControl.programName")
 
 # Run pipeline
-OPTIONS=" -d $@"
+OPTIONS=" -d -c $PIPELINE_CONFIG"
   
 # Set up the environment (information to propagate to the node scripts for monitoring and logging)
 export LOFAR_OBSID="$OBSID"
@@ -47,6 +88,8 @@ echo "Executing: ${PROGRAM_NAME} ${OPTIONS} ${PARSET}"
 ${PROGRAM_NAME} ${OPTIONS} ${PARSET}
 RESULT=$?
 
+# ======= Fini
+
 # Process the result
 setStatus.py -o $OBSID -s completing -b $SETSTATUS_BUS || true
 
@@ -60,3 +103,4 @@ fi
 
 # Propagate result to caller
 exit $RESULT
+
diff --git a/MAC/Services/src/PipelineControl.py b/MAC/Services/src/PipelineControl.py
index b8afe302d2a..c530cdbb22b 100755
--- a/MAC/Services/src/PipelineControl.py
+++ b/MAC/Services/src/PipelineControl.py
@@ -351,11 +351,12 @@ class PipelineControl(OTDBBusListener):
         " -e LUSER=$UID"
         " -v $HOME/.ssh:/home/lofar/.ssh:ro"
         " -e SLURM_JOB_ID=$SLURM_JOB_ID"
-        " runPipeline.sh {obsid} --config /opt/lofar/share/pipeline/pipeline.cfg.{cluster}"
+        " runPipeline.sh -o {obsid} -c /opt/lofar/share/pipeline/pipeline.cfg.{cluster} -b {status_bus}"
       .format(
         obsid = treeId,
         tag = parset.dockerTag(),
-        cluster = parset.processingCluster()
+        cluster = parset.processingCluster(),
+        status_bus = self.setStatus_busname,
       ),
 
       sbatch_params=sbatch_params
@@ -369,10 +370,11 @@ class PipelineControl(OTDBBusListener):
       "docker run --rm lofar-pipeline:{tag}"
         " --net=host"
         " -e LUSER=$UID"
-        " pipelineAborted.sh {obsid}"
+        " pipelineAborted.sh -o {obsid} -b {status_bus}"
       .format(
         obsid = treeId,
         tag = parset.dockerTag(),
+        status_bus = self.setStatus_busname,
       ),
 
       sbatch_params=[
-- 
GitLab