diff --git a/.gitattributes b/.gitattributes index 32f2277e1b6c6bcb6263c334a7167816feb2b061..07d87d47bd649b6e30bc04ecc440e43b91208b58 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1522,7 +1522,6 @@ CEP/Pipeline/recipes/sip/bin/long_baseline_pipeline.py eol=lf CEP/Pipeline/recipes/sip/bin/msss_calibrator_pipeline.py eol=lf CEP/Pipeline/recipes/sip/bin/msss_imager_pipeline.py eol=lf CEP/Pipeline/recipes/sip/bin/msss_target_pipeline.py eol=lf -CEP/Pipeline/recipes/sip/bin/pipelineAborted.sh eol=lf CEP/Pipeline/recipes/sip/bin/pulsar_pipeline.py -text CEP/Pipeline/recipes/sip/bin/runPipeline.sh eol=lf CEP/Pipeline/recipes/sip/bin/selfcal_imager_pipeline.py eol=lf diff --git a/CEP/Pipeline/recipes/sip/CMakeLists.txt b/CEP/Pipeline/recipes/sip/CMakeLists.txt index b034415be1db67761be55c121449e97ad864da7b..297a997162b28231fd5155ff07d976baa293f234 100644 --- a/CEP/Pipeline/recipes/sip/CMakeLists.txt +++ b/CEP/Pipeline/recipes/sip/CMakeLists.txt @@ -74,7 +74,6 @@ lofar_add_bin_scripts( bin/pulsar_pipeline.py bin/long_baseline_pipeline.py bin/selfcal_imager_pipeline.py - bin/pipelineAborted.sh bin/runPipeline.sh bin/startPython.sh bin/startPythonVersion.sh diff --git a/CEP/Pipeline/recipes/sip/bin/pipelineAborted.sh b/CEP/Pipeline/recipes/sip/bin/pipelineAborted.sh deleted file mode 100755 index bf835676ca8568aba692a5bb6769847054bb4c2d..0000000000000000000000000000000000000000 --- a/CEP/Pipeline/recipes/sip/bin/pipelineAborted.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash -e - -# Signals a specific obs id as ABORTED -# -# The following chain is executed: -# -# setStatus(ABORTED) -# -# Syntax: -# -# runPipeline.sh -o <obsid> || pipelineAborted.sh -o <obsid> - -# ======= Defaults - -# Obs ID -OBSID= - -# Queue on which to post status changes -SETSTATUS_BUS=lofar.otdb.command - -# ======= Parse command-line parameters - -function usage() { - echo "$0 -o OBSID [options]" - echo "" - echo " -o OBSID Task identifier" - echo " -B busname Bus name to post status changes on (default: $SETSTATUS_BUS)" - exit 1 -} - -while getopts "o:c:p:B:" opt; do - case $opt in - h) usage - ;; - o) OBSID="$OPTARG" - ;; - B) SETSTATUS_BUS="$OPTARG" - ;; - \?) error "Invalid option: -$OPTARG" - ;; - :) error "Option requires an argument: -$OPTARG" - ;; - esac -done -[ -z "$OBSID" ] && usage - -# ======= Run - -# Mark as aborted -setStatus.py -o $OBSID -s aborted -B $SETSTATUS_BUS || true - -exit 0 diff --git a/MAC/Services/src/PipelineControl.py b/MAC/Services/src/PipelineControl.py index 40517b11a80c7d974cea4cf13e8a655c38116614..e8447d68307a237df35599b153823064da56be1a 100755 --- a/MAC/Services/src/PipelineControl.py +++ b/MAC/Services/src/PipelineControl.py @@ -30,7 +30,7 @@ The execution chains are as follows: [SCHEDULED] -> PipelineControl schedules - runPipeline.sh <obsid> || pipelineAborted.sh <obsid> + runPipeline.sh <obsid> || setStatus.py -o <obsid> -s aborted using two SLURM jobs, guaranteeing that pipelineAborted.sh is called in the following circumstances: @@ -49,7 +49,7 @@ The execution chains are as follows: - (wrap up) - state <- [FINISHED] -(pipelineAborted.sh) -> Calls +(setStatus.py) -> Calls - state <- [ABORTED] ----------------------------- @@ -72,6 +72,7 @@ import subprocess import datetime import os import re +from socket import getfqdn import logging logger = logging.getLogger(__name__) @@ -378,18 +379,16 @@ class PipelineControl(OTDBBusListener): # Schedule pipelineAborted.sh logger.info("Scheduling SLURM job for pipelineAborted.sh") - slurm_cancel_job_id = self.slurm.submit("%s-aborted" % parset.slurmJobName(), + slurm_cancel_job_id = self.slurm.submit("%s-abort-trigger" % parset.slurmJobName(), - "docker run --rm" - " --net=host" - " -u $UID" - " -e LOFARENV={lofarenv}" - " {image}" - " pipelineAborted.sh -o {obsid} -B {status_bus}" + "ssh {myhostname} '" + "source {lofarroot}/lofarinit.sh && " + "setStatus.py -o {obsid} -s aborted -B {status_bus}" + "'" .format( - lofarenv = os.environ.get("LOFARENV", ""), + myhostname = getfqdn(), + lofarroot = os.environ.get("LOFARROOT", ""), obsid = otdbId, - image = parset.defaultDockerImage(), status_bus = self.otdb_service_busname, ), @@ -399,8 +398,8 @@ class PipelineControl(OTDBBusListener): "--dependency=afternotok:%s" % slurm_job_id, "--kill-on-invalid-dep=yes", "--requeue", - "--error=/data/log/pipelineAborted-%s.stderr" % (otdbId,), - "--output=/data/log/pipelineAborted-%s.log" % (otdbId,), + "--error=/data/log/abort-trigger-%s.stderr" % (otdbId,), + "--output=/data/log/abort-trigger-%s.log" % (otdbId,), ] ) logger.info("Scheduled SLURM job %s" % (slurm_cancel_job_id,))