Skip to content
Snippets Groups Projects
Commit 931cb33e authored by Ger van Diepen's avatar Ger van Diepen
Browse files

Task #2699 

Improved documentation; use -s instead of -p
parent b2f6723d
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
CEP/LMWCommon/src/cexecms
+ 45
− 49
View file @ 931cb33e
......@@ -7,60 +7,66 @@
showhelp()
{
echo ''
echo ' cexecms runs a command on cluster nodes for files matching'
echo ' the given file name glob pattern.'
echo ' cexecms runs a command or script on cluster nodes for files matching the'
echo ' given file name glob pattern. Placeholders in command or script are'
echo ' replaced by the actual file name.'
echo ''
echo ' usage:'
echo ' cexecms [-c cluster] command nameglob [arg1 arg2 ...]'
echo ' cexecms [-c cluster] [-d] [-s script] [-w workdir] command nameglob [arg1 arg2 ...]'
echo ''
echo ' -c cluster cluster name as defined for cexec'
echo ' -c cluster Cluster name as defined for cexec.'
echo ' default is lce: if run on an lfe node, otherwise locus:'
echo ' -d do a dryrun (i.e., do not execute, but only print the command)'
echo ' -i ids List of ids to replace <ID> in the nameglob argument'
echo ' A comma and/or blank can be used as separator'
echo ' -d Do a dryrun.'
echo ' (do not execute, but only print the command/script)'
echo ' -i ids List of ids to replace <ID> in the nameglob argument.'
echo ' A comma and/or blank can be used as separator.'
echo ' Ids can be given with parset range style (e.g. 33678..33683)'
echo ' In this way the command can be executed for multiple observations'
echo ' -p parset parset to be used by the command'
echo ' It must b accessible on all nodes in the cluster'
echo ' placeholders in the parset like <FN> are replaced'
echo ' The parset will be used as the first argument for the command'
echo ' -w workdir Working directory in remote process. Default is login directory'
echo ' command command to be executed remotely'
echo ' In this way the command can be executed for multiple observations.'
echo ' -s script Script or parset file to be used by the command.'
echo ' It must be accessible on all nodes in the cluster.'
echo ' Placeholders (like <FN>) in the file are replaced.'
echo ' It will be used as the first argument in the command.'
echo ' -w workdir Working directory in remote process. Default is login directory.'
echo ' command Command to be executed remotely.'
echo ' Quotes are needed if it contains spaces, etc.'
echo ' nameglob file name glob pattern (# is a shorthand for [0-9])'
echo ' E.g., one can use SB### for any subband'
echo ' arg1 arg2 .. Optional extra arguments given to the command'
echo ' Placeholders (like <FN>) in the command are replaced.'
echo ' nameglob File name glob pattern (# is a shorthand for [0-9]).'
echo ' E.g., one can use SB### meaning any subband.'
echo ' arg1 arg2 .. Optional extra arguments to be given to command.'
echo ''
echo ' Using cexec, the script cexecms-part is executed on the given cluster'
echo ' nodes. It looks for files matching the given file name glob pattern.'
echo ' The glob pattern can be bash-style, thus *, ?, [], and {} are possible.'
echo ' The given command will be executed for each matching file name.'
echo ' The actual file name can be made part of the command using:'
echo ' <FILENAME> or <FN> for the full file name'
echo ' <BASENAME> or <BN> for the basename part'
echo ' <DIRNAME> or <DN> for the directory part'
echo ' The actual file name can be made part of the command or script'
echo ' using one or more of the following placeholders:'
echo ' <FILENAME> or <FN> for the full file name'
echo ' <BASENAME> or <BN> for the basename part'
echo ' <DIRNAME> or <DN> for the directory part'
echo ' The first two can be followed by a . (e.g. <FN.>) meaning that the'
echo ' basename is used till the first dot (thus the extension is removed.)'
echo ' If no substitutions have been done, the command is executed like'
echo ' command filename arg1 arg2 ...'
echo ' otherwise like'
echo ''
echo ' If -s is given, the command is executed like:'
echo ' command script arg1 arg2 ..'
echo ' Otherwise, if no substitutions have been done the command is executed like:'
echo ' command <FN> arg1 arg2 ...'
echo ' Otherwise like'
echo ' command arg1 arg2...'
echo ''
echo ' For example:'
echo ' cexecms "ls -d" "/data/scratch/pipeline/L2011_22663/*"'
echo ' cexecms "ls -d" "/data/scratch/pipeline/L2011_22663/*"'
echo ' is a trivial example and could also be done with cexec. Note that'
echo ' cexecms "ls -d <FN>" "/data/scratch/pipeline/L2011_22663/*"'
echo ' cexecms "ls -d <FN>" "/data/scratch/pipeline/L2011_22663/*"'
echo ' is doing the same.'
echo ' The following example is more elaborate and creates a _sel.MS table'
echo ' in another directory for each MS of subband 000 till 099.'
echo ' cexecms "taql '"'"'select from <FN> where ANTENNA1 in [0,1,2]'
echo ' giving /data/scratch/diepen/<BN.>_sel.MS'"'"'"'
echo ' "/data/scratch/pipeline/L2011_22663/*SB0##*"'
echo ' cexecms "taql '"'"'select from <FN> where ANTENNA1 in [0,1,2]'
echo ' giving /data/scratch/diepen/<BN.>_sel.MS'"'"'"'
echo ' "/data/scratch/pipeline/L2011_22663/*SB0##*"'
echo ' Note that quotes have to be used abundantly.'
echo ''
echo ' Sometimes a command can be dangerous or take a long time to run.'
echo ' In such a case it makes sense to execute it first on a single subband'
echo ' using a glob pattern with a specific subband or by using cexecms-part'
echo ' on a remote node'.
echo ' In such a case it makes sense to execute it first with the -d option.'
echo ''
echo ' Note that the current environment (paths, etc.) is copied. You should'
echo ' have done "use LofIm" if you need LofIm in the (remote) command.'
......@@ -76,7 +82,7 @@ if test $# -lt 2; then
exit 1
fi
cluster=
parset=
script=
ids=
dryrun=
workdir=
......@@ -87,9 +93,9 @@ do
shift
cluster="$1"
shift
elif test "$1" = "-p"; then
elif test "$1" = "-s"; then
shift
parset="$1"
script="$1"
shift
elif test "$1" = "-i"; then
shift
......@@ -115,16 +121,6 @@ do
esac
fi
done
if test "$1" = "-p"; then
shift
parset="$1"
shift
fi
if test "$1" = "-c"; then
shift
cluster="$1"
shift
fi
if test $# -lt 2; then
showhelp
exit 1
......@@ -188,18 +184,18 @@ nameglobnew=`echo "$nameglob" | sed \
-e 's%{%:LBRACE:%g' \
-e 's%}%:RBRACE:%g'`
# Add -p is parset is used.
# Add -s is script is used.
# Same for -w.
test "$parset" = "" || parset="-p $parset"
test "$script" = "" || script="-s $script"
test "$workdir" = "" || workdir="-w $workdir"
# Do a single run if no ids given, otherwise iterate over all ids.
if test "$ids" = ""; then
cexec $cluster $pgmpath/cexecms-part $dryrun -e "$envfile" "$parset" "$workdir" "'$commandnew'" "$nameglobnew" "$@"
cexec $cluster $pgmpath/cexecms-part $dryrun -e "$envfile" "$script" "$workdir" "'$commandnew'" "$nameglobnew" "$@"
else
for id in $ids
do
nm=`echo "$nameglobnew" | sed -e "s%<ID>%$id%g"`
cexec $cluster $pgmpath/cexecms-part $dryrun -e "$envfile" "$parset" "$workdir" "'$commandnew'" "$nm" "$@"
cexec $cluster $pgmpath/cexecms-part $dryrun -e "$envfile" "$script" "$workdir" "'$commandnew'" "$nm" "$@"
done
fi
rm -f "$envfile"
CEP/LMWCommon/src/cexecms-part
+ 9
− 9
View file @ 931cb33e
......@@ -18,13 +18,13 @@
showhelp()
{
echo ' run like: cexecms-part [-d] [-e envfile] [-w workdir] [-p parset] command nameglob [arg1 arg2 ...]'
echo ' run like: cexecms-part [-d] [-e envfile] [-w workdir] [-s script] command nameglob [arg1 arg2 ...]'
echo ' see cexecms for more information'
}
# Handle possible options.
envfile=
parset=
script=
dryrun=0
workdir=
while [ $# != 0 ]
......@@ -33,9 +33,9 @@ do
shift
envfile="$1"
shift
elif test "$1" = "-p"; then
elif test "$1" = "-s"; then
shift
parset="$1"
script="$1"
shift
elif test "$1" = "-w"; then
shift
......@@ -121,7 +121,7 @@ command=`echo "$command" | sed \
-e 's%:DOLLAR:%$%g'`
# Process all files.
# Substitute placeholders in the command and possible parset.
# Substitute placeholders in the command and possible script.
for fname in $names
do
# Form the various placeholder replacements.
......@@ -143,9 +143,9 @@ do
-e "s%<FILENAME\.>%$fnamed%g" \
-e "s%<DIRNAME\.>%$dnamed%g" \
-e "s%<BASENAME\.>%$bnamed%g"`
if test "$parset" != ""; then
# A parset is given, replace placeholders in there as well.
psname=$HOME/`basename $parset`-$USER-$$
if test "$script" != ""; then
# A script is given, replace placeholders in there as well.
psname=$HOME/`basename $script`-$USER-$$
sed \
-e "s%<FN>%$fname%g" \
-e "s%<DN>%$dname%g" \
......@@ -159,7 +159,7 @@ do
-e "s%<FILENAME\.>%$fnamed%g" \
-e "s%<DIRNAME\.>%$dnamed%g" \
-e "s%<BASENAME\.>%$bnamed%g" \
$parset > $psname
$script > $psname
if test $dryrun = 1; then
echo "Dryrun: " $commandnew $psname "$@"
cat $psname
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment