From 5c4619bc4a19cb7080e0daf04cbb29f6c2e1949c Mon Sep 17 00:00:00 2001 From: Jan David Mol <mol@astron.nl> Date: Thu, 29 Mar 2012 14:08:23 +0000 Subject: [PATCH] Task #2669: Major python environment script cleanup --- .gitattributes | 2 - MAC/APL/Appl_Controller/startBGL.sh | 18 ++- MAC/APL/Appl_Controller/stopBGL.sh | 3 +- RTCP/Run/src/LOFAR/Core.py | 181 ---------------------------- RTCP/Run/src/LOFAR/Locations.py | 25 ---- RTCP/Run/src/LOFAR/Parset.py | 55 ++++++--- RTCP/Run/src/RTCP.parset | 8 -- RTCP/Run/src/locations.sh.in | 52 +++++++- RTCP/Run/src/runParset.py | 129 -------------------- RTCP/Run/src/watchlogs.sh | 2 +- 10 files changed, 97 insertions(+), 378 deletions(-) delete mode 100644 RTCP/Run/src/LOFAR/Core.py delete mode 100755 RTCP/Run/src/runParset.py diff --git a/.gitattributes b/.gitattributes index 7b4ed0c0e36..335a7cf01ab 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2593,7 +2593,6 @@ RTCP/Run/src/CNProcessing.sh -text RTCP/Run/src/IONProcessing.sh -text RTCP/Run/src/LOFAR/CMakeLists.txt -text RTCP/Run/src/LOFAR/CommandClient.py -text -RTCP/Run/src/LOFAR/Core.py -text RTCP/Run/src/LOFAR/Locations.py -text RTCP/Run/src/LOFAR/LogValidators.py -text RTCP/Run/src/LOFAR/Logger.py -text @@ -2614,7 +2613,6 @@ RTCP/Run/src/generate_OLAP.parset.pl -text RTCP/Run/src/gracefullyStopBGProcessing.sh -text RTCP/Run/src/locations.sh.in -text RTCP/Run/src/multitail-olap.conf -text -RTCP/Run/src/runParset.py -text RTCP/Run/src/tools/CMakeLists.txt -text RTCP/Run/src/tools/analyzer.c -text RTCP/Run/src/tools/common.c -text diff --git a/MAC/APL/Appl_Controller/startBGL.sh b/MAC/APL/Appl_Controller/startBGL.sh index 0578437b233..9349e0f7d5a 100755 --- a/MAC/APL/Appl_Controller/startBGL.sh +++ b/MAC/APL/Appl_Controller/startBGL.sh @@ -1,3 +1,4 @@ +#!/bin/bash # startBGL.sh jobName partition executable workingDir paramfile noNodes # # jobName @@ -15,15 +16,20 @@ JOBNAME=$1 PARSET=$4 OBSID=$5 -. /opt/lofar/etc/BlueGeneControl.conf +source /opt/lofar/bin/locations.sh -# Remove values which runParset should derive +( +# Remove values which should be derived sed -i 's/.*OLAP.CNProc.integrationSteps.*//' $PARSET sed -i 's/.*OLAP.IONProc.integrationSteps.*//' $PARSET -( -echo "OLAP.IONProc.PLC_controlled = T" -) >> $PARSET +# Convert keys where needed +/opt/lofar/bin/LOFAR/Parset.py $PARSET /opt/lofar/etc/OLAP.parset <(echo "$EXTRA_KEYS") > $IONPROC_PARSET + +# Distribute the parset to the storage nodes +cp $IONPROC_PARSET $STORAGE_PARSET # Inject the parset into the correlator -/opt/lofar/bin/runParset.py -P $PARTITION parset=$PARSET >>/opt/lofar/log/run.runParset.py.log 2>&1 & +/opt/lofar/bin/commandOLAP.py -P $PARTITION parset $IONPROC_PARSET + +) >> $LOGDIR/runParset.py.log 2>&1 diff --git a/MAC/APL/Appl_Controller/stopBGL.sh b/MAC/APL/Appl_Controller/stopBGL.sh index c384294e994..59c3a4a11cd 100755 --- a/MAC/APL/Appl_Controller/stopBGL.sh +++ b/MAC/APL/Appl_Controller/stopBGL.sh @@ -1,3 +1,4 @@ +#!/bin/bash # stopAP.sh partition jobName # # jobName The name of the job @@ -8,7 +9,7 @@ #killjob $1 `cat $2.jobID` #rm -f $2.pid $2.ps -. /opt/lofar/etc/BlueGeneControl.conf +source /opt/lofar/bin/locations.sh JOB=$1 OBSID=$2 diff --git a/RTCP/Run/src/LOFAR/Core.py b/RTCP/Run/src/LOFAR/Core.py deleted file mode 100644 index 38ee7c39321..00000000000 --- a/RTCP/Run/src/LOFAR/Core.py +++ /dev/null @@ -1,181 +0,0 @@ -#!/usr/bin/python - -import Logger -from logging import debug,info,warning,error,critical -from Locations import Locations -from ObservationID import ObservationID -from Parset import Parset -from Stations import Stations,overrideRack - -def buildParset( parset = None, args = "", olapparset = "OLAP.parset", partition = None ): - """ - Adjust and augment the keys of a parset for use in OLAP. - - parset: if defined, a parset object which already contains the keys read from the parset files - args: a string of arguments (parset=RTCP.parset,start=+60, etc, see the __main__ function below) - olapparset: the filename of OLAP.parset (contains station positions and delays) - partition: the BG/P partition that will be used (if none, the partition has to be specified in the parset) - """ - - # valid observation parameters - validObsParams = [ - "parset", "output", "stations", "tcp", "null", - "start", "stop", "run", - "clock", "integration", - "nostorage" - ] - - def isValidObsParam( key ): - return key in validObsParams or "." in key - - # default observation parameters - - defaultObsParams = { - "parset": "RTCP.parset", - "start": "+15", - "output": Locations.files["parset"], - } - defaultRunTime = "00:01:00" - - # define no default run or stop time in defaultObsParams, - # otherwise it will always overrule the parset - assert "stop" not in defaultObsParams - assert "run" not in defaultObsParams - - # always assume the stations will be sending to the rack we will be using - if partition and partition != "R00R01": - overrideRack( Stations, int(partition[2]) ) - - # ========== Observations - obs = args - info( "===== Parsing observation %s =====" % (obs,) ) - - # parse and check the observation parameters - def splitparam( s ): - """ Convert a parameter which is either 'key=value' or 'key' into a - key,value tuple. """ - - if "=" in s: - return s.split("=",1) - - return (s,None) - - obsparams = defaultObsParams.copy() - obsparams.update( dict( map( splitparam, obs.split(",") ) ) ) - - for p in obsparams: - if p and not isValidObsParam( p ): - critical("Unknown observation parameter '%s'" % (p,)) - - if parset is None: - parset = Parset() - - if "parset" not in obsparams: - critical("Observation '%s' does not define a parset file." % (obs,)) - - # read the parset file - def addParset( f ): - info( "Reading parset %s..." % (f,) ) - parset.readFile( Locations.resolvePath( f ) ) - - try: - addParset( olapparset ) - addParset( obsparams["parset"] ) - except IOError,e: - critical("Cannot read parset file: %s" % (e,)) - - # parse specific parset values from the command line (all options containing ".") - # apply them so that we will parse them instead of the parset values - for k,v in obsparams.iteritems(): - if "." not in k: - continue - - parset.parse( "%s=%s" % (k,v) ) - - # reserve an observation id - parset.postRead() - - if parset.getObsID(): - info( "Distilled observation ID %s from parset." % (parset.getObsID(),) ) - else: - try: - obsid = ObservationID().generateID() - except IOError,e: - critical("Could not generate observation ID: %s" % (e,)) - - parset.setObsID( obsid ) - - info( "Generated observation ID %s" % (obsid,) ) - - # override parset with command-line values - if partition: - parset.setPartition( partition ) - else: - parset.setPartition( parset.distillPartition() ) - - # set stations - if "stations" in obsparams: - stationList = Stations.parse( obsparams["stations"] ) - - parset.forceStations( stationList ) - else: - stationList = parset.stations - - if "tcp" in obsparams: - # turn all inputs into tcp:* - def tcpify( input ): - if input.startswith("tcp:") or input.startswith("file:"): - return input - elif input.startswith("udp:"): - return "tcp:"+input[4:] - else: - return "tcp:"+input - - for s in stationList: - s.inputs = map( tcpify, s.inputs ) - - if "null" in obsparams: - # turn all inputs into null: - for s in stationList: - s.inputs = ["null:"] * len(s.inputs) - - parset.setStations( stationList ) - - if "nostorage" in obsparams: - parset.disableStorage() - - # set runtime - if "start" in obsparams and "stop" in obsparams: - parset.setStartStopTime( obsparams["start"], obsparams["stop"] ) - elif "start" in obsparams and "run" in obsparams: - parset.setStartRunTime( obsparams["start"], obsparams["run"] ) - elif "Observation.startTime" in parset and "Observation.stopTime" in parset: - parset.setStartStopTime( parset["Observation.startTime"], parset["Observation.stopTime"] ) - else: - parset.setStartRunTime( obsparams["start"], defaultRunTime ) - - info( "Running from %s to %s." % (parset["Observation.startTime"], parset["Observation.stopTime"] ) ) - info( "Partition = %s" % (parset.partition,) ) - info( "Storage Nodes = %s" % (", ".join(parset.storagenodes),) ) - info( "Stations = %s" % (", ".join([s.name for s in parset.stations]),) ) - - # set a few other options - configmap = { - "clock": parset.setClock, - "integration": parset.setIntegrationTime, - } - - for k,v in configmap.iteritems(): - if k in obsparams: - v( obsparams[k] ) - - # reapply them in case they got overwritten - for k,v in obsparams.iteritems(): - if "." not in k: - continue - - parset.parse( "%s=%s" % (k,v) ) - - parset.setFilename( Locations.resolvePath( obsparams["output"], parset ) ) - - return parset diff --git a/RTCP/Run/src/LOFAR/Locations.py b/RTCP/Run/src/LOFAR/Locations.py index a06903fa61f..ea53095c6d8 100644 --- a/RTCP/Run/src/LOFAR/Locations.py +++ b/RTCP/Run/src/LOFAR/Locations.py @@ -25,33 +25,8 @@ class Locations: self.files = { # where configuration files are kept "configdir": "%s/etc" % (lofarRoot(),), - - # location of the observation id counter - "nextmsnumber": "/globalhome/lofarsystem/log/nextMSNumber", } - if isProduction(): - self.files.update( { - # the location of the Storage executable - "storage": "/data/home/lofarsys/production/lofar/bin/Storage_main", - - # where to save the parset - "parset": "/globalhome/lofarsystem/parsets/L${OBSID}.parset", # for communication with Storage and offline pipelines - "parset-ion": "/bghome0/lofarsys/parsets/L${OBSID}.parset", # for communication to the IO nodes - } ) - else: - self.files.update( { - # the base directory most paths will be related to - "basedir": "%s/projects/LOFAR" % (homeDir(),), - - # the location of the Storage executable - "storage": "${BASEDIR}/installed/gnu_opt/bin/Storage_main", - - # where to save the parset - "parset": "${BASEDIR}/parsets/L${OBSID}.parset", # for communication with Storage and offline pipelines - "parset-ion": "${BASEDIR}/parsets/L${OBSID}.parset", # for communication with the I/O nodes - } ) - def resolvePath(self,path,parset=None): """ Resolve a path by replacing ${BASEDIR} by self["basedir"], etc. diff --git a/RTCP/Run/src/LOFAR/Parset.py b/RTCP/Run/src/LOFAR/Parset.py index 5823924058f..c19fbf79b80 100644 --- a/RTCP/Run/src/LOFAR/Parset.py +++ b/RTCP/Run/src/LOFAR/Parset.py @@ -12,8 +12,7 @@ import getpass import os from itertools import count from Partitions import PartitionPsets -from Locations import Locations -from Stations import Stations +from Stations import Stations, overrideRack from RingCoordinates import RingCoordinates from util.dateutil import parse,format,parseDuration,timestamp from logging import error,warn @@ -120,6 +119,9 @@ class Parset(util.Parset.Parset): if partition: self.setPartition( partition ) + if self.partition and self.partition != "R00R01": + overrideRack( Stations, int(partition[2]) ) + # storage nodes storagenodes = self.distillStorageNodes() or [] self.setStorageNodes( storagenodes ) @@ -341,28 +343,12 @@ class Parset(util.Parset.Parset): self["Observation.DataProducts.Output_Beamformed.enabled"] = True - def addStorageKeys(self): - self["OLAP.Storage.userName"] = getpass.getuser() - self["OLAP.Storage.sshIdentityFile"] = "%s/.ssh/id_rsa" % (os.environ["HOME"],) - self["OLAP.Storage.msWriter"] = Locations.resolvePath( Locations.files["storage"], self ) - self["OLAP.Storage.parsetFilename"] = self.filename - - self.setdefault("OLAP.Storage.AntennaSetsConf", "${STORAGE_CONFIGDIR}/AntennaSets.conf"); - self.setdefault("OLAP.Storage.AntennaFieldsDir", "${STORAGE_CONFIGDIR}/StaticMetaData"); - self.setdefault("OLAP.Storage.HBADeltasDir", "${STORAGE_CONFIGDIR}/StaticMetaData"); - - for p in ["OLAP.Storage.AntennaSetsConf", - "OLAP.Storage.AntennaFieldsDir", - "OLAP.Storage.HBADeltasDir"]: - self[p] = Locations.resolvePath( self[p], self ) - def preWrite(self): """ Derive some final keys and finalise any parameters necessary before writing the parset to disk. """ self.convertSASkeys(); self.addMissingKeys(); - self.addStorageKeys(); # Versioning info self["OLAP.BeamsAreTransposed"] = True @@ -930,6 +916,24 @@ if __name__ == "__main__": # parse the command line parser = OptionParser( "usage: %prog [options] parset [parset ...]" ) + opgroup = OptionGroup( parser, "Request" ) + opgroup.add_option( "-k", "--key", + dest="key", + type="string", + default="", + help="print the given key from the resulting parset" ) + opgroup.add_option( "-p", "--partition", + dest="partition", + type="string", + default=os.environ["PARTITION"] or "", + help="use this partition [%default%]" ) + opgroup.add_option( "-r", "--runtime", + dest="runtime", + type="string", + default="", + help="starttime,runtime" ) + parser.add_option_group( opgroup ) + # parse arguments (options, args) = parser.parse_args() @@ -942,8 +946,21 @@ if __name__ == "__main__": for files in args: parset.readFile( files ) + if options.partition: + parset.setPartition( options.partition ) + + if options.runtime: + starttime, runtime = options.runtime.split(",") + parset.setStartRunTime( starttime, runtime ) + parset.postRead() parset.preWrite() - parset.writeFile( "-" ) + + if options.key: + print parset[options.key] + else: + # default: print whole parset + parset.writeFile( "-" ) sys.exit(0) + diff --git a/RTCP/Run/src/RTCP.parset b/RTCP/Run/src/RTCP.parset index 82d606b9ed1..e5b906b712c 100644 --- a/RTCP/Run/src/RTCP.parset +++ b/RTCP/Run/src/RTCP.parset @@ -165,14 +165,6 @@ OLAP.maxNetworkDelay = 0.5 OLAP.OLAP_Conn.IONProc_Storage_Transport = TCP #OLAP.OLAP_Conn.IONProc_Storage_Ports = [6300..7291] # only define this if you want specific ports to be used -OLAP.Storage.AntennaSetsConf = ${STORAGE_CONFIGDIR}/AntennaSets.conf -OLAP.Storage.AntennaFieldsDir = ${STORAGE_CONFIGDIR}/StaticMetaData -OLAP.Storage.HBADeltasDir = ${STORAGE_CONFIGDIR}/StaticMetaData - -# ----- Global internals -OLAP.log2SasMac = F -OLAP.OLAP_Conn.log2SasMacOutputs = [tcp:0.0.0.0:24500] - # ----- Coordinates of common sources #Observation.Beam[0].angle1 = 0 # NCP #Observation.Beam[0].angle2 = 1.570796327 diff --git a/RTCP/Run/src/locations.sh.in b/RTCP/Run/src/locations.sh.in index bbc67bd3759..e1356004052 100644 --- a/RTCP/Run/src/locations.sh.in +++ b/RTCP/Run/src/locations.sh.in @@ -4,33 +4,73 @@ function isproduction() { TIMESTAMP=`date +%Y-%m-%d_%H%M%S` +if [ -z "$LOFARROOT" ] +then + LOFARROOT=/opt/lofar +fi + +if [ -z "$OBSID" ] +then + OBSID='${OBSID}' +fi + +ETCDIR=$LOFARROOT/etc + +source $ETCDIR/BlueGeneControl.conf + if isproduction then ISPRODUCTION=1 CNPROC=$HOME/production/lofar/bgp_cn/bin/CN_Processing IONPROC=$HOME/production/lofar/bgp_ion/bin/ION_Processing - ETCDIR=/opt/lofar/etc + STORAGE=/data/home/lofarsys/production/lofar/bin/Storage_main LOGDIR=/localhome/log LOGBACKUPDIR=/globalhome/lofarsystem/log-archive LOGPARAMS="-s ccu001:24500" + + IONPROC_PARSET="/bghome0/lofarsys/parsets/L$OBSID.parset" + STORAGE_PARSET="/globalhome/lofarsystem/parsets/L$OBSID.parset" + EXTRA_KEYS=" +OLAP.Storage.parsetFilename = $STORAGE_PARSET +OLAP.Storage.userName = lofarsys +OLAP.Storage.sshIdentityFile = /globalhome/lofarsystem/.ssh/id_rsa +OLAP.Storage.msWriter=$STORAGE +OLAP.Storage.AntennaSetsConf = /data/home/lofarsys/production/lofar/etc/AntennaSets.conf +OLAP.Storage.AntennaFieldsDir = /data/home/lofarsys/production/lofar/etc/StaticMetaData +OLAP.Storage.HBADeltasDir = /data/home/lofarsys/production/lofar/etc/StaticMetaData + +OLAP.IONProc.PLC_controlled = T +OLAP.CNProc.partition = $PARTITION + " else ISPRODUCTION=0 CNPROC=$HOME/projects/LOFAR/installed/bgpcn_opt/bin/CN_Processing IONPROC=$HOME/projects/LOFAR/installed/bgpion_opt/bin/ION_Processing - ETCDIR=@CMAKE_INSTALL_PREFIX@/etc + STORAGE=$HOME/projects/LOFAR/installed/gnu_opt/bin/Storage_main LOGDIR=$HOME/projects/LOFAR/log LOGBACKUPDIR=$LOGDIR LOGPARAMS="-v" + + IONPROC_PARSET="$LOGDIR/L$OBSID.parset" + STORAGE_PARSET="$LOGDIR/L$OBSID.parset" + EXTRA_KEYS=" +OLAP.Storage.parsetFilename = $STORAGE_PARSET +OLAP.Storage.userName = $USER +OLAP.Storage.sshIdentityFile = $HOME/.ssh/id_rsa +OLAP.Storage.msWriter = $STORAGE +OLAP.Storage.AntennaSetsConf = /data/home/lofarsys/production/lofar/etc/AntennaSets.conf +OLAP.Storage.AntennaFieldsDir = /data/home/lofarsys/production/lofar/etc/StaticMetaData +OLAP.Storage.HBADeltasDir = /data/home/lofarsys/production/lofar/etc/StaticMetaData + +OLAP.IONProc.PLC_controlled = F +OLAP.CNProc.partition = $PARTITION + " fi -if [ -d $ETCDIR ] -then - . $ETCDIR/BlueGeneControl.conf -fi function set_psetinfo() { # list both the partition directly (small partitions) and recursively (large partitions) to get all -32 subpartitions diff --git a/RTCP/Run/src/runParset.py b/RTCP/Run/src/runParset.py deleted file mode 100755 index 4b583522ef5..00000000000 --- a/RTCP/Run/src/runParset.py +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env python - -from LOFAR import Logger -from logging import debug,info,warning,error,critical -from LOFAR.Core import buildParset -from LOFAR.Parset import Parset -from util.Hosts import rsymlink -from LOFAR.Locations import Locations -from LOFAR.CommandClient import sendCommand -import sys -import socket - -if __name__ == "__main__": - from optparse import OptionParser,OptionGroup - import os - import time - - # parse the command line - parser = OptionParser( usage = """usage: %prog [options] observation [observation] ... - - 'observation' is a comma-separated list of the following options: - - parset=name (mandatory) the filename of the parset to read - output=name the filename of the parset to write (default: RCTP-(obsid).parset in latest logdir) - stations=xxx+yyy use stations xxx and yyy - tcp station input arrives over TCP - null station input is generated from null: - nostorage do not start storage processes - - start=xxx start/stop time. allowed syntax: - stop=xxx [YYYY-MM-DD] HH:MM[:SS] - timestamp - +seconds - +HH:MM[:SS] - run=xxx run time. allowed syntax: - HH:MM[:SS] - seconds - - clock=xxx use a station clock of xxx MHz - integration=xxx use xxx seconds of integration time on the IO node - - foo.bar=value override parset key 'foo.bar' with 'value'. - """ ) - - opgroup = OptionGroup(parser, "Output" ) - opgroup.add_option( "-v", "--verbose", - dest = "verbose", - action = "store_true", - default = False, - help = "be verbose [%default]" ) - opgroup.add_option( "-q", "--quiet", - dest = "quiet", - action = "store_true", - default = False, - help = "be quiet [%default]" ) - parser.add_option_group( opgroup ) - - hwgroup = OptionGroup(parser, "Settings" ) - hwgroup.add_option( "-O", "--olap-parset", - dest = "olapparset", - type = "string", - default = "%s/OLAP.parset" % (Locations.files["configdir"],), - help = "the parset containing station definitions [%default]" ) - hwgroup.add_option( "-P", "--partition", - dest = "partition", - type = "string", - help = "name of the BlueGene partition [%default]" ) - hwgroup.add_option( "-n", "--norun", - dest = "norun", - action = "store_true", - default = False, - help = "do not send the parset to the correlator" ) - parser.add_option_group( hwgroup ) - - # parse arguments - (options, args) = parser.parse_args() - - # ========== Global options - - if not args: - parser.print_help() - sys.exit(1) - - if options.verbose: - Logger.DEBUG = True - - if not options.quiet: - DEBUG = True - - Logger.initLogger() - - # read and convert parsets - parsets = [buildParset( None, arg, Locations.resolvePath( options.olapparset ), options.partition ) for arg in args] - for p in parsets: - p.preWrite() - p.check() - - # output them to stdout or file - info( "========== Saving parsets ==========" ) - for parset in parsets: - info( "Saving parset to %s" % (parset.filename,) ) - parset.save() - - if not options.norun: - """ - try: - obsDir = os.path.dirname( os.path.realpath( parset.filename ) ) - symlinkName = Locations.resolvePath( Locations.files["obssymlink"], parset ) - rsymlink( symlinkName, obsDir ) - - info( "Created symlink %s -> %s" % (symlinkName, obsDir) ) - except IOError,msg: - warning( "Failed to create symlink %s -> %s" % (symlinkName,obsDir) ) - except OSError,msg: - warning( "Failed to create symlink %s -> %s" % (symlinkName,obsDir) ) - """ - - # save in separate location for IO nodes, to prevent contention for NFS drives with other processes - parset.setFilename( Locations.resolvePath( Locations.files["parset-ion"], parset ) ) - info( "Saving parset to %s" % (parset.filename,) ) - parset.save() - - info( "Sending parset %s to the correlator on partition %s" % (parset.filename,parset.partition) ) - try: - sendCommand( options.partition, "parset %s" % (parset.filename,) ) - except socket.error,msg: - error( "Failed to connect to correlator on partition %s: %s" % (parset.partition,msg) ) - - info( "========== Done ==========" ) diff --git a/RTCP/Run/src/watchlogs.sh b/RTCP/Run/src/watchlogs.sh index 695a6c54fde..e89e632bcc4 100755 --- a/RTCP/Run/src/watchlogs.sh +++ b/RTCP/Run/src/watchlogs.sh @@ -14,7 +14,7 @@ ERRORLOGS= if [ $ISPRODUCTION -eq 1 ] then - for l in $LOGDIR/BlueGeneControl.log $LOGDIR/run.runParset.py.log + for l in $LOGDIR/BlueGeneControl.log $LOGDIR/runParset.py.log do echo Reading additional error log $l ERRORLOGS="$ERRORLOGS $FLAGS -cS olap -fr errors -I $l" -- GitLab