diff --git a/RTCP/Run/src/LOFAR/Partitions.py b/RTCP/Run/src/LOFAR/Partitions.py index e9749861b4211722a7f9b85bc1554cd2a9d22741..b6da86e2fad506f049d462f63075079c64c74e7f 100755 --- a/RTCP/Run/src/LOFAR/Partitions.py +++ b/RTCP/Run/src/LOFAR/Partitions.py @@ -96,11 +96,19 @@ def killJobs( partition ): """ Kill anything running on the partition. """ return SyncCommand( "%s | /usr/bin/grep %s | /usr/bin/awk '{ print $1; }' | /usr/bin/xargs -r bgkilljob" % (BGJOBS,partition,) ).isSuccess() - def freePartition( partition ): """ Free the given partition. """ return SyncCommand( "mpirun -partition %s -free wait" % (partition,) ).isSuccess() +def resetPartition( partition ): + """ Reset /dev/flatmem on all I/O nodes and kill all processes that we started. """ + success = True + + for node in PartitionPsets[partition]: + success = success and SyncCommand( "ssh -tq %s pkill IONProc ; pkill orted ; echo 1 > /proc/flatmem_reset" % (node,) ).isSuccess() + + return success + def allocatePartition( partition ): """ Allocate the given partition by running Hello World. """ return SyncCommand( "mpirun -partition %s -nofree -exe /bgsys/tools/hello" % (partition,), ["/dev/null"] ).isSuccess() @@ -160,6 +168,11 @@ if __name__ == "__main__": action = "store_true", default = False, help = "free the partition" ) + parser.add_option( "-r", "--reset", + dest = "reset", + action = "store_true", + default = False, + help = "reset the partition without freeing it" ) parser.add_option( "-s", "--steal", dest = "steal", action = "store_true", @@ -194,6 +207,10 @@ if __name__ == "__main__": print "Allocating %s..." % ( partition, ) errorOccured = allocatePartition( partition ) + if options.reset and not errorOccurred: + print "Resetting %s..." % ( partition, ) + errorOccured = resetPartition( partition ) + if options.steal and not errorOccurred: print "Taking over partition %s..." % ( partition, ) errorOccured = stealPartition( partition ) diff --git a/RTCP/Run/src/util/dateutil.py b/RTCP/Run/src/util/dateutil.py index 04d07324dd609b5b1b666cd65dc428b38adbd511..4c49716221270288fe1e4c82eeee8f1a20190527 100644 --- a/RTCP/Run/src/util/dateutil.py +++ b/RTCP/Run/src/util/dateutil.py @@ -35,7 +35,8 @@ def parse( str ): y,m,d = date.split("-") date_elements = [int(y),conv_month(m),int(d)] - time_elements = map( int, time.split(":") ) + h,m,s = time.split(":") + time_elements = [int(h),int(m),float(s)] return datetime.datetime( *(date_elements + time_elements) ) elif ":" in str: @@ -61,7 +62,7 @@ def parseDuration( str ): return toSeconds( *(time_elements) ) else: # a number of seconds - return int( str ) + return float( str ) def format( dt ): """ Convert either a datetime object or a timestamp to YYYY-MM-DD HH:MM:SS. """ @@ -74,7 +75,7 @@ def format( dt ): def timestamp( dt ): """ Returns the UTC timestamp corresponding to the provided datetime object. """ - return int(time.mktime( dt.utctimetuple() )) + return float(time.mktime( dt.utctimetuple() )) if __name__ == "__main__": """ Convert whatever parse() accepts into UNIX timestamps and a proper date/time string. """