From 93c03424db023696c59989c83fcbdd16da428735 Mon Sep 17 00:00:00 2001
From: Pieter Donker <donker@astron.nl>
Date: Mon, 2 Nov 2015 08:22:10 +0000
Subject: [PATCH] Task #8683: removed tile based rf-test and added extra
logging
---
LCU/checkhardware/checkHardware.py | 453 ++++++++++++++-------------
LCU/checkhardware/lib/general_lib.py | 13 +-
LCU/checkhardware/rtsm.py | 26 +-
LCU/checkhardware/showTestResult.py | 316 ++++++++++---------
4 files changed, 425 insertions(+), 383 deletions(-)
diff --git a/LCU/checkhardware/checkHardware.py b/LCU/checkhardware/checkHardware.py
index abc6fcd9f51..cda79248ee1 100755
--- a/LCU/checkhardware/checkHardware.py
+++ b/LCU/checkhardware/checkHardware.py
@@ -1,22 +1,23 @@
#!/usr/bin/python
+
info = ''' ----------------------------------------------------------------------------
Usage of arguments
-cf=fullfilename : full path and filename for the configurationfile to use.
-l=2 : set level to 2 (default level is 0)
- level 0 : manual checks, use keys listed below
+ level 0 : manual checks, use keys listed below
level 1..n : see checkhardware.conf file for checks done
-
+
To start a long check set number of runs or start and stop time, if start time
is not given the first run is started immediately
-r=1 : repeats, number of runs to do
-start=[date_time]: start time of first run, format [YYYYMMDD_HH:MM:SS]
-stop=[date_time] : stop time of last run, format [YYYYMMDD_HH:MM:SS]
-
+
Set logging level, can be: debug|info|warning|error
-ls=debug : print all information on screen, default=info
-lf=info : print debug|warning|error information to log file, default=debug
-
+
Select checks to do, can be combined with all levels
-s(rcumode) : signal check for rcumode
-sh(rcumode) : short test for rcumode 1..4
@@ -28,16 +29,16 @@ info = ''' ------------------------------------------------------------------
-e(rcumode)[=120] : do all HBA element tests, optional data time in seconds default = 10 sec.
-sn(rcumode) : HBA summator noise check.
-m(rcumode) : HBA modem check.
-
+
-rcu(mode) : do all rcu(mode) tests
-
+
-rv : RSP version check, always done
-tv : TBB version check, always done
-
+
-rbc : RSP board check, voltage and temperature
-spu : SPU voltage check
-tm : TBB memmory check
-
+
example : ./checkHardware.py -s5 -n5=180
----------------------------------------------------------------------------'''
@@ -69,22 +70,22 @@ os.umask(001)
logger = None
-rcu_keys = ('RCU1','RCU2','RCU3','RCU4','RCU5','RCU6','RCU7')
-rcu_m1_keys = ('O1','SP1','N1','S1','SH1','D1','F1')
-rcu_m2_keys = ('O2','SP2','N2','S2','SH2','D2','F2')
-rcu_m3_keys = ('O3','SP3','N3','S3','SH3','D3','F3')
-rcu_m4_keys = ('O4','SP4','N4','S4','SH4','D4','F4')
-rcu_m5_keys = ('M5','O5','SN5','SP5','N5','S5','E5')
-rcu_m6_keys = ('M6','O6','SN6','SP6','N6','S6','E6')
-rcu_m7_keys = ('M7','O7','SN7','SP7','N7','S7','E7')
+rcu_keys = ('RCU1', 'RCU2', 'RCU3', 'RCU4', 'RCU5', 'RCU6', 'RCU7')
+rcu_m1_keys = ('O1', 'SP1', 'N1', 'S1', 'SH1', 'D1', 'F1')
+rcu_m2_keys = ('O2', 'SP2', 'N2', 'S2', 'SH2', 'D2', 'F2')
+rcu_m3_keys = ('O3', 'SP3', 'N3', 'S3', 'SH3', 'D3', 'F3')
+rcu_m4_keys = ('O4', 'SP4', 'N4', 'S4', 'SH4', 'D4', 'F4')
+rcu_m5_keys = ('M5', 'O5', 'SN5', 'SP5', 'N5', 'S5', 'E5')
+rcu_m6_keys = ('M6', 'O6', 'SN6', 'SP6', 'N6', 'S6', 'E6')
+rcu_m7_keys = ('M7', 'O7', 'SN7', 'SP7', 'N7', 'S7', 'E7')
rcu_m12_keys = rcu_m1_keys + rcu_m2_keys
rcu_m34_keys = rcu_m3_keys + rcu_m4_keys
rcu_m567_keys = rcu_m5_keys + rcu_m6_keys + rcu_m7_keys
-rsp_keys = ('RV','SPU','RBC') + rcu_keys + rcu_m12_keys + rcu_m34_keys + rcu_m567_keys
-tbb_keys = ('TV','TM')
-control_keys = ('R','START','STOP')
+rsp_keys = ('RV', 'SPU', 'RBC') + rcu_keys + rcu_m12_keys + rcu_m34_keys + rcu_m567_keys
+tbb_keys = ('TV', 'TM')
+control_keys = ('R', 'START', 'STOP')
all_keys = control_keys + rsp_keys + tbb_keys
rsp_check = False
tbb_check = False
@@ -99,18 +100,19 @@ args['TV'] = '-'
def printHelp():
print info
+
# return readable info for test
def getTestInfo(key=''):
if key[-1] in '1234567':
test_name = ''
test = key[:-1]
mode = key[-1]
-
+
if mode in '1234':
ant_type = 'LBA'
else:
ant_type = 'HBA'
-
+
if test in ('O',):
test_name += 'Oscillation'
if test in ('SP',):
@@ -124,7 +126,7 @@ def getTestInfo(key=''):
if test in ('D',):
test_name += 'Down'
if test in ('F',):
- test_name += 'Flat'
+ test_name += 'Flat'
if test in ('S',):
test_name += 'RF'
if test in ('SN',):
@@ -132,16 +134,18 @@ def getTestInfo(key=''):
if test in ('E',):
test_name += 'Element'
if test in ('M',):
- test_name += 'Modem'
-
+ test_name += 'Modem'
+ if test in ('RCU',):
+ test_name += 'All tests'
+
return '%s mode-%c %s check' % (ant_type, mode, test_name)
-
+
if key in 'RV':
return 'RSP Version check'
if key in 'SPU':
return 'SPU check'
if key in 'RBC':
- return 'RSP board check'
+ return 'RSP board check'
if key in 'TV':
return 'TBB Version check'
if key in 'TM':
@@ -149,30 +153,31 @@ def getTestInfo(key=''):
if key in 'START':
return 'START checks'
if key in 'STOP':
- return 'STOP checks'
+ return 'STOP checks'
if key in 'R':
return 'Number of test repeats set to'
- return('')
-
+ return('')
+
+
def addToArgs(key, value):
if key == '':
return
global args, rsp_check, tbb_check
- if key in rsp_keys or key in tbb_keys or key in ('H','L','LS','LF','R','START','STOP'):
+ if key in rsp_keys or key in tbb_keys or key in ('H', 'L', 'LS', 'LF', 'R', 'START', 'STOP'):
if value != '-':
args[key] = value
- else:
+ else:
args[key] = '-'
-
+
if key in rsp_keys:
rsp_check = True
if key in tbb_keys:
tbb_check = True
else:
- sys.exit('Unknown key %s' %(key))
+ sys.exit('Unknown key %s' % (key))
return
-
-
+
+
def getArguments():
key = ''
value = '-'
@@ -186,15 +191,14 @@ def getArguments():
key, value = opt, '-'
addToArgs(key=key, value=value)
return
-
+
+
# get checklevel and set tests to do
def setLevelTests(conf):
- #global args
-
level = args.get('L', '0')
if level == '0':
return
- tests = conf.getStr('level-%s-tests' %(level)).split(',')
+ tests = conf.getStr('level-%s-tests' % (level)).split(',')
for tst in tests:
opt = tst.upper()
valpos = opt.find('=')
@@ -206,37 +210,38 @@ def setLevelTests(conf):
addToArgs(key=key, value=value)
return
+
# get and unpack configuration file
class cConfiguration:
def __init__(self):
self.conf = dict()
- if args.has_key('CF'):
+ if 'CF' in args:
filename = args.get('CF')
else:
filename = '/localhome/stationtest/config/checkHardware.conf'
-
+
f = open('/localhome/stationtest/config/checkHardware.conf', 'r')
data = f.readlines()
f.close()
for line in data:
- if line[0] in ('#','\n',' '):
+ if line[0] in ('#', '\n', ' '):
continue
if line.find('#') > 0:
line = line[:line.find('#')]
try:
key, value = line.strip().split('=')
- key = key.replace('_','-')
+ key = key.replace('_', '-')
self.conf[key] = value
except:
- print 'Not a valid configuration setting: %s' %(line)
+ print 'Not a valid configuration setting: %s' % (line)
- def getInt(self,key, default=0):
+ def getInt(self, key, default=0):
return (int(self.conf.get(key, str(default))))
- def getFloat(self,key, default=0.0):
+ def getFloat(self, key, default=0.0):
return (float(self.conf.get(key, str(default))))
- def getStr(self,key):
+ def getStr(self, key):
return (self.conf.get(key, ''))
@@ -263,15 +268,15 @@ def init_logging():
# create logger
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
-
- # check if log dir exist
+
+ # check if log dir exist
if not os.access(logPath, os.F_OK):
os.mkdir(logPath)
-
+
# create file handler
full_filename = os.path.join(logPath, 'checkHardware.log')
- #backup_filename = os.path.join(mainPath, 'checkHardware_bk.log')
- #sendCmd('cp', '%s %s' %(full_filename, backup_filename))
+ # backup_filename = os.path.join(mainPath, 'checkHardware_bk.log')
+ # sendCmd('cp', '%s %s' % (full_filename, backup_filename))
file_handler = logging.FileHandler(full_filename, mode='w')
formatter = logging.Formatter('%(asctime)s %(levelname)-8s %(message)s')
file_handler.setFormatter(formatter)
@@ -281,29 +286,31 @@ def init_logging():
if len(logger.handlers) == 1:
# create console handler
stream_handler = logging.StreamHandler()
- fmt = '%s %%(levelname)-8s %%(message)s' %(station)
+ fmt = '%s %%(levelname)-8s %%(message)s' % (station)
formatter = logging.Formatter(fmt)
stream_handler.setFormatter(formatter)
stream_handler.setLevel(log_levels[screen_log_level])
logger.addHandler(stream_handler)
return
+
def backupLogFiles():
- for nr in range(8,-1,-1):
+ for nr in range(8, -1, -1):
if nr == 0:
full_filename = os.path.join(logPath, 'checkHardware.log')
else:
- full_filename = os.path.join(logPath, 'checkHardware.log.%d' %(nr))
- full_filename_new = os.path.join(logPath, 'checkHardware.log.%d' %(nr+1))
+ full_filename = os.path.join(logPath, 'checkHardware.log.%d' % (nr))
+ full_filename_new = os.path.join(logPath, 'checkHardware.log.%d' % (nr+1))
if os.path.exists(full_filename):
os.rename(full_filename, full_filename_new)
- return
+ return
+
def waitForStart(start_datetime):
start_time = time.mktime(start_datetime.timetuple())
if start_time > time.time():
- logger.info('delayed start, sleep till %s' %(time.asctime(start_datetime.timetuple())))
-
+ logger.info('delayed start, sleep till %s' % (time.asctime(start_datetime.timetuple())))
+
while start_time > time.time():
wait_time = start_time - time.time()
sleep_time = min(wait_time, 3600.0)
@@ -313,13 +320,13 @@ def waitForStart(start_datetime):
def main():
getArguments()
- #print args
- if len(args) == 0 or args.has_key('H'):
+ # print args
+ if len(args) == 0 or 'H' in args:
printHelp()
sys.exit()
- backupLogFiles() # backup logfiles, max 10 logfiles .9 is the oldest
-
+ backupLogFiles() # backup logfiles, max 10 logfiles .9 is the oldest
+
init_logging()
init_lofar_lib()
init_test_db()
@@ -336,60 +343,58 @@ def main():
logger.info('== requested checks and settings ==')
logger.info('-'*40)
for i in all_keys:
- if args.has_key(i):
+ if i in args:
if args.get(i) == '-':
- logger.info(' %s' %(getTestInfo(i)))
+ logger.info(' %s' % (getTestInfo(i)))
else:
- logger.info(' %s, time = %s' %(getTestInfo(i), args.get(i)))
+ logger.info(' %s, time = %s' % (getTestInfo(i), args.get(i)))
logger.info('-'*40)
-
+
removeAllDataFiles()
# use format YYYYMMDD_HH:MM:SS
stop_time = -1
- if args.has_key('STOP'):
+ if 'STOP' in args:
stop = args.get('STOP')
if len(stop) != 17:
- sys.exit('wrong stoptime format must be YYYYMMDD_HH:MM:SS')
- stop_datetime = datetime.datetime(int(stop[:4]),int(stop[4:6]),int(stop[6:8]), \
- int(stop[9:11]),int(stop[12:14]),int(stop[15:]))
+ return 'wrong stoptime format must be YYYYMMDD_HH:MM:SS'
+ stop_datetime = datetime.datetime(int(stop[:4]), int(stop[4:6]), int(stop[6:8]),
+ int(stop[9:11]), int(stop[12:14]), int(stop[15:]))
stop_time = time.mktime(stop_datetime.timetuple())
-
- if args.has_key('START'):
+
+ if 'START' in args:
start = args.get('START')
if len(start) != 17:
- sys.exit('wrong starttime format must be YYYYMMDD_HH:MM:SS')
- start_datetime = datetime.datetime(int(start[:4]),int(start[4:6]),int(start[6:8]), \
- int(start[9:11]),int(start[12:14]),int(start[15:]))
+ return 'wrong starttime format must be YYYYMMDD_HH:MM:SS'
+ start_datetime = datetime.datetime(int(start[:4]), int(start[4:6]), int(start[6:8]),
+ int(start[9:11]), int(start[12:14]), int(start[15:]))
if (time.mktime(start_datetime.timetuple()) < time.time()):
- #print time.mktime(start_datetime.timetuple()), time.time()
+ # print time.mktime(start_datetime.timetuple()), time.time()
logger.error('Stop program, StartTime in past')
- sys.exit(2)
+ return 2
if(time.mktime(start_datetime.timetuple()) > stop_time):
logger.error('Stop program, stop before start')
- sys.exit(2)
+ return 2
waitForStart(start_datetime)
-
- logger.info('run checks till %s' %(time.asctime(stop_datetime.timetuple())))
-
+
+ logger.info('run checks till %s' % (time.asctime(stop_datetime.timetuple())))
+
start_time = time.gmtime()
# Read in RemoteStation.conf
ID, nRSP, nTBB, nLBL, nLBH, nHBA, HBA_SPLIT = readStationConfig()
# setup intern database with station layout
db = cDB(StID, nRSP, nTBB, nLBL, nLBH, nHBA, HBA_SPLIT)
-
-
+
if (stop_time > 0.0):
db.setTestEndTime((stop_time-120.0))
-
-
+
# set manualy marked bad antennas
global_log_dir = conf.getStr('log-dir-global')
host = getHostName()
if os.path.exists(global_log_dir):
full_filename = os.path.join(global_log_dir, 'bad_antenna_list.txt')
- logger.info('add bad_antenna_list data from file "%s" to db' %(full_filename))
+ logger.info('add bad_antenna_list data from file "%s" to db' % (full_filename))
f = open(full_filename, 'r')
data = f.readlines()
for line in data:
@@ -402,25 +407,24 @@ def main():
ant_type = ant[:3].strip().upper()
if ant_type == 'LBA':
ant_nr = int(ant[3:].strip())
- #print 'ant type=%s nr=%d' %(ant_type, ant_nr)
+ # print 'ant type=%s nr=%d' % (ant_type, ant_nr)
if ant_nr < nLBH:
db.lbh.ant[ant_nr].on_bad_list = 1
else:
db.lbl.ant[ant_nr-nLBH].on_bad_list = 1
elif ant_type == 'HBA':
ant_nr = int(ant[3:].strip())
- #print 'ant type=%s nr=%d' %(ant_type, ant_nr)
+ # print 'ant type=%s nr=%d' % (ant_type, ant_nr)
db.hba.tile[ant_nr].on_bad_list = 1
break
else:
- logger.warn('bad_antenna_list data from file "%s" not found' %(full_filename))
-
- #
+ logger.warn('bad_antenna_list data from file "%s" not found' % (full_filename))
+
db.script_versions = 'CHECK=%s,DB=%s,TEST=%s,SEARCH=%s,LOFAR=%s,GENERAL=%s' %\
(check_version, db_version, test_version, search_version, lofar_version, general_version)
db.check_start_time = time.gmtime()
-
- writeMessage('!!! This station will be in use for a test! Please do not use the station! (script version %s) !!!' %(check_version))
+
+ writeMessage('!!! This station will be in use for a test! Please do not use the station! (script version %s) !!!' % (check_version))
start_level, board_errors = swlevel()
sw_level, board_errors = swlevel(2)
if start_level == 1:
@@ -432,7 +436,7 @@ def main():
spu = cSPU(db)
# do RSP tests if requested
- if rsp_check == True:
+ if rsp_check is True:
# check if RSPDriver is running
if checkActiveRSPDriver() == False:
logger.warn('RSPDriver not running')
@@ -456,166 +460,174 @@ def main():
else:
rsp_ready = True
restarts = 2
-
- # if all rsp boards ready do all tests
+
+ # if all rsp boards ready do all tests
if rsp_ready:
- if args.has_key('RV'):
+ if 'RV' in args:
rsp.checkVersions(conf.getStr('bp-version'), conf.getStr('ap-version'))
-
+
resetRSPsettings()
-
+
lbl = cLBA(db, db.lbl)
lbh = cLBA(db, db.lbh)
hba = cHBA(db, db.hba)
-
- repeats = int(args.get('R','1'))
+
+ repeats = int(args.get('R', '1'))
repeat_cnt = 1
-
+
runtime = 0
db.tests = ''
while (repeat_cnt <= repeats) or ((stop_time > -1) and ((time.time() + runtime) < stop_time)):
-
+
try:
runstart = time.time()
if stop_time > -1:
- logger.info('\n=== Start testrun %d ===\n' %(repeat_cnt))
+ logger.info('\n=== Start testrun %d ===\n' % (repeat_cnt))
else:
- logger.info('\n=== Start testrun %d of %d ===\n' %(repeat_cnt, repeats))
-
- if args.has_key('SPU'):
+ logger.info('\n=== Start testrun %d of %d ===\n' % (repeat_cnt, repeats))
+
+ if 'SPU' in args:
spu.checkStatus()
-
- if args.has_key('RBC'):
+
+ if 'RBC' in args:
rsp.checkBoard()
# check if mode 1,2 is available on this station
if StID in CoreStations or StID in RemoteStations:
for mode in (1, 2):
lbl.reset()
- # do all rcumode 1,2 tests
+ # do all rcumode 1,2 tests
if 'RCU%d' % mode in args or 'SH%d' % mode in args:
- lbl.checkShort(mode=mode)
-
+ lbl.checkShort(mode=mode)
+
if 'RCU%d' % mode in args or 'F%d' % mode in args:
- lbl.checkFlat(mode=mode)
-
+ lbl.checkFlat(mode=mode)
+
if 'RCU%d' % mode in args or 'D%d' % mode in args:
- lbl.checkDown(mode=mode, subband=conf.getInt('lbl-test-sb',301))
-
+ lbl.checkDown(mode=mode, subband=conf.getInt('lbl-test-sb', 301))
+
if 'RCU%d' % mode in args or 'O%d' % mode in args:
lbl.checkOscillation(mode=mode)
-
+
if 'RCU%d' % mode in args or 'SP%d' % mode in args:
lbl.checkSpurious(mode=mode)
-
+
if 'RCU%d' % mode in args or 'N%d' % mode in args:
- if 'RCU%d' % mode in args or args.get('N%d' %(mode)) == '-':
+ if 'RCU%d' % mode in args or args.get('N%d' % (mode)) == '-':
recordtime = 120
else:
- recordtime = int(args.get('N%d' %(mode)))
- lbl.checkNoise(mode=mode,
- record_time=recordtime,
- low_deviation=conf.getFloat('lba-noise-min-deviation', -3.0),
- high_deviation=conf.getFloat('lba-noise-max-deviation', 2.5),
- max_diff=conf.getFloat('lba-noise-max-difference', 2.0))
-
+ recordtime = int(args.get('N%d' % (mode)))
+ lbl.checkNoise(
+ mode=mode,
+ record_time=recordtime,
+ low_deviation=conf.getFloat('lba-noise-min-deviation', -3.0),
+ high_deviation=conf.getFloat('lba-noise-max-deviation', 2.5),
+ max_diff=conf.getFloat('lba-noise-max-difference', 2.0))
+
if 'RCU%d' % mode in args or 'S%d' % mode in args:
- lbl.checkSignal(mode=mode,
- subband=conf.getInt('lbl-test-sb',301),
- min_signal=conf.getFloat('lba-rf-min-signal', 75.0),
- low_deviation=conf.getFloat('lba-rf-min-deviation', -2.0),
- high_deviation=conf.getFloat('lba-rf-max-deviation', 2.0))
-
+ lbl.checkSignal(
+ mode=mode,
+ subband=conf.getInt('lbl-test-sb', 301),
+ min_signal=conf.getFloat('lba-rf-min-signal', 75.0),
+ low_deviation=conf.getFloat('lba-rf-min-deviation', -2.0),
+ high_deviation=conf.getFloat('lba-rf-max-deviation', 2.0))
+
for mode in (3, 4):
lbh.reset()
# do all rcumode 3,4 tests
if 'RCU%d' % mode in args or 'SH%d' % mode in args:
- lbh.checkShort(mode=mode)
-
+ lbh.checkShort(mode=mode)
+
if 'RCU%d' % mode in args or 'F%d' % mode in args:
- lbh.checkFlat(mode=mode)
-
+ lbh.checkFlat(mode=mode)
+
if 'RCU%d' % mode in args or 'D%d' % mode in args:
- lbh.checkDown(mode=mode, subband=conf.getInt('lbh-test-sb',301))
+ lbh.checkDown(mode=mode, subband=conf.getInt('lbh-test-sb', 301))
if 'RCU%d' % mode in args or 'O%d' % mode in args:
lbh.checkOscillation(mode=mode)
-
+
if 'RCU%d' % mode in args or 'SP%d' % mode in args:
lbh.checkSpurious(mode=mode)
-
+
if 'RCU%d' % mode in args or 'N%d' % mode in args:
- if 'RCU%d' % mode in args or args.get('N%d' %(mode)) == '-':
+ if 'RCU%d' % mode in args or args.get('N%d' % (mode)) == '-':
recordtime = 120
else:
- recordtime = int(args.get('N%d' %(mode)))
- lbh.checkNoise(mode=mode,
- record_time=recordtime,
- low_deviation=conf.getFloat('lba-noise-min-deviation', -3.0),
- high_deviation=conf.getFloat('lba-noise-max-deviation', 2.5),
- max_diff=conf.getFloat('lba-noise-max-difference', 1.5))
-
+ recordtime = int(args.get('N%d' % (mode)))
+ lbh.checkNoise(
+ mode=mode,
+ record_time=recordtime,
+ low_deviation=conf.getFloat('lba-noise-min-deviation', -3.0),
+ high_deviation=conf.getFloat('lba-noise-max-deviation', 2.5),
+ max_diff=conf.getFloat('lba-noise-max-difference', 1.5))
+
if 'RCU%d' % mode in args or 'S%d' % mode in args:
- lbh.checkSignal(mode=mode,
- subband=conf.getInt('lbh-test-sb',301),
- min_signal=conf.getFloat('lba-rf-min-signal', 75.0),
- low_deviation=conf.getFloat('lba-rf-min-deviation', -2.0),
- high_deviation=conf.getFloat('lba-rf-max-deviation', 2.0))
-
+ lbh.checkSignal(
+ mode=mode,
+ subband=conf.getInt('lbh-test-sb', 301),
+ min_signal=conf.getFloat('lba-rf-min-signal', 75.0),
+ low_deviation=conf.getFloat('lba-rf-min-deviation', -2.0),
+ high_deviation=conf.getFloat('lba-rf-max-deviation', 2.0))
+
for mode in (5, 6, 7):
- # do all rcumode 5, 6, 7 tests
+ # do all rcumode 5, 6, 7 tests
hba.reset()
-
+
if 'RCU%d' % mode in args or 'M%d' % mode in args:
hba.checkModem(mode=mode)
hba.turnOffBadTiles()
-
+
if 'RCU%d' % mode in args or 'O%d' % mode in args:
hba.checkOscillation(mode=mode)
-
+
if 'RCU%d' % mode in args or 'SN%d' % mode in args:
hba.checkSummatorNoise(mode=mode)
-
+
if 'RCU%d' % mode in args or 'SP%d' % mode in args:
hba.checkSpurious(mode=mode)
-
+
if 'RCU%d' % mode in args or 'N%d' % mode in args:
- if 'RCU%d' % mode in args or args.get('N%d' %(mode)) == '-':
+ if 'RCU%d' % mode in args or args.get('N%d' % (mode)) == '-':
recordtime = 120
else:
- recordtime = int(args.get('N%d' %(mode)))
- hba.checkNoise(mode=mode,
- record_time=recordtime,
- low_deviation=conf.getFloat('hba-noise-min-deviation', -3.0),
- high_deviation=conf.getFloat('hba-noise-max-deviation', 2.5),
- max_diff=conf.getFloat('hba-noise-max-difference', 2.0))
-
-
- if 'RCU%d' % mode in args or 'S%d' % mode in args:
- hba.checkSignal(mode=mode, subband=conf.getInt('hba-test-sb',155),
- min_signal=conf.getFloat('hba-rf-min-signal', 80.0),
- low_deviation=conf.getFloat('hba-rf-min-deviation', -24.0),
- high_deviation=conf.getFloat('hba-rf-max-deviation', 12.0))
-
+ recordtime = int(args.get('N%d' % (mode)))
+ hba.checkNoise(
+ mode=mode,
+ record_time=recordtime,
+ low_deviation=conf.getFloat('hba-noise-min-deviation', -3.0),
+ high_deviation=conf.getFloat('hba-noise-max-deviation', 2.5),
+ max_diff=conf.getFloat('hba-noise-max-difference', 2.0))
+
+ # if 'RCU%d' % mode in args or 'S%d' % mode in args:
+ if 'S%d' % mode in args:
+ hba.checkSignal(
+ mode=mode,
+ subband=conf.getInt('hba-test-sb', 155),
+ min_signal=conf.getFloat('hba-rf-min-signal', 80.0),
+ low_deviation=conf.getFloat('hba-rf-min-deviation', -24.0),
+ high_deviation=conf.getFloat('hba-rf-max-deviation', 12.0))
+
runtime = (time.time() - runstart)
-
+
# All element test
if 'E%d' % mode in args:
- if args.get('E%d' %(mode)) == '-':
+ if args.get('E%d' % (mode)) == '-':
recordtime = 10
else:
- recordtime = int(args.get('E%d' %(mode)))
-
- hba.checkElements( mode=mode,
- record_time=recordtime,
- subband=conf.getInt('hba-test-sb',155),
- noise_low_deviation=conf.getFloat('ehba-noise-min-deviation', -3.0),
- noise_high_deviation=conf.getFloat('ehba-noise-max-deviation', 2.5),
- noise_max_diff=conf.getFloat('ehba-noise-max-difference', 1.5),
- rf_min_signal=conf.getFloat('ehba-rf-min-signal', 70.0),
- rf_low_deviation=conf.getFloat('ehba-rf-min-deviation', -24.0),
- rf_high_deviation=conf.getFloat('ehba-rf-max-deviation', 12.0))
-
+ recordtime = int(args.get('E%d' % (mode)))
+
+ hba.checkElements(
+ mode=mode,
+ record_time=recordtime,
+ subband=conf.getInt('hba-test-sb', 155),
+ noise_low_deviation=conf.getFloat('ehba-noise-min-deviation', -3.0),
+ noise_high_deviation=conf.getFloat('ehba-noise-max-deviation', 2.5),
+ noise_max_diff=conf.getFloat('ehba-noise-max-difference', 1.5),
+ rf_min_signal=conf.getFloat('ehba-rf-min-signal', 70.0),
+ rf_low_deviation=conf.getFloat('ehba-rf-min-deviation', -24.0),
+ rf_high_deviation=conf.getFloat('ehba-rf-max-deviation', 12.0))
+
# stop test if driver stopped
db.rsp_driver_down = not checkActiveRSPDriver()
if db.rsp_driver_down and (restarts > 0):
@@ -628,59 +640,68 @@ def main():
break
else:
time.sleep(30.0)
-
+
# one run done
repeat_cnt += 1
-
except:
logging.error('Caught %s', str(sys.exc_info()[0]))
logging.error(str(sys.exc_info()[1]))
logging.error('TRACEBACK:\n%s', traceback.format_exc())
logging.error('Aborting NOW')
- #logger.error('Program fault, RSP test (%s)' %(sys.exc_value))
- #raise
break
-
- db.rsp_driver_down = not checkActiveRSPDriver()
+
+ db.rsp_driver_down = not checkActiveRSPDriver()
if not db.rsp_driver_down:
resetRSPsettings()
-
+
# do TBB tests if requested
db.tbb_driver_down = not checkActiveTBBDriver()
- if (not db.tbb_driver_down) and (tbb_check == True):
+ if (not db.tbb_driver_down) and (tbb_check is True):
# wait for TBB boards ready
if waitTBBready(nTBB) == 1:
try:
- if args.has_key('TV'):
+ if 'TV' in args:
db.addTestDone('TV')
tbb.checkVersions(conf.getStr('tbbdriver-version'), conf.getStr('tbbctl-version'),
conf.getStr('tp-version'), conf.getStr('mp-version'))
- if args.has_key('TM'):
+ if 'TM' in args:
db.addTestDone('TM')
tbb.checkMemory()
except:
- logger.error('Program fault, TBB test (%s)' %(sys.exc_value))
+ logging.error('Program fault, TBB test')
+ logging.error('Caught %s', str(sys.exc_info()[0]))
+ logging.error(str(sys.exc_info()[1]))
+ logging.error('TRACEBACK:\n%s', traceback.format_exc())
+ logging.error('Aborting NOW')
+
db.check_stop_time = time.gmtime()
- # do db test and write result files to log directory
- log_dir = conf.getStr('log-dir-local')
- if os.path.exists(log_dir):
- logger.info('write result data')
- db.test(log_dir)
- else:
- logger.warn('not a valid log directory')
- if not db.rsp_driver_down:
- logger.info('Going back to swlevel %d' %(start_level))
- swlevel(start_level)
-
+ try:
+ # do db test and write result files to log directory
+ log_dir = conf.getStr('log-dir-local')
+ if os.path.exists(log_dir):
+ logger.info('write result data')
+ db.test(log_dir)
+ else:
+ logger.warn('not a valid log directory')
+ if not db.rsp_driver_down:
+ logger.info('Going back to swlevel %d' % (start_level))
+ swlevel(start_level)
+ # delete files from data directory
+ removeAllDataFiles()
+ except:
+ logging.error('Program fault, reporting and cleanup')
+ logging.error('Caught %s', str(sys.exc_info()[0]))
+ logging.error(str(sys.exc_info()[1]))
+ logging.error('TRACEBACK:\n%s', traceback.format_exc())
+ logging.error('Aborting NOW')
+
logger.info('Test ready.')
- writeMessage('!!! The test is ready and the station can be used again! !!!')
+ writeMessage('!!! The test is ready and the station can be used again! !!!')
- # delete files from data directory
- removeAllDataFiles()
- sys.exit(0)
+ return 0
if __name__ == '__main__':
- main()
+ sys.exit(main())
diff --git a/LCU/checkhardware/lib/general_lib.py b/LCU/checkhardware/lib/general_lib.py
index c1de15811ae..36d5de786fa 100644
--- a/LCU/checkhardware/lib/general_lib.py
+++ b/LCU/checkhardware/lib/general_lib.py
@@ -3,13 +3,17 @@ general script
"""
from subprocess import (Popen, PIPE)
+import traceback
import time
import os
+import sys
+import logging
general_version = '0913'
+logger = logging.getLogger()
def writeMessage(msg):
- res = sendCmd('wall', msg)
+ sendCmd('wall', msg)
return
# Return date string in the following format YYYYMMDD
@@ -39,10 +43,13 @@ def sendCmd(cmd='', args=''):
if len(so) != 0:
return (so)
else:
- print se
- return ('No Response')
+ return ('Error, %s' % se)
except:
+ logger.error('Caught %s', str(sys.exc_info()[0]))
+ logger.error(str(sys.exc_info()[1]))
+ logger.error('TRACEBACK:\n%s', traceback.format_exc())
return ('Exception Error')
+
return ('')
# Get Host name
diff --git a/LCU/checkhardware/rtsm.py b/LCU/checkhardware/rtsm.py
index 8b3a6573d1a..1f9d114fd00 100755
--- a/LCU/checkhardware/rtsm.py
+++ b/LCU/checkhardware/rtsm.py
@@ -8,7 +8,7 @@ import traceback
import os
import numpy as np
import time
-import datetime
+#import datetime
import logging
mainPath = r'/opt/stationtest'
@@ -241,8 +241,8 @@ class CSV:
return
@staticmethod
def writeSpectra(data, rcu, check):
- dumpTime = time.gmtime(CSV.record_timestamp)
- date_str = time.strftime("%Y%m%d", dumpTime)
+ #dumpTime = time.gmtime(CSV.record_timestamp)
+ #date_str = time.strftime("%Y%m%d", dumpTime)
full_filename = os.path.join(rtsmPath, CSV.filename)
@@ -536,10 +536,10 @@ class cDayInfo:
def getObsId():
- obs_start_str = ""
- obs_stop_str = ""
- obs_start_time = 0.0
- obs_stop_time = 0.0
+ #obs_start_str = ""
+ #obs_stop_str = ""
+ #obs_start_time = 0.0
+ #obs_stop_time = 0.0
obsids = ""
answer = sendCmd('swlevel')
if answer.find("ObsID") > -1:
@@ -596,10 +596,10 @@ class RecordBeamletStatistics(Thread):
def kill_recording(self):
if self.running:
logger.debug("kill recording beamlet statistics")
- response = sendCmd(cmd='killall', args='rspctl')
+ sendCmd(cmd='pkill', args='rspctl')
logger.debug("recording killed")
- self.running = False
- self.make_plots()
+ #self.running = False
+ #self.make_plots()
def make_plots(self):
if self.obsid:
@@ -624,7 +624,7 @@ def main():
obs_id = ""
active_obs_id = ""
rcumode = 0
- station = getHostName()
+ #station = getHostName()
DI = cDayInfo()
args = getArguments()
@@ -638,7 +638,7 @@ def main():
conf = cConfiguration()
- StID = getHostName()
+ #StID = getHostName()
logger.info('== Start rtsm (Real Time Station Monitor) ==')
@@ -704,7 +704,7 @@ def main():
# start recording beamlets
if not beamlet_recording:
if obs_start_time > 0.0 and time.time() >= obs_start_time:
- duration = obs_stop_time - time.time() - 5
+ duration = obs_stop_time - time.time() - 10
if duration > 2:
beamlet_recording = RecordBeamletStatistics()
beamlet_recording.set_obsid(active_obs_id)
diff --git a/LCU/checkhardware/showTestResult.py b/LCU/checkhardware/showTestResult.py
index 5687e12fe42..1900b69c923 100755
--- a/LCU/checkhardware/showTestResult.py
+++ b/LCU/checkhardware/showTestResult.py
@@ -8,28 +8,29 @@ import string
# import datetime
# import time
+logdir = ''
def print_help():
""" print help """
print "possible option for this script"
- print "-------------------------------"
+ print "--------------------------------------------------"
print "-h print this help screen"
- print "-d=2 show last 2 days"
- print "-f=full_path_filename"
- print "-------------------------------"
+ print "-L2=2 show last 2 checks from L2 file"
+ print "-S=2 show last 2 checks from S file"
+ print "-f=filename full_path_filename, L2 and S are ignored"
+ print " "
+ print " if no option is given the last check done is used"
+ print "--------------------------------------------------"
sys.exit(0)
-arg_nr = 1
args = dict()
-while arg_nr < len(sys.argv):
- if sys.argv[arg_nr][0] == '-':
- opt = sys.argv[arg_nr][1].upper()
- optval = '-'
- valpos = sys.argv[arg_nr].find('=')
- if valpos != -1:
- optval = sys.argv[arg_nr][valpos+1:]
- args[opt] = optval
- arg_nr += 1
+for argument in sys.argv[1:]:
+ if argument.startswith('-'):
+ if '=' in argument:
+ option, value = argument.strip()[1:].split('=')
+ args[option.strip().upper()] = value.strip()
+ else:
+ args[argument.strip()[1:].upper()] = '-'
if 'H' in args:
print_help()
@@ -48,6 +49,8 @@ station_id = getHostName().upper()
def main():
+ global logdir
+
""" main function """
fd = open(run_path+r'/checkHardware.conf', 'r')
data = fd.readlines()
@@ -56,161 +59,172 @@ def main():
if line.find('log-dir-local') != -1:
key, logdir = line.strip().split('=')
+ data_sets = []
+
if 'F' in args:
fullfilename = args.get('F')
else:
- if 'D' in args:
+ if 'L2' in args:
testfilename = '%s_L2_StationTestHistory.csv' % station_id
- else:
+ data_sets.append( ['L2', get_data(testfilename, int(args.get('L2', '1')))] )
+
+ if 'S' in args:
+ testfilename = '%s_S_StationTestHistory.csv' % station_id
+ data_sets.append( ['S', get_data(testfilename, int(args.get('S', '1')))] )
+
+ if not 'L2' in args and not 'S' in args:
testfilename = '%s_StationTest.csv' % station_id
-
- if os.path.exists(logdir):
- fullfilename = os.path.join(logdir, testfilename)
- else:
- print "not a valid log dir"
- sys.exit(-1)
-
- try:
- fd = open(fullfilename, 'r')
- data = fd.readlines()
- fd.close()
- except:
- print "%s not found in %s" % (testfilename, logdir)
- sys.exit(-1)
+ data_sets.append( ['', get_data(testfilename, 1)] )
rcu_x = rcu_y = 0
- print "\n" + "-" * 103
- print ">" * 36 + " LAST STATION-CHECK RESULT " + "<" * 36
- print "-" * 103
-
_part = ''
_part_nr = -1
- first_date = 0
- if 'D' in args:
- days = int(args.get('D'))
-
- days_cnt = 1
-
- first_date = data[-1].strip().split(',')[0]
- print first_date
- for i in range(len(data) - 1, -1, -1):
- line = data[i]
+ # print data for all sets
+ print "\n\n\n"
+ for check_type, data in data_sets:
+ message = "STATION-CHECK RESULTS %s for last %s checks" % (check_type, args.get('%s' % check_type, '1'))
+ banner_len = 100
+ msg_len = len(message)
+ print "-" * banner_len
+ print ">" * ((banner_len - msg_len - 6) / 2) + " %s " % message + "<" * ((banner_len - msg_len - 6) / 2)
+ print "-" * banner_len
+
+ check_nr = int(args.get('%s' % check_type, '1')) - 1
+ for line in data:
+ partnumber = -1
if line[0] == '#':
continue
- line_date = line.strip().split(',')[0]
- if line_date != first_date:
- first_date = line_date
- days_cnt += 1
- if days_cnt == days:
- break
- print first_date
-
- last_date = first_date
- for line in data:
- partnumber = -1
- if line[0] == '#':
- continue
-
- d = line.strip().split(',')
- if len(d) < 4:
- continue
- date = d[0]
- if 'D' in args:
- if last_date != date:
- print '\n'+'#'*103
- last_date = date
-
- if first_date != 0 and int(date) < int(first_date):
- continue
+ d = line.strip().split(',')
+ if len(d) < 4:
+ continue
+ date = d[0]
- part = d[1]
- if d[2] != '---':
- partnumber = int(d[2])
- if part == 'LBL':
- if partnumber < 48:
- print "ERROR: LBL %d NOT a legal partnumber" % partnumber
- rcu_x = 0
- rcu_y = 0
+ if 'STATION' in line:
+ if check_nr:
+ message = "= csv -%s- (last - %d) =" % (check_type, check_nr)
else:
- rcu_x = (partnumber - 48) * 2
- rcu_y = (partnumber - 48) * 2 + 1
- if part in ('LBH', 'HBA'):
- rcu_x = partnumber * 2
- rcu_y = partnumber * 2 + 1
-
- msg = d[3].strip()
- msg_info = string.join(d[4:], " ")
- keyvalue = dict()
- for i in range(4, len(d)):
- if d[i].find('=') != -1:
- key, valstr = d[i].split('=')
- vallist = valstr.split(' ')
- if len(vallist) == 1:
- keyvalue[key] = vallist[0]
- elif len(vallist) > 1:
- keyvalue[key] = vallist
- else:
- keyvalue[d[i]] = '-'
-
- if part == 'NFO':
- print_info(msg, keyvalue, msg_info)
-
- if part == 'SPU':
- if part != _part:
- _part = part
- hdr = "\n== SPU "
- print hdr + "=" * (104 - len(hdr))
- print_spu(partnumber, msg, keyvalue, msg_info)
-
- if part == 'RSP':
- if part != _part:
- _part = part
- hdr = "\n== RSP "
- print hdr + "=" * (104 - len(hdr))
- print_rsp(partnumber, msg, keyvalue)
-
- if part == 'TBB':
- if part != _part:
- _part = part
- hdr = "\n== TBB "
- print hdr + "="*(104-len(hdr))
- print_tbb(partnumber, msg, keyvalue)
-
- if part == 'RCU':
- if part != _part:
- _part = part
- hdr = "\n== RCU "
- print hdr + "=" * (104 - len(hdr))
- print_rcu(partnumber, msg, keyvalue)
-
- if part in ('LBL', 'LBH'):
- if part != _part:
- _part = part
+ message = "= csv -%s- (last) =" % check_type
+ print ' ' + '=' * len(message)
+ print ' ' + message
+ print ' ' + '=' * len(message)
+ check_nr -= 1
+
+ part = d[1]
+ if d[2] != '---':
+ partnumber = int(d[2])
if part == 'LBL':
- hdr = "\n== LBA Low "
+ if partnumber < 48:
+ print "ERROR: LBL %d NOT a legal partnumber" % partnumber
+ rcu_x = 0
+ rcu_y = 0
+ else:
+ rcu_x = (partnumber - 48) * 2
+ rcu_y = (partnumber - 48) * 2 + 1
+ if part in ('LBH', 'HBA'):
+ rcu_x = partnumber * 2
+ rcu_y = partnumber * 2 + 1
+
+ msg = d[3].strip()
+ msg_info = string.join(d[4:], " ")
+ keyvalue = dict()
+ for i in range(4, len(d)):
+ if d[i].find('=') != -1:
+ key, valstr = d[i].split('=')
+ vallist = valstr.split(' ')
+ if len(vallist) == 1:
+ keyvalue[key] = vallist[0]
+ elif len(vallist) > 1:
+ keyvalue[key] = vallist
else:
- hdr = "\n== LBA High "
- print hdr + "=" * (104 - len(hdr))
- print_lba(partnumber, msg, keyvalue, rcu_x, rcu_y)
-
- if part == 'HBA':
- if part != _part:
- _part = part
- hdr = "\n== HBA "
- print hdr + "=" * (104 - len(hdr))
-
- if partnumber != -1 and partnumber != _part_nr:
- _part_nr = partnumber
- header = "Tile %d (RCU %d/%d)" % (partnumber, rcu_x, rcu_y)
- print "\n-- %s %s" % (header, '-' * (99 - len(header)))
-
- print_hba(partnumber, msg, keyvalue, rcu_x, rcu_y)
-
- print '\n' + '#' * 103
-
+ keyvalue[d[i]] = '-'
+
+ if part == 'NFO':
+ print_info(msg, keyvalue, msg_info)
+
+ if part == 'SPU':
+ if part != _part:
+ _part = part
+ hdr = "\n== SPU "
+ print hdr + "=" * (banner_len - len(hdr))
+ print_spu(partnumber, msg, keyvalue, msg_info)
+
+ if part == 'RSP':
+ if part != _part:
+ _part = part
+ hdr = "\n== RSP "
+ print hdr + "=" * (banner_len - len(hdr))
+ print_rsp(partnumber, msg, keyvalue)
+
+ if part == 'TBB':
+ if part != _part:
+ _part = part
+ hdr = "\n== TBB "
+ print hdr + "="*(banner_len - len(hdr))
+ print_tbb(partnumber, msg, keyvalue)
+
+ if part == 'RCU':
+ if part != _part:
+ _part = part
+ hdr = "\n== RCU "
+ print hdr + "=" * (banner_len - len(hdr))
+ print_rcu(partnumber, msg, keyvalue)
+
+ if part in ('LBL', 'LBH'):
+ if part != _part:
+ _part = part
+ if part == 'LBL':
+ hdr = "\n== LBA Low "
+ else:
+ hdr = "\n== LBA High "
+ print hdr + "=" * (banner_len - len(hdr))
+ print_lba(partnumber, msg, keyvalue, rcu_x, rcu_y)
+
+ if part == 'HBA':
+ if part != _part:
+ _part = part
+ hdr = "\n== HBA "
+ print hdr + "=" * (banner_len - len(hdr))
+
+ if partnumber != -1 and partnumber != _part_nr:
+ _part_nr = partnumber
+ header = "Tile %d (RCU %d/%d)" % (partnumber, rcu_x, rcu_y)
+ print "\n-- %s %s" % (header, '-' * (banner_len - len(header)))
+
+ print_hba(partnumber, msg, keyvalue, rcu_x, rcu_y)
+
+ #print '\n' + '#' * banner_len
+
+
+def get_data(filename, n_checks):
+ if not filename.startswith('/'):
+ if os.path.exists(logdir):
+ fullfilename = os.path.join(logdir, filename)
+ else:
+ print "not a valid log dir"
+ sys.exit(-1)
+ try:
+ fd = open(fullfilename, 'r')
+ data = fd.readlines()
+ fd.close()
+ except:
+ print "%s not found in %s" % (filename, logdir)
+ sys.exit(-1)
+
+ first_line = 0
+ check_cnt = 0
+ for i in range(len(data) - 1, -1, -1):
+ line = data[i]
+ if line[0] == '#':
+ continue
+ if 'STATION' in line:
+ first_line = i
+ check_cnt += 1
+ if check_cnt == n_checks:
+ break
+ return data[first_line:]
def print_info(msg, keyvalue, msg_info):
"""
--
GitLab