Newer
Older
#!/usr/bin/env python
#
# Run the tests to test a LOFAR station
# H. Meulman
# 24 sep: local log directory aangepast
# 27 sept: - Toevoeging delay voor tbbdriver polling
# - Aanzetten van LBA's m.b.v rspctl --aweights=8000,0
# 26 nov: Check op 160 MHZ en 200 MHZ clock.
# 18 jan 2011: Check op !="?" vervangen door =='LOCKED' in 160 en 200 MHz clock test
# 18 jan 2011: Local Log directory aangepast
# 19 jan 2011: clocktest in 160 en 200 MHz clock test over een 10 itteraties!
# 19 jan 2011: Diff test AP nummer word nu ook gelogged. 'sync' verwijderd
# 19 jan 2011: TBB versie aangepast naar 2.32
# 21 jan 2011: TBB Version foutmelding aangepast (AP/BP Error moet TP/BP Error zijn)
# 02 feb 2011: LBA down toe gevoegd!
# 18 mrt 2011: Automatiesche detectie Core- remote- en International station toegevoegd.
# 18 mrt 2011: Volgende testen aangepast zodat ze ook internationale stations kunnen testen:
# CheckRSPVersion, CheckTDSStatus160 en 200, CheckRSPStatus, CheckTBBVersion
# LBAtest(), HBAModemTest(), HBAtest() De laatste test werkt pas als transmitters geinstalleerd zijn.
# 18 mrt 2011: Als alle LBA's niet werken, wordt error gelogd. (average < 4000000)
# 30 mrt 2011: TBBversion_int.gold aangepast voor internationale stations.
# 7 sep 2011: Bug removed. On the remote stations LBA mode 1 will now also be tested.
# added CS028 and CS031
# 22 nov 2011 TBB versie aanpassen naar 2.39
# todo:
# - Als meer dan 10 elementen geen rf signaal hebben, keur dan hele tile af
# - als beamserver weer goed werkt deze weer toevoegen aan LBA test
# - =='LOCKED' in 160 en 200 MHz clock test over een aantal keren!
# Loggen absolute waarden van alle antennes (LBH LBL eb HBA)
import sys
from optparse import OptionParser
import cli
import testlog
from time import localtime, strftime
import array
import os
import time
import commands
import operator
import math
import numpy
################################################################################
# Init
# Variables
debug=0
clkoffset=1
factor = 30 # station statistics fault window: Antenna average + and - factor = 100 +/- 30
InternationalStations = ('DE601C','DE602C','DE603C','DE604C','DE605C','FR606C','SE607C','UK608C')
RemoteStations = ('CS302C','RS106C','RS205C','RS208C','RS306C','RS307C','RS406C','RS503C')
CoreStations = ('CS001C','CS002C','CS003C','CS004C','CS005C','CS006C','CS007C','CS011C','CS013C','CS017C','CS021C','CS024C','CS026C','CS028C','CS030C','CS031','CS032C','CS101C','CS103C','CS201C','CS301C','CS401C','CS501C')
NoHBAelementtestPossible = ('DE601C','DE602C','DE603C','DE605C','FR606C','SE607C','UK608C')
HBASubband = dict( DE601C=155,\
DE602C=155,\
DE603C=284,\
DE605C=479,\
FR606C=155,\
SE607C=155,\
UK608C=155)
# Do not change:
Severity=0 # Severity (0='' 1=feature 2=minor 3=major 4=block 5=crash
Priority=0 # Priority (0=no 1=low 2=normal 3=high 4=urgent 5=immediate
SeverityLevel=('-- ','feature','minor ','Major ','BLOCK ','CRASH ')
PriorityLevel=('-- ','low ','normal ','High ','URGENT ','IMMEDIATE')
#print (SeverityLevel[Severity])
#print (PriorityLevel[Priority])
# Time
tm=strftime("%a, %d %b %Y %H:%M:%S", localtime()) # Determine system time
tme=strftime("_%b_%d_%Y_%H.%M", localtime()) # Time for fileheader History log file
# Determine station ID and station type
StationType = 0
Core = 1
Remote = 2
International = 3
StIDlist = os.popen3('hostname -s')[1].readlines() # Name of the station
StID = str(StIDlist[0].strip('\n'))
if debug: print ('StationID = %s' % StID)
if StID in InternationalStations: StationType = International # International station
if StID in RemoteStations: StationType = Remote # Remote Station
if StID in CoreStations: StationType = Core # Core Station
if debug: print ('StationType = %d' % StationType)
if StationType == 0: print ('Error: StationType = %d (Unknown station)' % StationType)
if StationType == International:
RSPgoldfile=('/misc/home/etc/stationtest/gold/rsp_version_int.gold')
TBBgoldfile=('/misc/home/etc/stationtest/gold/tbb_version_int.gold')
TDS=[0,4,8,12,16,20]
else:
RSPgoldfile=('/misc/home/etc/stationtest/gold/rsp_version.gold')
TBBgoldfile=('/misc/home/etc/stationtest/gold/tbb_version.gold')
TDS=[0,4,8]
TBBmgoldfile=('/misc/home/etc/stationtest/gold/tbb_memory.gold')
#LogPath=('/misc/home/log/')
TestLogPath=('/misc/home/log/') # Logging remote (on Kis001)
#TestLogPath=('/opt/stationtest/data/') # Logging local (on station)
#HistLogPath=('/opt/stationtest/data/') # Logging local (on station)
HistLogPath=('/localhome/stationtest/data/') # Logging local (on station)
TestlogName = ('%sstationtest_%s.tmp' % (TestLogPath, StID))
TestlogNameFinalized = ('%sstationtest_%s.log' % (TestLogPath, StID))
HistlogName = ('%sstationtest_%s%s.log' % (HistLogPath, StID, tme))
# Array om bij te houden welke Tiles niet RF getest hoeven worden omdat de modems niet werken.
if len(sys.argv) < 3 :
if StationType == International:
num_rcu=192
else:
num_rcu=96
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
else :
num_rcu = int(sys.argv[2])
ModemFail=[0 for i in range (num_rcu/2)]
if debug: print ModemFail
#print (TestlogName)
#print (TestlogNameFinalized)
# Parse command line for station ID
#
# -v 0 : only PASSED or FAILED
# -v 1 : overall title
# -v 11 : result per test
# -v 21 : title per test
op = OptionParser(usage='usage: python %prog [options]', version='%prog 0.1')
op.add_option('-v', type='int', dest='verbosity',
help='Verbosity level',default=11)
#op.add_option('-r', type='int', dest='rsp_nr',
# help='Provide number of rsp boards that will be used in this test',default=None)
#op.add_option('-t', type='int', dest='tbb_nr',
# help='Provide number of tbb boards that will be used in this test',default=None)
opts, args = op.parse_args()
opts.rsp_nr=12 # fixed number
opts.tbb_nr=6 # Fixed number
if (StationType == Core or StationType == Remote): # NL station doe have 12 rsp's and 6 TBB's
opts.rsp_nr=12 # fixed number
opts.tbb_nr=6 # Fixed number
noTBB=6
if StationType == International: # INT station doe have 24 rsp's and 12 TBB's
opts.rsp_nr=24 # fixed number
opts.tbb_nr=12 # Fixed number
noTBB=12
if debug: print ('RSPs = %d' % opts.rsp_nr)
if debug: print ('TBBs = %d' % opts.tbb_nr)
# - Option checks and/or reformatting
if opts.rsp_nr==None:
op.error('Option -r must specify the number of rsp boards')
if opts.tbb_nr==None:
op.error('Option -t must specify the number of tbb boards')
if opts.rsp_nr == 4:
RspBrd = 'rsp0,rsp1,rsp2,rsp3'
SubBrd = 'rsp0'
SubRck = 'sub0'
if opts.rsp_nr == 12:
RspBrd = 'rsp0,rsp1,rsp2,rsp3,rsp4,rsp5,rsp6,rsp7,rsp8,rsp9,rsp10,rsp11'
SubBrd = 'rsp0,rsp4,rsp8'
SubRck = 'sub0,sub1,sub2'
if opts.rsp_nr == 24:
RspBrd = 'rsp0,rsp1,rsp2,rsp3,rsp4,rsp5,rsp6,rsp7,rsp8,rsp9,rsp10,rsp11,rsp12,rsp13,rsp14,rsp15,rsp16,rsp17,rsp18,rsp19,rsp20,rsp21,rsp22,rsp23'
SubBrd = 'rsp0,rsp4,rsp8,rsp12,rsp16,rsp20'
SubRck = 'sub0,sub1,sub2,sub3,sub4,sub5'
if debug: print ('RspBrd = %s' % RspBrd)
# Define subrack testlog class for pass/fail and logging
vlev = opts.verbosity
testId = ''
appLev = False
logName = '/opt/stationtest/data/STAT-%05d-%05d.dat' % (opts.rsp_nr, opts.tbb_nr)
cli.command('rm -f /opt/stationtest/data/STAT-%05d-%05d.dat', appLev)
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
sr = testlog.Testlog(vlev, testId, logName)
sr.setResult('PASSED')
sr.setId('Station - ')
sr.appendLog(11,'')
sr.appendLog(1,'Station production test %s' % logName)
sr.appendLog(11,'')
# Define station testlog
st_log = file(TestlogName, 'w')
st_log.write('StID >: %s\n' % StID)
st_log.write('Lgfl >: %s\n' % TestlogNameFinalized)
st_log.write('Time >: %s\n' % tm)
#cli.command('rm -f %s' % logName, appLev)
#cli.command('rm -f /opt/stationtest/data/stationtest.log', appLev)
#sr.setId('')
#sr.appendLog(11,'')
#sr.appendLog(1,'Lgfl >: %s' % logNameFinalized)
#sr.appendLog(1,'StID >: %s' % StID)
#sr.appendLog(1,'Time >: %s' % tm)
#time.sleep(20)
################################################################################
# Function CheckTBB : CHeck if TBB's are running. The returned string
# "V 0.3 V 4.7 V 2.4 V 2.9" Shouls have 4 times 'V'
def CheckTBB():
SeverityOfThisTest=3
PriorityOfThisTest=3
global Severity
global Priority
print 'Checking TBB!!!'
print 'wait 60 sec'
time.sleep(60)
if debug: print int(len(os.popen3('tbbctl --version')[1].readlines()))
sr.setId('TBB >: ')
n=0 # Maximum itteration
while len(os.popen3('tbbctl --version')[1].readlines()) < 4:
print ('-'),
# if debug:
print ('Polling TBB Driver')
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
time.sleep(5)
n+=1
if n > 12:
sr.appendLog(11,'Error: TBB driver is not running or some TBBs not active')
sr.setResult('FAILED')
# store station testlog
if Severity<SeverityOfThisTest: Severity=SeverityOfThisTest
if Priority<PriorityOfThisTest: Priority=PriorityOfThisTest
st_log.write('TBB >: Sv=%s Pr=%s, Error: TBB driver is not running\n' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest]))
return
n=0 # Check till 4 V's per TBB
while n < 12: # maximum itterations
res2 = os.popen3('tbbctl --version')[1].readlines()
if debug:
for line in res2:
print ('%s' % line.rstrip('\n'))
#print ('res2 is: %s' % res2)
#print ('res2[9] is: %s' % res2[9])
print ('Itteration %d' % n)
else:
print '*',
cnt=0
TBBrange=range(noTBB)
for TBBnr in TBBrange:
cnt += res2[9+TBBnr].count('V') # count number of 'V's (Version)
if cnt == noTBB*4: # 4 per TBB
print "TBB's OK"
break
n+=1
time.sleep(5)
else:
for TBBnr in TBBrange:
if res2[9+TBBnr].count('V') != 4: # Log Errors
sr.appendLog(11,'Error: TBB :%s' % str(res2[9+TBBnr].strip('\n')))
sr.setResult('FAILED')
# store station testlog
if Severity<SeverityOfThisTest: Severity=SeverityOfThisTest
if Priority<PriorityOfThisTest: Priority=PriorityOfThisTest
st_log.write('TBB >: Sv=%s Pr=%s, Error: TBB : %s\n' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest], str(res2[9+TBBnr].strip('\n'))))
#print ('number of Vs is ', res2[9+TBBnr].count('V')),
#print (' Error in TBB : %s' % res2[9+TBBnr])
if debug:
print 'stopped Checking TBB'
print ("number of V's is %d" % cnt)
return
################################################################################
# Function Goto Swlevel 2
def GotoSwlevel2():
res = os.popen3('swlevel 1')[1].readlines()
if debug:
print('System is Going to swlevel 1')
# for line in res:
# print ('To swlevel 1 ', line) # werkt niet!!???
time.sleep(5)
# Set swlevel 2 if not running
#
res = os.popen3('swlevel')[1].readlines()
#print res[1]
if len(res) > 0:
for line in res:
if debug: print ('%s' % line.rstrip('\n'))
if line == ('2 : RSPDriver DOWN\n') or line == ('2 : TBBDriver DOWN\n'):
print 'System is Going to swlevel 2'
# errorprg = os.system('swlevel 2')
# if len(err) > 0:
# else:
# print fromprg.readlines()
res2 = os.popen3('swlevel 2')[1].readlines()
# print errorprg
print 'wait 120 sec'
if debug:
for line in res2:
print ('%s' % line.rstrip('\n'))
time.sleep(30)
res = os.popen3('rspctl --datastream=0')[1].readlines()
print res
time.sleep(90) # Tijdelijk toe gevoegd voor nieuwe tbbdriver. Deze loopt vast tijdens pollen
# CheckTBB() # Tijdelijk weg gelaten voor nieuwe tbbdriver. Deze loopt vast tijdens pollen
#fromprg.close()
break
return
#res.close()
################################################################################
# Check ntpd time demon
#
def CheckNtpd():
SeverityOfThisTest=3
PriorityOfThisTest=3
global Severity
global Priority
print ('Check of the Ntpd!')
sr.setId('Clock - ')
res = os.popen3('/usr/sbin/ntpq -p')[1].readlines()
#res = os.popen3('/opt/stationtest/test/timing/ntpd.sh')[1].readlines()
if debug:
for line in res:
print ('-%s' % line.rstrip('\n'))
#print ('res : %s' % res)
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
if len(res) > 0:
# print (res[3])
offset=0
for line in res:
if debug: print('line= %s' % line)
locallock=line.find('*LOCAL(0)')
if locallock==0: break
gpslock=line.find('*GPS_ONCORE(0)')
if gpslock==0:
offset=float((line.split())[8])
break
if debug:
# print ('res[3] is: %s' % res[3])
# print ('res[4] is: %s' % res[4])
print ('gpslock is %s' % gpslock)
print ('locallock is %s' % locallock)
print ('offset is %.3f' % offset)
if gpslock > -1:
if debug: print('GPS in Lock. OK')
else:
if locallock > -1:
sr.appendLog(11,'Clock locked on Local Clock!!')
if Severity<SeverityOfThisTest: Severity=SeverityOfThisTest
if Priority<PriorityOfThisTest: Priority=PriorityOfThisTest
st_log.write('Clock >: Sv=%s Pr=%s, Clock locked on Local Clock!\n' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest]))
sr.setResult('FAILED')
else:
sr.appendLog(11,'Clock out of sync!!')
if Severity<SeverityOfThisTest: Severity=SeverityOfThisTest
if Priority<PriorityOfThisTest: Priority=PriorityOfThisTest
st_log.write('Clock >: Sv=%s Pr=%s, Clock out of sync!\n' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest]))
sr.setResult('FAILED')
if offset < -clkoffset or offset > clkoffset:
if Severity<SeverityOfThisTest: Severity=SeverityOfThisTest
if Priority<PriorityOfThisTest: Priority=PriorityOfThisTest
st_log.write('Clock >: Sv=%s Pr=%s, Clock Offset to large : %.3f\n' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest], offset))
sr.appendLog(11,'Clock Offset to large : %.3f' % offset)
sr.setResult('FAILED')
else:
sr.appendLog(11,'no answer from ntpq!')
if Severity<SeverityOfThisTest: Severity=SeverityOfThisTest
if Priority<PriorityOfThisTest: Priority=PriorityOfThisTest
st_log.write('Clock >: Sv=%s Pr=%s, no answer from ntpq!\n' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest]))
sr.setResult('FAILED')
return
################################################################################
# Function Check RSP status bytes
#
def CheckRSPStatus():
# debug = 1
SeverityOfThisTest=2
PriorityOfThisTest=2
global Severity
global Priority
sr.setId('RSPst >: ')
print ('Check RSP Status')
OutputClock,PLL160MHz,PLL200MHz=gettdstatus() # td-status
time.sleep(1)
res = os.popen3('rspctl --status')[1].readlines()
#print res[1]
linecount=0
if len(res) > 0:
for line in res:
sync=line.find('RSP[ 0] Sync')
if sync==0: break
linecount+=1
#print 'sync = ' + str(sync) + ' and linecount = ' + str(linecount)
for rsp in range(opts.rsp_nr):
# print res[linecount+rsp]
# x = res[linecount+rsp].split( )
# print res[linecount+rsp*5].lstrip('RSP').strip('[').split()
if debug:
print '\n',
print res[linecount+rsp*5],
for sync in range(1, 5):
dif = res[linecount+rsp*5+sync].lstrip('RSP').strip('[').strip(':').split()
if debug:
print ('Dif = %s' % dif)
#print str(linecount+rsp*5+sync),
#print dif[2]
if dif[2] not in ('0', '512'): # was ('0', '1', '512', '513'):
#if debug: print ('RSP : %d status error: sync = %d, diff = %d' % (int(rsp), int(sync), int(dif[2])))
sr.appendLog(11,'RSP : %d status error: sync = %d diff = %d' % (int(rsp), int(sync), int(dif[2])))
sr.setResult('FAILED')
if Severity<SeverityOfThisTest: Severity=SeverityOfThisTest
if Priority<PriorityOfThisTest: Priority=PriorityOfThisTest
st_log.write('RSPst >: Sv=%s Pr=%s, RSP : %d AP%d status error at %s MHz: diff = %d\n' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest], int(rsp), int(dif[1].strip(':')), OutputClock, int(dif[2])))
sr.setResult('FAILED')
#time.sleep(3)
# debug = 0
################################################################################
# Function check if clock 160 MHz is locked
#
def CheckTDSStatus160():
SeverityOfThisTest=2
PriorityOfThisTest=2
global Severity
global Priority
sr.setId('TDSst >: ')
# TDS=[0,4,8]
if debug: print('TDS = ',TDS)
if StationType == International:
LockCount160=[0 for i in range (21)]
else:
LockCount160=[0 for i in range (9)]
if debug: print('LockCount160 = ',LockCount160)
PLL160MHz = '?'
PLL200MHz = '?'
res = os.popen3('rspctl --clock=160')[1].readlines()
print ('Clock set to 160MHz')
time.sleep(1)
n=0 # Wait till clock set
while n < 15: # maximum itterations
OutputClock,PLL160MHz,PLL200MHz=gettdstatus() # td-status
if PLL160MHz=='LOCKED':
print ('Clock %s' %(PLL160MHz))
break
# print ('OutputClock = ',OutputClock)
# print ('PLL160MHz = ',PLL160MHz)
# print ('PLL200MHz = ',PLL200MHz)
n+=1
time.sleep(5)
if n==15:
print ('Clock never locked')
# if Severity<SeverityOfThisTest: Severity=SeverityOfThisTest
# if Priority<PriorityOfThisTest: Priority=PriorityOfThisTest
# st_log.write('TDSst >: Sv=%s Pr=%s, TDS : all @ 160MHz never locked: PLL200MHz = %s, PLL160MHz = %s, Output Clock = %s\n' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest], PLL200MHz, PLL160MHz, OutputClock))
sr.setResult('FAILED')
# if n < 15:
for TDSBrd in TDS:
# print('TDSBrd = ',TDSBrd)
LockCount160[TDSBrd]==0
if debug: print('LockCount160[%s] = %s' % (TDSBrd,LockCount160[TDSBrd]))
n=0 # Check if clock is LOCKED every 2 seconds for 10 times!
while n < 10:
n+=1
for TDSBrd in TDS:
valid=0
PLL160MHz = '?'
PLL200MHz = '?'
# print('TDSBrd = ',TDSBrd)
res = os.popen3('rspctl --tdstatus --sel=%s'%(TDSBrd))[1].readlines()
if debug: print res[0]
for line in res:
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
if line[0] == 'R':
valid=1
if debug: print ('valid tdstatus')
#print res[0].split()
if valid == 1:
for line in res:
if line[0] == 'R': # Check of regel geldig is!
header=line.replace('|',' ').split()
if debug: print ('header = ', header)
else: # Check of regel geldig is!
status=line.replace('|',' ').replace('not locked','notlocked').split()
if debug:
print ('status= ', status)
print ('OutputClock = ',status[2])
print ('PLL160MHz = ',status[4])
print ('PLL200MHz = ',status[5])
OutputClock = status[2]
PLL160MHz = status[4]
PLL200MHz = status[5]
if PLL160MHz <> 'LOCKED':
LockCount160[TDSBrd] += 1 # store station testlog
# print('LockCount160[TDSBrd] = ',LockCount160[TDSBrd])
if LockCount160[TDSBrd] == 1: # Store Error at the first time
print ('Clock 160MHz not locked')
if Severity<SeverityOfThisTest: Severity=SeverityOfThisTest
if Priority<PriorityOfThisTest: Priority=PriorityOfThisTest
st_log.write('TDSst >: Sv=%s Pr=%s, TDS : %s @ 160MHz not locked: PLL200MHz = %s, PLL160MHz = %s, Output Clock = %s\n' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest], TDSBrd, PLL200MHz, PLL160MHz, OutputClock))
sr.setResult('FAILED')
if (n==10 and LockCount160[TDSBrd]<>0): # Store number of Errors only at the last time first time
st_log.write('TDSlt >: Sv=%s Pr=%s, TDS : %s @ 160MHz Did go wrong %s out of 10 times\n' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest], TDSBrd, LockCount160[TDSBrd]))
time.sleep(1)
return
################################################################################
# Function check if clock 200 MHz is locked
#
def CheckTDSStatus200():
# debug = 1
SeverityOfThisTest=2
PriorityOfThisTest=2
global Severity
global Priority
sr.setId('TDSst >: ')
if debug: print('TDS = ',TDS)
if StationType == International:
LockCount200=[0 for i in range (21)]
else:
LockCount200=[0 for i in range (9)]
if debug: print('LockCount200 = ',LockCount200)
PLL160MHz = '?'
PLL200MHz = '?'
res = os.popen3('rspctl --clock=200')[1].readlines()
print ('Clock set to 200MHz')
time.sleep(1)
n=0 # Wait till clock set
while n < 15: # maximum itterations
OutputClock,PLL160MHz,PLL200MHz=gettdstatus() # td-status
if PLL200MHz=='LOCKED':
print ('Clock %s' %(PLL200MHz))
break
# print ('OutputClock = ',OutputClock)
# print ('PLL160MHz = ',PLL160MHz)
# print ('PLL200MHz = ',PLL200MHz)
n+=1
time.sleep(5)
if n==15:
print ('Clock never locked')
# if Severity<SeverityOfThisTest: Severity=SeverityOfThisTest
# if Priority<PriorityOfThisTest: Priority=PriorityOfThisTest
# st_log.write('TDSst >: Sv=%s Pr=%s, TDS : all @ 200MHz not locked: PLL200MHz = %s, PLL160MHz = %s, Output Clock = %s\n' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest], PLL200MHz, PLL160MHz, OutputClock))
sr.setResult('FAILED')
# if n < 15:
for TDSBrd in TDS:
# print('TDSBrd = ',TDSBrd)
LockCount200[TDSBrd]==0
# print('LockCount200[TDSBrd] = ',LockCount200[TDSBrd])
n=0 # Check if clock is LOCKED every 2 seconds for 10 times!
while n < 10:
n+=1
for TDSBrd in TDS:
valid=0
PLL160MHz = '?'
PLL200MHz = '?'
# print('TDSBrd = ',TDSBrd)
res = os.popen3('rspctl --tdstatus --sel=%s'%(TDSBrd))[1].readlines()
if debug: print res[0]
for line in res:
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
if line[0] == 'R':
valid=1
if debug: print ('valid tdstatus')
#print res[0].split()
if valid == 1:
for line in res:
if line[0] == 'R': # Check of regel geldig is!
header=line.replace('|',' ').split()
if debug: print ('header = ', header)
else: # Check of regel geldig is!
status=line.replace('|',' ').replace('not locked','notlocked').split()
if debug:
print ('status= ', status)
print ('OutputClock = ',status[2])
print ('PLL160MHz = ',status[4])
print ('PLL200MHz = ',status[5])
OutputClock = status[2]
PLL160MHz = status[4]
PLL200MHz = status[5]
if PLL200MHz <> 'LOCKED':
LockCount200[TDSBrd] += 1 # store station testlog
# print('LockCount200[TDSBrd] = ',LockCount200[TDSBrd])
if LockCount200[TDSBrd] == 1: # Store Error at the first time
print ('Clock 200MHz not locked')
if Severity<SeverityOfThisTest: Severity=SeverityOfThisTest
if Priority<PriorityOfThisTest: Priority=PriorityOfThisTest
st_log.write('TDSst >: Sv=%s Pr=%s, TDS : %s @ 200MHz not locked: PLL200MHz = %s, PLL160MHz = %s, Output Clock = %s\n' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest], TDSBrd, PLL200MHz, PLL160MHz, OutputClock))
sr.setResult('FAILED')
if (n==10 and LockCount200[TDSBrd]<>0): # Store number of Errors only at the last time first time
st_log.write('TDSlt >: Sv=%s Pr=%s, TDS : %s @ 200MHz Did go wrong %s out of 10 times\n' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest], TDSBrd, LockCount200[TDSBrd]))
time.sleep(1)
# debug = 0
################################################################################
# Function get the TD status
#
def gettdstatus():
res = os.popen3('rspctl --tdstatus --sel=0')[1].readlines()
for line in res:
if line[0] == 'R': # Check of regel geldig is!
header=line.replace('|',' ').split()
#print ('header = ', header)
else: # Check of regel geldig is!
status=line.replace('|',' ').replace('not locked','notlocked').split()
#print ('status= ', status)
#print ('OutputClock = ',status[2])
#print ('PLL160MHz = ',status[4])
if debug: print ('PLL160MHz = %s, PLL200MHz = %s' % (status[4],status[5]))
OutputClock = status[2]
PLL160MHz = status[4]
PLL200MHz = status[5]
return OutputClock,PLL160MHz,PLL200MHz
################################################################################
# Function make RSP Version gold
#
def makeRSPVersionGold():
res = os.popen3('rspctl --version')[1].readlines()
time.sleep(3)
if StationType == International:
f_log = file('/misc/home/etc/stationtest/gold/rsp_version-int.gold', 'w')
else:
f_log = file('/misc/home/etc/stationtest/gold/rsp_version.gold', 'w')
for line in res:
print ('Res = ', line)
f_log.write(line)
print ('RSP Version Gold file has been made!')
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
return
################################################################################
# Function read RSP Version gold
#
def readRSPVersionGold():
f=open(RSPgoldfile,'rb')
# if debug:
# for line in f:
# print ('Res = ', line)
return f
################################################################################
# Function Check RSP Version
#
def CheckRSPVersion():
SeverityOfThisTest=2
PriorityOfThisTest=2
global Severity
global Priority
sr.setId('RSPver>: ')
sr.appendLog(21,'')
sr.appendLog(21,'### Verify LCU - RSP ethernet link by getting the RSP version info')
sr.appendLog(21,'')
print ('Check RSP Version')
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
# RSPgold=readRSPVersionGold()
RSPgold = open(RSPgoldfile,'r').readlines() # Read RSP Version gold
RSPversion = os.popen3('rspctl --version')[1].readlines() # Get RSP Versions
# res = cli.command('./rsp_version.sh')
if debug:
for RSPnumber in range(len(RSPgold)):
if RSPgold[RSPnumber] == RSPversion[RSPnumber]: print ('RSP OK = ', RSPnumber)
else: print ('RSPNOK = ', RSPnumber)
# store subreck testlog
for RSPnumber in range(len(RSPgold)):
if RSPgold[RSPnumber] != RSPversion[RSPnumber]:
sr.appendLog(11,'>>> RSP version test went wrong')
#sr.appendLog(11,'CLI:')
#sr.appendLog(11,res,1,1,1)
sr.appendLog(11,'Result:')
for line in RSPversion:
#print ('%s' % line.rstrip('\n'))
sr.appendLog(11,'%s' % line.rstrip('\n'))
sr.appendLog(11,'Expected:')
for line in RSPgold:
#print ('%s' % line.rstrip('\n'))
sr.appendLog(11,'%s' % line.rstrip('\n'))
sr.setResult('FAILED')
break
# store station testlog
for RSPnumber in range(len(RSPgold)):
if RSPgold[RSPnumber] != RSPversion[RSPnumber]:
if Severity<SeverityOfThisTest: Severity=SeverityOfThisTest
if Priority<PriorityOfThisTest: Priority=PriorityOfThisTest
st_log.write('RSPver>: Sv=%s Pr=%s, BP/AP Error! %s' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest], RSPversion[RSPnumber]))
sr.setResult('FAILED')
if debug: print ('RSPNOK = ', RSPnumber)
return
################################################################################
# Function make TBB Version gold
#
def makeTBBVersionGold():
res = os.popen3('tbbctl --version')[1].readlines()
time.sleep(3)
f_log = file(TBBgoldfile, 'w')
for line in res:
print ('Res = ', line)
f_log.write(line)
print ('TBB Version Gold file has been made!')
return
################################################################################
# Function Check TBB Version
#
def CheckTBBVersion():
SeverityOfThisTest=2
PriorityOfThisTest=2
global Severity
global Priority
sr.setId('TBBver>: ')
sr.appendLog(21,'')
sr.appendLog(21,'### Verify LCU - TBB ethernet link by getting the TBB version info')
sr.appendLog(21,'')
TBBgold = open(TBBgoldfile,'r').readlines() # Read TBB Version gold
TBBversion = os.popen3('tbbctl --version')[1].readlines() # Get TBB Versions
time.sleep(1)
if len(TBBversion) < 4:
# store station testlog
if Severity<SeverityOfThisTest: Severity=SeverityOfThisTest
if Priority<PriorityOfThisTest: Priority=PriorityOfThisTest
st_log.write('TBB >: Sv=%s Pr=%s, Error: TBB driver is not running\n' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest]))
print ('Returned message from TBBversion: %s' % TBBversion)
return
if debug:
print ('TBBgold: %s' % TBBgold)
print ('TBBversion: %s' % TBBversion)
for TBBnumber in range(len(TBBgold)):
if TBBgold[TBBnumber] == TBBversion[TBBnumber]: print ('TBB OK = ', TBBnumber)
else: print ('TBBNOK = ', TBBnumber)
# store subreck testlog
for TBBnumber in range(len(TBBgold)):
if TBBgold[TBBnumber] != TBBversion[TBBnumber]:
sr.appendLog(11,'>>> TBB version test went wrong')
#sr.appendLog(11,'CLI:')
#sr.appendLog(11,res,1,1,1)
sr.appendLog(11,'Result:')
for line in TBBversion:
#print ('%s' % line.rstrip('\n'))
sr.appendLog(11,'%s' % line.rstrip('\n'))
sr.appendLog(11,'Expected:')
for line in TBBgold:
#print ('%s' % line.rstrip('\n'))
sr.appendLog(11,'%s' % line.rstrip('\n'))
sr.setResult('FAILED')
break
# store station testlog
for TBBnumber in range(len(TBBgold)):
if TBBgold[TBBnumber] != TBBversion[TBBnumber]:
if Severity<SeverityOfThisTest: Severity=SeverityOfThisTest
if Priority<PriorityOfThisTest: Priority=PriorityOfThisTest
st_log.write('TBBver>: Sv=%s Pr=%s, TP/MP Error! %s' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest], TBBversion[TBBnumber]))
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
sr.setResult('FAILED')
if debug: print ('TBBNOK = ', TBBnumber)
return
################################################################################
# Function Check TBB Version Eventueel nog toevoegen!
#
#sr.setId('TBB version - ')
#sr.appendLog(21,'')
#sr.appendLog(21,'### Verify LCU - TBB ethernet link by getting the TBB version info')
#sr.appendLog(21,'')
#res = cli.command('./tbb_version.sh')
#if res.find('OK')==-1:
# sr.appendLog(11,'>>> TBB version test went wrong')
# sr.appendLog(11,'CLI:')
# sr.appendLog(11,res,1,1,1)
# sr.appendLog(11,'Result:')
# sr.appendFile(11,'tbb_version.log')
# sr.appendLog(11,'Expected:')
# sr.appendFile(11,'gold/tbb_version.gold')
# sr.setResult('FAILED')
#else:
# sr.appendLog(11,'>>> TBB version test went OK')
################################################################################
# Function make TBB Memory gold
#
def makeTBBMemGold():
res = os.popen3('./tbb_memory.sh')[1].readlines()
time.sleep(3)
f_log = file(TBBmgoldfile, 'w')
for line in res:
print ('Res = ', line)
f_log.write(line)
print ('TBB Memory Gold file has been made!')
return
################################################################################
# Function Check TBB Memory
#
def CheckTBBMemory():
SeverityOfThisTest=2
PriorityOfThisTest=2
global Severity
global Priority
sr.setId('TBBmem>: ')
sr.appendLog(21,'')
sr.appendLog(21,'### Verify TBB memory modules on the TBB')
sr.appendLog(21,'')
print ('TBB Memory check')
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
TBBmgold = open(TBBmgoldfile,'r').readlines() # Read TBB Memory gold
TBBmem = os.popen3('./tbb_memory.sh')[1].readlines() # Start TBB memory test
# res = cli.command('./tbb_version.sh')
if debug:
for TBBnumber in range(len(TBBmgold)):
if TBBmgold[TBBnumber] == TBBmem[TBBnumber]: print ('TBB OK = ', TBBnumber)
else: print ('TBBNOK = ', TBBnumber)
# store subreck testlog
# for TBBnumber in range(len(TBBgold)):
# if TBBgold[TBBnumber] != TBBversion[TBBnumber]:
# sr.appendLog(11,'>>> TBB version test went wrong')
# #sr.appendLog(11,'CLI:')
# #sr.appendLog(11,res,1,1,1)
# sr.appendLog(11,'Result:')
# for line in TBBversion:
# #print ('%s' % line.rstrip('\n'))
# sr.appendLog(11,'%s' % line.rstrip('\n'))
# sr.appendLog(11,'Expected:')
# for line in TBBgold:
# #print ('%s' % line.rstrip('\n'))
# sr.appendLog(11,'%s' % line.rstrip('\n'))
# sr.setResult('FAILED')
# break
# store station testlog
for TBBnumber in range(len(TBBmgold)):
if TBBmgold[TBBnumber] != TBBmem[TBBnumber]:
if Severity<SeverityOfThisTest: Severity=SeverityOfThisTest
if Priority<PriorityOfThisTest: Priority=PriorityOfThisTest
st_log.write('TBBmem>: Sv=%s Pr=%s, BP/AP Error! %s' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest], TBBmem[TBBnumber]))
sr.setResult('FAILED')
if debug: print ('TBBNOK = ', TBBnumber)
return
################################################################################
# Function Check TBB Memory Nog testen met defect TBB board!
#
def CheckTBBMemoryOrg():
sr.setId('TBBmem>: ')
sr.appendLog(21,'### Verify TBB memory modules on the TBB')
# linecount=0
# TBBmemory = os.popen3('./tbb_memory')[1].readlines() # Get RSP Versions
# if len(TBBmemory) > 0:
# for line in TBBmemory:
# lineContainingTBBmemory=line.find('TBB memory')
# if lineContainingTBBmemory==0: break
# linecount+=1
#
# if debug:
#
# for tbb in range(opts.tbb_nr):
# print TBBmemory[linecount+tbb]
#
# return
res = cli.command('./tbb_memory.sh')
if res.find('wrong')==-1:
if debug: print(11,'>>> TBB memory test went OK')
else:
sr.appendLog(11,'>>> TBB memory test went wrong')
sr.appendLog(11,'CLI:')
sr.appendLog(11,res,1,1,1)
sr.appendLog(11,'Result:')
sr.appendFile(11,'tbb_memory.log')
sr.appendLog(11,'Expected:')
sr.appendFile(11,'gold/tbb_memory.gold')
sr.setResult('FAILED')
return
################################################################################
# Function Check TBB Size Nog testen met defect TBB board!
#
def CheckTBBSizetmp():
SeverityOfThisTest=2
PriorityOfThisTest=2
global Severity
global Priority
TBBsgold = open(TBBsgoldfile,'r').readlines() # Read TBB Memory gold
TBBsze = os.popen3('./tbb_size.sh')[1].readlines() # Start TBB memory test
# res = cli.command('./tbb_version.sh')
if debug:
for TBBnumber in range(len(TBBsgold)):
if TBBsgold[TBBnumber] == TBBsze[TBBnumber]: print ('TBB OK = ', TBBnumber)
else: print ('TBBNOK = ', TBBnumber)
# store station testlog
for TBBnumber in range(len(TBBsgold)):
if TBBsgold[TBBnumber] != TBBsze[TBBnumber]:
if Severity<SeverityOfThisTest: Severity=SeverityOfThisTest
if Priority<PriorityOfThisTest: Priority=PriorityOfThisTest
st_log.write('TBBsze>: Sv=%s Pr=%s, TBBSize Error! %s' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest], TBBsze[TBBnumber]))
sr.setResult('FAILED')
if debug: print ('TBBNOK = ', TBBnumber)
return
################################################################################
# Function Check TBB Size Nog testen met defect TBB board!
#
def CheckTBBSize():
SeverityOfThisTest=2
PriorityOfThisTest=2
global Severity
global Priority
sr.setId('TBBsze>: ')
sr.appendLog(21,'### Verify the size of the TBB memory modules')
res = cli.command('./tbb_size.sh')
if res.find('wrong')==-1:
#sr.appendLog(11,'>>> TBB size test went OK')
if debug: print(11,'>>> TBB size test went OK')
else:
sr.appendLog(11,'>>> TBB size test went wrong')
sr.appendLog(11,'CLI:')
sr.appendLog(11,res,1,1,1)
sr.appendLog(11,'Result:')
sr.appendFile(11,'tbb_size.log')
sr.appendLog(11,'Expected:')
sr.appendFile(11,'gold/tbb_size.gold')
sr.setResult('FAILED')
# store station testlog
if Severity<SeverityOfThisTest: Severity=SeverityOfThisTest
if Priority<PriorityOfThisTest: Priority=PriorityOfThisTest
st_log.write('TBBsze>: Sv=%s Pr=%s, TBB size test went wrong!\n' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest]))
return
################################################################################
# Function Pseudo Random TBB Test Nog testen met defect TBB board!
#
def PseudoRandomTBBTest():
sr.setId('PsRndT>: ')
sr.appendLog(21,'### Verify the RCU - RSP - TBB LVDS interfaces by capturing pseudo random data on TBB')
res = cli.command('./tbb_prbs_tester.sh')
if res.find('wrong')==-1:
#sr.appendLog(11,'>>> RCU - RSP - TBB LVDS interfaces test went OK')
if debug: print(11,'>>> RCU - RSP - TBB LVDS interfaces test went OK')
else:
sr.appendLog(11,'>>> RCU - RSP - TBB LVDS interfaces went wrong')
sr.appendLog(11,'CLI:')
sr.appendLog(11,res,1,1,1)
sr.setResult('FAILED')
return
################################################################################
# Function CHeck SPU status Nog testen met defect SPU board!
#
def CheckSPUStatus():
SeverityOfThisTest=2
PriorityOfThisTest=2
global Severity
global Priority
sr.setId('SPUst >: ')
sr.setId('SPU status - ')
sr.appendLog(21,'### Verify the RSP - SPU I2C interface by reading the SPU sensor data')