stationtest.py

#!/usr/bin/python

#
# Run the tests to test a LOFAR station
# H. Meulman
# Version 0.4                26-nov-2010

# 24 sep: local log directory aangepast
# 27 sept: 	- Toevoeging delay voor tbbdirver polling
#		- Aanzetten van LBA's m.b.v rspctl --aweights=8000,0
# 26 nov: Check op 160 MHZ en 200 MHZ clock.

# todo:
# - Als meer dan 10 elementen geen rf signaal hebben, keur dan hele tile af
# - als beamserver weer goed werkt deze weer toevoegen aan LBA test
# - Local Log directory aanpassen

import sys
from optparse import OptionParser
import cli
import testlog
from time import localtime, strftime
import array
import os
import time
import commands
import operator
import math
import numpy

################################################################################
# Init

# Variables
debug=0
clkoffset=1
noTBB=6
factor = 30	# station statistics fault window: Antenna average + and - factor = 100 +/- 30

# Do not change:
Severity=0	# Severity (0='' 1=feature 2=minor 3=major 4=block 5=crash
Priority=0	# Priority (0=no 1=low 2=normal 3=high 4=urgent 5=immediate
SeverityLevel=('--     ','feature','minor  ','Major  ','BLOCK  ','CRASH  ')
PriorityLevel=('--       ','low      ','normal   ','High     ','URGENT   ','IMMEDIATE')
#print (SeverityLevel[Severity])
#print (PriorityLevel[Priority])

# Path
RSPgoldfile=('/misc/home/etc/stationtest/gold/rsp_version.gold')
TBBgoldfile=('/misc/home/etc/stationtest/gold/tbb_version.gold')
TBBmgoldfile=('/misc/home/etc/stationtest/gold/tbb_memory.gold')
#LogPath=('/misc/home/log/')
TestLogPath=('/misc/home/log/')	# Logging remote (on Kis001)
#TestLogPath=('/opt/stationtest/data/')	# Logging local (on station)
#HistLogPath=('/opt/stationtest/data/')	# Logging local (on station)
HistLogPath=('/localhome/log/')	# Logging local (on station)


tm=strftime("%a, %d %b %Y %H:%M:%S", localtime())	# Determine system time
tme=strftime("_%b_%d_%Y_%H.%M", localtime())		# Time for fileheader History log file
StIDlist = os.popen3('hostname -s')[1].readlines()	# Name of the station
StID = str(StIDlist[0].strip('\n'))
if debug: print ('StationID = %s' % StID)

TestlogName = ('%sstationtest_%s.tmp' % (TestLogPath, StID))
TestlogNameFinalized = ('%sstationtest_%s.log' % (TestLogPath, StID))
HistlogName = ('%sstationtest_%s%s.log' % (HistLogPath, StID, tme))

# Array om bij te houden welke Tiles niet RF getest hoeven worden omdat de modems niet werken.
if len(sys.argv) < 3 :
        num_rcu=96
else :
	num_rcu = int(sys.argv[2])
ModemFail=[0 for i in range (num_rcu/2)]
if debug: print ModemFail

#print (TestlogName)
#print (TestlogNameFinalized)

# Parse command line for station ID
#
# -v  0 : only PASSED or FAILED
# -v  1 : overall title
# -v 11 : result per test
# -v 21 : title per test

op = OptionParser(usage='usage: python %prog [options]', version='%prog 0.1')

op.add_option('-v', type='int', dest='verbosity',
  help='Verbosity level',default=11)

#op.add_option('-r', type='int', dest='rsp_nr',
#  help='Provide number of rsp boards that will be used in this test',default=None)
#op.add_option('-t', type='int', dest='tbb_nr',
#  help='Provide number of tbb boards that will be used in this test',default=None)

opts, args = op.parse_args()
opts.rsp_nr=12		# fixed number
opts.tbb_nr=6		# Fixed number (make autodetect later!!)

# - Option checks and/or reformatting
if opts.rsp_nr==None:
  op.error('Option -r must specify the number of rsp boards')
if opts.tbb_nr==None:
  op.error('Option -t must specify the number of tbb boards')
if opts.rsp_nr == 4:
        RspBrd = 'rsp0,rsp1,rsp2,rsp3'
        SubBrd = 'rsp0'
        SubRck = 'sub0'      
if opts.rsp_nr == 12:
        RspBrd = 'rsp0,rsp1,rsp2,rsp3,rsp4,rsp5,rsp6,rsp7,rsp8,rsp9,rsp10,rsp11'
        SubBrd = 'rsp0,rsp4,rsp8'
        SubRck = 'sub0,sub1,sub2'  
if opts.rsp_nr == 24:
        RspBrd = 'rsp0,rsp1,rsp2,rsp3,rsp4,rsp5,rsp6,rsp7,rsp8,rsp9,rsp10,rsp11,rsp12,rsp13,rsp14,rsp15,rsp16,rsp17,rsp18,rsp19,rsp20,rsp21,rsp22,rsp23'
        SubBrd = 'rsp0,rsp4,rsp8,rsp12,rsp16,rsp20'
        SubRck = 'sub0,sub1,sub2,sub3,sub4,sub5'

	
# Define subrack testlog class for pass/fail and logging
vlev = opts.verbosity
testId = ''
appLev = False
logName = '/opt/stationtest/data/SUBR-%05d-%05d.dat' % (opts.rsp_nr, opts.tbb_nr)
cli.command('rm -f /opt/stationtest/data/SUBR-%05d-%05d.dat', appLev) 
sr = testlog.Testlog(vlev, testId, logName)

sr.setResult('PASSED')

sr.setId('Station - ')
sr.appendLog(11,'')
sr.appendLog(1,'Station production test %s' % logName)
sr.appendLog(11,'')

# Define station testlog
st_log = file(TestlogName, 'w')
st_log.write('StID  >: %s\n' % StID)
st_log.write('Lgfl  >: %s\n' % TestlogNameFinalized)
st_log.write('Time  >: %s\n' % tm)
#cli.command('rm -f %s' % logName, appLev)
#cli.command('rm -f /opt/stationtest/data/stationtest.log', appLev)
#sr.setId('')
#sr.appendLog(11,'')
#sr.appendLog(1,'Lgfl  >: %s' % logNameFinalized)
#sr.appendLog(1,'StID  >: %s' % StID)
#sr.appendLog(1,'Time  >: %s' % tm)

#time.sleep(20)

################################################################################
# Function CheckTBB : CHeck if TBB's are running. The returned string 
# "V 0.3  V 4.7  V 2.4  V 2.9" Shouls have 4 times 'V'

def CheckTBB():
    SeverityOfThisTest=3
    PriorityOfThisTest=3

    global Severity
    global Priority
    print 'Checking TBB!!!'
#    print 'wait 20 sec'
#    time.sleep(20)
    if debug: print int(len(os.popen3('tbbctl --version')[1].readlines()))
    sr.setId('TBB   >: ')
    n=0 # Maximum itteration
    while len(os.popen3('tbbctl --version')[1].readlines()) < 4:
        print ('-'),
	if debug: print ('Polling TBB Driver')
        time.sleep(5)
	n+=1
	if n > 12:
		sr.appendLog(11,'Error: TBB driver is not running or some TBBs not active')
		sr.setResult('FAILED')
# store station testlog			
		if Severity<SeverityOfThisTest: Severity=SeverityOfThisTest
		if Priority<PriorityOfThisTest: Priority=PriorityOfThisTest
		st_log.write('TBB   >: Sv=%s Pr=%s, Error: TBB driver is not running\n' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest]))
		return

    n=0                                                   # Check till 4 V's per TBB
    while n < 12:                                               # maximum itterations
        res2 = os.popen3('tbbctl --version')[1].readlines()
        if debug:
	    for line in res2:
	        print ('%s' % line.rstrip('\n'))
            #print ('res2 is: %s' % res2)
            #print ('res2[9] is: %s' % res2[9])
            print ('Itteration %d' % n)
        else:
            print '*',
        cnt=0
        TBBrange=range(noTBB)
        for TBBnr in TBBrange:
            cnt += res2[9+TBBnr].count('V')         # count number of 'V's (Version)
        if cnt == noTBB*4:                                               # 4 per TBB
            print "TBB's OK"
            break
        n+=1
        time.sleep(5)
    else:
        for TBBnr in TBBrange:
            if res2[9+TBBnr].count('V') != 4:                           # Log Errors
                sr.appendLog(11,'Error: TBB :%s' % str(res2[9+TBBnr].strip('\n')))
		sr.setResult('FAILED')
# store station testlog			
		if Severity<SeverityOfThisTest: Severity=SeverityOfThisTest
		if Priority<PriorityOfThisTest: Priority=PriorityOfThisTest
		st_log.write('TBB   >: Sv=%s Pr=%s, Error: TBB : %s\n' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest], str(res2[9+TBBnr].strip('\n'))))
                #print ('number of Vs is ', res2[9+TBBnr].count('V')),
                #print (' Error in TBB : %s' % res2[9+TBBnr])
    if debug:
        print 'stopped Checking TBB'
        print ("number of V's is %d" % cnt)
    return

################################################################################
# Function Goto Swlevel 2

def GotoSwlevel2():
	res = os.popen3('swlevel 1')[1].readlines()
	if debug:
		print('System is Going to swlevel 1')
#		for line in res:
#			print ('To swlevel 1 ', line)	# werkt niet!!???
	time.sleep(5)

# Set swlevel 2 if not running
#
	res = os.popen3('swlevel')[1].readlines()
#print res[1]
	if len(res) > 0:
		for line in res:
			if debug: print ('%s' % line.rstrip('\n'))
			if line == ('2 : RSPDriver                 DOWN\n') or line == ('2 : TBBDriver                 DOWN\n'):
				print 'System is Going to swlevel 2'

#            errorprg = os.system('swlevel 2')
 #           if len(err) > 0:
 #           else:
#            print fromprg.readlines()

				res2 = os.popen3('swlevel 2')[1].readlines()
#            print  errorprg
				print 'wait 30 sec'
				if debug:
					for line in res2:
						print ('%s' % line.rstrip('\n'))
				time.sleep(30)
				time.sleep(90)  # Tijdelijk toe gevoegd voor nieuwe tbbdriver. Deze loopt vast tijdens pollen
#				CheckTBB()	# Tijdelijk weg gelaten voor nieuwe tbbdriver. Deze loopt vast tijdens pollen
#fromprg.close()
				break
	return
#res.close()

################################################################################
# Check ntpd time demon
#
def CheckNtpd():
	SeverityOfThisTest=3
	PriorityOfThisTest=3
	
	global Severity
	global Priority
	sr.setId('Clock   - ')
	res = os.popen3('/usr/sbin/ntpq -p')[1].readlines()
	#res = os.popen3('/opt/stationtest/test/timing/ntpd.sh')[1].readlines()
	if debug:
		for line in res:
	        	print ('-%s' % line.rstrip('\n'))
	#print ('res : %s' % res)

	if len(res) > 0:
#		print (res[3])
		offset=0
		for line in res:
			if debug: print('line= %s' % line)
			locallock=line.find('*LOCAL(0)')
			if locallock==0: break
			gpslock=line.find('*GPS_ONCORE(0)')
			if gpslock==0:
				offset=float((line.split())[8])
				break
		if debug:
#			print ('res[3] is: %s' % res[3])
#			print ('res[4] is: %s' % res[4])
			print ('gpslock is %s' % gpslock)
			print ('locallock is %s' % locallock)
			print ('offset is %.3f' % offset)

		if gpslock > -1:
	        	if debug: print('GPS in Lock. OK')
		else:
			if locallock > -1:
				sr.appendLog(11,'Clock locked on Local Clock!!')
				if Severity<SeverityOfThisTest: Severity=SeverityOfThisTest
				if Priority<PriorityOfThisTest: Priority=PriorityOfThisTest
				st_log.write('Clock >: Sv=%s Pr=%s, Clock locked on Local Clock!\n' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest]))
				sr.setResult('FAILED')
			else: 
				sr.appendLog(11,'Clock out of sync!!')
				if Severity<SeverityOfThisTest: Severity=SeverityOfThisTest
				if Priority<PriorityOfThisTest: Priority=PriorityOfThisTest
				st_log.write('Clock >: Sv=%s Pr=%s, Clock out of sync!\n' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest]))
				sr.setResult('FAILED')
		if offset < -clkoffset or offset > clkoffset:
			if Severity<SeverityOfThisTest: Severity=SeverityOfThisTest
			if Priority<PriorityOfThisTest: Priority=PriorityOfThisTest
			st_log.write('Clock >: Sv=%s Pr=%s, Clock Offset to large : %.3f\n' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest], offset))
			sr.appendLog(11,'Clock Offset to large : %.3f' % offset)
			sr.setResult('FAILED')
	else:
		sr.appendLog(11,'no answer from ntpq!')
		if Severity<SeverityOfThisTest: Severity=SeverityOfThisTest
		if Priority<PriorityOfThisTest: Priority=PriorityOfThisTest
		st_log.write('Clock >: Sv=%s Pr=%s, no answer from ntpq!\n' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest]))
		sr.setResult('FAILED')
	return

################################################################################
# Function Check RSP status bytes
#
def CheckRSPStatus():
	SeverityOfThisTest=2
	PriorityOfThisTest=2
	
	global Severity
	global Priority
	
	sr.setId('RSPst >: ')
	OutputClock,PLL160MHz,PLL200MHz=gettdstatus() # td-status
	time.sleep(1)
	res = os.popen3('rspctl --status')[1].readlines()
	#print res[1]
	linecount=0
	if len(res) > 0:
		for line in res:
			sync=line.find('RSP[ 0] Sync')
			if sync==0: break
			linecount+=1
	#print 'sync = ' + str(sync) + ' and linecount = ' + str(linecount)
	for rsp in range(opts.rsp_nr):
	#	print res[linecount+rsp]
	#	x = res[linecount+rsp].split( )
	#	print res[linecount+rsp*5].lstrip('RSP').strip('[').split()
		if debug: 
			print '\n',
			print res[linecount+rsp*5],
		for sync in range(1, 5):
			dif = res[linecount+rsp*5+sync].lstrip('RSP').strip('[').split()
			if debug:
				print ('Dif = %s' % dif)
				#print str(linecount+rsp*5+sync),
				#print dif[2]
			if dif[2] not in ('0', '1', '512', '513'):
				#if debug: print ('RSP : %d status error:  sync = %d, diff = %d' % (int(rsp), int(sync), int(dif[2])))
				sr.appendLog(11,'RSP : %d status error:  sync = %d diff = %d' % (int(rsp), int(sync), int(dif[2])))
				sr.setResult('FAILED')
				
				if Severity<SeverityOfThisTest: Severity=SeverityOfThisTest
				if Priority<PriorityOfThisTest: Priority=PriorityOfThisTest
				st_log.write('RSPst >: Sv=%s Pr=%s, RSP : %d status error at %s MHz:  sync = %d diff = %d\n' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest], int(rsp), OutputClock, int(sync), int(dif[2])))
				sr.setResult('FAILED')
			
	#time.sleep(3)
	return
				
################################################################################
# Function check if clock 160 MHz is locked
#
def CheckTDSStatus160():
	SeverityOfThisTest=2
	PriorityOfThisTest=2
	
	global Severity
	global Priority

	sr.setId('TDSst >: ')
	TDS=['0','4','8']
	PLL160MHz = '?'
	PLL200MHz = '?'
	res = os.popen3('rspctl --clock=160')[1].readlines()
	print ('Clock set to 160MHz')
	time.sleep(1)
	n=0                                                   # Wait till clock set
    	while n < 15:                                         # maximum itterations
		OutputClock,PLL160MHz,PLL200MHz=gettdstatus() # td-status
		if PLL160MHz!='?': 
			print ('Clock %s' %(PLL160MHz))
			break
		n+=1
		time.sleep(5)
		if n==15:
			print ('Clock not locked')
			if Severity<SeverityOfThisTest: Severity=SeverityOfThisTest
			if Priority<PriorityOfThisTest: Priority=PriorityOfThisTest
			st_log.write('TDSst >: Sv=%s Pr=%s, TDS : all @ 160MHz not locked:  PLL200MHz = %s, PLL160MHz = %s, Output Clock = %s\n' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest], PLL200MHz, PLL160MHz, OutputClock))
			sr.setResult('FAILED')
		
	for TDSBrd in TDS:
		valid=0
		PLL160MHz = '?'
		PLL200MHz = '?'
		#print('TDSBrd = ',TDSBrd)
		res = os.popen3('rspctl --tdstatus --sel=%s'%(TDSBrd))[1].readlines()
		if debug: print res[0]
		for line in res:
			if line[0] == 'R': 
				valid=1
				if debug: print ('valid tdstatus')
		#print res[0].split()
		if valid == 1: 
			for line in res:
				if line[0] == 'R':		# Check of regel geldig is!
					header=line.replace('|',' ').split()
					if debug: print ('header = ', header)
				else:		# Check of regel geldig is!
					status=line.replace('|',' ').replace('not locked','notlocked').split()
					if debug: 
						print ('status= ', status)
						print ('OutputClock = ',status[2])
						print ('PLL160MHz = ',status[4])
						print ('PLL200MHz = ',status[5])
					OutputClock = status[2]
					PLL160MHz = status[4]
					PLL200MHz = status[5]
			if PLL160MHz <> 'LOCKED':
# store subreck testlog	
#			sr.appendLog(11,'TDS : %d not locked:  PLL200MHz = %d PLL160MHz = %d Output Clock = %d' % (TDS, PLL200MHz, PLL160MHz, OutputClock))
#			sr.setResult('FAILED')
# store station testlog		
				print ('Clock 160MHz not locked')
				if Severity<SeverityOfThisTest: Severity=SeverityOfThisTest
				if Priority<PriorityOfThisTest: Priority=PriorityOfThisTest
				st_log.write('TDSst >: Sv=%s Pr=%s, TDS : %s @ 160MHz not locked:  PLL200MHz = %s, PLL160MHz = %s, Output Clock = %s\n' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest], TDSBrd, PLL200MHz, PLL160MHz, OutputClock))
				sr.setResult('FAILED')
	return
				
################################################################################
# Function check if clock 200 MHz is locked
#
def CheckTDSStatus200():
	SeverityOfThisTest=2
	PriorityOfThisTest=2
	
	global Severity
	global Priority

	sr.setId('TDSst >: ')
	TDS=['0','4','8']
	PLL160MHz = '?'
	PLL200MHz = '?'
	res = os.popen3('rspctl --clock=200')[1].readlines()
	print ('Clock set to 200MHz')
	time.sleep(1)
	n=0                                                   # Wait till clock set
    	while n < 15:                                         # maximum itterations
		OutputClock,PLL160MHz,PLL200MHz=gettdstatus() # td-status
		if PLL200MHz!='?': 
			print ('Clock %s' %(PLL200MHz))
			break
		print ('OutputClock = ',OutputClock)
		print ('PLL160MHz = ',OutputClock)
		print ('PLL200MHz = ',OutputClock)
		n+=1
		time.sleep(5)
		if n==15:
			print ('Clock not locked')
			if Severity<SeverityOfThisTest: Severity=SeverityOfThisTest
			if Priority<PriorityOfThisTest: Priority=PriorityOfThisTest
			st_log.write('TDSst >: Sv=%s Pr=%s, TDS : all @ 200MHz not locked:  PLL200MHz = %s, PLL160MHz = %s, Output Clock = %s\n' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest], PLL200MHz, PLL160MHz, OutputClock))
			sr.setResult('FAILED')
		
	for TDSBrd in TDS:
		valid=0
		PLL160MHz = '?'
		PLL200MHz = '?'
		#print('TDSBrd = ',TDSBrd)
		res = os.popen3('rspctl --tdstatus --sel=%s'%(TDSBrd))[1].readlines()
		if debug: print res[0]
		for line in res:
			if line[0] == 'R': 
				valid=1
				if debug: print ('valid tdstatus')
		#print res[0].split()
		if valid == 1: 
			for line in res:
				if line[0] == 'R':		# Check of regel geldig is!
					header=line.replace('|',' ').split()
					if debug: print ('header = ', header)
				else:		# Check of regel geldig is!
					status=line.replace('|',' ').replace('not locked','notlocked').split()
					if debug: 
						print ('status= ', status)
						print ('OutputClock = ',status[2])
						print ('PLL160MHz = ',status[4])
						print ('PLL200MHz = ',status[5])
					OutputClock = status[2]
					PLL160MHz = status[4]
					PLL200MHz = status[5]
			if PLL200MHz <> 'LOCKED':
# store subreck testlog	
#			sr.appendLog(11,'TDS : %d not locked:  PLL200MHz = %d PLL160MHz = %d Output Clock = %d' % (TDS, PLL200MHz, PLL160MHz, OutputClock))
#			sr.setResult('FAILED')
# store station testlog		
				print ('Clock 200MHz not locked')
				if Severity<SeverityOfThisTest: Severity=SeverityOfThisTest
				if Priority<PriorityOfThisTest: Priority=PriorityOfThisTest
				st_log.write('TDSst >: Sv=%s Pr=%s, TDS : %s @ 200MHz not locked:  PLL200MHz = %s, PLL160MHz = %s, Output Clock = %s\n' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest], TDSBrd, PLL200MHz, PLL160MHz, OutputClock))
				sr.setResult('FAILED')
	return
################################################################################
# Function get the TD status
#
def gettdstatus():
	res = os.popen3('rspctl --tdstatus --sel=0')[1].readlines()
	for line in res:
		if line[0] == 'R':		# Check of regel geldig is!
			header=line.replace('|',' ').split()
			#print ('header = ', header)
		else:		# Check of regel geldig is!
			status=line.replace('|',' ').replace('not locked','notlocked').split()
			#print ('status= ', status)
			#print ('OutputClock = ',status[2])
			#print ('PLL160MHz = ',status[4])
			if debug: print ('PLL200MHz = ',status[5])
			OutputClock = status[2]
			PLL160MHz = status[4]
			PLL200MHz = status[5]
	return OutputClock,PLL160MHz,PLL200MHz
				
################################################################################
# Function make RSP Version gold
#
def makeRSPVersionGold():
	res = os.popen3('rspctl --version')[1].readlines()
	time.sleep(3)
	f_log = file('/misc/home/etc/stationtest/gold/rsp_version.gold', 'w')
	for line in res:
		print ('Res = ', line)
	        f_log.write(line)
	return

################################################################################
# Function read RSP Version gold
#
def readRSPVersionGold():
	f=open(RSPgoldfile,'rb')
#	if debug: 
#		for line in f:
#			print ('Res = ', line)
	return f

################################################################################
# Function Check RSP Version
#
def CheckRSPVersion():
	SeverityOfThisTest=2
	PriorityOfThisTest=2
	
	global Severity
	global Priority
	
	sr.setId('RSPver>: ')
	sr.appendLog(21,'')
	sr.appendLog(21,'### Verify LCU - RSP ethernet link by getting the RSP version info')
	sr.appendLog(21,'')	
#	RSPgold=readRSPVersionGold()
	RSPgold = open(RSPgoldfile,'r').readlines()			# Read RSP Version gold
	RSPversion = os.popen3('rspctl --version')[1].readlines()	# Get RSP Versions
#	res = cli.command('./rsp_version.sh')
	if debug:
		for RSPnumber in range(len(RSPgold)):
			if RSPgold[RSPnumber] == RSPversion[RSPnumber]: print ('RSP OK = ', RSPnumber)
			else: print ('RSPNOK = ', RSPnumber)
# store subreck testlog			
	for RSPnumber in range(len(RSPgold)):
		if RSPgold[RSPnumber] != RSPversion[RSPnumber]: 
			sr.appendLog(11,'>>> RSP version test went wrong')
			#sr.appendLog(11,'CLI:')
			#sr.appendLog(11,res,1,1,1)
			sr.appendLog(11,'Result:')
			for line in RSPversion:
				#print ('%s' % line.rstrip('\n'))
				sr.appendLog(11,'%s' % line.rstrip('\n'))
			sr.appendLog(11,'Expected:')
			for line in RSPgold:
				#print ('%s' % line.rstrip('\n'))
				sr.appendLog(11,'%s' % line.rstrip('\n'))
			sr.setResult('FAILED')
			break
		
# store station testlog	
	for RSPnumber in range(len(RSPgold)):
		if RSPgold[RSPnumber] != RSPversion[RSPnumber]:
			if Severity<SeverityOfThisTest: Severity=SeverityOfThisTest
			if Priority<PriorityOfThisTest: Priority=PriorityOfThisTest
			st_log.write('RSPver>: Sv=%s Pr=%s, BP/AP Error! %s' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest], RSPversion[RSPnumber]))
			sr.setResult('FAILED')
			if debug: print ('RSPNOK = ', RSPnumber)
	return

################################################################################
# Function make TBB Version gold
#
def makeTBBVersionGold():
	res = os.popen3('tbbctl --version')[1].readlines()
	time.sleep(3)
	f_log = file(TBBgoldfile, 'w')
	for line in res:
		print ('Res = ', line)
	        f_log.write(line)
	return
	
################################################################################
# Function Check TBB Version
#
def CheckTBBVersion():
	SeverityOfThisTest=2
	PriorityOfThisTest=2
	
	global Severity
	global Priority
	
	sr.setId('TBBver>: ')
	sr.appendLog(21,'')
	sr.appendLog(21,'### Verify LCU - TBB ethernet link by getting the TBB version info')
	sr.appendLog(21,'')	

	TBBgold = open(TBBgoldfile,'r').readlines()			# Read TBB Version gold
	TBBversion = os.popen3('tbbctl --version')[1].readlines()	# Get TBB Versions
#	res = cli.command('./tbb_version.sh')
	if len(TBBversion) < 4:
	# store station testlog			
		if Severity<SeverityOfThisTest: Severity=SeverityOfThisTest
		if Priority<PriorityOfThisTest: Priority=PriorityOfThisTest
		st_log.write('TBB   >: Sv=%s Pr=%s, Error: TBB driver is not running\n' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest]))
		return
			
	if debug:
		print ('TBBgold: %s' % TBBgold)
		print ('TBBgold: %s' % TBBversion)
		for TBBnumber in range(len(TBBgold)):
			if TBBgold[TBBnumber] == TBBversion[TBBnumber]: print ('TBB OK = ', TBBnumber)
			else: print ('TBBNOK = ', TBBnumber)
# store subreck testlog			
	for TBBnumber in range(len(TBBgold)):
		if TBBgold[TBBnumber] != TBBversion[TBBnumber]: 
			sr.appendLog(11,'>>> TBB version test went wrong')
			#sr.appendLog(11,'CLI:')
			#sr.appendLog(11,res,1,1,1)
			sr.appendLog(11,'Result:')
			for line in TBBversion:
				#print ('%s' % line.rstrip('\n'))
				sr.appendLog(11,'%s' % line.rstrip('\n'))
			sr.appendLog(11,'Expected:')
			for line in TBBgold:
				#print ('%s' % line.rstrip('\n'))
				sr.appendLog(11,'%s' % line.rstrip('\n'))
			sr.setResult('FAILED')
			break
		
# store station testlog	
	for TBBnumber in range(len(TBBgold)):
		if TBBgold[TBBnumber] != TBBversion[TBBnumber]:
			if Severity<SeverityOfThisTest: Severity=SeverityOfThisTest
			if Priority<PriorityOfThisTest: Priority=PriorityOfThisTest
			st_log.write('TBBver>: Sv=%s Pr=%s, BP/AP Error! %s' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest], TBBversion[TBBnumber]))
			sr.setResult('FAILED')
			if debug: print ('TBBNOK = ', TBBnumber)
	return

################################################################################
# Function Check TBB Version				Eventueel nog toevoegen!
#
#sr.setId('TBB version - ')
#sr.appendLog(21,'')
#sr.appendLog(21,'### Verify LCU - TBB ethernet link by getting the TBB version info')
#sr.appendLog(21,'')
#res = cli.command('./tbb_version.sh')
#if res.find('OK')==-1:
#  sr.appendLog(11,'>>> TBB version test went wrong')
#  sr.appendLog(11,'CLI:')
#  sr.appendLog(11,res,1,1,1)
#  sr.appendLog(11,'Result:')
#  sr.appendFile(11,'tbb_version.log')
#  sr.appendLog(11,'Expected:')
#  sr.appendFile(11,'gold/tbb_version.gold')
#  sr.setResult('FAILED')
#else:
#  sr.appendLog(11,'>>> TBB version test went OK')

################################################################################
# Function make TBB Memory gold
#
def makeTBBMemGold():
	res = os.popen3('./tbb_memory.sh')[1].readlines()
	time.sleep(3)
	f_log = file(TBBmgoldfile, 'w')
	for line in res:
		print ('Res = ', line)
	        f_log.write(line)
	return
	
################################################################################
# Function Check TBB Memory
#
def CheckTBBMemory():
	SeverityOfThisTest=2
	PriorityOfThisTest=2
	
	global Severity
	global Priority
	
	sr.setId('TBBmem>: ')
	sr.appendLog(21,'')
	sr.appendLog(21,'### Verify TBB memory modules on the TBB')
	sr.appendLog(21,'')	

	TBBmgold = open(TBBmgoldfile,'r').readlines()		# Read TBB Memory gold
	TBBmem = os.popen3('./tbb_memory.sh')[1].readlines()	# Start TBB memory test
#	res = cli.command('./tbb_version.sh')
	if debug:
		for TBBnumber in range(len(TBBmgold)):
			if TBBmgold[TBBnumber] == TBBmem[TBBnumber]: print ('TBB OK = ', TBBnumber)
			else: print ('TBBNOK = ', TBBnumber)
# store subreck testlog			
#	for TBBnumber in range(len(TBBgold)):
#		if TBBgold[TBBnumber] != TBBversion[TBBnumber]: 
#			sr.appendLog(11,'>>> TBB version test went wrong')
#			#sr.appendLog(11,'CLI:')
#			#sr.appendLog(11,res,1,1,1)
#			sr.appendLog(11,'Result:')
#			for line in TBBversion:
#				#print ('%s' % line.rstrip('\n'))
#				sr.appendLog(11,'%s' % line.rstrip('\n'))
#			sr.appendLog(11,'Expected:')
#			for line in TBBgold:
#				#print ('%s' % line.rstrip('\n'))
#				sr.appendLog(11,'%s' % line.rstrip('\n'))
#			sr.setResult('FAILED')
#			break
		
# store station testlog	
	for TBBnumber in range(len(TBBmgold)):
		if TBBmgold[TBBnumber] != TBBmem[TBBnumber]:
			if Severity<SeverityOfThisTest: Severity=SeverityOfThisTest
			if Priority<PriorityOfThisTest: Priority=PriorityOfThisTest
			st_log.write('TBBmem>: Sv=%s Pr=%s, BP/AP Error! %s' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest], TBBmem[TBBnumber]))
			sr.setResult('FAILED')
			if debug: print ('TBBNOK = ', TBBnumber)
	return

################################################################################
# Function Check TBB Memory			Nog testen met defect TBB board!
#
def CheckTBBMemoryOrg():
	sr.setId('TBBmem>: ')
	sr.appendLog(21,'### Verify TBB memory modules on the TBB')
#	linecount=0
#	TBBmemory = os.popen3('./tbb_memory')[1].readlines()	# Get RSP Versions
#	if len(TBBmemory) > 0:
#		for line in TBBmemory:
#			lineContainingTBBmemory=line.find('TBB memory')
#			if lineContainingTBBmemory==0: break
#			linecount+=1
#			
#	if debug:
#		
#	for tbb in range(opts.tbb_nr):
#		print TBBmemory[linecount+tbb]
#
#	return
			
	res = cli.command('./tbb_memory.sh')
	if res.find('wrong')==-1:
		if debug: print(11,'>>> TBB memory test went OK')
	else:
		sr.appendLog(11,'>>> TBB memory test went wrong')
		sr.appendLog(11,'CLI:')
		sr.appendLog(11,res,1,1,1)
		sr.appendLog(11,'Result:')
		sr.appendFile(11,'tbb_memory.log')
		sr.appendLog(11,'Expected:')
		sr.appendFile(11,'gold/tbb_memory.gold')
		sr.setResult('FAILED')
	return

################################################################################
# Function Check TBB Size			Nog testen met defect TBB board!
#
def CheckTBBSizetmp():
	SeverityOfThisTest=2
	PriorityOfThisTest=2
	
	global Severity
	global Priority
	
	TBBsgold = open(TBBsgoldfile,'r').readlines()		# Read TBB Memory gold
	TBBsze = os.popen3('./tbb_size.sh')[1].readlines()	# Start TBB memory test
#	res = cli.command('./tbb_version.sh')
	if debug:
		for TBBnumber in range(len(TBBsgold)):
			if TBBsgold[TBBnumber] == TBBsze[TBBnumber]: print ('TBB OK = ', TBBnumber)
			else: print ('TBBNOK = ', TBBnumber)
# store station testlog	
	for TBBnumber in range(len(TBBsgold)):
		if TBBsgold[TBBnumber] != TBBsze[TBBnumber]:
			if Severity<SeverityOfThisTest: Severity=SeverityOfThisTest
			if Priority<PriorityOfThisTest: Priority=PriorityOfThisTest
			st_log.write('TBBsze>: Sv=%s Pr=%s, TBBSize Error! %s' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest], TBBsze[TBBnumber]))
			sr.setResult('FAILED')
			if debug: print ('TBBNOK = ', TBBnumber)
	return	
	
################################################################################
# Function Check TBB Size			Nog testen met defect TBB board!
#
def CheckTBBSize():
	SeverityOfThisTest=2
	PriorityOfThisTest=2
	
	global Severity
	global Priority
	
	sr.setId('TBBsze>: ')
	sr.appendLog(21,'### Verify the size of the TBB memory modules')
	res = cli.command('./tbb_size.sh')
	if res.find('wrong')==-1:
		#sr.appendLog(11,'>>> TBB size test went OK')
		if debug: print(11,'>>> TBB size test went OK')
	else:
		sr.appendLog(11,'>>> TBB size test went wrong')
		sr.appendLog(11,'CLI:')
		sr.appendLog(11,res,1,1,1)
		sr.appendLog(11,'Result:')
		sr.appendFile(11,'tbb_size.log')
		sr.appendLog(11,'Expected:')
		sr.appendFile(11,'gold/tbb_size.gold')
		sr.setResult('FAILED')

# store station testlog			
		if Severity<SeverityOfThisTest: Severity=SeverityOfThisTest
		if Priority<PriorityOfThisTest: Priority=PriorityOfThisTest
		st_log.write('TBBsze>: Sv=%s Pr=%s, TBB size test went wrong!\n' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest]))
	return

################################################################################
# Function Pseudo Random TBB Test		Nog testen met defect TBB board!
#
def PseudoRandomTBBTest():
	sr.setId('PsRndT>: ')
	sr.appendLog(21,'### Verify the RCU - RSP - TBB LVDS interfaces by capturing pseudo random data on TBB')
	res = cli.command('./tbb_prbs_tester.sh')
	if res.find('wrong')==-1:
		#sr.appendLog(11,'>>> RCU - RSP - TBB LVDS interfaces test went OK')
		if debug: print(11,'>>> RCU - RSP - TBB LVDS interfaces test went OK')
	else:
		sr.appendLog(11,'>>> RCU - RSP - TBB LVDS interfaces went wrong')
		sr.appendLog(11,'CLI:')
		sr.appendLog(11,res,1,1,1)
		sr.setResult('FAILED')
	return

################################################################################
# Function CHeck SPU status			Nog testen met defect SPU board!
#
def CheckSPUStatus():
	SeverityOfThisTest=2
	PriorityOfThisTest=2
	
	global Severity
	global Priority
	
	sr.setId('SPUst >: ')
	sr.setId('SPU status - ')
	sr.appendLog(21,'### Verify the RSP - SPU I2C interface by reading the SPU sensor data')
	res = cli.command('python i2c_spu.py --sub %s --rep 1 -v 11' %(SubRck,))
	res = cli.command('python i2c_spu.py --sub sub0,sub1,sub2')
	if res.find('FAILED')==-1:
		#sr.appendLog(11,'>>> RSP - SPU I2c interface test went OK')
		if debug: print(11,'>>> RSP - SPU I2c interface test went OK')
	else:
		sr.appendLog(11,'>>> RSP - SPU I2c interface test went wrong')
#		sr.appendLog(11,'CLI:')
#		sr.appendLog(11,res,1,1,1)
#		sr.appendLog(11,'Result:')
		sr.appendFile(11,'spustat.log')
		sr.setResult('FAILED')
		
# store station testlog			
		if Severity<SeverityOfThisTest: Severity=SeverityOfThisTest
		if Priority<PriorityOfThisTest: Priority=PriorityOfThisTest
		st_log.write('SPUst >: Sv=%s Pr=%s, RSP - SPU I2c interface test went wrong!\n' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest]))
	return


################################################################################
# Function CHeck RSP TD interface		Nog testen met defect interface!
#
def CheckRSPTdI2C():
	SeverityOfThisTest=2
	PriorityOfThisTest=2
	
	global Severity
	global Priority
	
	sr.setId('RSPTD >: ')
	sr.appendLog(21,'### Verify the RSP - TD I2C interface by reading the TD sensor data')
	res = cli.command('python i2c_td.py --brd %s' %(SubBrd,))
	if debug: print('res = %s' % res)
	if res.find('FAILED')==-1:
		#sr.appendLog(11,'>>> RSP - TD I2c interface test went OK')
		if debug: print(11,'>>> RSP - TD I2c interface test went OK')
	else:
		sr.appendLog(11,'>>> RSP - TD I2c interface test went wrong')
#		sr.appendLog(11,'CLI:')
#		sr.appendLog(11,res,1,1,1)
#		sr.appendLog(11,'Result:')
		sr.appendFile(11,'tdstat.log')
		sr.setResult('FAILED')
		
# store station testlog			
		if Severity<SeverityOfThisTest: Severity=SeverityOfThisTest
		if Priority<PriorityOfThisTest: Priority=PriorityOfThisTest
		st_log.write('RSPTD >: Sv=%s Pr=%s, RSP - TD I2c interface test went wrong!\n' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest]))
	return

################################################################################
# Function Built in self test RSP		  Nog testen op een defecte RSP!
#
def Bist():
	SeverityOfThisTest=2
	PriorityOfThisTest=2
	
	global Severity
	global Priority
	
	sr.setId('Bist  >: ')
	sr.appendLog(21,'### Build In Self Test (BIST)')
	res = cli.command('python verify.py --brd %s --rep 1 -v 21 --te tc/bist.py' %(RspBrd,))
	if debug: print('res = %s' % res)
	if res.find('wrong')==-1:
		#sr.appendLog(11,'>>> BIST went OK')
		if debug: print(11,'>>> BIST went OK')
		sr.appendLog(21,'tc/bist.log')
	else:
		sr.appendLog(11,'>>> BIST went wrong')
#		sr.appendLog(11,'CLI:')
#		sr.appendLog(11,res,1,1,1)
		sr.appendLog(11,'tc/bist.log')
		sr.appendLog('FAILED')
		
# store station testlog			
		if Severity<SeverityOfThisTest: Severity=SeverityOfThisTest
		if Priority<PriorityOfThisTest: Priority=PriorityOfThisTest
		st_log.write('Bist  >: Sv=%s Pr=%s, BIST went wrong!\n' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest]))
	return

################################################################################
# Function Pseudo Random RSP Test		  Nog testen op een defecte RSP!
#
def PseudoRandomRSPTest():
	sr.setId('PsRndR>: ')
	sr.appendLog(21,'### Verify the RCU -> RSP LVDS interfaces by capturing pseudo random data on RSP')
	res = cli.command('python verify.py --brd %s --fpga blp0,blp1,blp2,blp3 --pol x,y --rep 1 -v 11 --te tc/prsg.py' %(RspBrd,)) 
	if res.find('wrong')==-1:
		#sr.appendLog(11,'>>> RCU-RSP interface test went OK')
		if debug: print(11,'>>> RCU-RSP interface test went OK')
		sr.appendFile(21,'tc/prsg.log')
	else:
		sr.appendLog(11,'>>> RCU-RSP interface test went wrong')
#		sr.appendLog(11,'CLI:')
#		sr.appendLog(11,res,1,1,1)
		sr.appendFile(11,'tc/prsg.log')
		sr.setResult('FAILED')
	return

################################################################################
# Function RCU - HBA modem test                       Nog testen met defecte RCU
#
def RCUHBAModemTest():
	sr.setId('RCUHBm>: ')
	sr.appendLog(21,'### Verify the control modem on the RCU')
	res = cli.command('python verify.py --brd %s --fpga blp0,blp1,blp2,blp3 --rep 1 -v 11 --te tc/hba_client.py --client_access r --client_reg version --data 10' %(RspBrd,)) 
	if res.find('wrong')==-1:
		#sr.appendLog(11,'>>> RCU-HBA modem test went OK')
		if debug: print(11,'>>> RCU-HBA modem test went OK')
		sr.appendFile(21,'tc/hba_client.log')
	else:
		sr.appendLog(11,'>>> RCU-HBA modem test went wrong')
#		sr.appendLog(11,'CLI:')
#		sr.appendLog(11,res,1,1,1)
		sr.appendFile(11,'tc/hba_client.log')
		sr.setResult('FAILED')
		
# store station testlog	
		if Severity<SeverityOfThisTest: Severity=SeverityOfThisTest
		if Priority<PriorityOfThisTest: Priority=PriorityOfThisTest
		st_log.write('RCUHBm>: Sv=%s Pr=%s, RCU-HBA modem test went wrong!\n' % (SeverityLevel[SeverityOfThisTest], PriorityLevel[PriorityOfThisTest]))