diff --git a/.gitattributes b/.gitattributes
index ebef7bfa1d0b16e37bf47959d2f3624ae1abf80c..ff7a4e0019ed9cfd4bd565b4c0bb9dce133addd8 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1966,6 +1966,7 @@ LCU/StationTest/xc_160_verify.sh eol=lf
LCU/StationTest/xc_200_setup.sh eol=lf
LCU/StationTest/xc_200_verify.sh eol=lf
LCU/checkhardware/check_hardware.py -text
+LCU/checkhardware/checkhardware_lib/CMakeLists.txt -text
LCU/checkhardware/checkhardware_lib/__init__.py -text
LCU/checkhardware/checkhardware_lib/data.py -text
LCU/checkhardware/checkhardware_lib/db.py -text
@@ -1977,6 +1978,7 @@ LCU/checkhardware/checkhardware_lib/lofar.py -text
LCU/checkhardware/checkhardware_lib/reporting.py -text
LCU/checkhardware/checkhardware_lib/rsp.py -text
LCU/checkhardware/checkhardware_lib/settings.py -text
+LCU/checkhardware/checkhardware_lib/spectrum_checks/CMakeLists.txt -text
LCU/checkhardware/checkhardware_lib/spectrum_checks/__init__.py -text
LCU/checkhardware/checkhardware_lib/spectrum_checks/cable_reflection.py -text
LCU/checkhardware/checkhardware_lib/spectrum_checks/down.py -text
@@ -2046,6 +2048,11 @@ LCU/checkhardware/do_station_test.sh -text svneol=unset#application/x-shellscrip
LCU/checkhardware/rtsm.py -text
LCU/checkhardware/show_bad_spectra.py -text
LCU/checkhardware/show_test_result.py -text
+LCU/checkhardware/test/CMakeLists.txt -text
+LCU/checkhardware/test/t_check_hardware.py -text
+LCU/checkhardware/test/t_check_hardware.run -text
+LCU/checkhardware/test/t_check_hardware.sh -text
+LCU/checkhardware/test/test-check_hardware.conf -text
LCU/checkhardware/update_pvss.py -text
LTA/LTAIngest/LTAIngestClient/bin/CMakeLists.txt -text
LTA/LTAIngest/LTAIngestClient/bin/ingestaddjobstoqueue -text
diff --git a/LCU/checkhardware/CMakeLists.txt b/LCU/checkhardware/CMakeLists.txt
index a94754b9dcd79d8ac79b0748e765220cc2d76098..2bddc75a90d43b66a6d55ce8d89da0d463cf4aaf 100644
--- a/LCU/checkhardware/CMakeLists.txt
+++ b/LCU/checkhardware/CMakeLists.txt
@@ -1,6 +1,15 @@
# $Id$
-lofar_package(checkhardware 1.0)
+lofar_package(checkhardware 1.0 DEPENDS PyCommon)
+include(PythonInstall)
+
+# install for testing in cmake
+set(_py_files
+ check_hardware.py
+)
+
+python_install(${_py_files} DESTINATION lofar/lcu/checkhardware)
+
# Install files matching regex pattern in current directory and below
install(DIRECTORY .
@@ -14,3 +23,9 @@ install(DIRECTORY config/
USE_SOURCE_PERMISSIONS
FILES_MATCHING REGEX "(\\.conf)$"
PATTERN ".svn" EXCLUDE)
+
+
+add_subdirectory(test)
+add_subdirectory(checkhardware_lib)
+
+
diff --git a/LCU/checkhardware/check_hardware.py b/LCU/checkhardware/check_hardware.py
index 9dc57811d14e6ab9b14886dde096c0d544f5bf00..333afef267e223f80d9237de9f207b7f94e252c8 100755
--- a/LCU/checkhardware/check_hardware.py
+++ b/LCU/checkhardware/check_hardware.py
@@ -51,6 +51,13 @@ from time import sleep
import datetime
from socket import gethostname
import logging
+from signal import SIGABRT, SIGINT, SIGTERM, signal
+import atexit
+from subprocess import Popen, check_call, CalledProcessError, STDOUT, check_output
+from functools import partial
+
+# FIXME: There is _way_ too much going on here outside a function, including things that might fail (like path checks)
+# FIXME: emoving hard dependencies on station environment
os.umask(001)
@@ -283,6 +290,148 @@ def wait_for_start(start_datetime):
return
+def stop_test_signal(cmd):
+ logger.info("Stopping test signal.")
+
+ # try to execute command to stop test signal
+ try:
+ check_call(cmd, shell=True)
+ except CalledProcessError as ex:
+ logger.error(("Could not stop the test signal! Non-zero return code from start_cmd (%s)." % cmd), ex)
+ raise
+
+def stop_test_signal_and_exit(cmd, *optargs):
+ """
+ Signal handler that exits with the return code of a passed POSIX signal after executing the provided command.
+
+ :param cmd: The command to stop the test signal
+ :param optargs: The intercepted POSIX signal
+ """
+ stop_test_signal(cmd)
+ exit_without_triggering_handler(cmd, *optargs)
+
+
+def exit_without_triggering_handler(cmd, *optargs):
+ """
+ :param cmd: The command to stop the test signal
+ :param optargs: The intercepted POSIX signal
+ """
+
+ # try to get correct return code
+ logger.info('Now exiting.')
+ try:
+ ret_code = int(optargs[0]) # use POSIX signal code
+ os._exit(ret_code) # sys.exit() won't work here, we don't want to trigger our handler again
+ # (hm, we could actually call sys.exit and just trigger the atexit handler, but this is more explicit and keeps
+ # things operational independently.)
+ except:
+ os._exit(1)
+
+
+def register_exit_handler(cmd):
+ """
+ execute stop_cmd when script exits normally or with Exception
+ :param cmd: the command to execute
+ """
+ # execute stop_cmd when script exits normally
+ atexit.register(stop_test_signal, cmd)
+
+
+def register_signal_handlers(cmd):
+ """
+ execute stop_cmd when script is terminated externally
+ :param cmd: the command to execute
+ """
+ # execute stop_cmd when script exits normally
+ atexit.register(stop_test_signal, cmd)
+
+ # handle POSIX signals
+ for sig in (SIGABRT, SIGINT, SIGTERM):
+ signal(sig, partial(stop_test_signal_and_exit, cmd))
+
+
+def start_watchdog_daemon(pid, cmd):
+ """
+ Start a daemon that sits and waits for this script to terminate and then execute the provided command.
+ We cannot handle SIGKILL / kill -9 from inside the script, so we have to handle that case this way. This may be
+ a bit --wait for it-- overkill (hah!) and I don't see why this would be needed under normal circumstances, but
+ nonetheless, since this was requested on the ticket, here we go.
+ :param cmd: command as shell-executable string
+ """
+ daemon_cmd = 'while ps -p %s > /dev/null; do sleep 1; done; %s' % (pid, cmd)
+ Popen(daemon_cmd, stdout=open('/dev/null', 'w'), stderr=STDOUT, shell=True, preexec_fn=os.setpgrp)
+
+
+def safely_start_test_signal(start_cmd, stop_cmd):
+ """
+ This will start start_cmd and set things up in a way that stop_cmd is executed in case the check_hardware script
+ either exits regularly or gets killed for some reason by a POSIX signal. stop_cmd might be executed repeatedly
+ under circumstances.
+ :param start_cmd: the command to start as shell-executable string
+ :param stop_cmd: the command to stop on exit as shell-executable string
+ """
+
+ # start signal
+ try:
+ check_call(start_cmd, shell=True)
+ except CalledProcessError as ex:
+ logger.error("Could not start the test signal! Non-zero return code from start_cmd (%s)." % start_cmd, ex)
+ raise
+
+ # set things up sp signal is stopped when check_hardware terminates
+ register_signal_handlers(stop_cmd)
+ register_exit_handler(stop_cmd)
+ start_watchdog_daemon(os.getpid(), stop_cmd) # this alone would actually be sufficient
+
+
+def safely_start_test_signal_from_ParameterSet(settings):
+ '''
+ :param settings: A settings.ParameterSet (e.g. obtained through TestSettings.group)
+ '''
+ try:
+ start_cmd = settings.parset['testsignal']['start-cmd']
+ stop_cmd = settings.parset['testsignal']['stop-cmd']
+ logger.info('Test signal start/stop settings found. (%s // %s)' % (start_cmd, stop_cmd))
+
+ # start signal:
+ safely_start_test_signal(start_cmd, stop_cmd)
+
+ try:
+ status_cmd = settings.parset['testsignal']['status-cmd']
+ ok_status = settings.parset['testsignal']['ok-status']
+ logger.info('Test signal status settings found. (%s // %s)' % (status_cmd, ok_status))
+
+ # wait for signal status to be ok:
+ wait_for_test_signal_status(status_cmd, ok_status)
+
+ except KeyError:
+ logger.info('No test signal status settings found.')
+
+ except KeyError:
+ logger.info('No test signal settings found.')
+
+
+def wait_for_test_signal_status(status_cmd, status, retry_limit=30):
+ """
+ :param status_cmd: command to get test signal status
+ :param status: the command output to wait for
+ :param retry_limit: raise RunTimeError after this many status_cmd that did not return status
+ """
+ logger.info("Waiting for '%s' to return '%s'" % (status_cmd, status))
+ out = None
+ for _ in range(retry_limit):
+ out = check_output(status_cmd, shell=True)
+ out = out.strip()
+ if out == status:
+ logger.info("Status ok.")
+ return status
+ else:
+ logger.info('Wrong status: %s != %s. Try again...'% (out, status))
+ time.sleep(1)
+
+ raise RuntimeError("Timed out. Last response was '%s'" % out)
+
+
def main():
global station_name
get_arguments()
@@ -501,6 +650,7 @@ def main():
lbh.check_rf_power(mode=mode, parset=settings)
for mode in (5, 6, 7):
+
# do all rcumode 5, 6, 7 tests
hba = HBA(db)
tile_settings = conf.group('rcumode.%d.tile' % mode)
@@ -528,6 +678,7 @@ def main():
# if 'RCU%d' % mode in args or 'S%d' % mode in args:
if 'S%d' % mode in args:
+ safely_start_test_signal_from_ParameterSet(tile_settings)
hba.check_rf_power(mode=mode, parset=tile_settings)
runtime = (time.time() - runstart)
@@ -538,12 +689,12 @@ def main():
recordtime = 4
else:
recordtime = int(args.get('E%d' % mode))
-
+ safely_start_test_signal_from_ParameterSet(elem_settings)
hba.check_elements(mode=mode, record_time=recordtime, parset=elem_settings)
# stop test if driver stopped
db.rsp_driver_down = not check_active_rspdriver()
- if db.rsp_driver_down and (restarts > 0):
+ if db.rsp_driver_down and (restarts > 0): # FIXME 'restarts' is undefined at this point?!
restarts -= 1
reset_48_volt()
time.sleep(30.0)
@@ -634,6 +785,7 @@ def main():
return 0
-
if __name__ == '__main__':
sys.exit(main())
+
+
diff --git a/LCU/checkhardware/checkhardware_lib/CMakeLists.txt b/LCU/checkhardware/checkhardware_lib/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..e5ca32db72ca295e7ab1b50ca18625a046363936
--- /dev/null
+++ b/LCU/checkhardware/checkhardware_lib/CMakeLists.txt
@@ -0,0 +1,21 @@
+# $Id: CMakeLists.txt $
+
+set(_py_files
+ __init__.py
+ data.py
+ db.py
+ general.py
+ hardware_tests.py
+ hba.py
+ lba.py
+ lofar.py
+ reporting.py
+ rsp.py
+ settings.py
+ spu.py
+ tbb.py
+)
+
+python_install(${_py_files} DESTINATION lofar/lcu/checkhardware/checkhardware_lib)
+
+add_subdirectory(spectrum_checks)
\ No newline at end of file
diff --git a/LCU/checkhardware/checkhardware_lib/__init__.py b/LCU/checkhardware/checkhardware_lib/__init__.py
index 837db9972b7db5382bdac83aedf7ae111f0a03dc..2a187de83473ab54a4bdf06f0d8c95801b081094 100644
--- a/LCU/checkhardware/checkhardware_lib/__init__.py
+++ b/LCU/checkhardware/checkhardware_lib/__init__.py
@@ -3,7 +3,7 @@
from general import *
from lofar import *
-from settings import TestSettings
+from settings import TestSettings, ParameterSet
from db import DB, db_version
from reporting import make_report
from spu import SPU
diff --git a/LCU/checkhardware/checkhardware_lib/spectrum_checks/CMakeLists.txt b/LCU/checkhardware/checkhardware_lib/spectrum_checks/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..2aaae58c10fc901e76fbb974969cc8e6e7155f17
--- /dev/null
+++ b/LCU/checkhardware/checkhardware_lib/spectrum_checks/CMakeLists.txt
@@ -0,0 +1,19 @@
+# $Id: CMakeLists.txt $
+
+set(_py_files
+ __init__.py
+ cable_reflections.py
+ down.py
+ down_old.py
+ flat.py
+ noise.py
+ oscillation.py
+ peakslib.py
+ rf_power.py
+ short.py
+ spurious.py
+ summator_noise.py
+ tools.py
+)
+
+python_install(${_py_files} DESTINATION lofar/lcu/checkhardware/checkhardware_lib/spectrum_checks)
diff --git a/LCU/checkhardware/config/FR606-check_hardware.conf b/LCU/checkhardware/config/FR606-check_hardware.conf
index f72e209b6823125c736aa0d727c6712495c40913..045a901b766ecbdbec7488484f28d86e559aa2fb 100755
--- a/LCU/checkhardware/config/FR606-check_hardware.conf
+++ b/LCU/checkhardware/config/FR606-check_hardware.conf
@@ -29,8 +29,8 @@ station= FR606C
always= RV, TV, RBC, TBC
list.0=
list.1= SPU,TM,RCU3,RCU5
-list.2= SPU,TM,RCU3,M5,SN5,O5,N5,SP5,S7,E7
-list.3= S3
+list.2= SPU,TM,RCU3,M5,SN5,O5,N5,SP5,S5,E5
+list.3= S5,E5
[spu]
temperature.min= 10.0
@@ -131,10 +131,10 @@ short.mean-pwr.min= 55.0
short.mean-pwr.max= 61.0
flat.mean-pwr.min= 61.0
flat.mean-pwr.max= 64.5
-rf.subbands= 105
+rf.subbands= 256
rf.min-sb-pwr= 65.0
-rf.negative-deviation= -24.0
-rf.positive-deviation= 12.0
+rf.negative-deviation= -28.0
+rf.positive-deviation= 16.0
noise.negative-deviation= -3.0
noise.positive-deviation= 1.5
noise.max-difference= 1.5
@@ -147,12 +147,16 @@ oscillation.min-peak-pwr= 6.0
oscillation.passband= 1:511
spurious.min-peak-pwr= 3.0
spurious.passband= 1:511
+testsignal.start-cmd= echo set_config 150.0 -10 | nc ncu 8093
+testsignal.stop-cmd= echo bye | nc ncu 8093
+testsignal.status-cmd= echo get_config | nc ncu 8093 | grep Frequency
+testsignal.ok-status= Frequency: 150 MHz Power level: -10 dBm RF: ON
[rcumode.5.element]
-rf.subbands= 105
+rf.subbands= 256
rf.min-sb-pwr= 65.0
-rf.negative-deviation= -24.0
-rf.positive-deviation= 12.0
+rf.negative-deviation= -30.0
+rf.positive-deviation= 16.0
noise.negative-deviation= -3.0
noise.positive-deviation= 1.5
noise.max-difference= 1.5
@@ -161,6 +165,10 @@ oscillation.min-peak-pwr= 6.0
oscillation.passband= 1:511
spurious.min-peak-pwr= 3.0
spurious.passband= 1:511
+testsignal.start-cmd= echo set_config 150.0 0 | nc ncu 8093
+testsignal.stop-cmd= echo bye | nc ncu 8093
+testsignal.status-cmd= echo get_config | nc ncu 8093 | grep Frequency
+testsignal.ok-status= Frequency: 150 MHz Power level: 0 dBm RF: ON
[rcumode.6.tile]
short.mean-pwr.min= 55.0
diff --git a/LCU/checkhardware/test/CMakeLists.txt b/LCU/checkhardware/test/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..acc3d69163f87e951e5c5ec2da0e9e7405a45bcb
--- /dev/null
+++ b/LCU/checkhardware/test/CMakeLists.txt
@@ -0,0 +1,5 @@
+# $Id: CMakeLists.txt 33404 2016-01-27 10:21:14Z jkuensem $
+
+include(LofarCTest)
+
+lofar_add_test(t_check_hardware)
diff --git a/LCU/checkhardware/test/t_check_hardware.py b/LCU/checkhardware/test/t_check_hardware.py
new file mode 100644
index 0000000000000000000000000000000000000000..54e5d8c78d1d8b0d5824826cb44200b0f929e1bf
--- /dev/null
+++ b/LCU/checkhardware/test/t_check_hardware.py
@@ -0,0 +1,429 @@
+import unittest
+import os
+from mock import MagicMock, patch, call
+import sys
+import logging
+import subprocess
+import signal
+import atexit
+import time
+
+logger = logging.getLogger(__name__)
+
+# mock out check for existing log directory on script import as module
+os.access = MagicMock(return_value=True)
+
+# mock out modules with relative imports (that only work with the namespace when executed as a script)
+# FIXME: make sure that absolute imports are ok and don't break things in production.
+# FIXME: ...Then fix the implementation and remove this mock so we can test those modules.
+with patch.dict('sys.modules', **{
+ 'checkhardware_lib': MagicMock(),
+ 'checkhardware_lib.data': MagicMock(),
+ 'checkhardware_lib.rsp': MagicMock(),
+ 'cable_reflection': MagicMock(),
+ 'logging': MagicMock(),
+}):
+ # import these here so we can before mock out checks on station environment
+ import lofar.lcu.checkhardware.check_hardware as check_hardware
+ from lofar.lcu.checkhardware.checkhardware_lib import TestSettings, ParameterSet
+ check_hardware.logger = logger # override logger to handle logging output here
+
+
+class TestCheckHardware(unittest.TestCase):
+
+ def setUp(self):
+ logger.info(">>>---> %s <---<<<" % self._testMethodName)
+
+ # mock exit call to not actually exit the test
+ os._exit = MagicMock()
+ # we don't want to actually call anything
+ check_hardware.check_call = MagicMock()
+
+ self.elem_settings_no_testsignal = ParameterSet()
+ self.elem_settings_no_testsignal.parset = {'spurious': {'min-peak-pwr': '3.0',
+ 'passband': '1:511'},
+ 'rf': {'negative-deviation': '-24.0',
+ 'subbands': '105',
+ 'min-sb-pwr': '65.0',
+ 'positive-deviation': '12.0'},
+ 'noise': {'negative-deviation': '-3.0',
+ 'max-difference': '1.5',
+ 'positive-deviation': '1.5',
+ 'passband': '1:511'},
+ 'oscillation': {'min-peak-pwr': '6.0',
+ 'passband': '1:511'}}
+
+ self.elem_settings_testsignal = ParameterSet()
+ self.elem_settings_testsignal.parset = {'spurious': {'min-peak-pwr': '3.0',
+ 'passband': '1:511'},
+ 'rf': {'negative-deviation': '-24.0',
+ 'subbands': '105',
+ 'min-sb-pwr': '65.0',
+ 'positive-deviation': '12.0'},
+ 'noise': {'negative-deviation': '-3.0',
+ 'max-difference': '1.5',
+ 'positive-deviation': '1.5',
+ 'passband': '1:511'},
+ 'testsignal': {'start-cmd': 'echo set config 56.0 -10 | nc ncu 8093',
+ 'stop-cmd': 'echo bye | nc ncu 8093'},
+ 'oscillation': {'min-peak-pwr': '6.0',
+ 'passband': '1:511'}}
+
+ self.elem_settings_testsignal_with_status = ParameterSet()
+ self.elem_settings_testsignal_with_status.parset = {'spurious': {'min-peak-pwr': '3.0',
+ 'passband': '1:511'},
+ 'rf': {'negative-deviation': '-24.0',
+ 'subbands': '105',
+ 'min-sb-pwr': '65.0',
+ 'positive-deviation': '12.0'},
+ 'noise': {'negative-deviation': '-3.0',
+ 'max-difference': '1.5',
+ 'positive-deviation': '1.5',
+ 'passband': '1:511'},
+ 'testsignal': {'start-cmd': 'echo set config 56.0 -10 | nc ncu 8093',
+ 'stop-cmd': 'echo bye | nc ncu 8093',
+ 'status-cmd': "echo 'Frequency: 56 MHz Power level: -10 dBm RF: ON'",
+ 'ok-status': "Frequency: 56 MHz Power level: -10 dBm RF: ON"},
+ 'oscillation': {'min-peak-pwr': '6.0',
+ 'passband': '1:511'}}
+
+
+ def test_safely_start_test_signal(self):
+ """ Verify that the provided command is executed and handlers are registered correctly"""
+
+ # test value
+ start_cmd = 'echo "Start the signal!"'
+ stop_cmd = 'echo "Stop the signal!"'
+
+ # setup test
+ with patch.object(check_hardware, 'register_exit_handler'), \
+ patch.object(check_hardware, 'register_signal_handlers'), \
+ patch.object(check_hardware, 'start_watchdog_daemon'):
+
+ # trigger action
+ check_hardware.safely_start_test_signal(start_cmd, stop_cmd)
+
+ # assert correct behavior
+ check_hardware.check_call.assert_called_with(start_cmd, shell=True)
+ check_hardware.register_exit_handler.assert_called_with(stop_cmd)
+ check_hardware.register_signal_handlers.assert_called_with(stop_cmd)
+ check_hardware.start_watchdog_daemon.assert_called_with(os.getpid(), stop_cmd)
+
+ def test_safely_start_test_signal_logs_and_reraises_CalledProcessError(self):
+ """ Verify that the provided command is executed and handlers are registered correctly"""
+
+ # test value
+ start_cmd = 'echo "Start the signal!"'
+ stop_cmd = 'echo "Stop the signal!"'
+
+ # setup test
+ with patch.object(check_hardware, 'register_exit_handler'), \
+ patch.object(check_hardware, 'register_signal_handlers'), \
+ patch.object(check_hardware, 'start_watchdog_daemon'), \
+ patch.object(check_hardware, 'check_call', MagicMock(side_effect=subprocess.CalledProcessError('', ''))), \
+ patch.object(check_hardware.logger, 'error'):
+
+ with self.assertRaises(subprocess.CalledProcessError):
+ # trigger action
+ check_hardware.safely_start_test_signal(start_cmd, stop_cmd)
+
+ # assert correct behavior
+ check_hardware.logger.error.assert_called()
+
+ def test_safely_start_test_signal_from_ParameterSet_turns_signal_and_waits_for_status_correctly(self):
+ """ Verify that the commands from ParameterSet are passed on to safely_start_test_signal and wait_for_test_signal_status"""
+
+ # test value
+ start_cmd = 'echo set config 56.0 -10 | nc ncu 8093'
+ stop_cmd = 'echo bye | nc ncu 8093'
+ expected_status_cmd = "echo 'Frequency: 56 MHz Power level: -10 dBm RF: ON'"
+ expected_ok_status = "Frequency: 56 MHz Power level: -10 dBm RF: ON"
+
+ # setup test
+ with patch.object(check_hardware, 'safely_start_test_signal'), \
+ patch.object(check_hardware, 'wait_for_test_signal_status'):
+
+ # trigger action
+ check_hardware.safely_start_test_signal_from_ParameterSet(self.elem_settings_testsignal_with_status)
+
+ # assert correct behavior
+ check_hardware.safely_start_test_signal.assert_called_with(start_cmd, stop_cmd)
+ check_hardware.wait_for_test_signal_status.assert_called_with(expected_status_cmd, expected_ok_status)
+
+ def test_safely_start_test_signal_from_ParameterSet_does_nothing_when_no_stationsignal_keys_in_ParameterSet(self):
+ """ Verify that the commands from ParameterSet are passed on to safely_start_test_signal and wait_for_test_signal_status is not called"""
+
+ # setup test
+ with patch.object(check_hardware, 'safely_start_test_signal'), \
+ patch.object(check_hardware, 'wait_for_test_signal_status'):
+
+ # trigger action
+ check_hardware.safely_start_test_signal_from_ParameterSet(self.elem_settings_no_testsignal)
+
+ # assert correct behavior
+ check_hardware.safely_start_test_signal.assert_not_called()
+ check_hardware.wait_for_test_signal_status.assert_not_called()
+
+ def test_safely_start_test_signal_from_ParameterSet_only_starts_signal_when_no_status_keys_in_ParameterSet(self):
+ """ Verify that safely_start_test_signal and wait_for_test_signal_status are not called"""
+
+ # test value
+ start_cmd = 'echo set config 56.0 -10 | nc ncu 8093'
+ stop_cmd = 'echo bye | nc ncu 8093'
+
+ # setup test
+ with patch.object(check_hardware, 'safely_start_test_signal'), \
+ patch.object(check_hardware, 'wait_for_test_signal_status'):
+ # trigger action
+ check_hardware.safely_start_test_signal_from_ParameterSet(self.elem_settings_testsignal)
+
+ # assert correct behavior
+ check_hardware.safely_start_test_signal.assert_called_with(start_cmd, stop_cmd)
+ check_hardware.wait_for_test_signal_status.assert_not_called()
+
+ def test_stop_test_signal(self):
+ """ Verify that the provided command is executed """
+
+ # test value
+ cmd = 'echo "Stop the signal! 1"'
+
+ # trigger action
+ check_hardware.stop_test_signal(cmd)
+
+ # assert correct behavior
+ os._exit.assert_not_called()
+ check_hardware.check_call.assert_called_with(cmd, shell=True) # command is executed
+
+ def test_stop_test_signal_and_exit_defaults_to_code_1(self):
+ """ Verify that the provided command is executed and os._exit is called with correct return code """
+
+ # test value
+ cmd = 'echo "Stop the signal! 2"'
+
+ # trigger action
+ check_hardware.stop_test_signal_and_exit(cmd)
+
+ # assert correct behavior
+ os._exit.assert_called_with(1) # exit code correct
+ check_hardware.check_call.assert_called_with(cmd, shell=True) # command is executed
+
+ def test_stop_test_signal_and_exit_handles_signal_correctly(self):
+ """ Verify that the provided command is executed and os._exit is called with correct return code """
+
+ # test value
+ cmd = 'echo "Stop the signal! 2"'
+ signal_code = 42
+
+ # trigger action
+ check_hardware.stop_test_signal_and_exit(cmd, signal_code, KeyboardInterrupt())
+
+ # assert correct behavior
+ os._exit.assert_called_with(signal_code) # exit code correct
+ check_hardware.check_call.assert_called_with(cmd, shell=True) # command is executed
+
+ def test_wait_for_test_signal_status_waits_for_correct_status(self):
+ """ Verify that the provided command is executed and os._exit is called with correct return code """
+
+ # test value
+ status_cmd = 'mockme'
+ responses = ['ne', 'ja\n', 'ne']
+ waitfor = 'ja'
+
+ with patch.object(check_hardware, 'check_output', MagicMock(side_effect=responses)),\
+ patch('time.sleep'):
+
+ # trigger action
+ check_hardware.wait_for_test_signal_status(status_cmd, waitfor)
+
+ # assert correct behavior
+ check_hardware.check_output.called_with(status_cmd, shell=True) # command is executed
+ self.assertEqual(check_hardware.check_output.call_count, 2)
+
+ def test_wait_for_test_signal_status_raises_RuntimeError_when_retry_limit_reached(self):
+ """ Verify that the provided command is executed and os._exit is called with correct return code """
+
+ # test value
+ limit=15
+ status_cmd = 'mockme'
+ responses = ['ne'] * limit # only 30 are read
+ responses.append('ja')
+ waitfor = 'ja'
+
+ with patch.object(check_hardware, 'check_output', MagicMock(side_effect=responses)),\
+ patch('time.sleep'):
+
+ # trigger action
+
+ with self.assertRaises(RuntimeError):
+ check_hardware.wait_for_test_signal_status(status_cmd, waitfor, retry_limit=limit)
+
+ # assert correct behavior
+ check_hardware.check_output.called_with(status_cmd, shell=True) # command is executed
+ self.assertEqual(check_hardware.check_output.call_count, limit)
+
+
+ def test_register_signal_handlers_stops_test_signal_on_POSIX_signal(self):
+ """ Verify that the provided command is executed and os._exit is called with correct return code """
+
+ # test value
+ cmd = 'echo "Stop the signal! 3"'
+
+ # register handlers we want to test
+ check_hardware.register_signal_handlers(cmd)
+
+ # trigger action:
+ pid = os.getpid()
+ os.kill(pid, signal.SIGINT) # code 2
+ os.kill(pid, signal.SIGABRT) # code 6
+ os.kill(pid, signal.SIGTERM) # code 15
+
+ # assert correct behavior
+ os._exit.assert_has_calls([call(2), call(6), call(15)]) # all signal error codes correct
+ check_hardware.check_call.assert_called_with(cmd, shell=True) # command is executed
+
+ def test_register_exit_handler_stops_test_signal_on_normal_exit(self):
+ """ Verify that the provided command is executed and os._exit is called with correct return code """
+
+ # This test turned out nastier than expected.
+ # The problem is that we cannot catch the SystemExit within the test, because the atexit hooks only fire after
+ # the test exits (even after tearDownClass), so we will get a stacktrace printed, but cmake won't count the
+ # assert failures as failure of the test.
+ # Note: As long as we use the watchdog, this is redundant anyway and we could also change the implementation
+ # to explicitely turn the test signal off before it exits and test for that instead.
+ # But who wants the easy way out, right? ;)
+ # FIXME: Find a way to make sure this test fails if the assertion fails or find a smarter way to test this.
+
+ # test value
+ cmd = 'echo "Stop the signal! 4"'
+
+ # assert correct behavior
+ def assert_on_exit():
+ logger.info('>>>----> Asserting on exit!')
+ check_hardware.check_call.assert_called_with(cmd, shell=True) # command is executed
+
+ # register a handler to trigger the assert.
+ atexit.register(assert_on_exit)
+
+ # register handlers we want to test
+ check_hardware.register_exit_handler(cmd)
+
+ # The test will now regularly exit with code 0, hopefully triggering all these hooks
+
+
+ def test_start_watchdog_daemon_stops_test_signal_when_provided_pid_is_killed(self):
+ """ Verify that the provided command is executed when watched process dies """
+
+ tmpfile = "/tmp/t_checkhardware.%s" % time.time()
+
+ # test value
+ good_message = 'Stop the signal! 5'
+ cmd = 'echo "%s" > %s' % (good_message, tmpfile)
+
+ # start dummy process
+ p = subprocess.Popen(['sleep', '120'])
+
+ # start watchdog for dummy process
+ check_hardware.start_watchdog_daemon(p.pid, cmd)
+
+ # kill dummy
+ os.kill(p.pid, signal.SIGKILL)
+ os.wait()
+
+ # check temporary file to confirm the watchdog command has been executed
+ for i in range (30):
+ if os.path.isfile(tmpfile):
+ break
+ time.sleep(1)
+ self.assertTrue(os.path.isfile(tmpfile))
+ with open(tmpfile) as f:
+ lines = f.read().split('\n')
+ self.assertTrue(good_message in lines) # cmd has been executed
+
+ os.remove(tmpfile)
+
+
+ # FIXME: Move this to t_settings once that exists
+ def test_settings_parset_raises_KeyError_when_accessing_missing_key(self):
+
+ # assert KeyError if setting not there
+ with self.assertRaises(KeyError):
+ logger.info(self.elem_settings_no_testsignal.parset['testsignal']['status-cmd'])
+
+
+ # FIXME: Move this to t_settings once that exists
+ def test_settings_contains_testsignal_commands_from_config_file(self):
+
+ # test_values
+ expected_start_cmd = "echo set config 56.0 -10 | nc ncu 8093"
+ expected_stop_cmd = "echo bye | nc ncu 8093"
+ expected_status_cmd = "echo 'Frequency: 56 MHz Power level: -10 dBm RF: ON'"
+ expected_ok_status = "Frequency: 56 MHz Power level: -10 dBm RF: ON"
+
+ # read settings
+ f = os.environ.get('srcdir')+'/test-check_hardware.conf'
+ settings = TestSettings(filename=f)
+ elem_settings = settings.group('rcumode.5.element')
+ start_cmd = elem_settings.parset['testsignal']['start-cmd']
+ stop_cmd = elem_settings.parset['testsignal']['stop-cmd']
+ status_cmd = elem_settings.parset['testsignal']['status-cmd']
+ ok_status = elem_settings.parset['testsignal']['ok-status']
+
+ # assert correct values
+ self.assertEqual(start_cmd, expected_start_cmd)
+ self.assertEqual(stop_cmd, expected_stop_cmd)
+ self.assertEqual(status_cmd, expected_status_cmd)
+ self.assertEqual(ok_status, expected_ok_status)
+
+ #@unittest.skip('disabled due to fork bomb behavior')
+ def test_main_turns_signal_with_commands_from_settings(self):
+
+ # test values
+ expected_start_cmd = "echo set config 56.0 -10 | nc ncu 8093"
+ expected_stop_cmd = "echo bye | nc ncu 8093"
+ expected_status_cmd = "echo 'Frequency: 56 MHz Power level: -10 dBm RF: ON'"
+ expected_ok_status = "Frequency: 56 MHz Power level: -10 dBm RF: ON"
+
+ # setup tests
+ # todo: mock the ParameterSet instead, once the imports are resolved and this can be done straight-forward
+ check_hardware.conf_file = r'test-check_hardware.conf'
+ check_hardware.confpath = os.environ.get('srcdir')+'/'
+
+ # pretend to be a station
+ # FIXME: correct behavior of mocked-out parts should be covered by additional tests
+ # FIXME: why is all this actually necessary when I only run an element test?
+ with patch.object(check_hardware, 'read_station_config', MagicMock(return_value=(1, 1, 1, 1, 1, 1, 1))), \
+ patch.object(check_hardware, 'safely_start_test_signal'), \
+ patch.object(check_hardware, 'wait_for_test_signal_status'), \
+ patch.object(check_hardware, 'swlevel', MagicMock(return_value=(5, None))), \
+ patch.object(check_hardware, 'rspctl'), \
+ patch.object(check_hardware, 'RSP'), \
+ patch.object(check_hardware, 'check_active_boards', MagicMock(return_value=(1, 1))), \
+ patch.object(check_hardware, 'check_active_tbbdriver', MagicMock(return_value=True)), \
+ patch.object(check_hardware, 'check_active_rspdriver', MagicMock(return_value=True)), \
+ patch.object(check_hardware, 'reset_rsp_settings'), \
+ patch.object(check_hardware, 'HBA'), \
+ patch.object(check_hardware, 'reset_48_volt'), \
+ patch.object(check_hardware, 'tbbctl'), \
+ patch.object(os, 'listdir'), \
+ patch.object(os, 'remove'): # I'm scared...
+
+ # patch arguments: pretend script was started with these.
+ # -TST (test mode)
+ # -e5: (element test in mode 5)
+ # Names optimized for disk space
+ testargs = ["check_hardware.py", '-TST', '-e5', '-s5']
+ with patch.object(sys, 'argv', testargs):
+ # trigger action
+ check_hardware.main() # Warning: Something acts as a fork bomb when mocks are not setup properly!
+
+ check_hardware.safely_start_test_signal.assert_called_with(expected_start_cmd, expected_stop_cmd)
+ check_hardware.wait_for_test_signal_status.assert_called_with(expected_status_cmd, expected_ok_status)
+ self.assertEqual(check_hardware.safely_start_test_signal.call_count, 2)
+ self.assertEqual(check_hardware.wait_for_test_signal_status.call_count, 2)
+
+
+if __name__ == "__main__":
+ logger.level = logging.DEBUG
+ stream_handler = logging.StreamHandler(sys.stdout)
+ logger.addHandler(stream_handler)
+ unittest.main()
diff --git a/LCU/checkhardware/test/t_check_hardware.run b/LCU/checkhardware/test/t_check_hardware.run
new file mode 100755
index 0000000000000000000000000000000000000000..3a348cbba93a2ac84832d7c0579197fa21577f4e
--- /dev/null
+++ b/LCU/checkhardware/test/t_check_hardware.run
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# Run the unit test
+source python-coverage.sh
+python_coverage_test "checkhardware*" t_check_hardware.py
diff --git a/LCU/checkhardware/test/t_check_hardware.sh b/LCU/checkhardware/test/t_check_hardware.sh
new file mode 100755
index 0000000000000000000000000000000000000000..43193a4253494dc8d06e0dac425a066d4a5f80b4
--- /dev/null
+++ b/LCU/checkhardware/test/t_check_hardware.sh
@@ -0,0 +1,2 @@
+#!/bin/sh
+./runctest.sh t_check_hardware
diff --git a/LCU/checkhardware/test/test-check_hardware.conf b/LCU/checkhardware/test/test-check_hardware.conf
new file mode 100644
index 0000000000000000000000000000000000000000..4b7c43ad06d8ed9d560c4c77dc641de6a4a2c085
--- /dev/null
+++ b/LCU/checkhardware/test/test-check_hardware.conf
@@ -0,0 +1,255 @@
+#
+# configuration file for check_hardware.py
+#
+
+[configuration]
+version= 00.01
+station= FR606C
+
+# checks to do if '-l=x' argument is given, all checks behind list.x are executed
+# always checks will be done always
+#
+# S(rcumode) : signal check for rcumode (also down and flat-check in rcumode 1..4).
+# O(rcumode) : oscillation check for rcumode.
+# SP(rcumode) : spurious check for rcumode.
+# N(rcumode)[= 300]: noise check for rcumode, optional data time in seconds
+# default data time= 120 sec.
+# E(rcumode)[= 60] : do all RCU5 element tests, optional data time in seconds.
+# default data time= 10 sec.
+# M(rcumode) : do modem
+# SN(rcumode) : do summator noise
+#
+# RCU(mode) : do all rcu checks for given mode, no element tests done.
+#
+# RBC : RSP voltage/temperature check
+# TBC : TBB voltage/temperature check
+# SPU : SPU voltage
+# TM : TBB memmory
+[check]
+always=
+list.0=
+list.1= SPU,TM,RCU3,RCU5
+list.2= SPU,TM,RCU3,M5,SN5,O5,N5,SP5,S7,E7
+list.3= S3
+
+[spu]
+temperature.min= 10.0
+temperature.max= 35.0
+voltage.3_3.min= 3.1
+voltage.3_3.max= 3.4
+voltage.3_3.max-drop= 0.3
+voltage.5_0.min= 4.5
+voltage.5_0.max= 5.0
+voltage.5_0.max-drop= 0.3
+voltage.8_0.min= 7.4
+voltage.8_0.max= 8.0
+voltage.8_0.max-drop= 0.3
+voltage.48_0.min= 43.0
+voltage.48_0.max= 48.0
+voltage.48_0.max-drop= 2.0
+
+[tbb]
+version.tp= 2.4
+version.mp= 3.0
+temperature.min= 10.0
+temperature.max= 45.0
+temperature.tp.min= 10.0
+temperature.tp.max= 75.0
+temperature.tp.max_delta= 10.0
+temperature.mp.min= 10.0
+temperature.mp.max= 75.0
+temperature.mp.max_delta= 10.0
+voltage.1_2.min= 1.1
+voltage.1_2.max= 1.3
+voltage.2_5.min= 2.4
+voltage.2_5.max= 2.6
+voltage.3_3.min= 3.1
+voltage.3_3.max= 3.4
+
+[rsp]
+version.ap= 8.2
+version.bp= 8.2
+temperature.min= 10.0
+temperature.max= 50.0
+temperature.ap.min= 10.0
+temperature.ap.max= 80.0
+temperature.ap.max_delta= 10.0
+temperature.bp.min= 10.0
+temperature.bp.max= 80.0
+temperature.bp.max_delta= 10.0
+voltage.1_2.min= 1.1
+voltage.1_2.max= 1.3
+voltage.2_5.min= 2.4
+voltage.2_5.max= 2.6
+voltage.3_3.min= 3.1
+voltage.3_3.max= 3.4
+
+[rcumode.1-3]
+short.mean-pwr.min= 55.0
+short.mean-pwr.max= 61.0
+flat.mean-pwr.min= 61.0
+flat.mean-pwr.max= 64.5
+rf.subbands= 301
+rf.min-sb-pwr= 75.0
+rf.negative-deviation= -3.0
+rf.positive-deviation= 3.0
+noise.negative-deviation= -2.5
+noise.positive-deviation= 2.5
+noise.max-difference= 1.5
+noise.passband= 1:511
+oscillation.min-peak-pwr= 6.0
+oscillation.passband= 1:511
+cable-reflection.min-peak-pwr= 0.8
+cable-reflection.passband= 1:511
+spurious.min-peak-pwr= 3.0
+spurious.passband= 1:511
+down.passband= 231:371
+
+[rcumode.2-4]
+short.mean-pwr.min= 55.0
+short.mean-pwr.max= 61.0
+flat.mean-pwr.min= 61.0
+flat.mean-pwr.max= 64.5
+rf.subbands= 301
+rf.min-sb-pwr= 75.0
+rf.negative-deviation= -3.0
+rf.positive-deviation= 3.0
+noise.negative-deviation= -2.5
+noise.positive-deviation= 2.5
+noise.max-difference= 1.5
+noise.passband= 1:511
+oscillation.min-peak-pwr= 6.0
+oscillation.passband= 1:511
+cable-reflection.min-peak-pwr= 0.8
+cable-reflection.passband= 1:511
+spurious.min-peak-pwr= 3.0
+spurious.passband= 1:511
+down.passband= 231:371
+
+[rcumode.5.tile]
+short.mean-pwr.min= 55.0
+short.mean-pwr.max= 61.0
+flat.mean-pwr.min= 61.0
+flat.mean-pwr.max= 64.5
+rf.subbands= 105
+rf.min-sb-pwr= 65.0
+rf.negative-deviation= -24.0
+rf.positive-deviation= 12.0
+noise.negative-deviation= -3.0
+noise.positive-deviation= 1.5
+noise.max-difference= 1.5
+noise.passband= 1:511
+summator-noise.min-peak-pwr= 1.2
+summator-noise.passband= 45:135,200:270
+cable-reflection.min-peak-pwr= 0.8
+cable-reflection.passband= 1:511
+oscillation.min-peak-pwr= 6.0
+oscillation.passband= 1:511
+spurious.min-peak-pwr= 3.0
+spurious.passband= 1:511
+testsignal.start-cmd= echo set config 56.0 -10 | nc ncu 8093
+testsignal.stop-cmd= echo bye | nc ncu 8093
+testsignal.status-cmd= echo 'Frequency: 56 MHz Power level: -10 dBm RF: ON'
+testsignal.ok-status= Frequency: 56 MHz Power level: -10 dBm RF: ON
+
+[rcumode.5.element]
+rf.subbands= 105
+rf.min-sb-pwr= 65.0
+rf.negative-deviation= -24.0
+rf.positive-deviation= 12.0
+noise.negative-deviation= -3.0
+noise.positive-deviation= 1.5
+noise.max-difference= 1.5
+noise.passband= 1:511
+oscillation.min-peak-pwr= 6.0
+oscillation.passband= 1:511
+spurious.min-peak-pwr= 3.0
+spurious.passband= 1:511
+testsignal.start-cmd= echo set config 56.0 -10 | nc ncu 8093
+testsignal.stop-cmd= echo bye | nc ncu 8093
+testsignal.status-cmd= echo 'Frequency: 56 MHz Power level: -10 dBm RF: ON'
+testsignal.ok-status= Frequency: 56 MHz Power level: -10 dBm RF: ON
+
+[rcumode.6.tile]
+short.mean-pwr.min= 55.0
+short.mean-pwr.max= 61.0
+flat.mean-pwr.min= 61.0
+flat.mean-pwr.max= 64.5
+rf.subbands= 105
+rf.min-sb-pwr= 65.0
+rf.negative-deviation= -24.0
+rf.positive-deviation= 12.0
+noise.negative-deviation= -3.0
+noise.positive-deviation= 1.5
+noise.max-difference= 1.5
+noise.passband= 1:511
+summator-noise.min-peak-pwr= 1.2
+summator-noise.passband= 45:135
+cable-reflection.min-peak-pwr= 0.8
+cable-reflection.passband= 1:511
+oscillation.min-peak-pwr= 6.0
+oscillation.passband= 1:511
+spurious.min-peak-pwr= 3.0
+spurious.passband= 1:511
+
+[rcumode.6.element]
+rf.subbands= 105
+rf.min-sb-pwr= 65.0
+rf.negative-deviation= -24.0
+rf.positive-deviation= 12.0
+noise.negative-deviation= -3.0
+noise.positive-deviation= 1.5
+noise.max-difference= 1.5
+noise.passband= 1:511
+oscillation.min-peak-pwr= 6.0
+oscillation.passband= 1:511
+spurious.min-peak-pwr= 3.0
+spurious.passband= 1:511
+testsignal.start-cmd= echo set config 127.5 0 | nc ncu 8093
+testsignal.stop-cmd= echo bye | nc ncu 8093
+
+[rcumode.7.tile]
+short.mean-pwr.min= 55.0
+short.mean-pwr.max= 61.0
+flat.mean-pwr.min= 61.0
+flat.mean-pwr.max= 64.5
+rf.subbands= 105
+rf.min-sb-pwr= 65.0
+rf.negative-deviation= -24.0
+rf.positive-deviation= 12.0
+noise.negative-deviation= -3.0
+noise.positive-deviation= 1.5
+noise.max-difference= 1.5
+noise.passband= 1:511
+summator-noise.min-peak-pwr= 1.2
+summator-noise.passband= 45:135
+cable-reflection.min-peak-pwr= 0.8
+cable-reflection.passband= 1:511
+oscillation.min-peak-pwr= 6.0
+oscillation.passband= 1:511
+spurious.min-peak-pwr= 3.0
+spurious.passband= 1:511
+
+[rcumode.7.element]
+rf.subbands= 105
+rf.min-sb-pwr= 65.0
+rf.negative-deviation= -24.0
+rf.positive-deviation= 12.0
+noise.negative-deviation= -3.0
+noise.positive-deviation= 3.0
+noise.max-difference= 1.5
+noise.passband= 1:511
+oscillation.min-peak-pwr= 6.0
+oscillation.passband= 1:511
+spurious.min-peak-pwr= 3.0
+spurious.passband= 1:511
+
+# General settings
+[paths]
+global-data= /globalhome/log/stationtest
+local-data= /opt/stationtest/data
+local-report-dir= /localhome/stationtest/data
+global-report-dir= /globalhome/log/stationtest
+
+[files]
+bad-antenna-list= /localhome/stationtest/data/bad_antenna_list.txt