Skip to content
Snippets Groups Projects
Commit cdeb5d50 authored by Eric Kooistra's avatar Eric Kooistra
Browse files

Added regression_test_vhdl key and test script...

Added regression_test_vhdl key and test script modelsim_regression_test_vhdl.py that works ok for dp, diag and mac_10g libraries.
parent a0fcc9b5
No related branches found
No related tags found
No related merge requests found
......@@ -41,6 +41,16 @@ test_bench_files =
tb/vhdl/tb_tb_mms_diag_block_gen.vhd
tb/vhdl/tb_diag_regression.vhd
regression_test_vhdl =
tb/vhdl/tb_diag_wg.vhd
tb/vhdl/tb_diag_wg_wideband.vhd
tb/vhdl/tb_diag_block_gen.vhd
tb/vhdl/tb_diag_frm_generator.vhd
tb/vhdl/tb_diag_frm_monitor.vhd
tb/vhdl/tb_tb_diag_rx_seq.vhd
tb/vhdl/tb_tb_mms_diag_seq.vhd
tb/vhdl/tb_tb_mms_diag_block_gen.vhd
[modelsim_project_file]
modelsim_copy_files =
......
......@@ -215,6 +215,11 @@ test_bench_files =
tb/vhdl/tb_tb_tb_dp_backpressure.vhd
tb/vhdl/tb_dp_offload_rx_filter.vhd
regression_test_vhdl =
tb/vhdl/tb_dp_latency_adapter.vhd
tb/vhdl/tb_dp_split.vhd
[modelsim_project_file]
......
......@@ -24,6 +24,9 @@ test_bench_files =
tb_tech_mac_10g.vhd
tb_tb_tech_mac_10g.vhd
regression_test_vhdl =
tb_tb_tech_mac_10g.vhd
[modelsim_project_file]
modelsim_copy_files =
......
#! /usr/bin/env python
###############################################################################
#
# Copyright (C) 2016
# ASTRON (Netherlands Institute for Radio Astronomy) <http://www.astron.nl/>
# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################
"""Regression test generation for simulating pure VHDL test benches with Modelsim.
Usage:
> python $RADIOHDL/tools/oneclick/base/modelsim_regression_test_vhdl.py -h
--lib:
The --lib specifies which HDL libraries are used. If --lib is not specified
then all available HDL libraries are used.
Only the HDL libraries that have a 'regression_test_vhdl' key are processed
further.
--run:
Without --run only the test bench do files are created for the VHDL test bench
files that are listed at the 'regression_test_vhdl' key. With --run then
the created test bench do files are also ran and the transcript log is saved.
The do files and the log files are kept in the same build directory as the mpf
directory of the HDL library. From the log files the regression test also
reports a pass/fail result summary in modelsim_regression_test_vhdl.log.
"""
import common as cm
import hdl_config
import modelsim_config
import sys
import subprocess
import glob
import os.path
import time
###############################################################################
# Parse command line arguments
###############################################################################
hdl_args = hdl_config.HdlParseArgs(toolsetSelect=['unb1', 'unb2', 'unb2a'])
if hdl_args.verbosity>=1:
print ''
hdl_args.argparser.print_help()
###############################################################################
# Read the dictionary info from all HDL tool and library configuration files in the current directory and the sub directories
###############################################################################
msim = modelsim_config.ModelsimConfig(toolRootDir=os.path.expandvars('$RADIOHDL/tools'), libFileName='hdllib.cfg', toolFileName=hdl_args.toolFileName)
###############################################################################
# Get HDL library names for regression test
###############################################################################
lib_names = hdl_args.lib_names # Default use lib_names from command line
msim.check_library_names(lib_names) # Check that the provided lib_names indeed exist
if lib_names==[]:
lib_names=msim.lib_names # If no lib_names are provided then use all available HDL libraries
lib_dicts = msim.libs.get_dicts(key='hdl_lib_name', values=lib_names) # Get HDL libraries dicts
test_dicts = msim.libs.get_dicts(key='regression_test_vhdl', values=None, dicts=lib_dicts) # Get HDL libraries dicts with 'regression_test_vhdl' key
###############################################################################
# Create test bench do files in same build directory as where the mpf is
###############################################################################
project_sim_p_defaults, project_sim_p_search_libraries, project_sim_p_otherargs, project_sim_p_optimization = msim.simulation_configuration(list_mode=True)
do_subdir = 'regression_test_vhdl'
for lib_dict in cm.listify(test_dicts):
lib_name = lib_dict['hdl_lib_name']
mpf_path = msim.get_lib_build_dirs('sim', lib_dicts=lib_dict)
do_path = os.path.join(mpf_path, do_subdir)
cm.mkdir(do_path) # mkdir <mpf_path>/regression_test_vhdl, if it does not exist yet
for rm in glob.glob(os.path.join(do_path, '*.do')): os.remove(rm) # rm <mpf_path>/regression_test_vhdl/*.do
for rm in glob.glob(os.path.join(do_path, '*.log')): os.remove(rm) # rm <mpf_path>/regression_test_vhdl/*.log
test_bench_files = lib_dict['regression_test_vhdl'].split()
for tbf in test_bench_files:
tbf_name = os.path.basename(tbf)
tb_name = os.path.splitext(tbf_name)[0]
do_name = tb_name + '.do'
doPathName = os.path.join(do_path, do_name)
# Write separate do file for each test bench in the VHDL regression test of this library
with open(doPathName, 'w') as fp:
fp.write('# Created by modelsim_regression_test_vhdl.py\n')
fp.write('do $env(RADIOHDL)/tools/modelsim/commands.do\n')
fp.write('echo ">>> PROJECT LOAD %s"\n' % lib_name)
fp.write('lp %s\n' % lib_name)
fp.write('echo ">>> PROJECT MAKE %s"\n' % lib_name)
fp.write('mk all\n')
fp.write('echo ">>> SIMULATION LOAD %s"\n' % tb_name)
fp.write('vsim ')
for other_arg in project_sim_p_otherargs:
fp.write('%s ' % other_arg)
for search_lib in project_sim_p_search_libraries:
fp.write('-L %s ' % search_lib)
fp.write('-novopt work.%s\n' % tb_name)
fp.write('echo ">>> SIMULATION RUN %s"\n' % tb_name)
# Use onbreak to avoid that vsim halts when a simulation VHDL assert failure occurs
fp.write('onbreak {resume}\n')
# Use onerror to avoid that vsim halts when a simulation fatal error occurs
fp.write('onerror {quit}\n')
# Use when -label tb_end to stop a tb or multi tb_tb that cannot be stopped by stopping all clocks
fp.write("when -label tb_end {tb_end == '1'} {\n")
fp.write(' echo "End of simulation due to -lable";\n')
fp.write(' echo ">>> SIMULATION END %s (Now = $Now)";\n' % tb_name)
fp.write(' quit;\n')
fp.write('}\n')
# A multi tb_tb that cannot stop its clocks will have been stopped by a top level tb_end
# A multi tb_tb that can stop its clocks may not have a top level tb_end en will be stopped here
fp.write('run -all\n')
fp.write('echo "End of simulation due to stop toggling";\n')
fp.write('echo ">>> SIMULATION END %s (Now = $Now)"\n' % tb_name)
fp.write('quit\n')
###############################################################################
# Optionally run the test bench do files
###############################################################################
# Remarks on subprocess:
# . Use shell=True to be able to pass on the entire CLI command and have environment variable and wildcard expansion.
# . Use call to run the CLI command and capture the exit code and avoid that Python breaks with CalledProcessError on exit code > 0.
# . Exit code 0 is ok, exit code > 0 is some error, but for 'egrep' exit 1 means that no match was found for the expression.
# . If 'egrep' has exit 0 then a match was found for the expression and then check_output can be used to capture the match.
# . An alternative scheme is to use try-except subprocess.CalledProcessError to handle exit code > 0.
# Remarks using tb_end <='1' to end a simulation
# . The tb_end is or-ed with the clocks, so tb_end <='1' stops all toggling and this ends the tb (see any tb, e.g. tb_dp_split.vhd)
# . For components that have some self ocillation the toggling does not stop so then:
# - for manual tests in the GUI 'REPORT "Tb simulation finished." SEVERITY FAILURE' in the tb VHDL is used to stop the tb simulation
# - for regression tests the "when -lable tb_end='1'" in the do file is used to stop the tb simulation
# To make this scheme work it is neccessary to have some WAIT time between tb_end <='1' and the FAILURE.
# . For multi tb_tb there needs to be a top level tb_end signal, even if it is not used, to ensure that the "when -lable tb_end='1'"
# can be applied (see e.g. tb_tb_mms_diag_block_gen.vhd).
# . For multi tb_tb that rely on the FAILURE to end the simulation there is a top level tb_end_vec signal that needs to be used to
# raise a top level tb_end <= '1' when all sub tb have ended (see e.g. tb_tb_tech_mac_10g.vhd).
#
# Remarks on checking for 'Error' and 'Failure' to detect whether a tb has failed:
# Make sure the tb has some WAIT time between "tb_end <= '1'" and 'REPORT "Tb simulation finished." SEVERITY FAILURE'.
# The tb_end will stop the tb in the regression test (via when -lable tb_end='1') and the FAILURE will stop the tb
# in manual simulation in the GUI. The advantage is that the regression test can check also on 'Failure' to detect
# whether a tb has failed.
# Remarks on Modelsim simulation control:
# . Google search: modelsim stop simulation vhdl
#
# . How to stop the simulation in VHDL TB
#
# 1) The easiest way is to use an assert: assert false report "Simulation Finished" severity failure;
# 2) The "recommended" way to end a simulation when everything has gone correctly, is to halt all stimulus. This will be stopping
# the clock like, but also putting any input processes into a never ending wait if they do not use the generated clock.
# 3) In VHDL 2008, they have introduced and new package called env to the std library, with procedures called stop and finish,
# that act like $finish in verilog. This works, but is not used because the scheme with tb_end suffices:
#
# library std;
# use std.env.all;
#
# .......
# stop(0);
# --or
# finish(0);
#
# -- For both STOP and FINISH the STATUS values are those used
# -- in the Verilog $finish task
# -- 0 prints nothing
# -- 1 prints simulation time and location
# -- 2 prints simulation time, location, and statistics about
# -- the memory and CPU times used in simulation
# -- Other STATUS values are interpreted as 0.
if hdl_args.run:
start_time = time.time()
prev_time = start_time;
# Put modelsim_regression_test_vhdl.log file in tool build dir
build_main_dir, build_toolset_dir, build_tool_dir = msim.get_tool_build_dir('sim')
logFileName='modelsim_regression_test_vhdl.log'
logFileNamePath=os.path.join(build_main_dir, build_toolset_dir, build_tool_dir, logFileName)
nofTb = 0 # number of tb in regression test
nofFailed = 0 # number of tb in regression test that failed
# Open the log file and run the test bench do files
with open(logFileNamePath, 'w') as fp:
fp.write("# Created by modelsim_regression_test_vhdl.py using HDL library key 'regression_test_vhdl'\n")
fp.write('# Tested HDL libraries: %s\n' % msim.get_lib_names_from_lib_dicts(test_dicts))
fp.write('#\n')
for lb, lib_dict in enumerate(cm.listify(test_dicts)):
# Derive the do file names from the HDL library 'regression_test_vhdl' key
lib_name = lib_dict['hdl_lib_name']
fp.write('# %d: %s\n' % (lb, lib_name))
mpf_path = msim.get_lib_build_dirs('sim', lib_dicts=lib_dict)
transcriptPathName = os.path.join(mpf_path, 'transcript')
do_path = os.path.join(mpf_path, do_subdir)
test_bench_files = lib_dict['regression_test_vhdl'].split()
for tbf in test_bench_files:
tbf_name = os.path.basename(tbf)
tb_name = os.path.splitext(tbf_name)[0]
doName = tb_name + '.do'
doPathName = os.path.join(do_path, doName)
doLogName = tb_name + '.log'
doLogPathName = os.path.join(do_path, doLogName)
# Simulate the do file with Modelsim
vsim_cmd = 'cli_modelsim %s %s' % (hdl_args.toolset, doPathName)
call_status = subprocess.call(vsim_cmd, shell=True)
nofTb += 1
if call_status==0:
# Keep the transcript file in the library build directory
subprocess.call("cp %s %s" % (transcriptPathName, doLogPathName), shell=True)
# Log the simulation run time (use try-except to handle exit code > 0)
try:
sim_end = subprocess.check_output("egrep '>>> SIMULATION END' %s" % transcriptPathName, shell=True)
except subprocess.CalledProcessError:
fp.write('Error occured while running vsim for %s\n' % tb_name)
nofFailed += 1
else:
fp.write('%s' % sim_end)
# Log the simulation Errors if they occured (use subprocess.call-subprocess.check_output to handle exit code > 0)
grep_cmd = "egrep 'Error' %s" % transcriptPathName
grep_status = subprocess.call(grep_cmd, shell=True)
if grep_status==0:
sim_msg = subprocess.check_output(grep_cmd, shell=True)
fp.write('%s\n' % sim_msg)
nofFailed += 1
# Log the simulation Failures if they occured (use subprocess.call-subprocess.check_output to handle exit code > 0)
grep_cmd = "egrep 'Failure' %s" % transcriptPathName
grep_status = subprocess.call(grep_cmd, shell=True)
if grep_status==0:
sim_msg = subprocess.check_output(grep_cmd, shell=True)
fp.write('%s\n' % sim_msg)
nofFailed += 1
else:
fp.write('> Error occured while calling: %s\n' % vsim_cmd)
nofFailed += 1
# Log test time for this HDL library
cur_time = time.time()
run_time = cur_time-prev_time
prev_time = cur_time;
fp.write('# Test duration for library %s: %.1f seconds\n' % (lib_name, run_time))
fp.write('#\n')
fp.write('# Regression test summary:\n')
# Log overall PASSED or FAILED
if nofFailed==0:
fp.write('# * All %d VHDL test benches PASSED\n' % nofTb)
else:
fp.write('# * Out of %d VHDL test benches %d FAILED\n' % (nofTb, nofFailed))
# Log total test time
end_time = time.time()
run_time = end_time-start_time
fp.write('# * Total regression test duration: %.1f seconds\n' % run_time)
# Echo the log file
print '\n\ncat %s:\n' % logFileNamePath
subprocess.call('cat %s' % logFileNamePath, shell=True)
print '\n'
\ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment