diff --git a/libraries/base/diag/hdllib.cfg b/libraries/base/diag/hdllib.cfg index d3ab815ac0b476efe7d2efb32be5133f9ab5c4bc..cd67ae40a484636b2ddd5ec8477f6ce4b6dee16e 100644 --- a/libraries/base/diag/hdllib.cfg +++ b/libraries/base/diag/hdllib.cfg @@ -41,6 +41,16 @@ test_bench_files = tb/vhdl/tb_tb_mms_diag_block_gen.vhd tb/vhdl/tb_diag_regression.vhd +regression_test_vhdl = + tb/vhdl/tb_diag_wg.vhd + tb/vhdl/tb_diag_wg_wideband.vhd + tb/vhdl/tb_diag_block_gen.vhd + tb/vhdl/tb_diag_frm_generator.vhd + tb/vhdl/tb_diag_frm_monitor.vhd + tb/vhdl/tb_tb_diag_rx_seq.vhd + tb/vhdl/tb_tb_mms_diag_seq.vhd + tb/vhdl/tb_tb_mms_diag_block_gen.vhd + [modelsim_project_file] modelsim_copy_files = diff --git a/libraries/base/dp/hdllib.cfg b/libraries/base/dp/hdllib.cfg index b388ba2ffad4f52c0667bbae0e67be4d283b6a6a..30f064e2bcd3e448c2ebb7b4c208ac3c5fd432ae 100644 --- a/libraries/base/dp/hdllib.cfg +++ b/libraries/base/dp/hdllib.cfg @@ -215,6 +215,11 @@ test_bench_files = tb/vhdl/tb_tb_tb_dp_backpressure.vhd tb/vhdl/tb_dp_offload_rx_filter.vhd +regression_test_vhdl = + tb/vhdl/tb_dp_latency_adapter.vhd + tb/vhdl/tb_dp_split.vhd + + [modelsim_project_file] diff --git a/libraries/technology/mac_10g/hdllib.cfg b/libraries/technology/mac_10g/hdllib.cfg index a4b5a7432186bbb941fd864170c4de7edc350350..9f68232d9f3b08add32e09b2dc9d97c286540498 100644 --- a/libraries/technology/mac_10g/hdllib.cfg +++ b/libraries/technology/mac_10g/hdllib.cfg @@ -24,6 +24,9 @@ test_bench_files = tb_tech_mac_10g.vhd tb_tb_tech_mac_10g.vhd +regression_test_vhdl = + tb_tb_tech_mac_10g.vhd + [modelsim_project_file] modelsim_copy_files = diff --git a/tools/oneclick/base/modelsim_regression_test_vhdl.py b/tools/oneclick/base/modelsim_regression_test_vhdl.py new file mode 100644 index 0000000000000000000000000000000000000000..9c9b226240c26e3c53c1fd2ceb3d152c7a6e6e7b --- /dev/null +++ b/tools/oneclick/base/modelsim_regression_test_vhdl.py @@ -0,0 +1,274 @@ +#! /usr/bin/env python +############################################################################### +# +# Copyright (C) 2016 +# ASTRON (Netherlands Institute for Radio Astronomy) <http://www.astron.nl/> +# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +############################################################################### + +"""Regression test generation for simulating pure VHDL test benches with Modelsim. + + Usage: + > python $RADIOHDL/tools/oneclick/base/modelsim_regression_test_vhdl.py -h + + --lib: + The --lib specifies which HDL libraries are used. If --lib is not specified + then all available HDL libraries are used. + Only the HDL libraries that have a 'regression_test_vhdl' key are processed + further. + + --run: + Without --run only the test bench do files are created for the VHDL test bench + files that are listed at the 'regression_test_vhdl' key. With --run then + the created test bench do files are also ran and the transcript log is saved. + The do files and the log files are kept in the same build directory as the mpf + directory of the HDL library. From the log files the regression test also + reports a pass/fail result summary in modelsim_regression_test_vhdl.log. + +""" + +import common as cm +import hdl_config +import modelsim_config +import sys +import subprocess +import glob +import os.path +import time + + +############################################################################### +# Parse command line arguments +############################################################################### +hdl_args = hdl_config.HdlParseArgs(toolsetSelect=['unb1', 'unb2', 'unb2a']) +if hdl_args.verbosity>=1: + print '' + hdl_args.argparser.print_help() + + +############################################################################### +# Read the dictionary info from all HDL tool and library configuration files in the current directory and the sub directories +############################################################################### +msim = modelsim_config.ModelsimConfig(toolRootDir=os.path.expandvars('$RADIOHDL/tools'), libFileName='hdllib.cfg', toolFileName=hdl_args.toolFileName) + + +############################################################################### +# Get HDL library names for regression test +############################################################################### +lib_names = hdl_args.lib_names # Default use lib_names from command line +msim.check_library_names(lib_names) # Check that the provided lib_names indeed exist +if lib_names==[]: + lib_names=msim.lib_names # If no lib_names are provided then use all available HDL libraries +lib_dicts = msim.libs.get_dicts(key='hdl_lib_name', values=lib_names) # Get HDL libraries dicts +test_dicts = msim.libs.get_dicts(key='regression_test_vhdl', values=None, dicts=lib_dicts) # Get HDL libraries dicts with 'regression_test_vhdl' key + + +############################################################################### +# Create test bench do files in same build directory as where the mpf is +############################################################################### +project_sim_p_defaults, project_sim_p_search_libraries, project_sim_p_otherargs, project_sim_p_optimization = msim.simulation_configuration(list_mode=True) + +do_subdir = 'regression_test_vhdl' +for lib_dict in cm.listify(test_dicts): + lib_name = lib_dict['hdl_lib_name'] + mpf_path = msim.get_lib_build_dirs('sim', lib_dicts=lib_dict) + do_path = os.path.join(mpf_path, do_subdir) + cm.mkdir(do_path) # mkdir <mpf_path>/regression_test_vhdl, if it does not exist yet + for rm in glob.glob(os.path.join(do_path, '*.do')): os.remove(rm) # rm <mpf_path>/regression_test_vhdl/*.do + for rm in glob.glob(os.path.join(do_path, '*.log')): os.remove(rm) # rm <mpf_path>/regression_test_vhdl/*.log + test_bench_files = lib_dict['regression_test_vhdl'].split() + for tbf in test_bench_files: + tbf_name = os.path.basename(tbf) + tb_name = os.path.splitext(tbf_name)[0] + do_name = tb_name + '.do' + doPathName = os.path.join(do_path, do_name) + # Write separate do file for each test bench in the VHDL regression test of this library + with open(doPathName, 'w') as fp: + fp.write('# Created by modelsim_regression_test_vhdl.py\n') + fp.write('do $env(RADIOHDL)/tools/modelsim/commands.do\n') + fp.write('echo ">>> PROJECT LOAD %s"\n' % lib_name) + fp.write('lp %s\n' % lib_name) + fp.write('echo ">>> PROJECT MAKE %s"\n' % lib_name) + fp.write('mk all\n') + fp.write('echo ">>> SIMULATION LOAD %s"\n' % tb_name) + fp.write('vsim ') + for other_arg in project_sim_p_otherargs: + fp.write('%s ' % other_arg) + for search_lib in project_sim_p_search_libraries: + fp.write('-L %s ' % search_lib) + fp.write('-novopt work.%s\n' % tb_name) + fp.write('echo ">>> SIMULATION RUN %s"\n' % tb_name) + # Use onbreak to avoid that vsim halts when a simulation VHDL assert failure occurs + fp.write('onbreak {resume}\n') + # Use onerror to avoid that vsim halts when a simulation fatal error occurs + fp.write('onerror {quit}\n') + # Use when -label tb_end to stop a tb or multi tb_tb that cannot be stopped by stopping all clocks + fp.write("when -label tb_end {tb_end == '1'} {\n") + fp.write(' echo "End of simulation due to -lable";\n') + fp.write(' echo ">>> SIMULATION END %s (Now = $Now)";\n' % tb_name) + fp.write(' quit;\n') + fp.write('}\n') + # A multi tb_tb that cannot stop its clocks will have been stopped by a top level tb_end + # A multi tb_tb that can stop its clocks may not have a top level tb_end en will be stopped here + fp.write('run -all\n') + fp.write('echo "End of simulation due to stop toggling";\n') + fp.write('echo ">>> SIMULATION END %s (Now = $Now)"\n' % tb_name) + fp.write('quit\n') + + +############################################################################### +# Optionally run the test bench do files +############################################################################### + +# Remarks on subprocess: +# . Use shell=True to be able to pass on the entire CLI command and have environment variable and wildcard expansion. +# . Use call to run the CLI command and capture the exit code and avoid that Python breaks with CalledProcessError on exit code > 0. +# . Exit code 0 is ok, exit code > 0 is some error, but for 'egrep' exit 1 means that no match was found for the expression. +# . If 'egrep' has exit 0 then a match was found for the expression and then check_output can be used to capture the match. +# . An alternative scheme is to use try-except subprocess.CalledProcessError to handle exit code > 0. + +# Remarks using tb_end <='1' to end a simulation +# . The tb_end is or-ed with the clocks, so tb_end <='1' stops all toggling and this ends the tb (see any tb, e.g. tb_dp_split.vhd) +# . For components that have some self ocillation the toggling does not stop so then: +# - for manual tests in the GUI 'REPORT "Tb simulation finished." SEVERITY FAILURE' in the tb VHDL is used to stop the tb simulation +# - for regression tests the "when -lable tb_end='1'" in the do file is used to stop the tb simulation +# To make this scheme work it is neccessary to have some WAIT time between tb_end <='1' and the FAILURE. +# . For multi tb_tb there needs to be a top level tb_end signal, even if it is not used, to ensure that the "when -lable tb_end='1'" +# can be applied (see e.g. tb_tb_mms_diag_block_gen.vhd). +# . For multi tb_tb that rely on the FAILURE to end the simulation there is a top level tb_end_vec signal that needs to be used to +# raise a top level tb_end <= '1' when all sub tb have ended (see e.g. tb_tb_tech_mac_10g.vhd). +# +# Remarks on checking for 'Error' and 'Failure' to detect whether a tb has failed: +# Make sure the tb has some WAIT time between "tb_end <= '1'" and 'REPORT "Tb simulation finished." SEVERITY FAILURE'. +# The tb_end will stop the tb in the regression test (via when -lable tb_end='1') and the FAILURE will stop the tb +# in manual simulation in the GUI. The advantage is that the regression test can check also on 'Failure' to detect +# whether a tb has failed. + +# Remarks on Modelsim simulation control: +# . Google search: modelsim stop simulation vhdl +# +# . How to stop the simulation in VHDL TB +# +# 1) The easiest way is to use an assert: assert false report "Simulation Finished" severity failure; +# 2) The "recommended" way to end a simulation when everything has gone correctly, is to halt all stimulus. This will be stopping +# the clock like, but also putting any input processes into a never ending wait if they do not use the generated clock. +# 3) In VHDL 2008, they have introduced and new package called env to the std library, with procedures called stop and finish, +# that act like $finish in verilog. This works, but is not used because the scheme with tb_end suffices: +# +# library std; +# use std.env.all; +# +# ....... +# stop(0); +# --or +# finish(0); +# +# -- For both STOP and FINISH the STATUS values are those used +# -- in the Verilog $finish task +# -- 0 prints nothing +# -- 1 prints simulation time and location +# -- 2 prints simulation time, location, and statistics about +# -- the memory and CPU times used in simulation +# -- Other STATUS values are interpreted as 0. + +if hdl_args.run: + start_time = time.time() + prev_time = start_time; + # Put modelsim_regression_test_vhdl.log file in tool build dir + build_main_dir, build_toolset_dir, build_tool_dir = msim.get_tool_build_dir('sim') + logFileName='modelsim_regression_test_vhdl.log' + logFileNamePath=os.path.join(build_main_dir, build_toolset_dir, build_tool_dir, logFileName) + nofTb = 0 # number of tb in regression test + nofFailed = 0 # number of tb in regression test that failed + + # Open the log file and run the test bench do files + with open(logFileNamePath, 'w') as fp: + fp.write("# Created by modelsim_regression_test_vhdl.py using HDL library key 'regression_test_vhdl'\n") + fp.write('# Tested HDL libraries: %s\n' % msim.get_lib_names_from_lib_dicts(test_dicts)) + fp.write('#\n') + for lb, lib_dict in enumerate(cm.listify(test_dicts)): + # Derive the do file names from the HDL library 'regression_test_vhdl' key + lib_name = lib_dict['hdl_lib_name'] + fp.write('# %d: %s\n' % (lb, lib_name)) + mpf_path = msim.get_lib_build_dirs('sim', lib_dicts=lib_dict) + transcriptPathName = os.path.join(mpf_path, 'transcript') + do_path = os.path.join(mpf_path, do_subdir) + test_bench_files = lib_dict['regression_test_vhdl'].split() + for tbf in test_bench_files: + tbf_name = os.path.basename(tbf) + tb_name = os.path.splitext(tbf_name)[0] + doName = tb_name + '.do' + doPathName = os.path.join(do_path, doName) + doLogName = tb_name + '.log' + doLogPathName = os.path.join(do_path, doLogName) + + # Simulate the do file with Modelsim + vsim_cmd = 'cli_modelsim %s %s' % (hdl_args.toolset, doPathName) + call_status = subprocess.call(vsim_cmd, shell=True) + nofTb += 1 + if call_status==0: + # Keep the transcript file in the library build directory + subprocess.call("cp %s %s" % (transcriptPathName, doLogPathName), shell=True) + # Log the simulation run time (use try-except to handle exit code > 0) + try: + sim_end = subprocess.check_output("egrep '>>> SIMULATION END' %s" % transcriptPathName, shell=True) + except subprocess.CalledProcessError: + fp.write('Error occured while running vsim for %s\n' % tb_name) + nofFailed += 1 + else: + fp.write('%s' % sim_end) + # Log the simulation Errors if they occured (use subprocess.call-subprocess.check_output to handle exit code > 0) + grep_cmd = "egrep 'Error' %s" % transcriptPathName + grep_status = subprocess.call(grep_cmd, shell=True) + if grep_status==0: + sim_msg = subprocess.check_output(grep_cmd, shell=True) + fp.write('%s\n' % sim_msg) + nofFailed += 1 + # Log the simulation Failures if they occured (use subprocess.call-subprocess.check_output to handle exit code > 0) + grep_cmd = "egrep 'Failure' %s" % transcriptPathName + grep_status = subprocess.call(grep_cmd, shell=True) + if grep_status==0: + sim_msg = subprocess.check_output(grep_cmd, shell=True) + fp.write('%s\n' % sim_msg) + nofFailed += 1 + else: + fp.write('> Error occured while calling: %s\n' % vsim_cmd) + nofFailed += 1 + # Log test time for this HDL library + cur_time = time.time() + run_time = cur_time-prev_time + prev_time = cur_time; + fp.write('# Test duration for library %s: %.1f seconds\n' % (lib_name, run_time)) + fp.write('#\n') + + fp.write('# Regression test summary:\n') + # Log overall PASSED or FAILED + if nofFailed==0: + fp.write('# * All %d VHDL test benches PASSED\n' % nofTb) + else: + fp.write('# * Out of %d VHDL test benches %d FAILED\n' % (nofTb, nofFailed)) + + # Log total test time + end_time = time.time() + run_time = end_time-start_time + fp.write('# * Total regression test duration: %.1f seconds\n' % run_time) + + # Echo the log file + print '\n\ncat %s:\n' % logFileNamePath + subprocess.call('cat %s' % logFileNamePath, shell=True) + print '\n' + \ No newline at end of file