Select Git revision
developing_on_raspberrypi.md
Code owners
Assign users and groups as approvers for specific file changes. Learn more.
modelsim_regression_test_vhdl.py 39.54 KiB
#! /usr/bin/env python3
# ##########################################################################
# Copyright 2020
# ASTRON (Netherlands Institute for Radio Astronomy) <http://www.astron.nl/>
# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ##########################################################################
# ##########################################################################
# Author:
# . Pieter Donker (Based on svn regressiontest from Eric Kooistra)
# Purpose:
# . regressiontest of vhdl code blocks
# Description:
# . run all test benches available under the "regression_test_vhdl" key in hdllib.cfg files
# . usage:
# first source the "init_hdl.sh" file in "git/hdl dir"
# . ~/git/hdl/init_hdl.sh
# then run
# modelsim_regression_test_vhdl.py -h
#
# ##########################################################################
""" Regression test generation for simulating pure VHDL test benches with Modelsim.
Preconditions:
- source init_hdl.sh
> . $HOME/regtest/hdl/init_hdl.sh
- run compile_altera_simlibs
> compile_altera_simlibs BUILDSET
- run generate_ip_libs
> generate_ip_libs BUILDSET
- run modelsim_config and quartus_config
> modelsim_config BUILDSET
> quartus_config BUILDSET
Usage:
- running this script direct from a terminal:
> modelsim_regression_test_vhdl.py unb2b -p 3 # run regressiontest for all test benches in unb2b using 3 worker processes
--libnames:
The --libnames specifies which HDL libraries are used. If --libnames is not specified
then all available HDL libraries are used.
Only the HDL libraries that have a 'regression_test_vhdl' key are processed
further.
"""
import sys
import glob
import os.path
import multiprocessing
from time import time, sleep
from os import listdir
from datetime import timedelta
from argparse import ArgumentParser
from collections import OrderedDict
from copy import copy
import common_radiohdl as cm
from logger import MyLogger
from hdl_libraries_wizard import HdlLibrariesWizard
from terminal import Terminal
MAX_PROC = 4 # maximum number of processes to run, each process needs a modelsim license
MK_TIMEOUT = 20 * 60 # timeout in seconds (1x60 seconds = 1 min(s)) for each *_mk.do command
SIM_TIMEOUT = 3 * 3600 # timeout in seconds (1x3600 seconds = 1 hour(s)) for each *_sim.do command
def main(args, time_ordered_tb_dict):
n_proc = min(args.proc, MAX_PROC)
buildset = args.buildset
buildset_file = args.buildset_file
lib_names = args.libnames # Default use lib_names for regressiontest from command line
# remove modelsim log files before test
rm_list = glob.glob(os.path.join(args.log_path, "modelsim", "*.log"))
for rm in rm_list:
logger.debug("%s: remove '%s'", buildset, rm)
os.remove(rm)
# make queues for worker communication
# communication queue to control worker
control_queue = [multiprocessing.Queue() for i in range(n_proc)]
# communication queue to processes, holding tasks to process
task_queue = multiprocessing.Queue()
# communication queue from processes, holding response from processed tasks
response_queue = multiprocessing.Queue()
# start 'n_proc' tb worker processes
logger.info("Run all tests using %d workers", n_proc)
modelsim_log_path = os.path.join(args.log_path, "modelsim")
workers = []
for i in range(n_proc):
worker_id = i + 1
worker = TestBenchWorker(worker_id, control_queue[i], task_queue, response_queue, MK_TIMEOUT, SIM_TIMEOUT, modelsim_log_path)
workers.append(worker)
worker.start()
try:
run_start_time = time()
# get all task info
task_info, n_tb = generate_task_info_dict(buildset, buildset_file, lib_names)
make_all_do_paths(task_info)
# make all *.do files
make_do_files(task_info, task_queue, response_queue)
# keep worker 1 awake send others to sleep
for i in range(1, n_proc, 1):
control_queue[i].put("sleep")
sleep(1.0)
# make all libs using one process, running them together will crash the test
mk_rerun, mk_errors, mk_failed = make_all(task_info, task_queue, response_queue)
# wakeup all workers again
for i in range(n_proc):
control_queue[i].put("awake")
# simulate libs using all processes
tb_passed, tb_failed, tb_sim_errors = simulate_all_tb(task_info, task_queue, response_queue, time_ordered_tb_dict, mk_failed)
run_time = time() - run_start_time
logger.info("regressiontest done")
generate_email_report(args.log_path, n_tb, tb_failed, tb_passed, mk_errors, mk_rerun, tb_sim_errors, run_time)
except KeyboardInterrupt:
logger.info("stopped by user")
# Stop all tb workers processes
for i in range(n_proc):
control_queue[i].put("halt")
sleep(0.5)
for worker in workers:
worker.terminate()
return 0
def generate_task_info_dict(buildset, buildset_file, lib_names):
# Read the dictionary info from all HDL tool and library configuration files in the current directory and the sub directories
msim = ModelsimConfig(
toolRootDir=os.path.expandvars("$RADIOHDL_CONFIG"),
buildsetFile=buildset_file,
libFileName="hdllib.cfg",
)
# If user provided lib_names,
if lib_names is not None:
# check that the lib_names indeed exist
msim.check_library_names(lib_names)
else:
# If no lib_names are provided then use all available HDL libraries
lib_names = msim.lib_names
# Get HDL libraries dicts
lib_dicts = msim.libs.get_configfiles(key="hdl_lib_name", values=lib_names)
# Get HDL libraries dicts with 'regression_test_vhdl' key
test_dicts = msim.libs.get_configfiles(key="regression_test_vhdl", values=None, user_configfiles=lib_dicts)
# Open the log file and run the test bench do files
logger.info("Created by modelsim_regression_test_vhdl.py using HDL library key 'regression_test_vhdl'")
logger.info("Tested HDL libraries: %s", msim.get_lib_names_from_lib_dicts(test_dicts))
logger.debug("\nList of HDL libraries with 'regression_test_vhdl' key and the specified VHDL test benches:")
nof_lib = 0
n_tb = 0 # total number of tb in regression test
for lib_dict in test_dicts:
lib_name = lib_dict["hdl_lib_name"]
tb_files = lib_dict["regression_test_vhdl"].split()
if len(tb_files) == 0:
logger.debug("%-40s : -", lib_name)
else:
nof_lib += 1
for tbf in tb_files:
n_tb += 1
logger.debug("%-40s : %s", lib_name, tbf)
logger.info("The regression test contains %d HDL libraries and in total %d test benches.",
nof_lib,
n_tb)
# fill dictonary with al info used in the tasks
task_info = OrderedDict()
task_id = 0
for lib_dict in test_dicts:
lib_name = lib_dict["hdl_lib_name"]
project_sim_p_search_libraries, project_sim_p_otherargs = msim.simulation_configuration(
list_mode=True
)
tb_files = lib_dict["regression_test_vhdl"].split()
if len(tb_files) == 0:
tb_files = [None]
for tb_file in tb_files:
tb_name = None
if tb_file is not None:
tbf_name = os.path.basename(tb_file)
tb_name = os.path.splitext(tbf_name)[0]
task_info[task_id] = {
"task_id": task_id,
"task": None,
"lib_name": lib_name,
"buildset": buildset,
"mpf_path": msim.get_lib_build_dirs("sim", lib_dicts=lib_dict),
"project_sim_p_search_libraries": project_sim_p_search_libraries,
"project_sim_p_otherargs": project_sim_p_otherargs,
"tb_file": tb_file,
"tb_name": tb_name,
"n_make_runs": 0,
"n_simulate_runs": 0,
"remove_do_files": False,
"create_do_make_all_file": False,
"create_do_simulate_file": False,
}
task_id += 1
return task_info, n_tb
def make_all_do_paths(task_info):
for task_id in task_info:
mpf_path = task_info[task_id]["mpf_path"]
do_path = os.path.join(mpf_path, "regression_test_vhdl")
# mkdir <mpf_path>/regression_test_vhdl, if it does not exist yet
logger.info("Make dir %s", do_path)
cm.mkdir(do_path)
def make_do_files(task_info, task_queue, response_queue):
# 1) make all do files
open_tasks = []
remove_done = []
make_all_done = []
tb_file_done = []
for task_id in task_info:
lib_name = task_info[task_id]["lib_name"]
tb_file = task_info[task_id]["tb_file"]
tb_name = task_info[task_id]["tb_name"]
do_action = False
if lib_name not in remove_done:
logger.info("remove_do_files = True for %s", lib_name)
task_info[task_id]["remove_do_files"] = True
do_action = True
remove_done.append(lib_name)
if lib_name not in make_all_done:
logger.info("create_do_make_all_file = True for %s", lib_name)
task_info[task_id]["create_do_make_all_file"] = True
do_action = True
make_all_done.append(lib_name)
if (tb_file is not None) and (tb_file not in tb_file_done):
logger.info("create_do_simulate_file = True for %s, %s", lib_name, tb_name)
task_info[task_id]["create_do_simulate_file"] = True
do_action = True
tb_file_done.append(tb_file)
if do_action:
task_info[task_id]["task"] = "make_new_do_files"
task_queue.put(task_info[task_id])
open_tasks.append(task_id)
while open_tasks:
if response_queue.empty():
sleep(0.01)
else:
response = response_queue.get()
open_tasks.remove(response["task_id"])
def make_all(task_info, task_queue, response_queue):
open_tasks = []
make_all = []
for task_id in task_info:
lib_name = task_info[task_id]["lib_name"]
if lib_name not in make_all:
logger.info("add mk_all task for %s", lib_name)
task_info[task_id]["task"] = "mk_all"
task_info[task_id]["n_make_runs"] = 1
task_queue.put(task_info[task_id])
open_tasks.append(task_id)
make_all.append(lib_name)
mk_rerun = []
mk_errors = []
mk_failed = []
max_mk_runs = 2
while open_tasks:
if response_queue.empty():
sleep(0.1)
else:
response = response_queue.get()
task_id = response["task_id"]
open_tasks.remove(task_id)
if response["mk_errors"] != []:
task_info[task_id]["n_make_runs"] += 1
if task_info[task_id]["n_make_runs"] <= max_mk_runs:
task_queue.put(task_info[task_id])
open_tasks.append(task_id)
mk_rerun.extend(response["mk_errors"])
else:
mk_errors.extend(response["mk_errors"])
mk_failed.append(response["mk_errors"][0][0])
return mk_rerun, mk_errors, mk_failed
def simulate_all_tb(task_info, task_queue, response_queue, time_ordered_tb_dict, mk_failed):
tb_passed = []
tb_failed = []
tb_sim_errors = []
open_tasks = []
# check if all tasks are put in the time_ordered_tb_dict
for task_id in task_info:
tb_name = task_info[task_id]["tb_name"]
if tb_name is None:
continue
if tb_name not in time_ordered_tb_dict:
logger.info("Add missing tb: %s", tb_name)
time_ordered_tb_dict[tb_name] = 0.0
# run all simulations
for tb in time_ordered_tb_dict:
for task_id in task_info:
if task_info[task_id]["tb_name"] == tb:
lib_name = task_info[task_id]["lib_name"]
tb_file = task_info[task_id]["tb_file"]
tb_name = task_info[task_id]["tb_name"]
if tb_file is not None:
# if mk all failed, skip simulation for this lib and add test benches to the failed list
if lib_name in mk_failed:
tb_failed.append(tb_name)
logger.info("skip simulation for %s, mk failed", lib_name)
continue
logger.info("add simulate task for %s.%s", lib_name, tb_file)
task_info[task_id]["task"] = "simulate"
task_queue.put(task_info[task_id])
open_tasks.append(task_id)
# Wait till all simulate tasks are done
while open_tasks:
if response_queue.empty():
sleep(1.0)
else:
response = response_queue.get()
tb_passed.extend(response["passed"])
tb_failed.extend(response["failed"])
tb_sim_errors.extend(response["sim_errors"])
open_tasks.remove(response["task_id"])
logger.info("Total tasks done: %d/%d",
len(tb_failed) + len(tb_passed),
len(time_ordered_tb_dict))
return tb_passed, tb_failed, tb_sim_errors
def generate_email_report(
log_path, n_tb, tb_failed, tb_passed, tb_mk_errors, tb_mk_rerun, tb_sim_errors, run_time
):
filename = "modelsim_regressiontest_email.txt"
full_filename = os.path.join(log_path, filename)
with open(full_filename, "w") as fd:
fd.write(f"Regression test summary for {args.buildset}.\n")
fd.write("\n")
fd.write(f"Test duration in h:m:s = {timedelta(seconds=run_time)}\n") # Log total test time
fd.write("\n")
if len(tb_mk_errors) == 0:
fd.write("make SUCCESFULL for all libs.\n")
else:
fd.write(f"make FAILED for: {[i for i,j in tb_mk_errors]}.\n")
if len(tb_mk_rerun) > 0:
fd.write(f'WARNING, did run "make" two times for: {[i for i,j in tb_mk_rerun]}.\n')
fd.write("\n")
if n_tb == 0:
fd.write("FAILED because no VHDL test bench was simulated.\n")
elif len(tb_failed) == 0:
fd.write(f"All {len(tb_passed)} VHDL test benches PASSED\n")
else:
fd.write(f"Out of {n_tb} VHDL test benches {len(tb_failed)} FAILED\n")
fd.write("\n")
fd.write(f"Failed test benches: {tb_failed}\n")
fd.write("\n")
fd.write(f"Passed test benches: {tb_passed}\n")
fd.write("\n")
fd.write("first 10 errors for failed test benches\n")
for tb_name, errors in tb_mk_errors:
fd.write(f"{tb_name} [mk error]:\n")
for error in errors:
fd.write(f" - {error}\n")
fd.write("\n")
for tb_name, errors in tb_sim_errors:
fd.write(f"{tb_name} [sim error]:\n")
for error in errors:
fd.write(f" - {error}\n")
fd.write("\n")
class TestBenchWorker(multiprocessing.Process):
"""TestBenchWorker class is a process that will check for tasks to process in the in_queue"""
def __init__(self, worker_id, control_queue, in_queue, out_queue, max_mk_time, max_sim_time, log_path):
multiprocessing.Process.__init__(self)
self._id = worker_id
self._ctrl = control_queue
self._in = in_queue
self._out = out_queue
self._max_mk_time = max_mk_time
self._max_sim_time = max_sim_time
self._log_path = log_path
self._halt_time = -1
self._halt = False
self._sleep = False
self.buildset = None
self.task = None # 'create_do_files', 'mak_all', 'simulate'
self.remove_do_files_action = False
self.create_do_make_all_file_action = False
self.create_do_simulate_file_action = False
self.lib_name = None
self.mpf_path = None
self.do_path = None
self.tb_file = None
self.passed = []
self.failed = []
self.n_passed = 0
self.n_failed = 0
self.mk_errors = []
self.sim_errors = []
self.n_tb = 0
self.run_time = 0
sim_dir = os.path.join(os.getenv("HDL_BUILD_DIR"), f"sim_worker{self._id}")
if not os.path.exists(sim_dir):
os.mkdir(sim_dir)
os.environ["HDL_IOFILE_SIM_DIR"] = sim_dir
self.terminal = Terminal(print_stdout_on_timeout=True)
self.terminal.run_cmd(f"rm -rf {sim_dir}/*")
print(self.terminal.stdout())
# get do templates
tmpl_path = os.path.join(os.getenv("RADIOHDL_GEAR"), "regressiontest")
filename = os.path.join(tmpl_path, "do_mk_all.tmpl")
with open(filename, "r") as fd:
self.do_mk_all_tmpl = fd.read()
filename = os.path.join(tmpl_path, "do_simulation.tmpl")
with open(filename, "r") as fd:
self.do_simulation_tmpl = fd.read()
def run(self):
logger.debug("worker %d start waiting for tasks", self._id)
while not self._halt:
if not self._ctrl.empty():
ctrl = self._ctrl.get()
logger.debug("received ctrl command: %s", ctrl)
if ctrl == "sleep":
self._sleep = True
elif ctrl == "awake":
self._sleep = False
elif ctrl == "halt":
self.halt()
if self._sleep:
sleep(1.0)
continue
if self._in.empty():
sleep(0.1)
continue
start_time = time()
self.passed = []
self.failed = []
self.mk_errors = []
self.sim_errors = []
task = self._in.get()
self.task = task["task"]
self.task_id = task["task_id"]
self.lib_name = task["lib_name"]
self.mpf_path = task["mpf_path"]
self.tb_file = task["tb_file"]
self.tb_name = task["tb_name"]
self.buildset = task["buildset"]
self.project_sim_p_search_libraries = task["project_sim_p_search_libraries"]
self.project_sim_p_otherargs = task["project_sim_p_otherargs"]
self.remove_do_files_action = task["remove_do_files"]
self.create_do_make_all_action = task["create_do_make_all_file"]
self.create_do_simulate_action = task["create_do_simulate_file"]
if self.tb_name is None:
logger.info(f"worker {self._id} '{self.task}' for '{self.lib_name}'")
else:
logger.info(f"worker {self._id} '{self.task}' for '{self.tb_name}'")
# self.lib_name = self.lib_dict['hdl_lib_name']
self.do_path = os.path.join(self.mpf_path, "regression_test_vhdl")
self._halt_time = time() + self._max_sim_time
logger.debug("goto dir %s", self.do_path)
os.chdir(self.do_path)
if self.task == "make_new_do_files":
self.make_new_do_files()
elif self.task == "remove_do_files":
self.remove_do_files()
elif self.task == "create_do_make_all_file":
self.create_do_make_all_file()
elif self.task == "create_do_simulate_file":
self.create_do_simulate_file()
elif self.task == "mk_all":
self.run_mk_all()
elif self.task == "simulate":
self.run_simulation()
self.n_passed = len(self.passed)
self.n_failed = len(self.failed)
self.n_tb = self.n_passed + self.n_failed
if self.n_tb == 0:
logger.info("%s: has zero testbenches for regression test.", self.lib_name)
else:
logger.error('Not a valid task "%s"', self.task)
# Measure regression test time for this HDL library
self.run_time = time() - start_time
if self.tb_name is None:
logger.info(
f"worker {self._id} '{self.task}' done in '{timedelta(seconds=self.run_time)}'"
)
else:
logger.info(
f"worker {self._id} '{self.task}' '{self.tb_name}' done in '{timedelta(seconds=self.run_time)}'"
)
# fill and return response from task
response = {
"task_id": self.task_id,
"task": self.task,
"passed": self.passed,
"failed": self.failed,
"sim_errors": self.sim_errors,
"mk_errors": self.mk_errors,
}
self._out.put(response)
def halt(self):
self._halt = True
self.terminal.kill()
logger.info("stop worker %d", self._id)
def make_new_do_files(self):
if self.remove_do_files_action:
self.remove_do_files()
if self.create_do_make_all_action:
self.create_do_make_all_file()
if self.create_do_simulate_action:
self.create_do_simulate_file()
def remove_do_files(self):
"""
Remove all .do files.
"""
rm_list = glob.glob(os.path.join(self.do_path, "*.do"))
# logger.info("remove rm_list '%s'", str(rm_list))
for rm in rm_list:
try:
logger.debug("%s: remove '%s'", self.lib_name, rm)
os.remove(rm) # rm <mpf_path>/regression_test_vhdl/*.do
except FileNotFoundError:
logger.error("FileNotFoundError r632")
pass
def create_do_make_all_file(self):
"""
Create test bench do files in same build directory as where the mpf is
"""
# logger.debug("%s: create testbench do_make_all file", self.lib_name)
# make copy of template and replace values
do_file = copy(self.do_mk_all_tmpl)
do_file = do_file.replace(
"<tmpl file used by modelsim_regression_test_vhdl.py>",
"Created by modelsim_regression_test_vhdl.py",
)
do_file = do_file.replace("<lib_name>", self.lib_name)
# Write separate do file for each test bench in the VHDL regression test of this library
do_name = f"{self.lib_name}_mk_all.do"
do_pathname = os.path.join(self.do_path, do_name)
with open(do_pathname, "w") as fp:
fp.write(do_file)
return
def create_do_simulate_file(self):
"""
Create test bench do files in same build directory as where the mpf is
"""
if self.tb_file is None:
return
# logger.debug("%s: create testbench do simulate file for %s", self.lib_name, self.tb_file)
# make copy of template and replace values
do_file = copy(self.do_simulation_tmpl)
do_file = do_file.replace(
"<tmpl file used by modelsim_regression_test_vhdl.py>",
"Created by modelsim_regression_test_vhdl.py",
)
do_file = do_file.replace("<lib_name>", self.lib_name)
do_file = do_file.replace("<tb_name>", self.tb_name)
do_file = do_file.replace(
"<project_sim_p_otherargs>", " ".join(self.project_sim_p_otherargs)
)
do_file = do_file.replace(
"<project_sim_p_search_libraries>",
" ".join([f"-L {i}" for i in self.project_sim_p_search_libraries]),
)
# Write separate do file for each test bench in the VHDL regression test of this library
do_name = f"{self.tb_name}_simulate.do"
do_pathname = os.path.join(self.do_path, do_name)
with open(do_pathname, "w") as fp:
fp.write(do_file)
return
def run_mk_all(self):
"""
Run mk all on all libs
"""
logger.debug("%s: start making lib", self.lib_name)
this_mk_failed = False
do_name = f"{self.lib_name}_mk_all.do"
do_pathname = os.path.join(self.do_path, do_name)
# Simulate the do file with Modelsim
vsim_cmd = f"cli_modelsim {self.buildset} {do_pathname}"
logger.debug('%s: run mk_all do file for "%s"', self.lib_name, do_name)
run_exitcode = self.terminal.run_cmd(vsim_cmd, timeout=self._max_mk_time)
if self._halt:
return
if run_exitcode == 0:
do_log = self.terminal.stdout()
self.write_do_log(do_log, f"{self.lib_name}_mk")
# get vsim exitcode
vsim_exitcode = self.get_vsim_exit_code(do_log)
if vsim_exitcode:
vsim_error_str = self.get_vsim_exit_error(do_log)
logger.error(
"%s: vsim exitcode=%d, %s", self.lib_name, vsim_exitcode, vsim_error_str
)
this_mk_failed = True
else:
n_errors = 10
logger.debug("%s: mk_all ended", self.lib_name)
# if there is a Error or Failure add transcipt file to the log
if "Error:" in do_log: # or 'Failure:' in do_log:
logger.debug("worker %d found Errors: in do logging", self._id)
errors = self.get_test_errors(do_log, n_errors)
self.mk_errors.append([self.lib_name, errors]) # get first 10 errors
do_log_pathname = os.path.join(self._log_path, f"{self.lib_name}_mk.log")
error_str = "\n - ".join(errors)
logger.error(
"%s: first %d mk errors:\n - %s", self.lib_name, n_errors, error_str
)
logger.error(
'%s: mk all error, full output >> "%s"', self.lib_name, do_log_pathname
)
this_mk_failed = True
else:
logger.error("%s: Error occured while calling: %s", self.lib_name, vsim_cmd)
if self.terminal.exit_on_timeout():
self.mk_errors.append(
[self.lib_name, [f"timeout in {self._max_mk_time:3.0f} seconds"]]
) # get first 10 errors
if self.terminal.error():
logger.error("%s: cmd output stdout: \n%s", self.lib_name, self.terminal.stdout())
logger.error("%s: cmd output stderr: \n%s", self.lib_name, self.terminal.stderr())
this_mk_failed = True
if this_mk_failed:
self.failed.append(self.lib_name)
else:
self.passed.append(self.lib_name)
if self._halt:
return
logger.debug('worker %d run mk_all do file done for lib "%s"', self._id, self.lib_name)
def run_simulation(self):
"""
Remarks on Modelsim simulation control:
. Google search: modelsim stop simulation vhdl
. How to stop the simulation in VHDL TB
1) The easiest way is to use an assert: assert false report "Simulation Finished" severity failure;
2) The "recommended" way to end a simulation when everything has gone correctly, is to halt all stimulus. This will be stopping
the clock like, but also putting any input processes into a never ending wait if they do not use the generated clock.
3) In VHDL 2008, they have introduced and new package called env to the std library, with procedures called stop and finish,
that act like $finish in verilog. This works, but is not used because the scheme with tb_end suffices:
library std;
use std.env.all;
.......
stop(0);
--or
finish(0);
-- For both STOP and FINISH the STATUS values are those used
-- in the Verilog $finish task
-- 0 prints nothing
-- 1 prints simulation time and location
-- 2 prints simulation time, location, and statistics about
-- the memory and CPU times used in simulation
-- Other STATUS values are interpreted as 0.
"""
this_tb_failed = False
do_name = f"{self.tb_name}_simulate.do"
do_pathname = os.path.join(self.do_path, do_name)
# Simulate the do file with Modelsim
vsim_cmd = f"cli_modelsim {self.buildset} {do_pathname}"
logger.debug(f"{self.tb_name}: run modelsim test '{do_name}'")
run_exitcode = self.terminal.run_cmd(vsim_cmd, timeout=self._max_sim_time)
if self._halt:
return
logger.debug(f"{self.tb_name}: modelsim test '{do_name}' done")
if run_exitcode == 0:
do_log = self.terminal.stdout()
self.write_do_log(do_log, f"{self.tb_name}_sim")
# get vsim exitcode
vsim_exitcode = self.get_vsim_exit_code(do_log)
n_errors = 10
if vsim_exitcode:
vsim_error_str = self.get_vsim_exit_error(do_log)
logger.error(f"{self.tb_name}: vsim exitcode={vsim_exitcode}, {vsim_error_str}")
this_tb_failed = True
else:
# Check that the library compiled and the simulation ran (use try-except to handle exit code > 0)
if ">>> SIMULATION END" in do_log:
logger.debug(f"{self.tb_name}: simulation ended")
# if there is a Error or Failure add (part) of transcipt file to the log
if "Error:" in do_log: # or 'Failure:' in do_log:
logger.debug(f"worker {self._id} found Errors: in do logging")
errors = self.get_test_errors(do_log, n_errors)
self.sim_errors.append([self.tb_name, errors]) # get first 10 errors
do_log_pathname = os.path.join(self._log_path, f"{self.tb_name}_sim.log")
error_str = "\n - ".join(errors)
logger.error(f"{self.tb_name}: first {n_errors} simulation errors:\n - {error_str}")
logger.error(
f"{self.tb_name}: simulation error, full output >> '{do_log_pathname}'"
)
this_tb_failed = True
else:
logger.error(f"{self.tb_name}: Error occured while calling: {vsim_cmd}")
if self.terminal.exit_on_timeout():
self.sim_errors.append(
[self.tb_name, [f"timeout in {self._max_sim_time:3.0f} seconds"]]
) # get first 10 errors
if self.terminal.error():
logger.error(f"{self.tb_name}: cmd output: \n{self.terminal.stderr()}")
this_tb_failed = True
if this_tb_failed:
self.failed.append(self.tb_name)
else:
self.passed.append(self.tb_name)
if self._halt:
return
logger.debug(
'worker %d run do files done for lib:"%s", tb:"%s"',
self._id,
self.lib_name,
self.tb_name,
)
def get_vsim_exit_code(self, do_log):
"""
get exitcode from vsim logging and return exitcode if found else None
"""
find_str = "exitcode="
p1 = do_log.find(find_str)
p2 = do_log.find("\n", p1)
if p1 > -1 and p2 > p1:
exitcode = int(do_log[p1 + len(find_str) : p2], 10)
else:
exitcode = None
return exitcode
def get_vsim_exit_error(self, do_log):
"""
get error from vsim logging and return error if found else None
"""
p1 = do_log.find("Error ")
p2 = do_log.find("\n", p1)
if p1 > -1 and p2 > p1:
error = do_log[p1:p2]
else:
error = "Error not found"
return error
def get_test_errors(self, do_log, n_errors):
"""
get first n_errors from do_logging
"""
errors = []
for line in do_log.splitlines():
if "Error:" in line:
errors.append(line[line.find("Error:") :])
if len(errors) == n_errors:
break
return errors
def write_do_log(self, do_log, log_name):
logger.debug("%s: write do logging to file '%s'", self.lib_name, f"{log_name}.log")
# transcript_pathname = os.path.join(self.do_path, 'transcript')
do_log_pathname = os.path.join(self._log_path, f"{log_name}.log")
# os.rename(transcript_pathname, do_log_pathname)
with open(do_log_pathname, "w") as fd:
fd.write(do_log)
class ModelsimConfig(HdlLibrariesWizard):
def __init__(self, toolRootDir, buildsetFile, libFileName):
"""
Get Modelsim tool info from toolRootDir and all HDL library info from libRootDir.
This class uses the default keys and the keys from the libFileSections in the libFileName config file.
Arguments:
- toolRootDir : Root directory from where the hdl_buildset_<buildset>.cfg file is searched for.
- buildsetFile : Default HDL tools configuration file name
- libFileName : Default HDL library configuration file name
The libRootDir is defined in the hdl_buildset_<buildset>.cfg file and is the root directory from where the hdllib.cfg
files are searched for.
Files:
- hdl_buildset_<buildset>.cfg : HDL tool configuration dictionary file. One central file per buildset.
- hdllib.cfg : HDL library configuration dictionary file. One file for each HDL library.
- modelsim_project_files.txt
The modelsim_project_files.txt file is a dictionary file with the list the Modelsim project files for all HDL
libraries that were found in the libRootDir. The keys are the library names and the values are the paths to the
corresponding modelsim project files. The modelsim_project_files.txt file is created by
create_modelsim_project_files_file() and is read by the TCL commands.do file in Modelsim. Creating the file in
Python and then reading this in TCL makes the commands.do much simpler.
- <lib_name>.mpf : Modelsim project file for a certain HDL library based on the hdllib.cfg. The file is created by
create_modelsim_project_file().
- <lib_name>_lib_order.txt
The <lib_name>_lib_order.txt file contains the library compile order for a certain HDL library. The files are
created by create_lib_order_files() in the same build directory as where the Modelsim project file is stored.
The <lib_name>_lib_order.txt files are read by the TCL commands.do file in Modelsim. Creating the files in Python
and then reading them in TCL makes the commands.do much simpler.
"""
logger.debug("start ModelsimConfig.__init__()")
libFileSections = ["modelsim_project_file"]
HdlLibrariesWizard.__init__(
self, toolRootDir, buildsetFile, libFileName, libFileSections
)
logger.debug("ModelsimConfig.__init__() done")
def simulation_configuration(self, list_mode=False):
"""
Prepare settings for simulation configuration.
The output format is string or list, dependent on list_mode.
Return tuple of project_sim_p_search_libraries, project_sim_p_otherargs.
"""
# project_sim_p_search_libraries
if list_mode:
project_sim_p_search_libraries = self.buildset["modelsim_search_libraries"].split()
else:
project_sim_p_search_libraries = "-L {}"
if "modelsim_search_libraries" in self.buildset:
project_sim_p_search_libraries = "-L {"
for sl in self.buildset["modelsim_search_libraries"].split():
project_sim_p_search_libraries += sl
project_sim_p_search_libraries += " "
project_sim_p_search_libraries += "}"
# project_sim_p_otherargs
# Note:
# E.g. the vsim-8684 load warning does not occur when the simulation is loaded via double click, but it
# does occur when the simulation is relaoded via the command line, because in the command line history
# the +nowarn8684 is then for some reason not preserved by Modelsim.
otherargs = ""
otherargs = "+nowarn8684 +nowarn8683 -quiet"
otherargs = "+nowarn8684 +nowarn8683"
otherargs = "+nowarn8684 +nowarn8683 +nowarnTFMPC +nowarnPCDPC" # nowarn on verilog IP connection mismatch warnings
if list_mode:
project_sim_p_otherargs = otherargs.split()
else:
project_sim_p_otherargs = "OtherArgs {" + otherargs + "}"
return project_sim_p_search_libraries, project_sim_p_otherargs
def str2time(timestr):
tm = timestr.split(':')
return (int(tm[0]) * 60 * 60) + (int(tm[1]) * 60) + float(tm[2])
def read_timing_file(log_path):
ordered_tb_dict = OrderedDict()
timingfile = os.path.join(log_path, "simulation_time.txt")
try:
with open(timingfile, 'r') as fd:
data = fd.read()
except FileExistsError:
return ordered_tb_dict
for line in data.splitlines():
k, v = line.strip().split()
ordered_tb_dict[k] = float(v)
return ordered_tb_dict
def write_timing_file(log_path, ordered_tb_dict):
logfile = os.path.join(log_path, "modelsim_regressiontest_full.log")
with open(logfile, 'r') as fd:
data = fd.readlines()
timing = {}
# first add last timing
for k, v in ordered_tb_dict.items():
timing[k] = v
# add timing from last run
for line in data:
if "simulate" in line and "done in" in line:
ll = line.split()
timing[ll[7].replace("'", "")] = str2time(ll[10].replace("'", ""))
# sort on simulation time, in revered order
sorted_timing = sorted(timing.items(), key=lambda x:x[1], reverse=True)
# write timing to file for next run
timingfile = os.path.join(log_path, "simulation_time.txt")
with open(timingfile, 'w') as fd:
for k, v in sorted_timing:
fd.write(f"{k} {v}\n")
if __name__ == "__main__":
# get available buildsets
buildset_select = sorted(
[
cfgfile[13:-4]
for cfgfile in listdir(os.getenv("RADIOHDL_CONFIG"))
if cfgfile.startswith("hdl_buildset_") and cfgfile.endswith(".cfg")
]
)
# Parse command line arguments
argparser = ArgumentParser(description="Modelsim regression test.")
argparser.add_argument(
"buildset", choices=buildset_select, help="choose buildset %s" % (buildset_select)
)
argparser.add_argument("--libnames", nargs="*", help="select libs to check")
argparser.add_argument("-p", "--proc", type=int, default=1, help="number of processors to use")
argparser.add_argument(
"-v",
"--verbosity",
default="INFO",
help="stdout log level can be [ERROR | WARNING | INFO | DEBUG]",
)
args = argparser.parse_args()
args.buildset_file = "hdl_buildset_" + args.buildset + ".cfg"
args.log_path = os.path.join(os.getenv("RADIOHDL_LOG_DIR"), args.buildset)
my_logger = MyLogger(log_path=args.log_path, file_name="modelsim_regressiontest_full", name_size=15)
my_logger.set_file_log_level("DEBUG")
my_logger.set_stdout_log_level(args.verbosity)
logger = my_logger.get_logger()
logger.info("logger is started")
time_ordered_tb_dict = read_timing_file(args.log_path)
exit_code = main(args, time_ordered_tb_dict)
write_timing_file(args.log_path, time_ordered_tb_dict)
sys.exit(exit_code)