luke
/
iverilog
mirror of https://github.com/steveicarus/iverilog.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Cleanup python test script and add support for a suffix

This commit is contained in:
Cary R 2025-11-11 01:10:35 -08:00
parent a7502173d3
commit 87d9d0ac74
3 changed files with 122 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
ivtest/run_ivl.py
View File

@ -8,8 +8,10 @@ import os
import sys
import re
def assemble_iverilog_cmd(source: str, it_dir: str, args: list, outfile = "a.out") -> list:
res = ["iverilog", "-o", os.path.join("work", outfile)]
def assemble_iverilog_cmd(suffix: str, source: str, it_dir: str,
args: list, outfile = "a.out") -> list:
'''Build the iverilog command line'''
res = ["iverilog"+suffix, "-o", os.path.join("work", outfile)]
res += ["-D__ICARUS_UNSIZED__"]
res += args
src = os.path.join(it_dir, source)
@ -17,28 +19,29 @@ def assemble_iverilog_cmd(source: str, it_dir: str, args: list, outfile = "a.out
return res
def assemble_vvp_cmd(args: list = [], plusargs: list = []) -> list:
res = ["vvp"]
def assemble_vvp_cmd(suffix: str, args: list, plusargs: list) -> list:
'''Build the vvp command line'''
res = ["vvp"+suffix]
res = res + args
res.append(os.path.join("work", "a.out"))
res = res + plusargs
return res
def get_ivl_version () -> list:
def get_ivl_version (suffix: str) -> int:
'''Figure out the version of the installed iverilog compler.
The return value is a list of 2 numbers, the major and minor version
numbers, or None if the version string couldn't be found.'''
# Get the output from the "iverilog -V" command for the version string.
text = subprocess.check_output(["iverilog", "-V"])
text = subprocess.check_output(["iverilog"+suffix, "-V"])
match = re.search(b'Icarus Verilog version ([0-9]+)\\.([0-9]+)', text)
if not match:
return None
items = match.groups()
return [int(items[0]), int(items[1])]
return int(items[0])
def build_runtime(it_key: str) -> None:
'''Check and prepare the runtime environment for a test
@ -70,6 +73,8 @@ def build_runtime(it_key: str) -> None:
pass
def get_log_file(key: str, title: str, stream: str) -> str:
'''Get the name of the log file'''
# pylint: disable-next=consider-using-f-string
res = "{key}-{title}-{stream}.log".format(key=key, title=title, stream=stream)
return res
@ -93,42 +98,49 @@ def compare_files(log_path, gold_path):
If they differ, then write tou stdout a unified diff. In any case,
return True or False to indicate the results of the test.'''
with open(log_path, 'rt') as fd:
with open(log_path, 'rt', encoding='ascii') as fd:
a = fd.readlines()
# Allow to omit empty gold files
if os.path.exists(gold_path):
with open(gold_path, 'rt') as fd:
with open(gold_path, 'rt', encoding='ascii') as fd:
b = fd.readlines()
else:
b = []
flag = a == b
if not flag:
# pylint: disable-next=consider-using-f-string
print("{log} and {gold} differ:".format(log=log_path, gold=gold_path))
sys.stdout.writelines(difflib.unified_diff(a, b, log_path, gold_path))
return flag
def run_cmd(cmd: list) -> subprocess.CompletedProcess:
res = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
'''Run the given command'''
res = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False)
return res
def check_gold(it_key : str, it_gold : str, log_list : list) -> bool:
def check_gold(it_key: str, it_gold: str, log_list: list) -> bool:
'''Check if the log and gold file match'''
compared = True
for log_name in log_list:
# pylint: disable-next=consider-using-f-string
log_path = os.path.join("log", "{key}-{log}.log".format(key=it_key, log=log_name))
# pylint: disable-next=consider-using-f-string
gold_path = os.path.join("gold", "{gold}-{log}.gold".format(gold=it_gold, log=log_name))
compared = compared and compare_files(log_path, gold_path)
return compared
def run_CE(options : dict) -> list:
# pylint: disable-next=invalid-name
def run_CE(options: dict) -> list:
''' Run the compiler, and expect an error
In this case, we assert that the command fails to run and reports
an error. This is to check that invalid input generates errors.'''
suffix = options['suffix']
it_key = options['key']
it_dir = options['directory']
it_args = options['iverilog_args']
@ -136,7 +148,7 @@ def run_CE(options : dict) -> list:
build_runtime(it_key)
cmd = assemble_iverilog_cmd(options['source'], it_dir, it_args)
cmd = assemble_iverilog_cmd(suffix, options['source'], it_dir, it_args)
res = run_cmd(cmd)
log_results(it_key, "iverilog", res)
@ -151,7 +163,7 @@ def run_CE(options : dict) -> list:
else:
return [0, "Passed - CE"]
def check_run_outputs(options : dict, it_stdout : str, log_list : list) -> list:
def check_run_outputs(options: dict, it_stdout: str, log_list: list) -> list:
'''Check the output files, and return success for failed.
This function takes an options dictionary that describes the settings, and
@ -178,12 +190,13 @@ def check_run_outputs(options : dict, it_stdout : str, log_list : list) -> list:
diff_name2 = it_diff[1]
diff_skip = int(it_diff[2])
with open(diff_name1) as fd:
with open(diff_name1, 'rt', encoding='ascii') as fd:
# pylint: disable-next=unused-variable
for idx in range(diff_skip):
fd.readline()
diff_data1 = fd.read()
with open(diff_name2) as fd:
with open(diff_name2, 'rt', encoding='ascii') as fd:
for idx in range(diff_skip):
fd.readline()
diff_data2 = fd.read()
@ -191,7 +204,9 @@ def check_run_outputs(options : dict, it_stdout : str, log_list : list) -> list:
if diff_data1 == diff_data2:
return [0, "Passed"]
else:
return [1, "Failed - Files {name1} and {name2} differ.".format(name1=diff_name1, name2=diff_name2)]
# pylint: disable-next=consider-using-f-string
return [1, "Failed - Files {name1} and {name2} differ.".format(name1=diff_name1, \
name2=diff_name2)]
# Otherwise, look for the PASSED output string in stdout.
@ -204,7 +219,8 @@ def check_run_outputs(options : dict, it_stdout : str, log_list : list) -> list:
return [1, "Failed - No PASSED output, and no gold file"]
def do_run_normal_vlog95(options : dict, expected_fail : bool) -> list:
# pylint: disable-next=unused-argument
def do_run_normal_vlog95(options: dict, expected_fail: bool) -> list:
'''Run the iverilog and vvp commands.
In this case, run the compiler with the -tvlog95 flag to generate
@ -212,6 +228,7 @@ def do_run_normal_vlog95(options : dict, expected_fail : bool) -> list:
a vvp out. Run that vvp output to test the simulation results. Collect
the results and look for a "PASSED" string.'''
suffix = options['suffix']
it_key = options['key']
it_dir = options['directory']
it_iverilog_args = ["-tvlog95"] + options['iverilog_args']
@ -221,7 +238,7 @@ def do_run_normal_vlog95(options : dict, expected_fail : bool) -> list:
build_runtime(it_key)
# Run the first iverilog command, to generate the intermediate verilog
ivl1_cmd = assemble_iverilog_cmd(options['source'], it_dir, it_iverilog_args, "a.out.v")
ivl1_cmd = assemble_iverilog_cmd(suffix, options['source'], it_dir, it_iverilog_args, "a.out.v")
ivl1_res = run_cmd(ivl1_cmd)
log_results(it_key, "iverilog", ivl1_res)
@ -229,7 +246,7 @@ def do_run_normal_vlog95(options : dict, expected_fail : bool) -> list:
return [1, "Failed - Compile failed"]
# Run another iverilog command to compile the code generated from the first step.
ivl2_cmd = assemble_iverilog_cmd("a.out.v", "work", [ ], "a.out")
ivl2_cmd = assemble_iverilog_cmd(suffix, "a.out.v", "work", [ ], "a.out")
ivl2_res = run_cmd(ivl2_cmd)
log_results(it_key, "iverilog-vlog95", ivl2_res)
@ -237,9 +254,9 @@ def do_run_normal_vlog95(options : dict, expected_fail : bool) -> list:
return [1, "Failed - Compile of generated code failed"]
# Run the vvp command
vvp_cmd = assemble_vvp_cmd(it_vvp_args, it_vvp_args_extended)
vvp_cmd = assemble_vvp_cmd(suffix, it_vvp_args, it_vvp_args_extended)
vvp_res = run_cmd(vvp_cmd)
log_results(it_key, "vvp", vvp_res);
log_results(it_key, "vvp", vvp_res)
if vvp_res.returncode != 0:
return [1, "Failed - Vvp execution failed"]
@ -252,13 +269,14 @@ def do_run_normal_vlog95(options : dict, expected_fail : bool) -> list:
return check_run_outputs(options, it_stdout, log_list)
def do_run_normal(options : dict, expected_fail : bool) -> list:
def do_run_normal(options: dict, expected_fail: bool) -> list:
'''Run the iverilog and vvp commands.
In this case, run the compiler to generate a vvp output file, and
run the vvp command to actually execute the simulation. Collect
the results and look for a "PASSED" string.'''
suffix = options['suffix']
it_key = options['key']
it_dir = options['directory']
it_iverilog_args = options['iverilog_args']
@ -268,7 +286,7 @@ def do_run_normal(options : dict, expected_fail : bool) -> list:
build_runtime(it_key)
# Run the iverilog command
ivl_cmd = assemble_iverilog_cmd(options['source'], it_dir, it_iverilog_args)
ivl_cmd = assemble_iverilog_cmd(suffix, options['source'], it_dir, it_iverilog_args)
ivl_res = run_cmd(ivl_cmd)
log_results(it_key, "iverilog", ivl_res)
@ -276,9 +294,9 @@ def do_run_normal(options : dict, expected_fail : bool) -> list:
return [1, "Failed - Compile failed"]
# run the vvp command
vvp_cmd = assemble_vvp_cmd(it_vvp_args, it_vvp_args_extended)
vvp_cmd = assemble_vvp_cmd(suffix, it_vvp_args, it_vvp_args_extended)
vvp_res = run_cmd(vvp_cmd)
log_results(it_key, "vvp", vvp_res);
log_results(it_key, "vvp", vvp_res)
if vvp_res.returncode == 0 and expected_fail:
return [1, "Failed - Vvp execution did not fail, but was expted to fail."]
@ -293,14 +311,20 @@ def do_run_normal(options : dict, expected_fail : bool) -> list:
return check_run_outputs(options, it_stdout, log_list)
def run_normal(options : dict) -> list:
def run_normal(options: dict) -> list:
'''Run a normal test'''
return do_run_normal(options, False)
def run_EF(options : dict) -> list:
# pylint: disable-next=invalid-name
def run_EF(options: dict) -> list:
'''Run an expected fail test'''
return do_run_normal(options, True)
def run_normal_vlog95(options : dict) -> list:
def run_normal_vlog95(options: dict) -> list:
'''Run a vlog95 test'''
return do_run_normal_vlog95(options, False)
def run_EF_vlog95(options : dict) -> list:
# pylint: disable-next=invalid-name
def run_EF_vlog95(options: dict) -> list:
'''Run an expected fail vlog95 test'''
return do_run_normal_vlog95(options, True)

10
ivtest/test_lists.py
View File

@ -30,7 +30,7 @@ def read_list(fd) -> list:
in the list a list of tokens for the line. This is used by the read_lists
function.'''
build_list = list()
build_list = []
for line_raw in fd:
# Strip comments and leading/traling white space
idx = line_raw.find("#")
@ -56,18 +56,18 @@ def read_lists(paths: list) -> list:
of the test file lists is important, as is the order of tests
within each list file.'''
tests_list = list()
tests_list = []
for path in paths:
with open(path, "r") as fd:
with open(path, "rt", encoding='ascii') as fd:
tests_list += read_list(fd)
# The loop above creates a tests_list to list all of the tests in the
# order that they appear. Now we go though the list in order and eliminate
# duplictes by test name. This allows that lists might override tests that
# are already declared.
tests_dict = dict()
tests_dict = {}
for item in tests_list:
tests_dict[item[0]] = item;
tests_dict[item[0]] = item
# Convert the result to a sorted list, and return that.
tests_list = list(tests_dict.values())

99
ivtest/vvp_reg.py
View File

@ -12,16 +12,13 @@ Usage:
'''
import sys
# The docopt library is not available on msys2 or cygwin installations, so
# skip it completely on win32/cygwin platforms.
if sys.platform != 'win32' and sys.platform != 'cygwin':
from docopt import docopt
import test_lists
import json
import argparse
import test_lists
import run_ivl
def process_test(item: list) -> str:
def process_test(suffix: str, item: list) -> str:
'''Process a single test
This takes in the list of tokens from the tests list file, and converts
@ -30,7 +27,7 @@ def process_test(item: list) -> str:
# This is the name of the test, and the name of the main sorce file
it_key = item[0]
test_path = item[1]
with open(test_path, 'rt') as fd:
with open(test_path, 'rt', encoding='ascii') as fd:
it_dict = json.load(fd)
# Get the test type from the json configuration.
@ -38,14 +35,15 @@ def process_test(item: list) -> str:
# Wrap all of this into an options dictionary for ease of handling.
it_options = {
'key' : it_key,
'type' : it_type,
'iverilog_args' : it_dict.get('iverilog-args', [ ]),
'directory' : "ivltests",
'source' : it_dict['source'],
'modulename' : None,
'gold' : it_dict.get('gold', None),
'diff' : None,
'suffix' : suffix,
'key' : it_key,
'type' : it_type,
'iverilog_args' : it_dict.get('iverilog-args', [ ]),
'directory' : "ivltests",
'source' : it_dict['source'],
'modulename' : None,
'gold' : it_dict.get('gold', None),
'diff' : None,
'vvp_args' : it_dict.get('vvp-args', [ ]),
'vvp_args_extended' : it_dict.get('vvp-args-extended', [ ])
}
@ -69,6 +67,7 @@ def process_test(item: list) -> str:
res = run_ivl.run_EF_vlog95(it_options)
else:
# pylint: disable-next=consider-using-f-string
res = "{key}: I don't understand the test type ({type}).".format(key=it_key, type=it_type)
raise Exception(res)
@ -76,42 +75,70 @@ def process_test(item: list) -> str:
if __name__ == "__main__":
print("Running tests on platform: {platform}".format(platform=sys.platform))
if sys.platform == 'win32' or sys.platform == 'cygwin':
args = { "<list-paths>" : [] }
else:
args = docopt(__doc__)
argp = argparse.ArgumentParser(description='')
argp.add_argument('--suffix', type=str, default='',
help='The Icarus executable suffix, default "%(default)s".')
argp.add_argument('--strict', action='store_true',
help='Force strict standard compliance, default "%(default)s".')
argp.add_argument('--with-valgrind', action='store_true',
help='Run the test suite with valgrind, default "%(default)s".')
argp.add_argument('--force-sv', action='store_true',
help='Force tests to be run as SystemVerilog, default "%(default)s".')
argp.add_argument('--vlog95', action='store_true',
help='Convert tests to Verilog 95 and then run, default "%(default)s".')
argp.add_argument('files', nargs='*', type=str, default=['regress-vvp.list'],
help='File(s) containing a list of the tests to run, default "%(default)s".')
args = argp.parse_args()
# This returns [13, 0] or similar
ivl_version = run_ivl.get_ivl_version()
ivl_version_major = ivl_version[0]
print("Icarus Verilog version: {ivl_version}".format(ivl_version=ivl_version_major))
if args.strict:
print('Sorry: Forcing strict compatiability is not currently supported!')
sys.exit(1)
if args.with_valgrind:
print('Sorry: Running with valgrind is not currently supported!')
sys.exit(1)
if args.force_sv:
print('Sorry: Forcing SystemVerilog is not currently supported!')
sys.exit(1)
if args.vlog95:
print('Sorry: Converting to Verilog-95 and running is not currently supported!')
sys.exit(1)
# Select the lists to use. If any list paths are given on the command
# line, then use only those. Otherwise, use a default list.
list_paths = args["<list-paths>"]
if len(list_paths) == 0:
list_paths = list()
list_paths += ["regress-vvp.list"]
# This returns 13 or similar
ivl_version = run_ivl.get_ivl_version(args.suffix)
print("Use lists: {list_paths}".format(list_paths=list_paths))
print("Running compiler/VVP tests for Icarus Verilog ", end='')
# pylint: disable-next=consider-using-f-string
print("version: {ver}".format(ver=ivl_version), end='')
if args.suffix:
# pylint: disable-next=consider-using-f-string
print(", suffix: {suffix}".format(suffix=args.suffix), end='')
# FIXME: add strict, force SV and with valgrind
print("")
# pylint: disable-next=consider-using-f-string
print("Using list(s): {files}".format(files=', '.join(args.files)))
print("-" * 76)
# Read the list files, to get the tests.
tests_list = test_lists.read_lists(list_paths)
tests_list = test_lists.read_lists(args.files)
# We need the width of the widest key so that we can figure out
# how to align the key:result columns.
# pylint: disable-next=invalid-name
width = 0
for cur in tests_list:
if len(cur[0]) > width:
width = len(cur[0])
# pylint: disable-next=invalid-name
error_count = 0
for cur in tests_list:
result = process_test(cur)
result = process_test(args.suffix, cur)
error_count += result[0]
# pylint: disable-next=consider-using-f-string
print("{name:>{width}}: {result}".format(name=cur[0], width=width, result=result[1]))
print("===================================================")
print("Test results: Ran {ran}, Failed {failed}.".format(ran=len(tests_list), failed=error_count))
exit(error_count)
print("=" * 76)
# pylint: disable-next=consider-using-f-string
print("Test results: Ran {ran}, Failed {failed}.".format(ran=len(tests_list), \
failed=error_count))
sys.exit(error_count)