From 1c6f0e768ac811421764fcd3120540c90395d270 Mon Sep 17 00:00:00 2001 From: Cary R Date: Sat, 22 Nov 2025 13:23:49 -0800 Subject: [PATCH] Update vvp_reg.py to support strict, force-sv and with-valgrind --- .../gold/br_gh1087a1-fsv-iverilog-stderr.gold | 2 + ivtest/gold/br_gh13a_std-vvp-stdout.gold | 2 + ivtest/gold/br_gh383d_std-vvp-stdout.gold | 3 + ivtest/gold/ca_time_real_std-vvp-stdout.gold | 5 + ivtest/gold/delayed_sfunc_std-vvp-stdout.gold | 4 + ivtest/gold/param-width_std-vvp-stdout.gold | 4 + .../gold/parameter_type_std-vvp-stdout.gold | 8 + ivtest/gold/pr1701890_std-vvp-stdout.gold | 1 + ivtest/gold/pr1864110a_std-vvp-stdout.gold | 1 + ivtest/gold/pr1864110b_std-vvp-stdout.gold | 3 + ivtest/gold/pr1864115_std-vvp-stdout.gold | 2 + ivtest/regress-vvp.list | 36 +-- ivtest/run_ivl.py | 215 +++++++++--------- ivtest/vvp_reg.py | 158 +++++++++---- ivtest/vvp_tests/br_gh1087a1.json | 5 +- ivtest/vvp_tests/br_gh1143e.json | 6 +- ivtest/vvp_tests/br_gh1143f.json | 6 +- ivtest/vvp_tests/br_gh1143g.json | 6 +- ivtest/vvp_tests/br_gh1143h.json | 6 +- ivtest/vvp_tests/br_gh13a.json | 5 +- ivtest/vvp_tests/br_gh383d.json | 5 +- ivtest/vvp_tests/ca_time_real.json | 5 +- ivtest/vvp_tests/delayed_sfunc.json | 5 +- ivtest/vvp_tests/localparam_type.json | 5 +- ivtest/vvp_tests/param-width.json | 5 +- ivtest/vvp_tests/parameter_type.json | 5 +- ivtest/vvp_tests/pr1388974.json | 5 +- ivtest/vvp_tests/pr1701890.json | 5 +- ivtest/vvp_tests/pr1864110a.json | 5 +- ivtest/vvp_tests/pr1864110b.json | 5 +- ivtest/vvp_tests/pr1864115.json | 5 +- ivtest/vvp_tests/pr903.json | 5 +- 32 files changed, 353 insertions(+), 185 deletions(-) create mode 100644 ivtest/gold/br_gh1087a1-fsv-iverilog-stderr.gold create mode 100644 ivtest/gold/br_gh13a_std-vvp-stdout.gold create mode 100644 ivtest/gold/br_gh383d_std-vvp-stdout.gold create mode 100644 ivtest/gold/ca_time_real_std-vvp-stdout.gold create mode 100644 ivtest/gold/delayed_sfunc_std-vvp-stdout.gold create mode 100644 ivtest/gold/param-width_std-vvp-stdout.gold create mode 100644 ivtest/gold/parameter_type_std-vvp-stdout.gold create mode 100644 ivtest/gold/pr1701890_std-vvp-stdout.gold create mode 100644 ivtest/gold/pr1864110a_std-vvp-stdout.gold create mode 100644 ivtest/gold/pr1864110b_std-vvp-stdout.gold create mode 100644 ivtest/gold/pr1864115_std-vvp-stdout.gold diff --git a/ivtest/gold/br_gh1087a1-fsv-iverilog-stderr.gold b/ivtest/gold/br_gh1087a1-fsv-iverilog-stderr.gold new file mode 100644 index 000000000..b0ae793b0 --- /dev/null +++ b/ivtest/gold/br_gh1087a1-fsv-iverilog-stderr.gold @@ -0,0 +1,2 @@ +ivltests/br_gh1087a.v:3: error: Net `r` can not be of type `real`. +1 error(s) during elaboration. diff --git a/ivtest/gold/br_gh13a_std-vvp-stdout.gold b/ivtest/gold/br_gh13a_std-vvp-stdout.gold new file mode 100644 index 000000000..90aa7b3e8 --- /dev/null +++ b/ivtest/gold/br_gh13a_std-vvp-stdout.gold @@ -0,0 +1,2 @@ +0 +PASSED diff --git a/ivtest/gold/br_gh383d_std-vvp-stdout.gold b/ivtest/gold/br_gh383d_std-vvp-stdout.gold new file mode 100644 index 000000000..b08356e0b --- /dev/null +++ b/ivtest/gold/br_gh383d_std-vvp-stdout.gold @@ -0,0 +1,3 @@ +1 2 3 4 +4 3 2 1 +8 7 6 5 diff --git a/ivtest/gold/ca_time_real_std-vvp-stdout.gold b/ivtest/gold/ca_time_real_std-vvp-stdout.gold new file mode 100644 index 000000000..cd9f8cd4a --- /dev/null +++ b/ivtest/gold/ca_time_real_std-vvp-stdout.gold @@ -0,0 +1,5 @@ +0 +1 +0 +3 +0 diff --git a/ivtest/gold/delayed_sfunc_std-vvp-stdout.gold b/ivtest/gold/delayed_sfunc_std-vvp-stdout.gold new file mode 100644 index 000000000..4d6a07d77 --- /dev/null +++ b/ivtest/gold/delayed_sfunc_std-vvp-stdout.gold @@ -0,0 +1,4 @@ +0 0 x x 10 +1 0 4 4 10 +5 0 4 4 20 +6 0 5 5 20 diff --git a/ivtest/gold/param-width_std-vvp-stdout.gold b/ivtest/gold/param-width_std-vvp-stdout.gold new file mode 100644 index 000000000..bae403336 --- /dev/null +++ b/ivtest/gold/param-width_std-vvp-stdout.gold @@ -0,0 +1,4 @@ +1001 +001 +0101 +01 diff --git a/ivtest/gold/parameter_type_std-vvp-stdout.gold b/ivtest/gold/parameter_type_std-vvp-stdout.gold new file mode 100644 index 000000000..68d3e1733 --- /dev/null +++ b/ivtest/gold/parameter_type_std-vvp-stdout.gold @@ -0,0 +1,8 @@ +Implicit real: -1 +Implicit integer: -1 +Unsigned: 255 +Signed: -1 +Real: -1 +Real time: -1 +Integer: -1 +Time: 18446744073709551615 diff --git a/ivtest/gold/pr1701890_std-vvp-stdout.gold b/ivtest/gold/pr1701890_std-vvp-stdout.gold new file mode 100644 index 000000000..4d7f677a5 --- /dev/null +++ b/ivtest/gold/pr1701890_std-vvp-stdout.gold @@ -0,0 +1 @@ +rval1=1 rval2=2 rtval=1 diff --git a/ivtest/gold/pr1864110a_std-vvp-stdout.gold b/ivtest/gold/pr1864110a_std-vvp-stdout.gold new file mode 100644 index 000000000..9e74b2216 --- /dev/null +++ b/ivtest/gold/pr1864110a_std-vvp-stdout.gold @@ -0,0 +1 @@ +3 -3 diff --git a/ivtest/gold/pr1864110b_std-vvp-stdout.gold b/ivtest/gold/pr1864110b_std-vvp-stdout.gold new file mode 100644 index 000000000..ae46f1fe3 --- /dev/null +++ b/ivtest/gold/pr1864110b_std-vvp-stdout.gold @@ -0,0 +1,3 @@ +-3 3 +-4 4 +-6 6 diff --git a/ivtest/gold/pr1864115_std-vvp-stdout.gold b/ivtest/gold/pr1864115_std-vvp-stdout.gold new file mode 100644 index 000000000..1b917e756 --- /dev/null +++ b/ivtest/gold/pr1864115_std-vvp-stdout.gold @@ -0,0 +1,2 @@ +0 0 0 +2 4611686018427387904 2 diff --git a/ivtest/regress-vvp.list b/ivtest/regress-vvp.list index 107f68e07..c61513a08 100644 --- a/ivtest/regress-vvp.list +++ b/ivtest/regress-vvp.list @@ -16,9 +16,9 @@ automatic_error13 vvp_tests/automatic_error13.json bits4 vvp_tests/bits4.json bitsel11 vvp_tests/bitsel11.json br_gh13a vvp_tests/br_gh13a.json -br_gh13a-vlog95 vvp_tests/br_gh13a-vlog95.json +#br_gh13a-vlog95 vvp_tests/br_gh13a-vlog95.json br_gh99c vvp_tests/br_gh99c.json -br_gh99c-vlog95 vvp_tests/br_gh99c-vlog95.json +#br_gh99c-vlog95 vvp_tests/br_gh99c-vlog95.json br_gh230 vvp_tests/br_gh230.json br_gh383a vvp_tests/br_gh383a.json br_gh383b vvp_tests/br_gh383b.json @@ -77,16 +77,16 @@ case2-S vvp_tests/case2-S.json case3 vvp_tests/case3.json casex_synth vvp_tests/casex_synth.json cast_int_ams vvp_tests/cast_int_ams.json -cast_int_ams-vlog95 vvp_tests/cast_int_ams-vlog95.json +#cast_int_ams-vlog95 vvp_tests/cast_int_ams-vlog95.json cast_real_invalid1 vvp_tests/cast_real_invalid1.json cast_real_invalid2 vvp_tests/cast_real_invalid2.json cast_real_invalid3 vvp_tests/cast_real_invalid3.json cast_real_invalid4 vvp_tests/cast_real_invalid4.json comment1 vvp_tests/comment1.json constfunc4_ams vvp_tests/constfunc4_ams.json -constfunc4_ams-vlog95 vvp_tests/constfunc4_ams-vlog95.json +#constfunc4_ams-vlog95 vvp_tests/constfunc4_ams-vlog95.json constfunc6_ams vvp_tests/constfunc6_ams.json -constfunc6_ams-vlog95 vvp_tests/constfunc6_ams-vlog95.json +#constfunc6_ams-vlog95 vvp_tests/constfunc6_ams-vlog95.json constfunc16 vvp_tests/constfunc16.json constfunc17 vvp_tests/constfunc17.json constfunc18 vvp_tests/constfunc18.json @@ -119,7 +119,7 @@ early_sig_elab1 vvp_tests/early_sig_elab1.json early_sig_elab2 vvp_tests/early_sig_elab2.json early_sig_elab3 vvp_tests/early_sig_elab3.json eofmt_percent vvp_tests/eofmt_percent.json -eofmt_percent-vlog95 vvp_tests/eofmt_percent-vlog95.json +#eofmt_percent-vlog95 vvp_tests/eofmt_percent-vlog95.json fdisplay3 vvp_tests/fdisplay3.json final3 vvp_tests/final3.json fmonitor1 vvp_tests/fmonitor1.json @@ -127,7 +127,7 @@ fmonitor2 vvp_tests/fmonitor2.json fread-error vvp_tests/fread-error.json line_directive vvp_tests/line_directive.json localparam_type vvp_tests/localparam_type.json -localparam_type-vlog95 vvp_tests/localparam_type-vlog95.json +#localparam_type-vlog95 vvp_tests/localparam_type-vlog95.json macro_str_esc vvp_tests/macro_str_esc.json memsynth1 vvp_tests/memsynth1.json memsynth2 vvp_tests/memsynth2.json @@ -168,11 +168,11 @@ partsel_reversed_idx5 vvp_tests/partsel_reversed_idx5.json partsel_reversed_idx6 vvp_tests/partsel_reversed_idx6.json param_test3 vvp_tests/param_test3.json param-width vvp_tests/param-width.json -param-width-vlog95 vvp_tests/param-width-vlog95.json +#param-width-vlog95 vvp_tests/param-width-vlog95.json parameter_type vvp_tests/parameter_type.json -parameter_type-vlog95 vvp_tests/parameter_type-vlog95.json +#parameter_type-vlog95 vvp_tests/parameter_type-vlog95.json pr1388974 vvp_tests/pr1388974.json -pr1388974-vlog95 vvp_tests/pr1388974-vlog95.json +#pr1388974-vlog95 vvp_tests/pr1388974-vlog95.json pr1701890 vvp_tests/pr1701890.json pr1864110a vvp_tests/pr1864110a.json pr1864110b vvp_tests/pr1864110b.json @@ -180,17 +180,17 @@ pr1864115 vvp_tests/pr1864115.json pr1916261a vvp_tests/pr1916261a.json pr1916261a vvp_tests/pr1916261a.json pr2509349a vvp_tests/pr2509349a.json -pr2509349a-vlog95 vvp_tests/pr2509349a-vlog95.json +#pr2509349a-vlog95 vvp_tests/pr2509349a-vlog95.json pr2509349b vvp_tests/pr2509349b.json -pr2509349b-vlog95 vvp_tests/pr2509349b-vlog95.json +#pr2509349b-vlog95 vvp_tests/pr2509349b-vlog95.json pr2800985b vvp_tests/pr2800985b.json pr3270320_ams vvp_tests/pr3270320_ams.json pr903 vvp_tests/pr903.json -pr903-vlog95 vvp_tests/pr903-vlog95.json +#pr903-vlog95 vvp_tests/pr903-vlog95.json program2b vvp_tests/program2b.json -program2b-vlog95 vvp_tests/program2b-vlog95.json +#program2b-vlog95 vvp_tests/program2b-vlog95.json program3a vvp_tests/program3a.json -program3a-vlog95 vvp_tests/program3a-vlog95.json +#program3a-vlog95 vvp_tests/program3a-vlog95.json pv_wr_fn_vec2 vvp_tests/pv_wr_fn_vec2.json pv_wr_fn_vec4 vvp_tests/pv_wr_fn_vec4.json queue_fail vvp_tests/queue_fail.json @@ -293,11 +293,11 @@ value_range1 vvp_tests/value_range1.json value_range2 vvp_tests/value_range2.json value_range3 vvp_tests/value_range3.json vams_abs1 vvp_tests/vams_abs1.json -vams_abs1-vlog95 vvp_tests/vams_abs1-vlog95.json +#vams_abs1-vlog95 vvp_tests/vams_abs1-vlog95.json vams_abs2 vvp_tests/vams_abs2.json -vams_abs2-vlog95 vvp_tests/vams_abs2-vlog95.json +#vams_abs2-vlog95 vvp_tests/vams_abs2-vlog95.json vams_abs3 vvp_tests/vams_abs3.json -vams_abs3-vlog95 vvp_tests/vams_abs3-vlog95.json +#vams_abs3-vlog95 vvp_tests/vams_abs3-vlog95.json vardly_undefined_vec vvp_tests/vardly_undefined_vec.json va_math vvp_tests/va_math.json vvp_quiet_mode vvp_tests/vvp_quiet_mode.json diff --git a/ivtest/run_ivl.py b/ivtest/run_ivl.py index 4f9af2822..08f7c6c95 100644 --- a/ivtest/run_ivl.py +++ b/ivtest/run_ivl.py @@ -8,23 +8,26 @@ import os import sys import re -def assemble_iverilog_cmd(suffix: str, source: str, it_dir: str, - args: list, outfile = "a.out") -> list: +def assemble_iverilog_cmd(options: dict, cfg: dict, outfile: str) -> list: '''Build the iverilog command line''' - res = ["iverilog"+suffix, "-o", os.path.join("work", outfile)] - res += ["-D__ICARUS_UNSIZED__"] - res += args - src = os.path.join(it_dir, source) - res += [src] + res = [] + if cfg['with-valgrind']: + res += ["valgrind", "--trace-children=yes"] + res += ["iverilog"+cfg['suffix'], "-o", os.path.join("work", outfile)] + res += options['iverilog_args'] + res += [os.path.join(options['directory'], options['source'])] return res -def assemble_vvp_cmd(suffix: str, args: list, plusargs: list) -> list: +def assemble_vvp_cmd(options: dict, cfg: dict) -> list: '''Build the vvp command line''' - res = ["vvp"+suffix] - res = res + args + res = [] + if cfg['with-valgrind']: + res += ["valgrind", "--leak-check=full", "--show-reachable=yes"] + res += ["vvp"+cfg['suffix']] + res += options['vvp_args'] res.append(os.path.join("work", "a.out")) - res = res + plusargs + res += options['vvp_args_extended'] return res @@ -95,11 +98,14 @@ def compare_files(log_path, gold_path): '''Compare the log file and the gold file The files are read it, line at a time, and the lines are compared. - If they differ, then write tou stdout a unified diff. In any case, + If they differ, then write to stdout a unified diff. In any case, return True or False to indicate the results of the test.''' with open(log_path, 'rt', encoding='ascii') as fd: - a = fd.readlines() + a_raw = fd.readlines() + # Remove the valgrind lines from the log. They start with ==PID== + # and error messages are **PID** + a = list(filter(lambda item: not re.match('(==\\d+==)|(\\*\\*\\d+\\*\\*)', item), a_raw)) # Allow to omit empty gold files if os.path.exists(gold_path): @@ -108,47 +114,32 @@ def compare_files(log_path, gold_path): else: b = [] - flag = a == b - if not flag: + if a != b: # pylint: disable-next=consider-using-f-string print("{log} and {gold} differ:".format(log=log_path, gold=gold_path)) sys.stdout.writelines(difflib.unified_diff(a, b, log_path, gold_path)) + return False + + return True - return flag def run_cmd(cmd: list) -> subprocess.CompletedProcess: '''Run the given command''' res = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) return res -def check_gold(it_key: str, it_gold: str, log_list: list) -> bool: - '''Check if the log and gold file match''' - compared = True - for log_name in log_list: - # pylint: disable-next=consider-using-f-string - log_path = os.path.join("log", "{key}-{log}.log".format(key=it_key, log=log_name)) - # pylint: disable-next=consider-using-f-string - gold_path = os.path.join("gold", "{gold}-{log}.gold".format(gold=it_gold, log=log_name)) - compared = compared and compare_files(log_path, gold_path) - - return compared # pylint: disable-next=invalid-name -def run_CE(options: dict) -> list: +def run_CE(options: dict, cfg: dict) -> list: ''' Run the compiler, and expect an error In this case, we assert that the command fails to run and reports an error. This is to check that invalid input generates errors.''' - suffix = options['suffix'] it_key = options['key'] - it_dir = options['directory'] - it_args = options['iverilog_args'] - it_gold = options['gold'] - build_runtime(it_key) - cmd = assemble_iverilog_cmd(suffix, options['source'], it_dir, it_args) + cmd = assemble_iverilog_cmd(options, cfg, 'a.out') res = run_cmd(cmd) log_results(it_key, "iverilog", res) @@ -156,58 +147,70 @@ def run_CE(options: dict) -> list: if res.returncode == 0: return [1, "Failed - CE (no error reported)"] - elif res.returncode >= 256: + if res.returncode >= 256: return [1, "Failed - CE (execution error)"] - elif it_gold is not None and not check_gold(it_key, it_gold, log_list): + if options['gold'] is not None and not check_gold(options, log_list): return [1, "Failed - CE (Gold output doesn't match actual output.)"] - else: - return [0, "Passed - CE"] + return [0, "Passed - CE"] -def check_run_outputs(options: dict, it_stdout: str, log_list: list) -> list: - '''Check the output files, and return success for failed. + +def check_gold(options: dict, log_list: list) -> list: + '''Check if the log and gold file match''' + compared = True + for log_name in log_list: + # pylint: disable-next=consider-using-f-string + log_path = os.path.join("log", "{key}-{log}.log".format(key=options['key'], + log=log_name)) + # pylint: disable-next=consider-using-f-string + gold_path = os.path.join("gold", "{gold}-{log}.gold".format(gold=options['gold'], + log=log_name)) + compared = compared and compare_files(log_path, gold_path) + + if compared: + return [0, "Passed"] + return [1, "Failed - Gold output doesn't match actual output."] + + +def check_diff(fname1: str, fname2: str, skip: int, expected_fail: bool) -> list: + '''Check if the difference files match after skipping the number of + specified lines''' + with open(fname1, 'rt', encoding='ascii') as fd: + # pylint: disable-next=unused-variable + for idx in range(skip): + fd.readline() + data1 = fd.read() + + with open(fname2, 'rt', encoding='ascii') as fd: + # pylint: disable-next=unused-variable + for idx in range(skip): + fd.readline() + data2 = fd.read() + + if data1 == data2: + return [0, "Passed"] + if expected_fail: + return [0, "Passed - EF"] + # pylint: disable-next=consider-using-f-string + return [1, "Failed - Files {name1} and {name2} differ.".format(name1=fname1, + name2=fname2)] + +def check_run_outputs(options: dict, it_stdout: str, log_list: list, + expected_fail: bool) -> list: + '''Check the output files, and return success or failed. This function takes an options dictionary that describes the settings, and the output from the final command. This also takes a list of log files to check - there there are gold files present.''' + if there are gold files present.''' - # Get the options this step needs... - it_key = options['key'] - it_gold = options['gold'] - it_diff = options['diff'] - - if it_gold is not None: - compared = check_gold(it_key, it_gold, log_list) - - if compared: - return [0, "Passed"] - else: - return [1, "Failed - Gold output doesn't match actual output."] + # Check the results against the gold file if it exists. + if options['gold'] is not None: + return check_gold(options, log_list) # If there is a diff description, then compare named files instead of # the log and a gold file. + it_diff = options['diff'] if it_diff is not None: - diff_name1 = it_diff[0] - diff_name2 = it_diff[1] - diff_skip = int(it_diff[2]) - - with open(diff_name1, 'rt', encoding='ascii') as fd: - # pylint: disable-next=unused-variable - for idx in range(diff_skip): - fd.readline() - diff_data1 = fd.read() - - with open(diff_name2, 'rt', encoding='ascii') as fd: - for idx in range(diff_skip): - fd.readline() - diff_data2 = fd.read() - - if diff_data1 == diff_data2: - return [0, "Passed"] - else: - # pylint: disable-next=consider-using-f-string - return [1, "Failed - Files {name1} and {name2} differ.".format(name1=diff_name1, \ - name2=diff_name2)] - + return check_diff(it_diff[0], it_diff[1], it_diff[2], expected_fail) # Otherwise, look for the PASSED output string in stdout. for line in it_stdout.splitlines(): @@ -215,12 +218,14 @@ def check_run_outputs(options: dict, it_stdout: str, log_list: list) -> list: return [0, "Passed"] # If there is no PASSED output, and nothing else to check, then - # assume a failure. + # assume a failure unless a fail is expected. + if expected_fail: + return [0, "Passed - EF"] return [1, "Failed - No PASSED output, and no gold file"] # pylint: disable-next=unused-argument -def do_run_normal_vlog95(options: dict, expected_fail: bool) -> list: +def do_run_normal_vlog95(options: dict, cfg: dict, expected_fail: bool) -> list: '''Run the iverilog and vvp commands. In this case, run the compiler with the -tvlog95 flag to generate @@ -228,17 +233,12 @@ def do_run_normal_vlog95(options: dict, expected_fail: bool) -> list: a vvp out. Run that vvp output to test the simulation results. Collect the results and look for a "PASSED" string.''' - suffix = options['suffix'] it_key = options['key'] - it_dir = options['directory'] - it_iverilog_args = ["-tvlog95"] + options['iverilog_args'] - it_vvp_args = options['vvp_args'] - it_vvp_args_extended = options['vvp_args_extended'] - build_runtime(it_key) # Run the first iverilog command, to generate the intermediate verilog - ivl1_cmd = assemble_iverilog_cmd(suffix, options['source'], it_dir, it_iverilog_args, "a.out.v") + # FIXME: this doesn't work since -tvlog95 is not passed + ivl1_cmd = assemble_iverilog_cmd(options, cfg, "a.out.v") ivl1_res = run_cmd(ivl1_cmd) log_results(it_key, "iverilog", ivl1_res) @@ -246,7 +246,8 @@ def do_run_normal_vlog95(options: dict, expected_fail: bool) -> list: return [1, "Failed - Compile failed"] # Run another iverilog command to compile the code generated from the first step. - ivl2_cmd = assemble_iverilog_cmd(suffix, "a.out.v", "work", [ ], "a.out") + # FIXME: this doesn't currently work since the vlog95 path needs to be used + ivl2_cmd = assemble_iverilog_cmd(options, cfg, 'a.out') ivl2_res = run_cmd(ivl2_cmd) log_results(it_key, "iverilog-vlog95", ivl2_res) @@ -254,39 +255,33 @@ def do_run_normal_vlog95(options: dict, expected_fail: bool) -> list: return [1, "Failed - Compile of generated code failed"] # Run the vvp command - vvp_cmd = assemble_vvp_cmd(suffix, it_vvp_args, it_vvp_args_extended) + vvp_cmd = assemble_vvp_cmd(options, cfg) vvp_res = run_cmd(vvp_cmd) log_results(it_key, "vvp", vvp_res) if vvp_res.returncode != 0: - return [1, "Failed - Vvp execution failed"] + return [1, "Failed - vvp execution failed"] it_stdout = vvp_res.stdout.decode('ascii') log_list = ["iverilog-stdout", "iverilog-stderr", "iverilog-vlog95-stdout", "iverilog-vlog95-stderr", "vvp-stdout", "vvp-stderr"] - return check_run_outputs(options, it_stdout, log_list) + return check_run_outputs(options, it_stdout, log_list, expected_fail) -def do_run_normal(options: dict, expected_fail: bool) -> list: +def do_run_normal(options: dict, cfg: dict, expected_fail: bool) -> list: '''Run the iverilog and vvp commands. In this case, run the compiler to generate a vvp output file, and run the vvp command to actually execute the simulation. Collect the results and look for a "PASSED" string.''' - suffix = options['suffix'] it_key = options['key'] - it_dir = options['directory'] - it_iverilog_args = options['iverilog_args'] - it_vvp_args = options['vvp_args'] - it_vvp_args_extended = options['vvp_args_extended'] - build_runtime(it_key) # Run the iverilog command - ivl_cmd = assemble_iverilog_cmd(suffix, options['source'], it_dir, it_iverilog_args) + ivl_cmd = assemble_iverilog_cmd(options, cfg, 'a.out') ivl_res = run_cmd(ivl_cmd) log_results(it_key, "iverilog", ivl_res) @@ -294,37 +289,37 @@ def do_run_normal(options: dict, expected_fail: bool) -> list: return [1, "Failed - Compile failed"] # run the vvp command - vvp_cmd = assemble_vvp_cmd(suffix, it_vvp_args, it_vvp_args_extended) + vvp_cmd = assemble_vvp_cmd(options, cfg) vvp_res = run_cmd(vvp_cmd) log_results(it_key, "vvp", vvp_res) - if vvp_res.returncode == 0 and expected_fail: - return [1, "Failed - Vvp execution did not fail, but was expted to fail."] + if vvp_res.returncode != 0 and expected_fail: + return [0, "Passed - EF"] if vvp_res.returncode >= 256: - return [1, "Failed - Vvp execution error"] + return [1, "Failed - vvp execution error"] if vvp_res.returncode > 0 and vvp_res.returncode < 256 and not expected_fail: - return [1, "Failed - Vvp error, but expected to succeed"] + return [1, "Failed - vvp error, but expected to succeed"] it_stdout = vvp_res.stdout.decode('ascii') log_list = ["iverilog-stdout", "iverilog-stderr", "vvp-stdout", "vvp-stderr"] - return check_run_outputs(options, it_stdout, log_list) + return check_run_outputs(options, it_stdout, log_list, expected_fail) -def run_normal(options: dict) -> list: +def run_normal(options: dict, cfg: dict) -> list: '''Run a normal test''' - return do_run_normal(options, False) + return do_run_normal(options, cfg, False) # pylint: disable-next=invalid-name -def run_EF(options: dict) -> list: +def run_EF(options: dict, cfg: dict) -> list: '''Run an expected fail test''' - return do_run_normal(options, True) + return do_run_normal(options, cfg, True) -def run_normal_vlog95(options: dict) -> list: +def run_normal_vlog95(options: dict, cfg: dict) -> list: '''Run a vlog95 test''' - return do_run_normal_vlog95(options, False) + return do_run_normal_vlog95(options, cfg, False) # pylint: disable-next=invalid-name -def run_EF_vlog95(options: dict) -> list: +def run_EF_vlog95(options: dict, cfg: dict) -> list: '''Run an expected fail vlog95 test''' - return do_run_normal_vlog95(options, True) + return do_run_normal_vlog95(options, cfg, True) diff --git a/ivtest/vvp_reg.py b/ivtest/vvp_reg.py index 890c5712d..e4235d795 100755 --- a/ivtest/vvp_reg.py +++ b/ivtest/vvp_reg.py @@ -18,7 +18,64 @@ import test_lists import run_ivl -def process_test(suffix: str, item: list) -> str: +class InvalidTestType(Exception): + '''Exception to raise when the test type is not supported.''' + def __init__(self, test, ttype, msg='Invalid test type!'): + self.test = test + self.ttype = ttype + self.msg = msg + super().__init__(self.msg) + + def __str__(self): + # pylint: disable-next=consider-using-f-string + return "Given test type '{ttype}' for test {test}".format(ttype=self.ttype,test=self.test) + +class InvalidJSON(Exception): + '''Exception to raise if the JSON is not parsed properly.''' + def __init__(self, test, path, msg='Invalid JSON file!'): + self.test = test + self.path = path + self.msg = msg + super().__init__(self.msg) + + def __str__(self): + # pylint: disable-next=consider-using-f-string + res = "Unable to parse JSON file '{path}' for test {test}:".format(path=self.path, + test=self.test) + # pylint: disable-next=consider-using-f-string + res += "\n {msg}".format(msg=self.msg) + return res + + +def process_overrides(group: str, it_dict: dict, it_opts: dict): + '''Override the gold file or type if needed.''' + if group in it_dict: + overrides = ['gold', 'type'] + for override in overrides: + if override in it_dict[group]: + if override == 'gold' and it_dict[group][override] == "": + it_opts[override] = None + else: + it_opts[override] = it_dict[group][override] + + +def force_gen(it_opts: dict): + '''Remove the current generation and force it to the latest.''' + generations = ['-g2012', '-g2009', '-g2005-sv', + '-g2005', '-g2001-noconfig', '-g2001', '-g1995', + '-g2', '-g1'] + for gen in generations: + if gen in it_opts['iverilog_args']: + it_opts['iverilog_args'].remove(gen) + + if '-g2x' in it_opts['iverilog_args']: + idx_to_replace = it_opts['iverilog_args'].index('-g2x') + it_opts[idx_to_replace] = '-gicarus-misc' + + it_opts['iverilog_args'].insert(0, '-g2012') + + +def process_test(item: list, cfg: list) -> str: '''Process a single test This takes in the list of tokens from the tests list file, and converts @@ -28,16 +85,15 @@ def process_test(suffix: str, item: list) -> str: it_key = item[0] test_path = item[1] with open(test_path, 'rt', encoding='ascii') as fd: - it_dict = json.load(fd) - - # Get the test type from the json configuration. - it_type = it_dict['type'] + try: + it_dict = json.load(fd) + except json.decoder.JSONDecodeError as exception: + raise InvalidJSON(it_key, test_path, exception) from exception # Wrap all of this into an options dictionary for ease of handling. - it_options = { - 'suffix' : suffix, + it_opts = { 'key' : it_key, - 'type' : it_type, + 'type' : it_dict['type'], 'iverilog_args' : it_dict.get('iverilog-args', [ ]), 'directory' : "ivltests", 'source' : it_dict['source'], @@ -48,32 +104,70 @@ def process_test(suffix: str, item: list) -> str: 'vvp_args_extended' : it_dict.get('vvp-args-extended', [ ]) } + if cfg['strict']: + it_opts['iverilog_args'].append("-gstrict-expr-width") + it_opts['vvp_args_extended'].append("-compatible") + process_overrides('strict', it_dict, it_opts) + else: + it_opts['iverilog_args'].append('-D__ICARUS_UNSIZED__') + + if cfg['force-sv']: + force_gen(it_opts) + process_overrides('force-sv', it_dict, it_opts) + + # Get the overridden test type. + it_type = it_opts['type'] + if it_type == "NI": res = [0, "Not Implemented."] elif it_type == "normal": - res = run_ivl.run_normal(it_options) + res = run_ivl.run_normal(it_opts, cfg) elif it_type == "normal-vlog95": - res = run_ivl.run_normal_vlog95(it_options) + res = run_ivl.run_normal_vlog95(it_opts, cfg) elif it_type == "CE": - res = run_ivl.run_CE(it_options) + res = run_ivl.run_CE(it_opts, cfg) elif it_type == "EF": - res = run_ivl.run_EF(it_options) + res = run_ivl.run_EF(it_opts, cfg) elif it_type == "EF-vlog95": - res = run_ivl.run_EF_vlog95(it_options) + res = run_ivl.run_EF_vlog95(it_opts, cfg) else: - # pylint: disable-next=consider-using-f-string - res = "{key}: I don't understand the test type ({type}).".format(key=it_key, type=it_type) - raise Exception(res) + raise InvalidTestType(it_key, it_type) return res +def print_header(cfg: dict, files: list): + '''Print all the header information. ''' + # This returns 13 or similar + ivl_version = run_ivl.get_ivl_version(cfg['suffix']) + + print("Running compiler/VVP tests for Icarus Verilog ", end='') + # pylint: disable-next=consider-using-f-string + print("version: {ver}".format(ver=ivl_version), end='') + if cfg['suffix']: + # pylint: disable-next=consider-using-f-string + print(", suffix: {suffix}".format(suffix=cfg['suffix']), end='') + if cfg['strict']: + if cfg['force-sv']: + print(" (strict, force SV)", end='') + else: + print(" (strict)", end='') + elif cfg['force-sv']: + print(" (force SV)", end='') + if cfg['with-valgrind']: + print(" (valgrind)", end='') + print("") + # pylint: disable-next=consider-using-f-string + print("Using list(s): {files}".format(files=', '.join(files))) + print("-" * 76) + + if __name__ == "__main__": argp = argparse.ArgumentParser(description='') argp.add_argument('--suffix', type=str, default='', @@ -90,33 +184,19 @@ if __name__ == "__main__": help='File(s) containing a list of the tests to run, default "%(default)s".') args = argp.parse_args() - if args.strict: - print('Sorry: Forcing strict compatiability is not currently supported!') - sys.exit(1) - if args.with_valgrind: - print('Sorry: Running with valgrind is not currently supported!') - sys.exit(1) - if args.force_sv: - print('Sorry: Forcing SystemVerilog is not currently supported!') - sys.exit(1) + # FIXME: need to add real vlog95 support if args.vlog95: print('Sorry: Converting to Verilog-95 and running is not currently supported!') sys.exit(1) - # This returns 13 or similar - ivl_version = run_ivl.get_ivl_version(args.suffix) + ivl_cfg = { + 'suffix' : args.suffix, + 'strict' : args.strict, + 'with-valgrind' : args.with_valgrind, + 'force-sv' : args.force_sv + } - print("Running compiler/VVP tests for Icarus Verilog ", end='') - # pylint: disable-next=consider-using-f-string - print("version: {ver}".format(ver=ivl_version), end='') - if args.suffix: - # pylint: disable-next=consider-using-f-string - print(", suffix: {suffix}".format(suffix=args.suffix), end='') - # FIXME: add strict, force SV and with valgrind - print("") - # pylint: disable-next=consider-using-f-string - print("Using list(s): {files}".format(files=', '.join(args.files))) - print("-" * 76) + print_header(ivl_cfg, args.files) # Read the list files, to get the tests. tests_list = test_lists.read_lists(args.files) @@ -132,7 +212,7 @@ if __name__ == "__main__": # pylint: disable-next=invalid-name error_count = 0 for cur in tests_list: - result = process_test(args.suffix, cur) + result = process_test(cur, ivl_cfg) error_count += result[0] # pylint: disable-next=consider-using-f-string print("{name:>{width}}: {result}".format(name=cur[0], width=width, result=result[1])) diff --git a/ivtest/vvp_tests/br_gh1087a1.json b/ivtest/vvp_tests/br_gh1087a1.json index 748a7d7e5..6e57fcc22 100644 --- a/ivtest/vvp_tests/br_gh1087a1.json +++ b/ivtest/vvp_tests/br_gh1087a1.json @@ -1,6 +1,9 @@ { "type" : "CE", "source" : "br_gh1087a.v", + "iverilog-args" : [ "-gno-xtypes" ], "gold" : "br_gh1087a1", - "iverilog-args" : [ "-gno-xtypes" ] + "force-sv" : { + "gold" : "br_gh1087a1-fsv" + } } diff --git a/ivtest/vvp_tests/br_gh1143e.json b/ivtest/vvp_tests/br_gh1143e.json index 83c311796..7baeb4f69 100644 --- a/ivtest/vvp_tests/br_gh1143e.json +++ b/ivtest/vvp_tests/br_gh1143e.json @@ -2,5 +2,9 @@ "type" : "CE", "source" : "br_gh1143a.v", "iverilog-args" : [ "-g2009" ], - "gold" : "br_gh1143e" + "gold" : "br_gh1143e", + "force-sv" : { + "type" : "normal", + "gold" : "" + } } diff --git a/ivtest/vvp_tests/br_gh1143f.json b/ivtest/vvp_tests/br_gh1143f.json index ef90be248..88f93d8a7 100644 --- a/ivtest/vvp_tests/br_gh1143f.json +++ b/ivtest/vvp_tests/br_gh1143f.json @@ -2,5 +2,9 @@ "type" : "CE", "source" : "br_gh1143b.v", "iverilog-args" : [ "-g2009" ], - "gold" : "br_gh1143f" + "gold" : "br_gh1143f", + "force-sv" : { + "type" : "normal", + "gold" : "" + } } diff --git a/ivtest/vvp_tests/br_gh1143g.json b/ivtest/vvp_tests/br_gh1143g.json index 11c3a3c73..95145c714 100644 --- a/ivtest/vvp_tests/br_gh1143g.json +++ b/ivtest/vvp_tests/br_gh1143g.json @@ -2,5 +2,9 @@ "type" : "CE", "source" : "br_gh1143c.v", "iverilog-args" : [ "-g2009" ], - "gold" : "br_gh1143g" + "gold" : "br_gh1143g", + "force-sv" : { + "type" : "normal", + "gold" : "" + } } diff --git a/ivtest/vvp_tests/br_gh1143h.json b/ivtest/vvp_tests/br_gh1143h.json index 2f2315ebd..32ee25e03 100644 --- a/ivtest/vvp_tests/br_gh1143h.json +++ b/ivtest/vvp_tests/br_gh1143h.json @@ -2,5 +2,9 @@ "type" : "CE", "source" : "br_gh1143d.v", "iverilog-args" : [ "-g2009" ], - "gold" : "br_gh1143h" + "gold" : "br_gh1143h", + "force-sv" : { + "type" : "normal", + "gold" : "" + } } diff --git a/ivtest/vvp_tests/br_gh13a.json b/ivtest/vvp_tests/br_gh13a.json index 3be446b17..d34efc78b 100644 --- a/ivtest/vvp_tests/br_gh13a.json +++ b/ivtest/vvp_tests/br_gh13a.json @@ -1,5 +1,8 @@ { "type" : "normal", "source" : "br_gh13a.v", - "gold" : "br_gh13a" + "gold" : "br_gh13a", + "strict" : { + "gold" : "br_gh13a_std" + } } diff --git a/ivtest/vvp_tests/br_gh383d.json b/ivtest/vvp_tests/br_gh383d.json index 789417aff..95de900a4 100644 --- a/ivtest/vvp_tests/br_gh383d.json +++ b/ivtest/vvp_tests/br_gh383d.json @@ -2,5 +2,8 @@ "type" : "normal", "source" : "br_gh383d.v", "iverilog-args" : [ "-g2012" ], - "gold" : "br_gh383d" + "gold" : "br_gh383d", + "strict" : { + "gold" : "br_gh383d_std" + } } diff --git a/ivtest/vvp_tests/ca_time_real.json b/ivtest/vvp_tests/ca_time_real.json index 8e0ad33fc..82e12b21d 100644 --- a/ivtest/vvp_tests/ca_time_real.json +++ b/ivtest/vvp_tests/ca_time_real.json @@ -1,5 +1,8 @@ { "type" : "normal", "source" : "ca_time_real.v", - "gold" : "ca_time_real" + "gold" : "ca_time_real", + "strict" : { + "gold" : "ca_time_real_std" + } } diff --git a/ivtest/vvp_tests/delayed_sfunc.json b/ivtest/vvp_tests/delayed_sfunc.json index a0ef1bf69..29dbb8da8 100644 --- a/ivtest/vvp_tests/delayed_sfunc.json +++ b/ivtest/vvp_tests/delayed_sfunc.json @@ -1,6 +1,9 @@ { "type" : "normal", "source" : "delayed_sfunc.v", + "iverilog-args" : [ "-gspecify" ], "gold" : "delayed_sfunc", - "iverilog-args" : [ "-gspecify" ] + "strict" : { + "gold" : "delayed_sfunc_std" + } } diff --git a/ivtest/vvp_tests/localparam_type.json b/ivtest/vvp_tests/localparam_type.json index f52acfc7c..4b777ff98 100644 --- a/ivtest/vvp_tests/localparam_type.json +++ b/ivtest/vvp_tests/localparam_type.json @@ -1,5 +1,8 @@ { "type" : "normal", "source" : "localparam_type.v", - "gold" : "parameter_type" + "gold" : "parameter_type", + "strict" : { + "gold" : "parameter_type_std" + } } diff --git a/ivtest/vvp_tests/param-width.json b/ivtest/vvp_tests/param-width.json index b6e0a210a..92c6dad7a 100644 --- a/ivtest/vvp_tests/param-width.json +++ b/ivtest/vvp_tests/param-width.json @@ -1,5 +1,8 @@ { "type" : "normal", "source" : "param-width.v", - "gold" : "param-width" + "gold" : "param-width", + "strict" : { + "gold" : "param-width_std" + } } diff --git a/ivtest/vvp_tests/parameter_type.json b/ivtest/vvp_tests/parameter_type.json index 4e28a984a..fe0793c19 100644 --- a/ivtest/vvp_tests/parameter_type.json +++ b/ivtest/vvp_tests/parameter_type.json @@ -1,5 +1,8 @@ { "type" : "normal", "source" : "parameter_type.v", - "gold" : "parameter_type" + "gold" : "parameter_type", + "strict" : { + "gold" : "parameter_type_std" + } } diff --git a/ivtest/vvp_tests/pr1388974.json b/ivtest/vvp_tests/pr1388974.json index fac4e13d4..710580af2 100644 --- a/ivtest/vvp_tests/pr1388974.json +++ b/ivtest/vvp_tests/pr1388974.json @@ -1,4 +1,7 @@ { "type" : "normal", - "source" : "pr1388974.v" + "source" : "pr1388974.v", + "strict" : { + "type" : "EF" + } } diff --git a/ivtest/vvp_tests/pr1701890.json b/ivtest/vvp_tests/pr1701890.json index b9b71010f..a159d28dd 100644 --- a/ivtest/vvp_tests/pr1701890.json +++ b/ivtest/vvp_tests/pr1701890.json @@ -1,5 +1,8 @@ { "type" : "normal", "source" : "pr1701890.v", - "gold" : "pr1701890" + "gold" : "pr1701890", + "strict" : { + "gold" : "pr1701890_std" + } } diff --git a/ivtest/vvp_tests/pr1864110a.json b/ivtest/vvp_tests/pr1864110a.json index ad44dda8b..287ed576d 100644 --- a/ivtest/vvp_tests/pr1864110a.json +++ b/ivtest/vvp_tests/pr1864110a.json @@ -1,5 +1,8 @@ { "type" : "normal", "source" : "pr1864110a.v", - "gold" : "pr1864110a" + "gold" : "pr1864110a", + "strict" : { + "gold" : "pr1864110a_std" + } } diff --git a/ivtest/vvp_tests/pr1864110b.json b/ivtest/vvp_tests/pr1864110b.json index 0f096e17e..bff88dbad 100644 --- a/ivtest/vvp_tests/pr1864110b.json +++ b/ivtest/vvp_tests/pr1864110b.json @@ -1,5 +1,8 @@ { "type" : "normal", "source" : "pr1864110b.v", - "gold" : "pr1864110b" + "gold" : "pr1864110b", + "strict" : { + "gold" : "pr1864110b_std" + } } diff --git a/ivtest/vvp_tests/pr1864115.json b/ivtest/vvp_tests/pr1864115.json index 9a389b4a6..0d8f115f4 100644 --- a/ivtest/vvp_tests/pr1864115.json +++ b/ivtest/vvp_tests/pr1864115.json @@ -1,5 +1,8 @@ { "type" : "normal", "source" : "pr1864115.v", - "gold" : "pr1864115" + "gold" : "pr1864115", + "strict" : { + "gold" : "pr1864115_std" + } } diff --git a/ivtest/vvp_tests/pr903.json b/ivtest/vvp_tests/pr903.json index fcdcb1b7c..27e6f551f 100644 --- a/ivtest/vvp_tests/pr903.json +++ b/ivtest/vvp_tests/pr903.json @@ -1,4 +1,7 @@ { "type" : "normal", - "source" : "pr903.v" + "source" : "pr903.v", + "strict" : { + "type" : "EF" + } }