Update vvp_reg.py to support strict, force-sv and with-valgrind

ivtest/gold/br_gh1087a1-fsv-iverilog-stderr.gold

@@ -0,0 +1,2 @@
+ivltests/br_gh1087a.v:3: error: Net `r` can not be of type `real`.
+1 error(s) during elaboration.

ivtest/gold/br_gh13a_std-vvp-stdout.gold

@@ -0,0 +1,2 @@
+0
+PASSED

ivtest/gold/br_gh383d_std-vvp-stdout.gold

@@ -0,0 +1,3 @@
+1 2 3 4
+4 3 2 1
+8 7 6 5

ivtest/gold/ca_time_real_std-vvp-stdout.gold

@@ -0,0 +1,5 @@
+0
+1
+0
+3
+0

ivtest/gold/delayed_sfunc_std-vvp-stdout.gold

@@ -0,0 +1,4 @@
+0 0         x          x          10
+1 0         4          4          10
+5 0         4          4          20
+6 0         5          5          20

ivtest/gold/param-width_std-vvp-stdout.gold

@@ -0,0 +1,4 @@
+1001
+001
+0101
+01

ivtest/gold/parameter_type_std-vvp-stdout.gold

@@ -0,0 +1,8 @@
+Implicit real: -1
+Implicit integer:          -1
+Unsigned: 255
+Signed:   -1
+Real: -1
+Real time: -1
+Integer:          -1
+Time: 18446744073709551615

ivtest/gold/pr1701890_std-vvp-stdout.gold

@@ -0,0 +1 @@
+rval1=1 rval2=2 rtval=1

ivtest/gold/pr1864110a_std-vvp-stdout.gold

@@ -0,0 +1 @@
+3 -3

ivtest/gold/pr1864110b_std-vvp-stdout.gold

@@ -0,0 +1,3 @@
+-3 3
+-4 4
+-6 6

ivtest/gold/pr1864115_std-vvp-stdout.gold

@@ -0,0 +1,2 @@
+0                    0 0
+2  4611686018427387904 2

ivtest/regress-vvp.list

@@ -16,9 +16,9 @@ automatic_error13		vvp_tests/automatic_error13.json
 bits4				vvp_tests/bits4.json
 bitsel11			vvp_tests/bitsel11.json
 br_gh13a			vvp_tests/br_gh13a.json
-br_gh13a-vlog95			vvp_tests/br_gh13a-vlog95.json
+#br_gh13a-vlog95			vvp_tests/br_gh13a-vlog95.json
 br_gh99c			vvp_tests/br_gh99c.json
-br_gh99c-vlog95			vvp_tests/br_gh99c-vlog95.json
+#br_gh99c-vlog95			vvp_tests/br_gh99c-vlog95.json
 br_gh230			vvp_tests/br_gh230.json
 br_gh383a			vvp_tests/br_gh383a.json
 br_gh383b			vvp_tests/br_gh383b.json
@@ -77,16 +77,16 @@ case2-S				vvp_tests/case2-S.json
 case3				vvp_tests/case3.json
 casex_synth			vvp_tests/casex_synth.json
 cast_int_ams			vvp_tests/cast_int_ams.json
-cast_int_ams-vlog95		vvp_tests/cast_int_ams-vlog95.json
+#cast_int_ams-vlog95		vvp_tests/cast_int_ams-vlog95.json
 cast_real_invalid1		vvp_tests/cast_real_invalid1.json
 cast_real_invalid2		vvp_tests/cast_real_invalid2.json
 cast_real_invalid3		vvp_tests/cast_real_invalid3.json
 cast_real_invalid4		vvp_tests/cast_real_invalid4.json
 comment1			vvp_tests/comment1.json
 constfunc4_ams			vvp_tests/constfunc4_ams.json
-constfunc4_ams-vlog95		vvp_tests/constfunc4_ams-vlog95.json
+#constfunc4_ams-vlog95		vvp_tests/constfunc4_ams-vlog95.json
 constfunc6_ams			vvp_tests/constfunc6_ams.json
-constfunc6_ams-vlog95		vvp_tests/constfunc6_ams-vlog95.json
+#constfunc6_ams-vlog95		vvp_tests/constfunc6_ams-vlog95.json
 constfunc16			vvp_tests/constfunc16.json
 constfunc17			vvp_tests/constfunc17.json
 constfunc18			vvp_tests/constfunc18.json
@@ -119,7 +119,7 @@ early_sig_elab1			vvp_tests/early_sig_elab1.json
 early_sig_elab2			vvp_tests/early_sig_elab2.json
 early_sig_elab3			vvp_tests/early_sig_elab3.json
 eofmt_percent			vvp_tests/eofmt_percent.json
-eofmt_percent-vlog95		vvp_tests/eofmt_percent-vlog95.json
+#eofmt_percent-vlog95		vvp_tests/eofmt_percent-vlog95.json
 fdisplay3			vvp_tests/fdisplay3.json
 final3				vvp_tests/final3.json
 fmonitor1			vvp_tests/fmonitor1.json
@@ -127,7 +127,7 @@ fmonitor2			vvp_tests/fmonitor2.json
 fread-error			vvp_tests/fread-error.json
 line_directive			vvp_tests/line_directive.json
 localparam_type			vvp_tests/localparam_type.json
-localparam_type-vlog95		vvp_tests/localparam_type-vlog95.json
+#localparam_type-vlog95		vvp_tests/localparam_type-vlog95.json
 macro_str_esc			vvp_tests/macro_str_esc.json
 memsynth1			vvp_tests/memsynth1.json
 memsynth2			vvp_tests/memsynth2.json
@@ -168,11 +168,11 @@ partsel_reversed_idx5		vvp_tests/partsel_reversed_idx5.json
 partsel_reversed_idx6		vvp_tests/partsel_reversed_idx6.json
 param_test3			vvp_tests/param_test3.json
 param-width			vvp_tests/param-width.json
-param-width-vlog95		vvp_tests/param-width-vlog95.json
+#param-width-vlog95		vvp_tests/param-width-vlog95.json
 parameter_type			vvp_tests/parameter_type.json
-parameter_type-vlog95		vvp_tests/parameter_type-vlog95.json
+#parameter_type-vlog95		vvp_tests/parameter_type-vlog95.json
 pr1388974			vvp_tests/pr1388974.json
-pr1388974-vlog95		vvp_tests/pr1388974-vlog95.json
+#pr1388974-vlog95		vvp_tests/pr1388974-vlog95.json
 pr1701890			vvp_tests/pr1701890.json
 pr1864110a			vvp_tests/pr1864110a.json
 pr1864110b			vvp_tests/pr1864110b.json
@@ -180,17 +180,17 @@ pr1864115			vvp_tests/pr1864115.json
 pr1916261a			vvp_tests/pr1916261a.json
 pr1916261a			vvp_tests/pr1916261a.json
 pr2509349a			vvp_tests/pr2509349a.json
-pr2509349a-vlog95		vvp_tests/pr2509349a-vlog95.json
+#pr2509349a-vlog95		vvp_tests/pr2509349a-vlog95.json
 pr2509349b			vvp_tests/pr2509349b.json
-pr2509349b-vlog95		vvp_tests/pr2509349b-vlog95.json
+#pr2509349b-vlog95		vvp_tests/pr2509349b-vlog95.json
 pr2800985b			vvp_tests/pr2800985b.json
 pr3270320_ams			vvp_tests/pr3270320_ams.json
 pr903				vvp_tests/pr903.json
-pr903-vlog95			vvp_tests/pr903-vlog95.json
+#pr903-vlog95			vvp_tests/pr903-vlog95.json
 program2b			vvp_tests/program2b.json
-program2b-vlog95		vvp_tests/program2b-vlog95.json
+#program2b-vlog95		vvp_tests/program2b-vlog95.json
 program3a			vvp_tests/program3a.json
-program3a-vlog95		vvp_tests/program3a-vlog95.json
+#program3a-vlog95		vvp_tests/program3a-vlog95.json
 pv_wr_fn_vec2			vvp_tests/pv_wr_fn_vec2.json
 pv_wr_fn_vec4			vvp_tests/pv_wr_fn_vec4.json
 queue_fail			vvp_tests/queue_fail.json
@@ -293,11 +293,11 @@ value_range1			vvp_tests/value_range1.json
 value_range2			vvp_tests/value_range2.json
 value_range3			vvp_tests/value_range3.json
 vams_abs1			vvp_tests/vams_abs1.json
-vams_abs1-vlog95		vvp_tests/vams_abs1-vlog95.json
+#vams_abs1-vlog95		vvp_tests/vams_abs1-vlog95.json
 vams_abs2			vvp_tests/vams_abs2.json
-vams_abs2-vlog95		vvp_tests/vams_abs2-vlog95.json
+#vams_abs2-vlog95		vvp_tests/vams_abs2-vlog95.json
 vams_abs3			vvp_tests/vams_abs3.json
-vams_abs3-vlog95		vvp_tests/vams_abs3-vlog95.json
+#vams_abs3-vlog95		vvp_tests/vams_abs3-vlog95.json
 vardly_undefined_vec		vvp_tests/vardly_undefined_vec.json
 va_math				vvp_tests/va_math.json
 vvp_quiet_mode			vvp_tests/vvp_quiet_mode.json

ivtest/run_ivl.py

@@ -8,23 +8,26 @@ import os
 import sys
 import re
 
-def assemble_iverilog_cmd(suffix: str, source: str, it_dir: str,
+def assemble_iverilog_cmd(options: dict, cfg: dict, outfile: str) -> list:
-                          args: list, outfile = "a.out") -> list:
     '''Build the iverilog command line'''
-    res = ["iverilog"+suffix, "-o", os.path.join("work", outfile)]
+    res = []
-    res += ["-D__ICARUS_UNSIZED__"]
+    if cfg['with-valgrind']:
-    res += args
+        res += ["valgrind", "--trace-children=yes"]
-    src = os.path.join(it_dir, source)
+    res += ["iverilog"+cfg['suffix'], "-o", os.path.join("work", outfile)]
-    res += [src]
+    res += options['iverilog_args']
+    res += [os.path.join(options['directory'], options['source'])]
     return res
 
 
-def assemble_vvp_cmd(suffix: str, args: list, plusargs: list) -> list:
+def assemble_vvp_cmd(options: dict, cfg: dict) -> list:
     '''Build the vvp command line'''
-    res = ["vvp"+suffix]
+    res = []
-    res = res + args
+    if cfg['with-valgrind']:
+        res += ["valgrind", "--leak-check=full", "--show-reachable=yes"]
+    res += ["vvp"+cfg['suffix']]
+    res += options['vvp_args']
     res.append(os.path.join("work", "a.out"))
-    res = res + plusargs
+    res += options['vvp_args_extended']
     return res
 
 
@@ -95,11 +98,14 @@ def compare_files(log_path, gold_path):
     '''Compare the log file and the gold file
 
     The files are read it, line at a time, and the lines are compared.
-    If they differ, then write tou stdout a unified diff. In any case,
+    If they differ, then write to stdout a unified diff. In any case,
     return True or False to indicate the results of the test.'''
 
     with open(log_path, 'rt', encoding='ascii') as fd:
-        a = fd.readlines()
+        a_raw = fd.readlines()
+    # Remove the valgrind lines from the log. They start with ==PID==
+    # and error messages are **PID**
+    a = list(filter(lambda item: not re.match('(==\\d+==)|(\\*\\*\\d+\\*\\*)', item), a_raw))
 
     # Allow to omit empty gold files
     if os.path.exists(gold_path):
@@ -108,47 +114,32 @@ def compare_files(log_path, gold_path):
     else:
         b = []
 
-    flag = a == b
+    if a != b:
-    if not flag:
         # pylint: disable-next=consider-using-f-string
         print("{log} and {gold} differ:".format(log=log_path, gold=gold_path))
         sys.stdout.writelines(difflib.unified_diff(a, b, log_path, gold_path))
+        return False
+
+    return True
 
-    return flag
 
 def run_cmd(cmd: list) -> subprocess.CompletedProcess:
     '''Run the given command'''
     res = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False)
     return res
 
-def check_gold(it_key: str, it_gold: str, log_list: list) -> bool:
-    '''Check if the log and gold file match'''
-    compared = True
-    for log_name in log_list:
-        # pylint: disable-next=consider-using-f-string
-        log_path = os.path.join("log", "{key}-{log}.log".format(key=it_key, log=log_name))
-        # pylint: disable-next=consider-using-f-string
-        gold_path = os.path.join("gold", "{gold}-{log}.gold".format(gold=it_gold, log=log_name))
-        compared = compared and compare_files(log_path, gold_path)
-
-    return compared
 
 # pylint: disable-next=invalid-name
-def run_CE(options: dict) -> list:
+def run_CE(options: dict, cfg: dict) -> list:
     ''' Run the compiler, and expect an error
 
     In this case, we assert that the command fails to run and reports
     an error. This is to check that invalid input generates errors.'''
 
-    suffix = options['suffix']
     it_key = options['key']
-    it_dir = options['directory']
-    it_args = options['iverilog_args']
-    it_gold = options['gold']
-
     build_runtime(it_key)
 
-    cmd = assemble_iverilog_cmd(suffix, options['source'], it_dir, it_args)
+    cmd = assemble_iverilog_cmd(options, cfg, 'a.out')
     res = run_cmd(cmd)
     log_results(it_key, "iverilog", res)
 
@@ -156,58 +147,70 @@ def run_CE(options: dict) -> list:
 
     if res.returncode == 0:
         return [1, "Failed - CE (no error reported)"]
-    elif res.returncode >= 256:
+    if res.returncode >= 256:
         return [1, "Failed - CE (execution error)"]
-    elif it_gold is not None and not check_gold(it_key, it_gold, log_list):
+    if options['gold'] is not None and not check_gold(options, log_list):
         return [1, "Failed - CE (Gold output doesn't match actual output.)"]
-    else:
     return [0, "Passed - CE"]
 
-def check_run_outputs(options: dict, it_stdout: str, log_list: list) -> list:
-    '''Check the output files, and return success for failed.
 
-    This function takes an options dictionary that describes the settings, and
+def check_gold(options: dict, log_list: list) -> list:
-    the output from the final command. This also takes a list of log files to check
+    '''Check if the log and gold file match'''
-    there there are gold files present.'''
+    compared = True
-
+    for log_name in log_list:
-    # Get the options this step needs...
+        # pylint: disable-next=consider-using-f-string
-    it_key = options['key']
+        log_path = os.path.join("log", "{key}-{log}.log".format(key=options['key'],
-    it_gold = options['gold']
+                                                                log=log_name))
-    it_diff = options['diff']
+        # pylint: disable-next=consider-using-f-string
-
+        gold_path = os.path.join("gold", "{gold}-{log}.gold".format(gold=options['gold'],
-    if it_gold is not None:
+                                                                    log=log_name))
-        compared = check_gold(it_key, it_gold, log_list)
+        compared = compared and compare_files(log_path, gold_path)
 
     if compared:
         return [0, "Passed"]
-        else:
     return [1, "Failed - Gold output doesn't match actual output."]
 
+
+def check_diff(fname1: str, fname2: str, skip: int, expected_fail: bool) -> list:
+    '''Check if the difference files match after skipping the number of
+       specified lines'''
+    with open(fname1, 'rt', encoding='ascii') as fd:
+        # pylint: disable-next=unused-variable
+        for idx in range(skip):
+            fd.readline()
+        data1 = fd.read()
+
+    with open(fname2, 'rt', encoding='ascii') as fd:
+        # pylint: disable-next=unused-variable
+        for idx in range(skip):
+            fd.readline()
+        data2 = fd.read()
+
+    if data1 == data2:
+        return [0, "Passed"]
+    if expected_fail:
+        return [0, "Passed - EF"]
+    # pylint: disable-next=consider-using-f-string
+    return [1, "Failed - Files {name1} and {name2} differ.".format(name1=fname1,
+                                                                  name2=fname2)]
+
+def check_run_outputs(options: dict, it_stdout: str, log_list: list,
+                      expected_fail: bool) -> list:
+    '''Check the output files, and return success or failed.
+
+    This function takes an options dictionary that describes the settings, and
+    the output from the final command. This also takes a list of log files to check
+    if there are gold files present.'''
+
+    # Check the results against the gold file if it exists.
+    if options['gold'] is not None:
+        return check_gold(options, log_list)
+
     # If there is a diff description, then compare named files instead of
     # the log and a gold file.
+    it_diff = options['diff']
     if it_diff is not None:
-        diff_name1 = it_diff[0]
+        return check_diff(it_diff[0], it_diff[1], it_diff[2], expected_fail)
-        diff_name2 = it_diff[1]
-        diff_skip = int(it_diff[2])
-
-        with open(diff_name1, 'rt', encoding='ascii') as fd:
-            # pylint: disable-next=unused-variable
-            for idx in range(diff_skip):
-                fd.readline()
-            diff_data1 = fd.read()
-
-        with open(diff_name2, 'rt', encoding='ascii') as fd:
-            for idx in range(diff_skip):
-                fd.readline()
-            diff_data2 = fd.read()
-
-        if diff_data1 == diff_data2:
-            return [0, "Passed"]
-        else:
-            # pylint: disable-next=consider-using-f-string
-            return [1, "Failed - Files {name1} and {name2} differ.".format(name1=diff_name1, \
-                                                                           name2=diff_name2)]
-
 
     # Otherwise, look for the PASSED output string in stdout.
     for line in it_stdout.splitlines():
@@ -215,12 +218,14 @@ def check_run_outputs(options: dict, it_stdout: str, log_list: list) -> list:
             return [0, "Passed"]
 
     # If there is no PASSED output, and nothing else to check, then
-    # assume a failure.
+    # assume a failure unless a fail is expected.
+    if expected_fail:
+        return [0, "Passed - EF"]
     return [1, "Failed - No PASSED output, and no gold file"]
 
 
 # pylint: disable-next=unused-argument
-def do_run_normal_vlog95(options: dict, expected_fail: bool) -> list:
+def do_run_normal_vlog95(options: dict, cfg: dict, expected_fail: bool) -> list:
     '''Run the iverilog and vvp commands.
 
     In this case, run the compiler with the -tvlog95 flag to generate
@@ -228,17 +233,12 @@ def do_run_normal_vlog95(options: dict, expected_fail: bool) -> list:
     a vvp out. Run that vvp output to test the simulation results. Collect
     the results and look for a "PASSED" string.'''
 
-    suffix = options['suffix']
     it_key = options['key']
-    it_dir = options['directory']
-    it_iverilog_args = ["-tvlog95"] + options['iverilog_args']
-    it_vvp_args = options['vvp_args']
-    it_vvp_args_extended = options['vvp_args_extended']
-
     build_runtime(it_key)
 
     # Run the first iverilog command, to generate the intermediate verilog
-    ivl1_cmd = assemble_iverilog_cmd(suffix, options['source'], it_dir, it_iverilog_args, "a.out.v")
+    # FIXME: this doesn't work since -tvlog95 is not passed
+    ivl1_cmd = assemble_iverilog_cmd(options, cfg, "a.out.v")
     ivl1_res = run_cmd(ivl1_cmd)
 
     log_results(it_key, "iverilog", ivl1_res)
@@ -246,7 +246,8 @@ def do_run_normal_vlog95(options: dict, expected_fail: bool) -> list:
         return [1, "Failed - Compile failed"]
 
     # Run another iverilog command to compile the code generated from the first step.
-    ivl2_cmd = assemble_iverilog_cmd(suffix, "a.out.v", "work", [ ], "a.out")
+    # FIXME: this doesn't currently work since the vlog95 path needs to be used
+    ivl2_cmd = assemble_iverilog_cmd(options, cfg, 'a.out')
     ivl2_res = run_cmd(ivl2_cmd)
 
     log_results(it_key, "iverilog-vlog95", ivl2_res)
@@ -254,39 +255,33 @@ def do_run_normal_vlog95(options: dict, expected_fail: bool) -> list:
         return [1, "Failed - Compile of generated code failed"]
 
     # Run the vvp command
-    vvp_cmd = assemble_vvp_cmd(suffix, it_vvp_args, it_vvp_args_extended)
+    vvp_cmd = assemble_vvp_cmd(options, cfg)
     vvp_res = run_cmd(vvp_cmd)
     log_results(it_key, "vvp", vvp_res)
 
     if vvp_res.returncode != 0:
-        return [1, "Failed - Vvp execution failed"]
+        return [1, "Failed - vvp execution failed"]
 
     it_stdout = vvp_res.stdout.decode('ascii')
     log_list = ["iverilog-stdout", "iverilog-stderr",
                 "iverilog-vlog95-stdout", "iverilog-vlog95-stderr",
                 "vvp-stdout", "vvp-stderr"]
 
-    return check_run_outputs(options, it_stdout, log_list)
+    return check_run_outputs(options, it_stdout, log_list, expected_fail)
 
 
-def do_run_normal(options: dict, expected_fail: bool) -> list:
+def do_run_normal(options: dict, cfg: dict, expected_fail: bool) -> list:
     '''Run the iverilog and vvp commands.
 
     In this case, run the compiler to generate a vvp output file, and
     run the vvp command to actually execute the simulation. Collect
     the results and look for a "PASSED" string.'''
 
-    suffix = options['suffix']
     it_key = options['key']
-    it_dir = options['directory']
-    it_iverilog_args = options['iverilog_args']
-    it_vvp_args = options['vvp_args']
-    it_vvp_args_extended = options['vvp_args_extended']
-
     build_runtime(it_key)
 
     # Run the iverilog command
-    ivl_cmd = assemble_iverilog_cmd(suffix, options['source'], it_dir, it_iverilog_args)
+    ivl_cmd = assemble_iverilog_cmd(options, cfg, 'a.out')
     ivl_res = run_cmd(ivl_cmd)
 
     log_results(it_key, "iverilog", ivl_res)
@@ -294,37 +289,37 @@ def do_run_normal(options: dict, expected_fail: bool) -> list:
         return [1, "Failed - Compile failed"]
 
     # run the vvp command
-    vvp_cmd = assemble_vvp_cmd(suffix, it_vvp_args, it_vvp_args_extended)
+    vvp_cmd = assemble_vvp_cmd(options, cfg)
     vvp_res = run_cmd(vvp_cmd)
     log_results(it_key, "vvp", vvp_res)
 
-    if vvp_res.returncode == 0 and expected_fail:
+    if vvp_res.returncode != 0 and expected_fail:
-        return [1, "Failed - Vvp execution did not fail, but was expted to fail."]
+        return [0, "Passed - EF"]
     if vvp_res.returncode >= 256:
-        return [1, "Failed - Vvp execution error"]
+        return [1, "Failed - vvp execution error"]
     if vvp_res.returncode > 0 and vvp_res.returncode < 256 and not expected_fail:
-        return [1, "Failed - Vvp error, but expected to succeed"]
+        return [1, "Failed - vvp error, but expected to succeed"]
 
     it_stdout = vvp_res.stdout.decode('ascii')
     log_list = ["iverilog-stdout", "iverilog-stderr",
                 "vvp-stdout", "vvp-stderr"]
 
-    return check_run_outputs(options, it_stdout, log_list)
+    return check_run_outputs(options, it_stdout, log_list, expected_fail)
 
-def run_normal(options: dict) -> list:
+def run_normal(options: dict, cfg: dict) -> list:
     '''Run a normal test'''
-    return do_run_normal(options, False)
+    return do_run_normal(options, cfg, False)
 
 # pylint: disable-next=invalid-name
-def run_EF(options: dict) -> list:
+def run_EF(options: dict, cfg: dict) -> list:
     '''Run an expected fail test'''
-    return do_run_normal(options, True)
+    return do_run_normal(options, cfg, True)
 
-def run_normal_vlog95(options: dict) -> list:
+def run_normal_vlog95(options: dict, cfg: dict) -> list:
     '''Run a vlog95 test'''
-    return do_run_normal_vlog95(options, False)
+    return do_run_normal_vlog95(options, cfg, False)
 
 # pylint: disable-next=invalid-name
-def run_EF_vlog95(options: dict) -> list:
+def run_EF_vlog95(options: dict, cfg: dict) -> list:
     '''Run an expected fail vlog95 test'''
-    return do_run_normal_vlog95(options, True)
+    return do_run_normal_vlog95(options, cfg, True)

ivtest/vvp_reg.py

@@ -18,7 +18,64 @@ import test_lists
 import run_ivl
 
 
-def process_test(suffix: str, item: list) -> str:
+class InvalidTestType(Exception):
+    '''Exception to raise when the test type is not supported.'''
+    def __init__(self, test, ttype, msg='Invalid test type!'):
+        self.test = test
+        self.ttype = ttype
+        self.msg = msg
+        super().__init__(self.msg)
+
+    def __str__(self):
+        # pylint: disable-next=consider-using-f-string
+        return "Given test type '{ttype}' for test {test}".format(ttype=self.ttype,test=self.test)
+
+class InvalidJSON(Exception):
+    '''Exception to raise if the JSON is not parsed properly.'''
+    def __init__(self, test, path, msg='Invalid JSON file!'):
+        self.test = test
+        self.path = path
+        self.msg = msg
+        super().__init__(self.msg)
+
+    def __str__(self):
+        # pylint: disable-next=consider-using-f-string
+        res = "Unable to parse JSON file '{path}' for test {test}:".format(path=self.path,
+                                                                           test=self.test)
+        # pylint: disable-next=consider-using-f-string
+        res += "\n    {msg}".format(msg=self.msg)
+        return res
+
+
+def process_overrides(group: str, it_dict: dict, it_opts: dict):
+    '''Override the gold file or type if needed.'''
+    if group in it_dict:
+        overrides = ['gold', 'type']
+        for override in overrides:
+            if override in it_dict[group]:
+                if override == 'gold' and it_dict[group][override] == "":
+                    it_opts[override] = None
+                else:
+                    it_opts[override] = it_dict[group][override]
+
+
+def force_gen(it_opts: dict):
+    '''Remove the current generation and force it to the latest.'''
+    generations = ['-g2012', '-g2009', '-g2005-sv',
+                   '-g2005', '-g2001-noconfig', '-g2001', '-g1995',
+                   '-g2', '-g1']
+    for gen in generations:
+        if gen in it_opts['iverilog_args']:
+            it_opts['iverilog_args'].remove(gen)
+
+    if '-g2x' in it_opts['iverilog_args']:
+        idx_to_replace = it_opts['iverilog_args'].index('-g2x')
+        it_opts[idx_to_replace] = '-gicarus-misc'
+
+    it_opts['iverilog_args'].insert(0, '-g2012')
+
+
+def process_test(item: list, cfg: list) -> str:
     '''Process a single test
 
     This takes in the list of tokens from the tests list file, and converts
@@ -28,16 +85,15 @@ def process_test(suffix: str, item: list) -> str:
     it_key = item[0]
     test_path = item[1]
     with open(test_path, 'rt', encoding='ascii') as fd:
+        try:
             it_dict = json.load(fd)
-
+        except json.decoder.JSONDecodeError as exception:
-    # Get the test type from the json configuration.
+            raise InvalidJSON(it_key, test_path, exception) from exception
-    it_type = it_dict['type']
 
     # Wrap all of this into an options dictionary for ease of handling.
-    it_options = {
+    it_opts = {
-        'suffix'            : suffix,
         'key'               : it_key,
-        'type'              : it_type,
+        'type'              : it_dict['type'],
         'iverilog_args'     : it_dict.get('iverilog-args', [ ]),
         'directory'         : "ivltests",
         'source'            : it_dict['source'],
@@ -48,32 +104,70 @@ def process_test(suffix: str, item: list) -> str:
         'vvp_args_extended' : it_dict.get('vvp-args-extended', [ ])
     }
 
+    if cfg['strict']:
+        it_opts['iverilog_args'].append("-gstrict-expr-width")
+        it_opts['vvp_args_extended'].append("-compatible")
+        process_overrides('strict', it_dict, it_opts)
+    else:
+        it_opts['iverilog_args'].append('-D__ICARUS_UNSIZED__')
+
+    if cfg['force-sv']:
+        force_gen(it_opts)
+        process_overrides('force-sv', it_dict, it_opts)
+
+    # Get the overridden test type.
+    it_type = it_opts['type']
+
     if it_type == "NI":
         res = [0, "Not Implemented."]
 
     elif it_type == "normal":
-        res = run_ivl.run_normal(it_options)
+        res = run_ivl.run_normal(it_opts, cfg)
 
     elif it_type == "normal-vlog95":
-        res = run_ivl.run_normal_vlog95(it_options)
+        res = run_ivl.run_normal_vlog95(it_opts, cfg)
 
     elif it_type == "CE":
-        res = run_ivl.run_CE(it_options)
+        res = run_ivl.run_CE(it_opts, cfg)
 
     elif it_type == "EF":
-        res = run_ivl.run_EF(it_options)
+        res = run_ivl.run_EF(it_opts, cfg)
 
     elif it_type == "EF-vlog95":
-        res = run_ivl.run_EF_vlog95(it_options)
+        res = run_ivl.run_EF_vlog95(it_opts, cfg)
 
     else:
-        # pylint: disable-next=consider-using-f-string
+        raise InvalidTestType(it_key, it_type)
-        res = "{key}: I don't understand the test type ({type}).".format(key=it_key, type=it_type)
-        raise Exception(res)
 
     return res
 
 
+def print_header(cfg: dict, files: list):
+    '''Print all the header information. '''
+    # This returns 13 or similar
+    ivl_version = run_ivl.get_ivl_version(cfg['suffix'])
+
+    print("Running compiler/VVP tests for Icarus Verilog ", end='')
+    # pylint: disable-next=consider-using-f-string
+    print("version: {ver}".format(ver=ivl_version), end='')
+    if cfg['suffix']:
+        # pylint: disable-next=consider-using-f-string
+        print(", suffix: {suffix}".format(suffix=cfg['suffix']), end='')
+    if cfg['strict']:
+        if cfg['force-sv']:
+            print(" (strict, force SV)", end='')
+        else:
+            print(" (strict)", end='')
+    elif cfg['force-sv']:
+        print(" (force SV)", end='')
+    if cfg['with-valgrind']:
+        print(" (valgrind)", end='')
+    print("")
+    # pylint: disable-next=consider-using-f-string
+    print("Using list(s): {files}".format(files=', '.join(files)))
+    print("-" * 76)
+
+
 if __name__ == "__main__":
     argp = argparse.ArgumentParser(description='')
     argp.add_argument('--suffix', type=str, default='',
@@ -90,33 +184,19 @@ if __name__ == "__main__":
                       help='File(s) containing a list of the tests to run, default "%(default)s".')
     args = argp.parse_args()
 
-    if args.strict:
+    # FIXME: need to add real vlog95 support
-        print('Sorry: Forcing strict compatiability is not currently supported!')
-        sys.exit(1)
-    if args.with_valgrind:
-        print('Sorry: Running with valgrind is not currently supported!')
-        sys.exit(1)
-    if args.force_sv:
-        print('Sorry: Forcing SystemVerilog is not currently supported!')
-        sys.exit(1)
     if args.vlog95:
         print('Sorry: Converting to Verilog-95 and running is not currently supported!')
         sys.exit(1)
 
-    # This returns 13 or similar
+    ivl_cfg = {
-    ivl_version = run_ivl.get_ivl_version(args.suffix)
+        'suffix'        : args.suffix,
+        'strict'        : args.strict,
+        'with-valgrind' : args.with_valgrind,
+        'force-sv'      : args.force_sv
+              }
 
-    print("Running compiler/VVP tests for Icarus Verilog ", end='')
+    print_header(ivl_cfg, args.files)
-    # pylint: disable-next=consider-using-f-string
-    print("version: {ver}".format(ver=ivl_version), end='')
-    if args.suffix:
-        # pylint: disable-next=consider-using-f-string
-        print(", suffix: {suffix}".format(suffix=args.suffix), end='')
-    # FIXME: add strict, force SV and with valgrind
-    print("")
-    # pylint: disable-next=consider-using-f-string
-    print("Using list(s): {files}".format(files=', '.join(args.files)))
-    print("-" * 76)
 
     # Read the list files, to get the tests.
     tests_list = test_lists.read_lists(args.files)
@@ -132,7 +212,7 @@ if __name__ == "__main__":
     # pylint: disable-next=invalid-name
     error_count = 0
     for cur in tests_list:
-        result = process_test(args.suffix, cur)
+        result = process_test(cur, ivl_cfg)
         error_count += result[0]
         # pylint: disable-next=consider-using-f-string
         print("{name:>{width}}: {result}".format(name=cur[0], width=width, result=result[1]))

ivtest/vvp_tests/br_gh1087a1.json

@@ -1,6 +1,9 @@
 {
     "type"   : "CE",
     "source" : "br_gh1087a.v",
+    "iverilog-args" : [ "-gno-xtypes" ],
     "gold"   : "br_gh1087a1",
-    "iverilog-args" : [ "-gno-xtypes" ]
+    "force-sv" : {
+        "gold"   : "br_gh1087a1-fsv"
+    }
 }

ivtest/vvp_tests/br_gh1143e.json

@@ -2,5 +2,9 @@
     "type"   : "CE",
     "source" : "br_gh1143a.v",
     "iverilog-args" : [ "-g2009" ],
-    "gold" : "br_gh1143e"
+    "gold"   : "br_gh1143e",
+    "force-sv" : {
+        "type"   : "normal",
+        "gold"   : ""
+    }
 }

ivtest/vvp_tests/br_gh1143f.json

@@ -2,5 +2,9 @@
     "type"   : "CE",
     "source" : "br_gh1143b.v",
     "iverilog-args" : [ "-g2009" ],
-    "gold" : "br_gh1143f"
+    "gold"   : "br_gh1143f",
+    "force-sv" : {
+        "type"   : "normal",
+        "gold"   : ""
+    }
 }

ivtest/vvp_tests/br_gh1143g.json

@@ -2,5 +2,9 @@
     "type"   : "CE",
     "source" : "br_gh1143c.v",
     "iverilog-args" : [ "-g2009" ],
-    "gold" : "br_gh1143g"
+    "gold"   : "br_gh1143g",
+    "force-sv" : {
+        "type"   : "normal",
+        "gold"   : ""
+    }
 }

ivtest/vvp_tests/br_gh1143h.json

@@ -2,5 +2,9 @@
     "type"   : "CE",
     "source" : "br_gh1143d.v",
     "iverilog-args" : [ "-g2009" ],
-    "gold" : "br_gh1143h"
+    "gold"   : "br_gh1143h",
+    "force-sv" : {
+        "type"   : "normal",
+        "gold"   : ""
+    }
 }

ivtest/vvp_tests/br_gh13a.json

@@ -1,5 +1,8 @@
 {
     "type"   : "normal",
     "source" : "br_gh13a.v",
-    "gold"   : "br_gh13a"
+    "gold"   : "br_gh13a",
+    "strict" : {
+        "gold"   : "br_gh13a_std"
+    }
 }

ivtest/vvp_tests/br_gh383d.json

@@ -2,5 +2,8 @@
     "type"   : "normal",
     "source" : "br_gh383d.v",
     "iverilog-args" : [ "-g2012" ],
-    "gold"   : "br_gh383d"
+    "gold"   : "br_gh383d",
+    "strict" : {
+        "gold"   : "br_gh383d_std"
+    }
 }

ivtest/vvp_tests/ca_time_real.json

@@ -1,5 +1,8 @@
 {
     "type"   : "normal",
     "source" : "ca_time_real.v",
-    "gold"   : "ca_time_real"
+    "gold"   : "ca_time_real",
+    "strict" : {
+        "gold"   : "ca_time_real_std"
+    }
 }

ivtest/vvp_tests/delayed_sfunc.json

@@ -1,6 +1,9 @@
 {
     "type"   : "normal",
     "source" : "delayed_sfunc.v",
+    "iverilog-args" : [ "-gspecify" ],
     "gold"   : "delayed_sfunc",
-    "iverilog-args" : [ "-gspecify" ]
+    "strict" : {
+        "gold"   : "delayed_sfunc_std"
+    }
 }

ivtest/vvp_tests/localparam_type.json

@@ -1,5 +1,8 @@
 {
     "type"   : "normal",
     "source" : "localparam_type.v",
-    "gold"   : "parameter_type"
+    "gold"   : "parameter_type",
+    "strict" : {
+        "gold"   : "parameter_type_std"
+    }
 }

ivtest/vvp_tests/param-width.json

@@ -1,5 +1,8 @@
 {
     "type"   : "normal",
     "source" : "param-width.v",
-    "gold"   : "param-width"
+    "gold"   : "param-width",
+    "strict" : {
+        "gold"   : "param-width_std"
+    }
 }

ivtest/vvp_tests/parameter_type.json

@@ -1,5 +1,8 @@
 {
     "type"   : "normal",
     "source" : "parameter_type.v",
-    "gold"   : "parameter_type"
+    "gold"   : "parameter_type",
+    "strict" : {
+        "gold"   : "parameter_type_std"
+    }
 }

ivtest/vvp_tests/pr1388974.json

@@ -1,4 +1,7 @@
 {
     "type"   : "normal",
-    "source" : "pr1388974.v"
+    "source" : "pr1388974.v",
+    "strict" : {
+        "type"   : "EF"
+    }
 }

ivtest/vvp_tests/pr1701890.json

@@ -1,5 +1,8 @@
 {
     "type"   : "normal",
     "source" : "pr1701890.v",
-    "gold"   : "pr1701890"
+    "gold"   : "pr1701890",
+    "strict" : {
+        "gold"   : "pr1701890_std"
+    }
 }

ivtest/vvp_tests/pr1864110a.json

@@ -1,5 +1,8 @@
 {
     "type"   : "normal",
     "source" : "pr1864110a.v",
-    "gold"   : "pr1864110a"
+    "gold"   : "pr1864110a",
+    "strict" : {
+        "gold"   : "pr1864110a_std"
+    }
 }

ivtest/vvp_tests/pr1864110b.json

@@ -1,5 +1,8 @@
 {
     "type"   : "normal",
     "source" : "pr1864110b.v",
-    "gold"   : "pr1864110b"
+    "gold"   : "pr1864110b",
+    "strict" : {
+        "gold"   : "pr1864110b_std"
+    }
 }

ivtest/vvp_tests/pr1864115.json

@@ -1,5 +1,8 @@
 {
     "type"   : "normal",
     "source" : "pr1864115.v",
-    "gold"   : "pr1864115"
+    "gold"   : "pr1864115",
+    "strict" : {
+        "gold"   : "pr1864115_std"
+    }
 }

ivtest/vvp_tests/pr903.json

@@ -1,4 +1,7 @@
 {
     "type"   : "normal",
-    "source" : "pr903.v"
+    "source" : "pr903.v",
+    "strict" : {
+        "type" : "EF"
+    }
 }