Cleanup python test script and add support for a suffix

ivtest/run_ivl.py

@@ -8,8 +8,10 @@ import os
 import sys
 import re
 
-def assemble_iverilog_cmd(source: str, it_dir: str, args: list, outfile = "a.out") -> list:
+def assemble_iverilog_cmd(suffix: str, source: str, it_dir: str,
-    res = ["iverilog", "-o", os.path.join("work", outfile)]
+                          args: list, outfile = "a.out") -> list:
+    '''Build the iverilog command line'''
+    res = ["iverilog"+suffix, "-o", os.path.join("work", outfile)]
     res += ["-D__ICARUS_UNSIZED__"]
     res += args
     src = os.path.join(it_dir, source)
@@ -17,28 +19,29 @@ def assemble_iverilog_cmd(source: str, it_dir: str, args: list, outfile = "a.out
     return res
 
 
-def assemble_vvp_cmd(args: list = [], plusargs: list = []) -> list:
+def assemble_vvp_cmd(suffix: str, args: list, plusargs: list) -> list:
-    res = ["vvp"]
+    '''Build the vvp command line'''
+    res = ["vvp"+suffix]
     res = res + args
     res.append(os.path.join("work", "a.out"))
     res = res + plusargs
     return res
 
 
-def get_ivl_version () -> list:
+def get_ivl_version (suffix: str) -> int:
     '''Figure out the version of the installed iverilog compler.
 
     The return value is a list of 2 numbers, the major and minor version
     numbers, or None if the version string couldn't be found.'''
 
     # Get the output from the "iverilog -V" command for the version string.
-    text = subprocess.check_output(["iverilog", "-V"])
+    text = subprocess.check_output(["iverilog"+suffix, "-V"])
     match = re.search(b'Icarus Verilog version ([0-9]+)\\.([0-9]+)', text)
     if not match:
         return None
 
     items = match.groups()
-    return [int(items[0]), int(items[1])]
+    return int(items[0])
 
 def build_runtime(it_key: str) -> None:
     '''Check and prepare the runtime environment for a test
@@ -70,6 +73,8 @@ def build_runtime(it_key: str) -> None:
         pass
 
 def get_log_file(key: str, title: str, stream: str) -> str:
+    '''Get the name of the log file'''
+    # pylint: disable-next=consider-using-f-string
     res = "{key}-{title}-{stream}.log".format(key=key, title=title, stream=stream)
     return res
 
@@ -93,42 +98,49 @@ def compare_files(log_path, gold_path):
     If they differ, then write tou stdout a unified diff. In any case,
     return True or False to indicate the results of the test.'''
 
-    with open(log_path, 'rt') as fd:
+    with open(log_path, 'rt', encoding='ascii') as fd:
         a = fd.readlines()
 
     # Allow to omit empty gold files
     if os.path.exists(gold_path):
-        with open(gold_path, 'rt') as fd:
+        with open(gold_path, 'rt', encoding='ascii') as fd:
             b = fd.readlines()
     else:
         b = []
 
     flag = a == b
     if not flag:
+        # pylint: disable-next=consider-using-f-string
         print("{log} and {gold} differ:".format(log=log_path, gold=gold_path))
         sys.stdout.writelines(difflib.unified_diff(a, b, log_path, gold_path))
 
     return flag
 
 def run_cmd(cmd: list) -> subprocess.CompletedProcess:
-    res = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    '''Run the given command'''
+    res = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False)
     return res
 
-def check_gold(it_key : str, it_gold : str, log_list : list) -> bool:
+def check_gold(it_key: str, it_gold: str, log_list: list) -> bool:
+    '''Check if the log and gold file match'''
     compared = True
     for log_name in log_list:
+        # pylint: disable-next=consider-using-f-string
         log_path = os.path.join("log", "{key}-{log}.log".format(key=it_key, log=log_name))
+        # pylint: disable-next=consider-using-f-string
         gold_path = os.path.join("gold", "{gold}-{log}.gold".format(gold=it_gold, log=log_name))
         compared = compared and compare_files(log_path, gold_path)
 
     return compared
 
-def run_CE(options : dict) -> list:
+# pylint: disable-next=invalid-name
+def run_CE(options: dict) -> list:
     ''' Run the compiler, and expect an error
 
     In this case, we assert that the command fails to run and reports
     an error. This is to check that invalid input generates errors.'''
 
+    suffix = options['suffix']
     it_key = options['key']
     it_dir = options['directory']
     it_args = options['iverilog_args']
@@ -136,7 +148,7 @@ def run_CE(options : dict) -> list:
 
     build_runtime(it_key)
 
-    cmd = assemble_iverilog_cmd(options['source'], it_dir, it_args)
+    cmd = assemble_iverilog_cmd(suffix, options['source'], it_dir, it_args)
     res = run_cmd(cmd)
     log_results(it_key, "iverilog", res)
 
@@ -151,7 +163,7 @@ def run_CE(options : dict) -> list:
     else:
         return [0, "Passed - CE"]
 
-def check_run_outputs(options : dict, it_stdout : str, log_list : list) -> list:
+def check_run_outputs(options: dict, it_stdout: str, log_list: list) -> list:
     '''Check the output files, and return success for failed.
 
     This function takes an options dictionary that describes the settings, and
@@ -178,12 +190,13 @@ def check_run_outputs(options : dict, it_stdout : str, log_list : list) -> list:
         diff_name2 = it_diff[1]
         diff_skip = int(it_diff[2])
 
-        with open(diff_name1) as fd:
+        with open(diff_name1, 'rt', encoding='ascii') as fd:
+            # pylint: disable-next=unused-variable
             for idx in range(diff_skip):
                 fd.readline()
             diff_data1 = fd.read()
 
-        with open(diff_name2) as fd:
+        with open(diff_name2, 'rt', encoding='ascii') as fd:
             for idx in range(diff_skip):
                 fd.readline()
             diff_data2 = fd.read()
@@ -191,7 +204,9 @@ def check_run_outputs(options : dict, it_stdout : str, log_list : list) -> list:
         if diff_data1 == diff_data2:
             return [0, "Passed"]
         else:
-            return [1, "Failed - Files {name1} and {name2} differ.".format(name1=diff_name1, name2=diff_name2)]
+            # pylint: disable-next=consider-using-f-string
+            return [1, "Failed - Files {name1} and {name2} differ.".format(name1=diff_name1, \
+                                                                           name2=diff_name2)]
 
 
     # Otherwise, look for the PASSED output string in stdout.
@@ -204,7 +219,8 @@ def check_run_outputs(options : dict, it_stdout : str, log_list : list) -> list:
     return [1, "Failed - No PASSED output, and no gold file"]
 
 
-def do_run_normal_vlog95(options : dict, expected_fail : bool) -> list:
+# pylint: disable-next=unused-argument
+def do_run_normal_vlog95(options: dict, expected_fail: bool) -> list:
     '''Run the iverilog and vvp commands.
 
     In this case, run the compiler with the -tvlog95 flag to generate
@@ -212,6 +228,7 @@ def do_run_normal_vlog95(options : dict, expected_fail : bool) -> list:
     a vvp out. Run that vvp output to test the simulation results. Collect
     the results and look for a "PASSED" string.'''
 
+    suffix = options['suffix']
     it_key = options['key']
     it_dir = options['directory']
     it_iverilog_args = ["-tvlog95"] + options['iverilog_args']
@@ -221,7 +238,7 @@ def do_run_normal_vlog95(options : dict, expected_fail : bool) -> list:
     build_runtime(it_key)
 
     # Run the first iverilog command, to generate the intermediate verilog
-    ivl1_cmd = assemble_iverilog_cmd(options['source'], it_dir, it_iverilog_args, "a.out.v")
+    ivl1_cmd = assemble_iverilog_cmd(suffix, options['source'], it_dir, it_iverilog_args, "a.out.v")
     ivl1_res = run_cmd(ivl1_cmd)
 
     log_results(it_key, "iverilog", ivl1_res)
@@ -229,7 +246,7 @@ def do_run_normal_vlog95(options : dict, expected_fail : bool) -> list:
         return [1, "Failed - Compile failed"]
 
     # Run another iverilog command to compile the code generated from the first step.
-    ivl2_cmd = assemble_iverilog_cmd("a.out.v", "work", [ ], "a.out")
+    ivl2_cmd = assemble_iverilog_cmd(suffix, "a.out.v", "work", [ ], "a.out")
     ivl2_res = run_cmd(ivl2_cmd)
 
     log_results(it_key, "iverilog-vlog95", ivl2_res)
@@ -237,9 +254,9 @@ def do_run_normal_vlog95(options : dict, expected_fail : bool) -> list:
         return [1, "Failed - Compile of generated code failed"]
 
     # Run the vvp command
-    vvp_cmd = assemble_vvp_cmd(it_vvp_args, it_vvp_args_extended)
+    vvp_cmd = assemble_vvp_cmd(suffix, it_vvp_args, it_vvp_args_extended)
     vvp_res = run_cmd(vvp_cmd)
-    log_results(it_key, "vvp", vvp_res);
+    log_results(it_key, "vvp", vvp_res)
 
     if vvp_res.returncode != 0:
         return [1, "Failed - Vvp execution failed"]
@@ -252,13 +269,14 @@ def do_run_normal_vlog95(options : dict, expected_fail : bool) -> list:
     return check_run_outputs(options, it_stdout, log_list)
 
 
-def do_run_normal(options : dict, expected_fail : bool) -> list:
+def do_run_normal(options: dict, expected_fail: bool) -> list:
     '''Run the iverilog and vvp commands.
 
     In this case, run the compiler to generate a vvp output file, and
     run the vvp command to actually execute the simulation. Collect
     the results and look for a "PASSED" string.'''
 
+    suffix = options['suffix']
     it_key = options['key']
     it_dir = options['directory']
     it_iverilog_args = options['iverilog_args']
@@ -268,7 +286,7 @@ def do_run_normal(options : dict, expected_fail : bool) -> list:
     build_runtime(it_key)
 
     # Run the iverilog command
-    ivl_cmd = assemble_iverilog_cmd(options['source'], it_dir, it_iverilog_args)
+    ivl_cmd = assemble_iverilog_cmd(suffix, options['source'], it_dir, it_iverilog_args)
     ivl_res = run_cmd(ivl_cmd)
 
     log_results(it_key, "iverilog", ivl_res)
@@ -276,9 +294,9 @@ def do_run_normal(options : dict, expected_fail : bool) -> list:
         return [1, "Failed - Compile failed"]
 
     # run the vvp command
-    vvp_cmd = assemble_vvp_cmd(it_vvp_args, it_vvp_args_extended)
+    vvp_cmd = assemble_vvp_cmd(suffix, it_vvp_args, it_vvp_args_extended)
     vvp_res = run_cmd(vvp_cmd)
-    log_results(it_key, "vvp", vvp_res);
+    log_results(it_key, "vvp", vvp_res)
 
     if vvp_res.returncode == 0 and expected_fail:
         return [1, "Failed - Vvp execution did not fail, but was expted to fail."]
@@ -293,14 +311,20 @@ def do_run_normal(options : dict, expected_fail : bool) -> list:
 
     return check_run_outputs(options, it_stdout, log_list)
 
-def run_normal(options : dict) -> list:
+def run_normal(options: dict) -> list:
+    '''Run a normal test'''
     return do_run_normal(options, False)
 
-def run_EF(options : dict) -> list:
+# pylint: disable-next=invalid-name
+def run_EF(options: dict) -> list:
+    '''Run an expected fail test'''
     return do_run_normal(options, True)
 
-def run_normal_vlog95(options : dict) -> list:
+def run_normal_vlog95(options: dict) -> list:
+    '''Run a vlog95 test'''
     return do_run_normal_vlog95(options, False)
 
-def run_EF_vlog95(options : dict) -> list:
+# pylint: disable-next=invalid-name
+def run_EF_vlog95(options: dict) -> list:
+    '''Run an expected fail vlog95 test'''
     return do_run_normal_vlog95(options, True)

ivtest/test_lists.py

@@ -30,7 +30,7 @@ def read_list(fd) -> list:
     in the list a list of tokens for the line. This is used by the read_lists
     function.'''
 
-    build_list = list()
+    build_list = []
     for line_raw in fd:
         # Strip comments and leading/traling white space
         idx = line_raw.find("#")
@@ -56,18 +56,18 @@ def read_lists(paths: list) -> list:
     of the test file lists is important, as is the order of tests
     within each list file.'''
 
-    tests_list = list()
+    tests_list = []
     for path in paths:
-        with open(path, "r") as fd:
+        with open(path, "rt", encoding='ascii') as fd:
             tests_list += read_list(fd)
 
     # The loop above creates a tests_list to list all of the tests in the
     # order that they appear. Now we go though the list in order and eliminate
     # duplictes by test name. This allows that lists might override tests that
     # are already declared.
-    tests_dict = dict()
+    tests_dict = {}
     for item in tests_list:
-        tests_dict[item[0]] = item;
+        tests_dict[item[0]] = item
 
     # Convert the result to a sorted list, and return that.
     tests_list = list(tests_dict.values())

ivtest/vvp_reg.py

@@ -12,16 +12,13 @@ Usage:
 '''
 
 import sys
-# The docopt library is not available on msys2 or cygwin installations, so
-# skip it completely on win32/cygwin platforms.
-if sys.platform != 'win32' and sys.platform != 'cygwin':
-    from docopt import docopt
-import test_lists
 import json
+import argparse
+import test_lists
 import run_ivl
 
 
-def process_test(item: list) -> str:
+def process_test(suffix: str, item: list) -> str:
     '''Process a single test
 
     This takes in the list of tokens from the tests list file, and converts
@@ -30,7 +27,7 @@ def process_test(item: list) -> str:
     # This is the name of the test, and the name of the main sorce file
     it_key = item[0]
     test_path = item[1]
-    with open(test_path, 'rt') as fd:
+    with open(test_path, 'rt', encoding='ascii') as fd:
         it_dict = json.load(fd)
 
     # Get the test type from the json configuration.
@@ -38,14 +35,15 @@ def process_test(item: list) -> str:
 
     # Wrap all of this into an options dictionary for ease of handling.
     it_options = {
-        'key'           : it_key,
+        'suffix'            : suffix,
-        'type'          : it_type,
+        'key'               : it_key,
-        'iverilog_args' : it_dict.get('iverilog-args', [ ]),
+        'type'              : it_type,
-        'directory'     : "ivltests",
+        'iverilog_args'     : it_dict.get('iverilog-args', [ ]),
-        'source'        : it_dict['source'],
+        'directory'         : "ivltests",
-        'modulename'    : None,
+        'source'            : it_dict['source'],
-        'gold'          : it_dict.get('gold', None),
+        'modulename'        : None,
-        'diff'          : None,
+        'gold'              : it_dict.get('gold', None),
+        'diff'              : None,
         'vvp_args'          : it_dict.get('vvp-args', [ ]),
         'vvp_args_extended' : it_dict.get('vvp-args-extended', [ ])
     }
@@ -69,6 +67,7 @@ def process_test(item: list) -> str:
         res = run_ivl.run_EF_vlog95(it_options)
 
     else:
+        # pylint: disable-next=consider-using-f-string
         res = "{key}: I don't understand the test type ({type}).".format(key=it_key, type=it_type)
         raise Exception(res)
 
@@ -76,42 +75,70 @@ def process_test(item: list) -> str:
 
 
 if __name__ == "__main__":
-    print("Running tests on platform: {platform}".format(platform=sys.platform))
+    argp = argparse.ArgumentParser(description='')
-    if sys.platform == 'win32' or sys.platform == 'cygwin':
+    argp.add_argument('--suffix', type=str, default='',
-        args = { "<list-paths>" : [] }
+                      help='The Icarus executable suffix, default "%(default)s".')
-    else:
+    argp.add_argument('--strict', action='store_true',
-        args = docopt(__doc__)
+                      help='Force strict standard compliance, default "%(default)s".')
+    argp.add_argument('--with-valgrind', action='store_true',
+                      help='Run the test suite with valgrind, default "%(default)s".')
+    argp.add_argument('--force-sv', action='store_true',
+                      help='Force tests to be run as SystemVerilog, default "%(default)s".')
+    argp.add_argument('--vlog95', action='store_true',
+                      help='Convert tests to Verilog 95 and then run, default "%(default)s".')
+    argp.add_argument('files', nargs='*', type=str, default=['regress-vvp.list'],
+                      help='File(s) containing a list of the tests to run, default "%(default)s".')
+    args = argp.parse_args()
 
-    # This returns [13, 0] or similar
+    if args.strict:
-    ivl_version = run_ivl.get_ivl_version()
+        print('Sorry: Forcing strict compatiability is not currently supported!')
-    ivl_version_major = ivl_version[0]
+        sys.exit(1)
-    print("Icarus Verilog version: {ivl_version}".format(ivl_version=ivl_version_major))
+    if args.with_valgrind:
+        print('Sorry: Running with valgrind is not currently supported!')
+        sys.exit(1)
+    if args.force_sv:
+        print('Sorry: Forcing SystemVerilog is not currently supported!')
+        sys.exit(1)
+    if args.vlog95:
+        print('Sorry: Converting to Verilog-95 and running is not currently supported!')
+        sys.exit(1)
 
-    # Select the lists to use. If any list paths are given on the command
+    # This returns 13 or similar
-    # line, then use only those. Otherwise, use a default list.
+    ivl_version = run_ivl.get_ivl_version(args.suffix)
-    list_paths = args["<list-paths>"]
-    if len(list_paths) == 0:
-        list_paths = list()
-        list_paths += ["regress-vvp.list"]
 
-    print("Use lists: {list_paths}".format(list_paths=list_paths))
+    print("Running compiler/VVP tests for Icarus Verilog ", end='')
+    # pylint: disable-next=consider-using-f-string
+    print("version: {ver}".format(ver=ivl_version), end='')
+    if args.suffix:
+        # pylint: disable-next=consider-using-f-string
+        print(", suffix: {suffix}".format(suffix=args.suffix), end='')
+    # FIXME: add strict, force SV and with valgrind
+    print("")
+    # pylint: disable-next=consider-using-f-string
+    print("Using list(s): {files}".format(files=', '.join(args.files)))
+    print("-" * 76)
 
     # Read the list files, to get the tests.
-    tests_list = test_lists.read_lists(list_paths)
+    tests_list = test_lists.read_lists(args.files)
 
     # We need the width of the widest key so that we can figure out
     # how to align the key:result columns.
+    # pylint: disable-next=invalid-name
     width = 0
     for cur in tests_list:
         if len(cur[0]) > width:
             width = len(cur[0])
 
+    # pylint: disable-next=invalid-name
     error_count = 0
     for cur in tests_list:
-        result = process_test(cur)
+        result = process_test(args.suffix, cur)
         error_count += result[0]
+        # pylint: disable-next=consider-using-f-string
         print("{name:>{width}}: {result}".format(name=cur[0], width=width, result=result[1]))
 
-    print("===================================================")
+    print("=" * 76)
-    print("Test results: Ran {ran}, Failed {failed}.".format(ran=len(tests_list), failed=error_count))
+    # pylint: disable-next=consider-using-f-string
-    exit(error_count)
+    print("Test results: Ran {ran}, Failed {failed}.".format(ran=len(tests_list), \
+                                                             failed=error_count))
+    sys.exit(error_count)