From 4ea442b2d92cfb5fe17895617c9abe5965af7ebe Mon Sep 17 00:00:00 2001 From: Stephen Williams Date: Fri, 30 Dec 2022 17:56:44 -0500 Subject: [PATCH 1/2] ivtest: Create the vvp_reg.py test script This creates a new test suite driver and infrastructure for going forward. This cleans up the descriptions for tests, and is more extensible. Include documentation for the new engine, so that new tests can go here instead of the old test suite. --- .github/test.sh | 2 + .github/workflows/test.yml | 9 +- Documentation/developer/index.rst | 1 + Documentation/developer/regression_tests.rst | 73 ++++++ ivtest/.gitignore | 1 + .../gold/macro_str_esc-iverilog-stderr.gold | 0 .../gold/macro_str_esc-iverilog-stdout.gold | 0 ivtest/gold/macro_str_esc-vvp-stderr.gold | 0 ...esc.gold => macro_str_esc-vvp-stdout.gold} | 0 ivtest/regress-ivl1.list | 1 - ivtest/regress-vvp.list | 4 + ivtest/run_ivl.py | 242 ++++++++++++++++++ ivtest/test_lists.py | 76 ++++++ ivtest/vvp_reg.py | 110 ++++++++ ivtest/vvp_tests/README.txt | 11 + ivtest/vvp_tests/case2-S.json | 5 + ivtest/vvp_tests/macro_str_esc.json | 5 + 17 files changed, 537 insertions(+), 3 deletions(-) create mode 100644 Documentation/developer/regression_tests.rst create mode 100644 ivtest/gold/macro_str_esc-iverilog-stderr.gold create mode 100644 ivtest/gold/macro_str_esc-iverilog-stdout.gold create mode 100644 ivtest/gold/macro_str_esc-vvp-stderr.gold rename ivtest/gold/{macro_str_esc.gold => macro_str_esc-vvp-stdout.gold} (100%) create mode 100644 ivtest/regress-vvp.list create mode 100644 ivtest/run_ivl.py create mode 100644 ivtest/test_lists.py create mode 100755 ivtest/vvp_reg.py create mode 100644 ivtest/vvp_tests/README.txt create mode 100644 ivtest/vvp_tests/case2-S.json create mode 100644 ivtest/vvp_tests/macro_str_esc.json diff --git a/.github/test.sh b/.github/test.sh index 7b5916db1..a49bdfb3c 100755 --- a/.github/test.sh +++ b/.github/test.sh @@ -11,4 +11,6 @@ perl vvp_reg.pl || status=1 perl vpi_reg.pl || status=1 +python3 vvp_reg.py || status=1 + exit $status diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b59e5b581..1431ecf4d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,6 +22,7 @@ jobs: - name: Install dependencies run: | brew install bison + pip3 install docopt - name: Build, check and install run: | @@ -52,7 +53,7 @@ jobs: - name: Install dependencies run: | sudo apt update -qq - sudo apt install -y make g++ git bison flex gperf libreadline-dev autoconf python3-sphinx + sudo apt install -y make g++ git bison flex gperf libreadline-dev autoconf python3-sphinx python3-docopt - name: Build, check and install run: | @@ -99,8 +100,11 @@ jobs: git base-devel gperf + python-pip mingw-w64-${{ matrix.arch }}-toolchain + - uses: actions/setup-python@v4 + - name: Build and check run: | cd msys2 @@ -110,7 +114,8 @@ jobs: run: pacman -U --noconfirm msys2/*.zst - name: Test - run: ./.github/test.sh + run: | + ./.github/test.sh - uses: actions/upload-artifact@v2 with: diff --git a/Documentation/developer/index.rst b/Documentation/developer/index.rst index 3be5a5337..d9a4689bb 100644 --- a/Documentation/developer/index.rst +++ b/Documentation/developer/index.rst @@ -9,5 +9,6 @@ Icarus Verilog. :maxdepth: 1 getting_started + regression_tests version_stamps diff --git a/Documentation/developer/regression_tests.rst b/Documentation/developer/regression_tests.rst new file mode 100644 index 000000000..c443613ee --- /dev/null +++ b/Documentation/developer/regression_tests.rst @@ -0,0 +1,73 @@ + +The Regression Test Suite +========================= + +Icarus Verilog development includes a regression test suite that is included +along with the source. The "ivtest" directory contains the regression test +suite, and this suite is used by the github actions as continuous integration +to make sure the code is always going forward. + +NOTE: There are scripts written in perl to run the regression tests, but they +are bing gradually replaced with a newer set of scripts. It is the newer +method that is described here. + +Test Descriptions +----------------- + +Regression tests are listed in the regress-vvp.list file. Each line lists the +name of the test and the path to the dest description. The list file is +therefore pretty simple, and all the description of the test is in the +description file: + +.. code-block:: console + + macro_str_esc vvp_tests/macro_str_esc.json + +The "name" is a simple name, and the test-description-file is the path (relative +the ivtest directory) to the description file. A simple test description file +is a JSON file, like this: + +.. code-block:: java + + { + "type" : "normal", + "source" : "macro_str_esc.v", + "gold" : "macro_str_esc" + } + +This description file contains all the information that the vvp_reg.py script +needs to run the regression test. The sections below describe the keys and +values in the description file dictionary. + +source (required) +^^^^^^^^^^^^^^^^^ +This specifies the name of the source file. The file is actually to be found +in the ivltests/ directory. + + +type (required) +^^^^^^^^^^^^^^^ + +This describes the kind of test to run. The valid values are: + +* **normal** - Compile the source using the iverilog compiler vvp target, and if + that succeeds execute it using the vvp command. If there is no gold file + specified, then look for an output line with the "PASSED" string. + +* **NI** - Mark the test as not implemented. The test will be skipped without + running or reporting an error. + +* **CE** - Compile, but expect the compiler to fail + +* **EF** - Compile and run, burt expect the run time to fail. + +gold (optional) +^^^^^^^^^^^^^^^ + +If this is specified, it replaces the "Passed" condition with a comparison of +the output with a gold file. The argument is the name of the gold file set, +which will be found in the "gold/" directory. The name here is actually the +basename of the gold files, with separate actual gold files for the iverilog +and vvp stderr and stdout. For example, if a "normal" test includes a gold +file, then the program is compiled and run, and the outputs are compared with +the gold file to make sure it ran properly. diff --git a/ivtest/.gitignore b/ivtest/.gitignore index 21462303a..e5a4b3d37 100644 --- a/ivtest/.gitignore +++ b/ivtest/.gitignore @@ -10,6 +10,7 @@ log/ work/ vpi_log/ vhdl/ +__pycache__/ # The normal regression output files. diff --git a/ivtest/gold/macro_str_esc-iverilog-stderr.gold b/ivtest/gold/macro_str_esc-iverilog-stderr.gold new file mode 100644 index 000000000..e69de29bb diff --git a/ivtest/gold/macro_str_esc-iverilog-stdout.gold b/ivtest/gold/macro_str_esc-iverilog-stdout.gold new file mode 100644 index 000000000..e69de29bb diff --git a/ivtest/gold/macro_str_esc-vvp-stderr.gold b/ivtest/gold/macro_str_esc-vvp-stderr.gold new file mode 100644 index 000000000..e69de29bb diff --git a/ivtest/gold/macro_str_esc.gold b/ivtest/gold/macro_str_esc-vvp-stdout.gold similarity index 100% rename from ivtest/gold/macro_str_esc.gold rename to ivtest/gold/macro_str_esc-vvp-stdout.gold diff --git a/ivtest/regress-ivl1.list b/ivtest/regress-ivl1.list index effab3aac..cbf094f98 100644 --- a/ivtest/regress-ivl1.list +++ b/ivtest/regress-ivl1.list @@ -75,7 +75,6 @@ #------------------------------------------------------------------------------ # Escaped defines `` -macro_str_esc normal ivltests gold=macro_str_esc.gold macro_with_args normal ivltests gold=macro_with_args.gold mcl1 normal ivltests gold=mcl1.gold pr622 normal ivltests gold=pr622.gold diff --git a/ivtest/regress-vvp.list b/ivtest/regress-vvp.list new file mode 100644 index 000000000..624b80d54 --- /dev/null +++ b/ivtest/regress-vvp.list @@ -0,0 +1,4 @@ + +# Test list files are a list of test names and the json that +# describes the test. +macro_str_esc vvp_tests/macro_str_esc.json diff --git a/ivtest/run_ivl.py b/ivtest/run_ivl.py new file mode 100644 index 000000000..6e5821bc4 --- /dev/null +++ b/ivtest/run_ivl.py @@ -0,0 +1,242 @@ +'''Functions for running Icarus Verilog + +''' + +import subprocess +import difflib +import os +import sys +import re + +def assemble_iverilog_cmd(source: str, it_dir: str, args: list) -> list: + res = ["iverilog", "-o", os.path.join("work", "a.out")] + res += ["-D__ICARUS_UNSIZED__"] + res += args + src = os.path.join(it_dir, source) + res += [src] + return res + + +def assemble_vvp_cmd(args: list = [], plusargs: list = []) -> list: + res = ["vvp", os.path.join("work", "a.out")] + return res + + +def get_ivl_version () -> list: + '''Figure out the version of the installed iverilog compler. + + The return value is a list of 2 numbers, the major and minor version + numbers, or None if the version string couldn't be found.''' + + # Get the output from the "iverilog -V" command for the version string. + text = subprocess.check_output(["iverilog", "-V"]) + match = re.search(b'Icarus Verilog version ([0-9]+)\.([0-9]+)', text) + if not match: + return None + + return [int(match[1]), int(match[2])] + +def build_runtime(it_key: str) -> None: + '''Check and prepare the runtime environment for a test + + This is called in front of tests to make sure that the directory + structure is correct, and common temp files that might linger from + a previous run are removed.''' + + try: + os.mkdir("log") + except FileExistsError: + pass + + try: + os.remove(os.path.join("log", it_key + ".log")) + except FileNotFoundError: + pass + + try: + os.mkdir("work") + except FileExistsError: + pass + + try: + os.remove(os.path.join("work", "a.out")) + except FileNotFoundError: + pass + +def log_results(key, title, res) -> None: + ''' Write results into log files. + + Generate a log file with the name of the key and title, and + put the stdout and stderr into separate files.''' + + with open(os.path.join("log", f"{key}-{title}-stdout.log"), 'wb') as fd: + fd.write(res.stdout) + + with open(os.path.join("log", f"{key}-{title}-stderr.log"), 'wb') as fd: + fd.write(res.stderr) + + +def compare_files(log_path, gold_path): + '''Compare the log file and the gold file + + The files are read it, line at a time, and the lines are compared. + If they differ, then write tou stdout a unified diff. In any case, + return True or False to indicate the results of the test.''' + + with open(log_path, 'rt') as fd: + a = fd.readlines() + with open(gold_path, 'rt') as fd: + b = fd.readlines() + + flag = a == b + if not flag: + print(f"{log_path} and {gold_path} differ:") + sys.stdout.writelines(difflib.unified_diff(a, b, log_path, gold_path)) + + return flag + + +def run_CE(options : dict) -> list: + ''' Run the compiler, and expect an error + + In this case, we assert that the command fails to run and reports + an error. This is to check that invalid input generates errors.''' + + it_key = options['key'] + it_dir = options['directory'] + it_args = options['iverilog_args'] + + build_runtime(it_key) + + cmd = assemble_iverilog_cmd(options['source'], it_dir, it_args) + res = subprocess.run(cmd, capture_output=True) + log_results(it_key, "iverilog", res) + + if res.returncode == 0: + return [1, "Failed - CE (no error reported)"] + elif res.returncode >= 256: + return [1, "Failed - CE (execution error)"] + else: + return [0, "Passed - CE"] + + +def do_run_normal(options : dict, expected_fail : bool) -> list: + '''Run the iverilog and vvp commands. + + In this case, run the compiler to generate a vvp output file, and + run the vvp command to actually execute the simulation. Collect + the results and look for a "PASSED" string.''' + + it_key = options['key'] + it_dir = options['directory'] + it_iverilog_args = options['iverilog_args'] + it_gold = options['gold'] + it_diff = options['diff'] + + build_runtime(it_key) + + # Run the iverilog command + ivl_cmd = assemble_iverilog_cmd(options['source'], it_dir, it_iverilog_args) + ivl_res = subprocess.run(ivl_cmd, capture_output=True) + + log_results(it_key, "iverilog", ivl_res) + if ivl_res.returncode != 0: + return [1, "Failed - Compile failed"] + + # run the vvp command + vvp_cmd = assemble_vvp_cmd() + vvp_res = subprocess.run(vvp_cmd, capture_output=True) + log_results(it_key, "vvp", vvp_res); + + if vvp_res.returncode != 0: + return [1, "Failed - Vvp execution failed"] + + it_stdout = vvp_res.stdout.decode('ascii') + + # If there is a gold file configured, the test result depends on + # the outputs matching the gold file. + if it_gold is not None: + + compared = True + + log_path = os.path.join("log", f"{it_key}-iverilog-stdout.log") + gold_path = os.path.join("gold", f"{it_gold}-iverilog-stdout.gold") + compared_ivl_stdout = compare_files(log_path, gold_path) + compared = compared and compared_ivl_stdout + + log_path = os.path.join("log", f"{it_key}-iverilog-stderr.log") + gold_path = os.path.join("gold", f"{it_gold}-iverilog-stderr.gold") + compared_ivl_stderr = compare_files(log_path, gold_path) + compared = compared and compared_ivl_stderr + + log_path = os.path.join("log", f"{it_key}-vvp-stdout.log") + gold_path = os.path.join("gold", f"{it_gold}-vvp-stdout.gold") + compared_vvp_stdout = compare_files(log_path, gold_path) + compared = compared and compared_vvp_stdout + + log_path = os.path.join("log", f"{it_key}-vvp-stderr.log") + gold_path = os.path.join("gold", f"{it_gold}-vvp-stderr.gold") + compared_vvp_stderr = compare_files(log_path, gold_path) + compared = compared and compared_vvp_stderr + + if expected_fail: + if compared: + return [1, "Failed - Passed, but expected failure"] + else: + return [0, "Passed - Expected fail"] + else: + if compared: + return [0, "Passed"] + else: + return [1, "Failed - Gold output doesn't match actual output."] + + + # If there is a diff description, then compare named files instead of + # the log and a gold file. + if it_diff is not None: + diff_name1 = it_diff[0] + diff_name2 = it_diff[1] + diff_skip = int(it_diff[2]) + + with open(diff_name1) as fd: + for idx in range(diff_skip): + fd.readline() + diff_data1 = fd.read() + + with open(diff_name2) as fd: + for idx in range(diff_skip): + fd.readline() + diff_data2 = fd.read() + + if expected_fail: + if diff_data1 == diff_data2: + return [1, "Failed - Passed, but expected failure"] + else: + return [0, "Passed"] + else: + if diff_data1 == diff_data2: + return [0, "Passed"] + else: + return [1, f"Failed - Files {diff_name1} and {diff_name2} differ."] + + # Otherwise, look for the PASSED output string in stdout. + for line in it_stdout.splitlines(): + if line == "PASSED": + if expected_fail: + return [1, "Failed - Passed, but expected failure"] + else: + return [0, "Passed"] + + # If there is no PASSED output, and nothing else to check, then + # assume a failure. + if expected_fail: + return [0, "Passed"] + else: + return [1, "Failed - No PASSED output, and no gold file"] + + +def run_normal(options : dict) -> list: + return do_run_normal(options, False) + +def run_EF(options : dict) -> list: + return do_run_normal(options, True) diff --git a/ivtest/test_lists.py b/ivtest/test_lists.py new file mode 100644 index 000000000..13c966fff --- /dev/null +++ b/ivtest/test_lists.py @@ -0,0 +1,76 @@ + +'''Functions for processing test list files + +The read_lists() function is the main export of this package. This function +takes a list of file names, reads all the test items in the files, and +puts the result into a sorted list. + +The tests list file is formatted like so: + + + +The is the name of the test. This is used to generate the source file +name for the test program. + +The is the name of a subdirectory were we search for the test. +So for example, if ==foo and ==bar, then the Verilog source +file will be inferred to be bar/foo.v. + +The is the test type. + +The field sets up how the tests will be checked. Things like gold +files and working directories are given here. +''' + +def read_list(fd) -> list: + '''Return a list of test items (each in list form) from the file. + + The input fd is the file opened in text mode. This function will read + the file, a line at a time, and make a list of lists, with each list + in the list a list of tokens for the line. This is used by the read_lists + function.''' + + build_list = list() + for line_raw in fd: + # Strip comments and leading/traling white space + idx = line_raw.find("#") + if idx < 0: + idx = len(line_raw) + line = line_raw[0:idx].strip() + + # Split into tokens + line_list = line.split() + if len(line_list) == 0: + continue + + build_list.append(line_list) + + return build_list + + +def read_lists(paths: list) -> list: + '''Read the paths in the list, and return the list of tests. + + The input is a list of list file names, and the result is a list + of all the tests, sorted, and with duplicates resolved. The order + of the test file lists is important, as is the order of tests + within each list file.''' + + tests_list = list() + for path in paths: + with open(path, "r") as fd: + tests_list += read_list(fd) + + # The loop above creates a tests_list to list all of the tests in the + # order that they appear. Now we go though the list in order and eliminate + # duplictes by test name. This allows that lists might override tests that + # are already declared. + tests_dict = dict() + for item in tests_list: + tests_dict[item[0]] = item; + + # Convert the result to a sorted list, and return that. + tests_list = list(tests_dict.values()) + tests_list.sort() + + return tests_list diff --git a/ivtest/vvp_reg.py b/ivtest/vvp_reg.py new file mode 100755 index 000000000..a8f47d365 --- /dev/null +++ b/ivtest/vvp_reg.py @@ -0,0 +1,110 @@ +#! python3 +''' +Usage: + vvp_reg + vvp_reg ... + + is a list of files in the current working directory that + each contain a list of tests. By convention, the file has the + suffix ".list". The files will be processed in order, so tests + can be overridden if listed twice. If no files are given, a + default list is used. +''' + +import sys +# It appears that docopt doesn't work on msys2 installations, so +# skip it completely on win32 platforms. +if sys.platform != 'win32': + from docopt import docopt +import test_lists +import json +import run_ivl + + +def process_test(item: list) -> str: + '''Process a single test + + This takes in the list of tokens from the tests list file, and converts + them (interprets them) to a collection of values.''' + + # This is the name of the test, and the name of the main sorce file + it_key = item[0] + test_path = item[1] + with open(test_path, 'rt') as fd: + it_dict = json.load(fd) + + # Get the test type from the json configuration. + it_type = it_dict['type'] + + # Wrap all of this into an options dictionary for ease of handling. + it_options = { + 'key' : it_key, + 'type' : it_type, + 'iverilog_args' : it_dict.get('iverilog-args', [ ]), + 'directory' : "ivltests", + 'source' : it_dict['source'], + 'modulename' : None, + 'gold' : it_dict.get('gold', None), + 'diff' : None + } + + if it_type == "NI": + res = [0, "Not Implemented."] + + elif it_type == "normal": + res = run_ivl.run_normal(it_options) + + elif it_type == "CE": + res = run_ivl.run_CE(it_options) + + elif it_type == "EF": + res = run_ivl.run_EF(it_options) + + else: + res = f"{it_key}: I don't understand the test type ({it_type})." + raise Exception(res) + + return res + + +if __name__ == "__main__": + print(f"Running tests on platform: {sys.platform}") + if sys.platform == 'win32': + args = { "" : [] } + else: + args = docopt(__doc__) + + # This returns [13, 0] or similar + ivl_version = run_ivl.get_ivl_version() + ivl_version_major = ivl_version[0] + print(f"Icarus Verilog version: {ivl_version_major}") + + # Select the lists to use. If any list paths are given on the command + # line, then use only those. Otherwise, use a default list. + list_paths = args[""] + if len(list_paths) == 0: + list_paths = list() + list_paths += ["regress-vvp.list"] + + print(f"Use lists: {list_paths}") + + # Read the list files, to get the tests. + tests_list = test_lists.read_lists(list_paths) + + # We need the width of the widest key so that we can figure out + # how to align the key:result columns. + width = 0 + for cur in tests_list: + if len(cur[0]) > width: + width = len(cur[0]) + + error_count = 0 + for cur in tests_list: + result = process_test(cur) + error_count += result[0] + print(f"{cur[0]:>{width}}: {result[1]}") + + print("===================================================") + print(f"Test results: Ran {len(tests_list)}, Failed {error_count}.") + exit(error_count) + diff --git a/ivtest/vvp_tests/README.txt b/ivtest/vvp_tests/README.txt new file mode 100644 index 000000000..9c760de9a --- /dev/null +++ b/ivtest/vvp_tests/README.txt @@ -0,0 +1,11 @@ + +This directory contains configurations for thet tests that test the iverilog +compiler with the vvp simulation engine. Eash test file is actually a JSON +file that calls out the test type, names the source file, the gold file, any +command argument flags. + +{ + "type" : "normal", + "source" : "macro_str_esc.v", + "gold" : "macro_str_esc" +} diff --git a/ivtest/vvp_tests/case2-S.json b/ivtest/vvp_tests/case2-S.json new file mode 100644 index 000000000..4bab5fe3a --- /dev/null +++ b/ivtest/vvp_tests/case2-S.json @@ -0,0 +1,5 @@ +{ + "type" : "normal", + "source" : "case2.v", + "iverilog-args" : [ "-S" ] +} diff --git a/ivtest/vvp_tests/macro_str_esc.json b/ivtest/vvp_tests/macro_str_esc.json new file mode 100644 index 000000000..09c7bd5e5 --- /dev/null +++ b/ivtest/vvp_tests/macro_str_esc.json @@ -0,0 +1,5 @@ +{ + "type" : "normal", + "source" : "macro_str_esc.v", + "gold" : "macro_str_esc" +} From 3085bd6845d5babba42e95fa744c38aef0f51680 Mon Sep 17 00:00:00 2001 From: Stephen Williams Date: Sat, 11 Feb 2023 18:35:32 -0800 Subject: [PATCH 2/2] ivtest: Move some tests to the revress-vvp list Move some tests from the old structure to the new more descriptive test suite. Remove them from the old lists. --- Documentation/developer/regression_tests.rst | 6 +++++ ivtest/ivltests/struct_packed_write_read.v | 24 +++++++++---------- ivtest/regress-fsv.list | 3 --- ivtest/regress-sv.list | 1 - ivtest/regress-synth.list | 16 ------------- ivtest/regress-v10.list | 11 +-------- ivtest/regress-v11.list | 7 ------ ivtest/regress-v12.list | 7 ------ ivtest/regress-v13.list | 7 ------ ivtest/regress-vvp.list | 24 ++++++++++++++++++- ivtest/sv_regress.list | 2 -- ivtest/vvp_tests/array_packed_write_read.json | 5 ++++ ivtest/vvp_tests/case1.json | 4 ++++ ivtest/vvp_tests/case2.json | 4 ++++ ivtest/vvp_tests/case3.json | 4 ++++ ivtest/vvp_tests/casex_synth.json | 5 ++++ ivtest/vvp_tests/dffsynth-S.json | 5 ++++ ivtest/vvp_tests/dffsynth.json | 4 ++++ ivtest/vvp_tests/dffsynth10.json | 5 ++++ ivtest/vvp_tests/dffsynth11.json | 5 ++++ ivtest/vvp_tests/dffsynth2.json | 5 ++++ ivtest/vvp_tests/dffsynth3.json | 5 ++++ ivtest/vvp_tests/dffsynth4.json | 5 ++++ ivtest/vvp_tests/dffsynth5.json | 5 ++++ ivtest/vvp_tests/dffsynth6.json | 5 ++++ ivtest/vvp_tests/dffsynth7.json | 5 ++++ ivtest/vvp_tests/dffsynth8.json | 5 ++++ ivtest/vvp_tests/dffsynth9.json | 5 ++++ ivtest/vvp_tests/memsynth1.json | 5 ++++ .../vvp_tests/struct_packed_write_read.json | 5 ++++ .../vvp_tests/struct_packed_write_read2.json | 5 ++++ 31 files changed, 138 insertions(+), 66 deletions(-) create mode 100644 ivtest/vvp_tests/array_packed_write_read.json create mode 100644 ivtest/vvp_tests/case1.json create mode 100644 ivtest/vvp_tests/case2.json create mode 100644 ivtest/vvp_tests/case3.json create mode 100644 ivtest/vvp_tests/casex_synth.json create mode 100644 ivtest/vvp_tests/dffsynth-S.json create mode 100644 ivtest/vvp_tests/dffsynth.json create mode 100644 ivtest/vvp_tests/dffsynth10.json create mode 100644 ivtest/vvp_tests/dffsynth11.json create mode 100644 ivtest/vvp_tests/dffsynth2.json create mode 100644 ivtest/vvp_tests/dffsynth3.json create mode 100644 ivtest/vvp_tests/dffsynth4.json create mode 100644 ivtest/vvp_tests/dffsynth5.json create mode 100644 ivtest/vvp_tests/dffsynth6.json create mode 100644 ivtest/vvp_tests/dffsynth7.json create mode 100644 ivtest/vvp_tests/dffsynth8.json create mode 100644 ivtest/vvp_tests/dffsynth9.json create mode 100644 ivtest/vvp_tests/memsynth1.json create mode 100644 ivtest/vvp_tests/struct_packed_write_read.json create mode 100644 ivtest/vvp_tests/struct_packed_write_read2.json diff --git a/Documentation/developer/regression_tests.rst b/Documentation/developer/regression_tests.rst index c443613ee..8f8eec7db 100644 --- a/Documentation/developer/regression_tests.rst +++ b/Documentation/developer/regression_tests.rst @@ -71,3 +71,9 @@ basename of the gold files, with separate actual gold files for the iverilog and vvp stderr and stdout. For example, if a "normal" test includes a gold file, then the program is compiled and run, and the outputs are compared with the gold file to make sure it ran properly. + +iverilog-args (optional) +^^^^^^^^^^^^^^^^^^^^^^^^ + +If this is specified, it is a list of strings that are passed as arguments to +the iverilog command line. diff --git a/ivtest/ivltests/struct_packed_write_read.v b/ivtest/ivltests/struct_packed_write_read.v index 435d20020..8073130e7 100644 --- a/ivtest/ivltests/struct_packed_write_read.v +++ b/ivtest/ivltests/struct_packed_write_read.v @@ -62,20 +62,20 @@ module test; if (word_sw3 !== 16'b01111111_11111111) begin $display("FAILED -- word_sw3 = 'b%b", word_sw3 ); err=1; end // access to parts of structure elements if (word_ep0 !== 16'bxxxxxxxx_xxxxxxxx) begin $display("FAILED -- word_ep0 = 'b%b", word_ep0 ); err=1; end - if (word_ep1 !== 16'bxxxx1111_xxxx0000) begin $display("FAILED -- word_ep1 = 'b%b", word_ep1 ); err=1; end - if (word_ep1.high !== 8'bxxxx1111 ) begin $display("FAILED -- word_ep1.high = 'b%b", word_ep1.high); err=1; end - if (word_ep1.low !== 8'bxxxx0000 ) begin $display("FAILED -- word_ep1.low = 'b%b", word_ep1.low ); err=1; end - if (word_ep2 !== 16'bxxxx1111_xxxx0000) begin $display("FAILED -- word_ep2 = 'b%b", word_ep2 ); err=1; end - if (word_ep2.high !== 8'bxxxx1111 ) begin $display("FAILED -- word_ep2.high = 'b%b", word_ep2.high); err=1; end - if (word_ep2.low !== 8'bxxxx0000 ) begin $display("FAILED -- word_ep2.low = 'b%b", word_ep2.low ); err=1; end - if (word_ep3 !== 16'bxxxx0111_xxxx0000) begin $display("FAILED -- word_ep3 = 'b%b", word_ep3 ); err=1; end - if (word_ep3.high !== 8'bxxxx0111 ) begin $display("FAILED -- word_ep3.high = 'b%b", word_ep3.high); err=1; end - if (word_ep3.low !== 8'bxxxx0000 ) begin $display("FAILED -- word_ep3.low = 'b%b", word_ep3.low ); err=1; end + if (word_ep1 !== 16'bzzzz1111_zzzz0000) begin $display("FAILED -- word_ep1 = 'b%b", word_ep1 ); err=1; end + if (word_ep1.high !== 8'bzzzz1111 ) begin $display("FAILED -- word_ep1.high = 'b%b", word_ep1.high); err=1; end + if (word_ep1.low !== 8'bzzzz0000 ) begin $display("FAILED -- word_ep1.low = 'b%b", word_ep1.low ); err=1; end + if (word_ep2 !== 16'bzzzz1111_zzzz0000) begin $display("FAILED -- word_ep2 = 'b%b", word_ep2 ); err=1; end + if (word_ep2.high !== 8'bzzzz1111 ) begin $display("FAILED -- word_ep2.high = 'b%b", word_ep2.high); err=1; end + if (word_ep2.low !== 8'bzzzz0000 ) begin $display("FAILED -- word_ep2.low = 'b%b", word_ep2.low ); err=1; end + if (word_ep3 !== 16'bzzzz0111_zzzz0000) begin $display("FAILED -- word_ep3 = 'b%b", word_ep3 ); err=1; end + if (word_ep3.high !== 8'bzzzz0111 ) begin $display("FAILED -- word_ep3.high = 'b%b", word_ep3.high); err=1; end + if (word_ep3.low !== 8'bzzzz0000 ) begin $display("FAILED -- word_ep3.low = 'b%b", word_ep3.low ); err=1; end // access to parts of the whole structure if (word_sp0 !== 16'bxxxxxxxx_xxxxxxxx) begin $display("FAILED -- word_sp0 = 'b%b", word_sp0 ); err=1; end - if (word_sp1 !== 16'bxxxx1111_1111xxxx) begin $display("FAILED -- word_sp1 = 'b%b", word_sp1 ); err=1; end - if (word_sp2 !== 16'bxxxx1111_1111xxxx) begin $display("FAILED -- word_sp2 = 'b%b", word_sp2 ); err=1; end - if (word_sp3 !== 16'bxxxx0111_1111xxxx) begin $display("FAILED -- word_sp3 = 'b%b", word_sp3 ); err=1; end + if (word_sp1 !== 16'bzzzz1111_1111zzzz) begin $display("FAILED -- word_sp1 = 'b%b", word_sp1 ); err=1; end + if (word_sp2 !== 16'bzzzz1111_1111zzzz) begin $display("FAILED -- word_sp2 = 'b%b", word_sp2 ); err=1; end + if (word_sp3 !== 16'bzzzz0111_1111zzzz) begin $display("FAILED -- word_sp3 = 'b%b", word_sp3 ); err=1; end if (!err) $display("PASSED"); end diff --git a/ivtest/regress-fsv.list b/ivtest/regress-fsv.list index 9424f42eb..4bf61e81c 100644 --- a/ivtest/regress-fsv.list +++ b/ivtest/regress-fsv.list @@ -99,9 +99,6 @@ br_gh306b normal ivltests case5-syn-fail normal ivltests casesynth7 normal ivltests casesynth8 normal ivltests -dffsynth normal ivltests -dffsynth8 normal ivltests -memsynth1 normal ivltests memsynth2 normal ivltests memsynth3 normal ivltests memsynth5 normal ivltests diff --git a/ivtest/regress-sv.list b/ivtest/regress-sv.list index a888672ba..9b8eaf01c 100644 --- a/ivtest/regress-sv.list +++ b/ivtest/regress-sv.list @@ -516,7 +516,6 @@ struct_packed_queue_fail CE,-g2009 ivltests struct_packed_sysfunct normal,-g2009 ivltests struct_packed_sysfunct2 normal,-g2009 ivltests struct_packed_uarray_fail CE,-g2009 ivltests -struct_packed_write_read2 normal,-g2009 ivltests struct_invalid_member CE,-g2009 ivltests gold=struct_invalid_member.gold struct_signed normal,-g2009 ivltests sv-constants normal,-g2005-sv ivltests diff --git a/ivtest/regress-synth.list b/ivtest/regress-synth.list index 7096ceab0..f925122d9 100644 --- a/ivtest/regress-synth.list +++ b/ivtest/regress-synth.list @@ -94,9 +94,6 @@ br_gh99x normal ivltests br_gh115 normal ivltests br_gh306a CE ivltests br_gh306b CE ivltests -case1 normal ivltests -case2 normal ivltests -case3 normal ivltests case4 normal ivltests case5 normal ivltests case5-syn-fail CE ivltests @@ -112,22 +109,10 @@ casesynth6 normal ivltests casesynth7 normal ivltests gold=casesynth7.gold casesynth8 CE ivltests casesynth9 normal ivltests -casex_synth normal ivltests condit1 normal ivltests conditsynth1 normal ivltests conditsynth2 normal ivltests conditsynth3 normal ivltests -dffsynth normal ivltests -dffsynth2 normal ivltests -dffsynth3 normal ivltests -dffsynth4 normal ivltests -dffsynth5 normal ivltests -dffsynth6 normal ivltests -dffsynth7 normal ivltests -dffsynth8 CE ivltests -dffsynth9 normal ivltests -dffsynth10 normal ivltests -dffsynth11 normal ivltests ff_dual_enable normal ivltests for_loop_synth normal ivltests for_loop_synth2 normal ivltests @@ -138,7 +123,6 @@ inside_synth2 normal ivltests inside_synth3 normal ivltests land5 normal ivltests lcatsynth normal ivltests -memsynth1 normal ivltests memsynth2 normal ivltests memsynth3 normal ivltests memsynth4 normal ivltests diff --git a/ivtest/regress-v10.list b/ivtest/regress-v10.list index c2221d5a1..524b2a293 100644 --- a/ivtest/regress-v10.list +++ b/ivtest/regress-v10.list @@ -289,8 +289,6 @@ br_gh115 CE,-S ivltests basiclatch normal ivltests blocksynth2 normal ivltests blocksynth3 normal ivltests -case1 normal ivltests -case2 normal ivltests case4 normal ivltests case5 normal ivltests case5-syn-fail normal ivltests @@ -299,20 +297,13 @@ casesynth1 normal ivltests casesynth2 normal ivltests casesynth3 normal ivltests casesynth7 NI -casex_synth normal ivltests condit1 normal ivltests conditsynth1 normal ivltests conditsynth2 normal ivltests conditsynth3 normal ivltests -dffsynth normal ivltests -dffsynth3 normal ivltests -dffsynth4 normal ivltests -dffsynth9 normal ivltests -dffsynth10 normal ivltests -dffsynth11 normal ivltests + inside_synth normal ivltests inside_synth3 normal ivltests -memsynth1 normal ivltests memsynth2 normal ivltests memsynth3 normal ivltests memsynth5 normal ivltests diff --git a/ivtest/regress-v11.list b/ivtest/regress-v11.list index 57fca1d72..259d48833 100644 --- a/ivtest/regress-v11.list +++ b/ivtest/regress-v11.list @@ -87,8 +87,6 @@ sv_default_port_value3 CE,-g2009 ivltests br_gh440 CE,-g2009 ivltests gold=br_gh440-v11.gold # v11 has incomplete synthesis support -dffsynth CE,-S ivltests -memsynth1 CE,-S ivltests memsynth2 CE,-S ivltests memsynth3 CE,-S ivltests memsynth5 CE,-S ivltests @@ -97,11 +95,6 @@ memsynth7 CE,-S ivltests memsynth9 CE,-S ivltests mix_reset CE,-S ivltests -# These tests pass, but synthesis is creating unnecessary latches. -case1 normal ivltests -case2 normal ivltests -casex_synth normal ivltests - # For V11 vvp does not fail for these tests automatic_error11 normal ivltests gold=automatic_error11.gold automatic_error12 normal ivltests gold=automatic_error12.gold diff --git a/ivtest/regress-v12.list b/ivtest/regress-v12.list index 8f09d06f0..766b2eeef 100644 --- a/ivtest/regress-v12.list +++ b/ivtest/regress-v12.list @@ -68,8 +68,6 @@ # # v11 has incomplete synthesis support -dffsynth CE,-S ivltests -memsynth1 CE,-S ivltests memsynth2 CE,-S ivltests memsynth3 CE,-S ivltests memsynth5 CE,-S ivltests @@ -77,8 +75,3 @@ memsynth6 CE,-S ivltests memsynth7 CE,-S ivltests memsynth9 CE,-S ivltests mix_reset CE,-S ivltests - -# These tests pass, but synthesis is creating unnecessary latches. -case1 normal ivltests -case2 normal ivltests -casex_synth normal ivltests diff --git a/ivtest/regress-v13.list b/ivtest/regress-v13.list index 3a773a9c3..d62d6b1af 100644 --- a/ivtest/regress-v13.list +++ b/ivtest/regress-v13.list @@ -68,8 +68,6 @@ # # v13 has incomplete synthesis support -dffsynth CE,-S ivltests -memsynth1 CE,-S ivltests memsynth2 CE,-S ivltests memsynth3 CE,-S ivltests memsynth5 CE,-S ivltests @@ -77,8 +75,3 @@ memsynth6 CE,-S ivltests memsynth7 CE,-S ivltests memsynth9 CE,-S ivltests mix_reset CE,-S ivltests - -# These tests pass, but synthesis is creating unnecessary latches. -case1 normal ivltests -case2 normal ivltests -casex_synth normal ivltests diff --git a/ivtest/regress-vvp.list b/ivtest/regress-vvp.list index 624b80d54..eb22d756a 100644 --- a/ivtest/regress-vvp.list +++ b/ivtest/regress-vvp.list @@ -1,4 +1,26 @@ # Test list files are a list of test names and the json that # describes the test. -macro_str_esc vvp_tests/macro_str_esc.json + +array_packed_write_read vvp_tests/array_packed_write_read.json +case1 vvp_tests/case1.json +case2 vvp_tests/case2.json +case2-S vvp_tests/case2-S.json +case3 vvp_tests/case3.json +casex_synth vvp_tests/casex_synth.json +dffsynth vvp_tests/dffsynth.json +dffsynth-S vvp_tests/dffsynth-S.json +dffsynth2 vvp_tests/dffsynth2.json +dffsynth3 vvp_tests/dffsynth3.json +dffsynth4 vvp_tests/dffsynth4.json +dffsynth5 vvp_tests/dffsynth5.json +dffsynth6 vvp_tests/dffsynth6.json +dffsynth7 vvp_tests/dffsynth7.json +dffsynth8 vvp_tests/dffsynth8.json +dffsynth9 vvp_tests/dffsynth9.json +dffsynth10 vvp_tests/dffsynth10.json +dffsynth11 vvp_tests/dffsynth11.json +macro_str_esc vvp_tests/macro_str_esc.json +memsynth1 vvp_tests/memsynth1.json +struct_packed_write_read vvp_tests/struct_packed_write_read.json +struct_packed_write_read2 vvp_tests/struct_packed_write_read2.json diff --git a/ivtest/sv_regress.list b/ivtest/sv_regress.list index dc834464a..88e5ffe0e 100644 --- a/ivtest/sv_regress.list +++ b/ivtest/sv_regress.list @@ -59,10 +59,8 @@ # gold or diff commands. # -struct_packed_write_read normal,-g2009 ivltests struct_packed_value_list normal,-g2009 ivltests -array_packed_write_read normal,-g2009 ivltests array_packed_value_list normal,-g2009 ivltests array_packed_sysfunct normal,-g2009 ivltests diff --git a/ivtest/vvp_tests/array_packed_write_read.json b/ivtest/vvp_tests/array_packed_write_read.json new file mode 100644 index 000000000..f83a222ff --- /dev/null +++ b/ivtest/vvp_tests/array_packed_write_read.json @@ -0,0 +1,5 @@ +{ + "type" : "normal", + "source" : "array_packed_write_read.v", + "iverilog-args" : [ "-g2009" ] +} diff --git a/ivtest/vvp_tests/case1.json b/ivtest/vvp_tests/case1.json new file mode 100644 index 000000000..0b64e7e35 --- /dev/null +++ b/ivtest/vvp_tests/case1.json @@ -0,0 +1,4 @@ +{ + "type" : "normal", + "source" : "case1.v" +} diff --git a/ivtest/vvp_tests/case2.json b/ivtest/vvp_tests/case2.json new file mode 100644 index 000000000..f01a5f78c --- /dev/null +++ b/ivtest/vvp_tests/case2.json @@ -0,0 +1,4 @@ +{ + "type" : "normal", + "source" : "case2.v" +} diff --git a/ivtest/vvp_tests/case3.json b/ivtest/vvp_tests/case3.json new file mode 100644 index 000000000..6b2bad14a --- /dev/null +++ b/ivtest/vvp_tests/case3.json @@ -0,0 +1,4 @@ +{ + "type" : "normal", + "source" : "case3.v" +} diff --git a/ivtest/vvp_tests/casex_synth.json b/ivtest/vvp_tests/casex_synth.json new file mode 100644 index 000000000..77925bcab --- /dev/null +++ b/ivtest/vvp_tests/casex_synth.json @@ -0,0 +1,5 @@ +{ + "type" : "normal", + "source" : "casex_synth.v", + "iverilog-args" : [ "-S" ] +} diff --git a/ivtest/vvp_tests/dffsynth-S.json b/ivtest/vvp_tests/dffsynth-S.json new file mode 100644 index 000000000..be08bda34 --- /dev/null +++ b/ivtest/vvp_tests/dffsynth-S.json @@ -0,0 +1,5 @@ +{ + "type" : "NI", + "source" : "dffsynth.v", + "iverilog-args" : [ "-S" ] +} diff --git a/ivtest/vvp_tests/dffsynth.json b/ivtest/vvp_tests/dffsynth.json new file mode 100644 index 000000000..c25531c2e --- /dev/null +++ b/ivtest/vvp_tests/dffsynth.json @@ -0,0 +1,4 @@ +{ + "type" : "normal", + "source" : "dffsynth.v" +} diff --git a/ivtest/vvp_tests/dffsynth10.json b/ivtest/vvp_tests/dffsynth10.json new file mode 100644 index 000000000..817d7a37b --- /dev/null +++ b/ivtest/vvp_tests/dffsynth10.json @@ -0,0 +1,5 @@ +{ + "type" : "normal", + "source" : "dffsynth10.v", + "iverilog-args" : [ "-S" ] +} diff --git a/ivtest/vvp_tests/dffsynth11.json b/ivtest/vvp_tests/dffsynth11.json new file mode 100644 index 000000000..519dfdcae --- /dev/null +++ b/ivtest/vvp_tests/dffsynth11.json @@ -0,0 +1,5 @@ +{ + "type" : "normal", + "source" : "dffsynth11.v", + "iverilog-args" : [ "-S" ] +} diff --git a/ivtest/vvp_tests/dffsynth2.json b/ivtest/vvp_tests/dffsynth2.json new file mode 100644 index 000000000..61718a896 --- /dev/null +++ b/ivtest/vvp_tests/dffsynth2.json @@ -0,0 +1,5 @@ +{ + "type" : "normal", + "source" : "dffsynth2.v", + "iverilog-args" : [ "-S" ] +} diff --git a/ivtest/vvp_tests/dffsynth3.json b/ivtest/vvp_tests/dffsynth3.json new file mode 100644 index 000000000..295dba54d --- /dev/null +++ b/ivtest/vvp_tests/dffsynth3.json @@ -0,0 +1,5 @@ +{ + "type" : "normal", + "source" : "dffsynth3.v", + "iverilog-args" : [ "-S" ] +} diff --git a/ivtest/vvp_tests/dffsynth4.json b/ivtest/vvp_tests/dffsynth4.json new file mode 100644 index 000000000..460354bfd --- /dev/null +++ b/ivtest/vvp_tests/dffsynth4.json @@ -0,0 +1,5 @@ +{ + "type" : "normal", + "source" : "dffsynth4.v", + "iverilog-args" : [ "-S" ] +} diff --git a/ivtest/vvp_tests/dffsynth5.json b/ivtest/vvp_tests/dffsynth5.json new file mode 100644 index 000000000..8e3519af8 --- /dev/null +++ b/ivtest/vvp_tests/dffsynth5.json @@ -0,0 +1,5 @@ +{ + "type" : "normal", + "source" : "dffsynth5.v", + "iverilog-args" : [ "-S" ] +} diff --git a/ivtest/vvp_tests/dffsynth6.json b/ivtest/vvp_tests/dffsynth6.json new file mode 100644 index 000000000..a240aa2bf --- /dev/null +++ b/ivtest/vvp_tests/dffsynth6.json @@ -0,0 +1,5 @@ +{ + "type" : "normal", + "source" : "dffsynth6.v", + "iverilog-args" : [ "-S" ] +} diff --git a/ivtest/vvp_tests/dffsynth7.json b/ivtest/vvp_tests/dffsynth7.json new file mode 100644 index 000000000..a0b0c0e4d --- /dev/null +++ b/ivtest/vvp_tests/dffsynth7.json @@ -0,0 +1,5 @@ +{ + "type" : "normal", + "source" : "dffsynth7.v", + "iverilog-args" : [ "-S" ] +} diff --git a/ivtest/vvp_tests/dffsynth8.json b/ivtest/vvp_tests/dffsynth8.json new file mode 100644 index 000000000..f5bdcbc65 --- /dev/null +++ b/ivtest/vvp_tests/dffsynth8.json @@ -0,0 +1,5 @@ +{ + "type" : "NI", + "source" : "dffsynth8.v", + "iverilog-args" : [ "-S" ] +} diff --git a/ivtest/vvp_tests/dffsynth9.json b/ivtest/vvp_tests/dffsynth9.json new file mode 100644 index 000000000..cfc0c8a97 --- /dev/null +++ b/ivtest/vvp_tests/dffsynth9.json @@ -0,0 +1,5 @@ +{ + "type" : "normal", + "source" : "dffsynth9.v", + "iverilog-args" : [ "-S" ] +} diff --git a/ivtest/vvp_tests/memsynth1.json b/ivtest/vvp_tests/memsynth1.json new file mode 100644 index 000000000..c25185d51 --- /dev/null +++ b/ivtest/vvp_tests/memsynth1.json @@ -0,0 +1,5 @@ +{ + "type" : "NI", + "source" : "memsynth1.v", + "iverilog-args" : [ "-S" ] +} diff --git a/ivtest/vvp_tests/struct_packed_write_read.json b/ivtest/vvp_tests/struct_packed_write_read.json new file mode 100644 index 000000000..766d7d0c0 --- /dev/null +++ b/ivtest/vvp_tests/struct_packed_write_read.json @@ -0,0 +1,5 @@ +{ + "type" : "normal", + "source" : "struct_packed_write_read.v", + "iverilog-args" : [ "-g2009" ] +} diff --git a/ivtest/vvp_tests/struct_packed_write_read2.json b/ivtest/vvp_tests/struct_packed_write_read2.json new file mode 100644 index 000000000..a7d60a6c6 --- /dev/null +++ b/ivtest/vvp_tests/struct_packed_write_read2.json @@ -0,0 +1,5 @@ +{ + "type" : "normal", + "source" : "struct_packed_write_read2.v", + "iverilog-args" : [ "-g2009" ] +}