diff --git a/.github/test.sh b/.github/test.sh index 7b5916db1..a49bdfb3c 100755 --- a/.github/test.sh +++ b/.github/test.sh @@ -11,4 +11,6 @@ perl vvp_reg.pl || status=1 perl vpi_reg.pl || status=1 +python3 vvp_reg.py || status=1 + exit $status diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b59e5b581..1431ecf4d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,6 +22,7 @@ jobs: - name: Install dependencies run: | brew install bison + pip3 install docopt - name: Build, check and install run: | @@ -52,7 +53,7 @@ jobs: - name: Install dependencies run: | sudo apt update -qq - sudo apt install -y make g++ git bison flex gperf libreadline-dev autoconf python3-sphinx + sudo apt install -y make g++ git bison flex gperf libreadline-dev autoconf python3-sphinx python3-docopt - name: Build, check and install run: | @@ -99,8 +100,11 @@ jobs: git base-devel gperf + python-pip mingw-w64-${{ matrix.arch }}-toolchain + - uses: actions/setup-python@v4 + - name: Build and check run: | cd msys2 @@ -110,7 +114,8 @@ jobs: run: pacman -U --noconfirm msys2/*.zst - name: Test - run: ./.github/test.sh + run: | + ./.github/test.sh - uses: actions/upload-artifact@v2 with: diff --git a/Documentation/developer/index.rst b/Documentation/developer/index.rst index 3be5a5337..d9a4689bb 100644 --- a/Documentation/developer/index.rst +++ b/Documentation/developer/index.rst @@ -9,5 +9,6 @@ Icarus Verilog. :maxdepth: 1 getting_started + regression_tests version_stamps diff --git a/Documentation/developer/regression_tests.rst b/Documentation/developer/regression_tests.rst new file mode 100644 index 000000000..8f8eec7db --- /dev/null +++ b/Documentation/developer/regression_tests.rst @@ -0,0 +1,79 @@ + +The Regression Test Suite +========================= + +Icarus Verilog development includes a regression test suite that is included +along with the source. The "ivtest" directory contains the regression test +suite, and this suite is used by the github actions as continuous integration +to make sure the code is always going forward. + +NOTE: There are scripts written in perl to run the regression tests, but they +are bing gradually replaced with a newer set of scripts. It is the newer +method that is described here. + +Test Descriptions +----------------- + +Regression tests are listed in the regress-vvp.list file. Each line lists the +name of the test and the path to the dest description. The list file is +therefore pretty simple, and all the description of the test is in the +description file: + +.. code-block:: console + + macro_str_esc vvp_tests/macro_str_esc.json + +The "name" is a simple name, and the test-description-file is the path (relative +the ivtest directory) to the description file. A simple test description file +is a JSON file, like this: + +.. code-block:: java + + { + "type" : "normal", + "source" : "macro_str_esc.v", + "gold" : "macro_str_esc" + } + +This description file contains all the information that the vvp_reg.py script +needs to run the regression test. The sections below describe the keys and +values in the description file dictionary. + +source (required) +^^^^^^^^^^^^^^^^^ +This specifies the name of the source file. The file is actually to be found +in the ivltests/ directory. + + +type (required) +^^^^^^^^^^^^^^^ + +This describes the kind of test to run. The valid values are: + +* **normal** - Compile the source using the iverilog compiler vvp target, and if + that succeeds execute it using the vvp command. If there is no gold file + specified, then look for an output line with the "PASSED" string. + +* **NI** - Mark the test as not implemented. The test will be skipped without + running or reporting an error. + +* **CE** - Compile, but expect the compiler to fail + +* **EF** - Compile and run, burt expect the run time to fail. + +gold (optional) +^^^^^^^^^^^^^^^ + +If this is specified, it replaces the "Passed" condition with a comparison of +the output with a gold file. The argument is the name of the gold file set, +which will be found in the "gold/" directory. The name here is actually the +basename of the gold files, with separate actual gold files for the iverilog +and vvp stderr and stdout. For example, if a "normal" test includes a gold +file, then the program is compiled and run, and the outputs are compared with +the gold file to make sure it ran properly. + +iverilog-args (optional) +^^^^^^^^^^^^^^^^^^^^^^^^ + +If this is specified, it is a list of strings that are passed as arguments to +the iverilog command line. diff --git a/ivtest/.gitignore b/ivtest/.gitignore index 21462303a..e5a4b3d37 100644 --- a/ivtest/.gitignore +++ b/ivtest/.gitignore @@ -10,6 +10,7 @@ log/ work/ vpi_log/ vhdl/ +__pycache__/ # The normal regression output files. diff --git a/ivtest/gold/macro_str_esc-iverilog-stderr.gold b/ivtest/gold/macro_str_esc-iverilog-stderr.gold new file mode 100644 index 000000000..e69de29bb diff --git a/ivtest/gold/macro_str_esc-iverilog-stdout.gold b/ivtest/gold/macro_str_esc-iverilog-stdout.gold new file mode 100644 index 000000000..e69de29bb diff --git a/ivtest/gold/macro_str_esc-vvp-stderr.gold b/ivtest/gold/macro_str_esc-vvp-stderr.gold new file mode 100644 index 000000000..e69de29bb diff --git a/ivtest/gold/macro_str_esc.gold b/ivtest/gold/macro_str_esc-vvp-stdout.gold similarity index 100% rename from ivtest/gold/macro_str_esc.gold rename to ivtest/gold/macro_str_esc-vvp-stdout.gold diff --git a/ivtest/ivltests/struct_packed_write_read.v b/ivtest/ivltests/struct_packed_write_read.v index 435d20020..8073130e7 100644 --- a/ivtest/ivltests/struct_packed_write_read.v +++ b/ivtest/ivltests/struct_packed_write_read.v @@ -62,20 +62,20 @@ module test; if (word_sw3 !== 16'b01111111_11111111) begin $display("FAILED -- word_sw3 = 'b%b", word_sw3 ); err=1; end // access to parts of structure elements if (word_ep0 !== 16'bxxxxxxxx_xxxxxxxx) begin $display("FAILED -- word_ep0 = 'b%b", word_ep0 ); err=1; end - if (word_ep1 !== 16'bxxxx1111_xxxx0000) begin $display("FAILED -- word_ep1 = 'b%b", word_ep1 ); err=1; end - if (word_ep1.high !== 8'bxxxx1111 ) begin $display("FAILED -- word_ep1.high = 'b%b", word_ep1.high); err=1; end - if (word_ep1.low !== 8'bxxxx0000 ) begin $display("FAILED -- word_ep1.low = 'b%b", word_ep1.low ); err=1; end - if (word_ep2 !== 16'bxxxx1111_xxxx0000) begin $display("FAILED -- word_ep2 = 'b%b", word_ep2 ); err=1; end - if (word_ep2.high !== 8'bxxxx1111 ) begin $display("FAILED -- word_ep2.high = 'b%b", word_ep2.high); err=1; end - if (word_ep2.low !== 8'bxxxx0000 ) begin $display("FAILED -- word_ep2.low = 'b%b", word_ep2.low ); err=1; end - if (word_ep3 !== 16'bxxxx0111_xxxx0000) begin $display("FAILED -- word_ep3 = 'b%b", word_ep3 ); err=1; end - if (word_ep3.high !== 8'bxxxx0111 ) begin $display("FAILED -- word_ep3.high = 'b%b", word_ep3.high); err=1; end - if (word_ep3.low !== 8'bxxxx0000 ) begin $display("FAILED -- word_ep3.low = 'b%b", word_ep3.low ); err=1; end + if (word_ep1 !== 16'bzzzz1111_zzzz0000) begin $display("FAILED -- word_ep1 = 'b%b", word_ep1 ); err=1; end + if (word_ep1.high !== 8'bzzzz1111 ) begin $display("FAILED -- word_ep1.high = 'b%b", word_ep1.high); err=1; end + if (word_ep1.low !== 8'bzzzz0000 ) begin $display("FAILED -- word_ep1.low = 'b%b", word_ep1.low ); err=1; end + if (word_ep2 !== 16'bzzzz1111_zzzz0000) begin $display("FAILED -- word_ep2 = 'b%b", word_ep2 ); err=1; end + if (word_ep2.high !== 8'bzzzz1111 ) begin $display("FAILED -- word_ep2.high = 'b%b", word_ep2.high); err=1; end + if (word_ep2.low !== 8'bzzzz0000 ) begin $display("FAILED -- word_ep2.low = 'b%b", word_ep2.low ); err=1; end + if (word_ep3 !== 16'bzzzz0111_zzzz0000) begin $display("FAILED -- word_ep3 = 'b%b", word_ep3 ); err=1; end + if (word_ep3.high !== 8'bzzzz0111 ) begin $display("FAILED -- word_ep3.high = 'b%b", word_ep3.high); err=1; end + if (word_ep3.low !== 8'bzzzz0000 ) begin $display("FAILED -- word_ep3.low = 'b%b", word_ep3.low ); err=1; end // access to parts of the whole structure if (word_sp0 !== 16'bxxxxxxxx_xxxxxxxx) begin $display("FAILED -- word_sp0 = 'b%b", word_sp0 ); err=1; end - if (word_sp1 !== 16'bxxxx1111_1111xxxx) begin $display("FAILED -- word_sp1 = 'b%b", word_sp1 ); err=1; end - if (word_sp2 !== 16'bxxxx1111_1111xxxx) begin $display("FAILED -- word_sp2 = 'b%b", word_sp2 ); err=1; end - if (word_sp3 !== 16'bxxxx0111_1111xxxx) begin $display("FAILED -- word_sp3 = 'b%b", word_sp3 ); err=1; end + if (word_sp1 !== 16'bzzzz1111_1111zzzz) begin $display("FAILED -- word_sp1 = 'b%b", word_sp1 ); err=1; end + if (word_sp2 !== 16'bzzzz1111_1111zzzz) begin $display("FAILED -- word_sp2 = 'b%b", word_sp2 ); err=1; end + if (word_sp3 !== 16'bzzzz0111_1111zzzz) begin $display("FAILED -- word_sp3 = 'b%b", word_sp3 ); err=1; end if (!err) $display("PASSED"); end diff --git a/ivtest/regress-fsv.list b/ivtest/regress-fsv.list index 9424f42eb..4bf61e81c 100644 --- a/ivtest/regress-fsv.list +++ b/ivtest/regress-fsv.list @@ -99,9 +99,6 @@ br_gh306b normal ivltests case5-syn-fail normal ivltests casesynth7 normal ivltests casesynth8 normal ivltests -dffsynth normal ivltests -dffsynth8 normal ivltests -memsynth1 normal ivltests memsynth2 normal ivltests memsynth3 normal ivltests memsynth5 normal ivltests diff --git a/ivtest/regress-ivl1.list b/ivtest/regress-ivl1.list index effab3aac..cbf094f98 100644 --- a/ivtest/regress-ivl1.list +++ b/ivtest/regress-ivl1.list @@ -75,7 +75,6 @@ #------------------------------------------------------------------------------ # Escaped defines `` -macro_str_esc normal ivltests gold=macro_str_esc.gold macro_with_args normal ivltests gold=macro_with_args.gold mcl1 normal ivltests gold=mcl1.gold pr622 normal ivltests gold=pr622.gold diff --git a/ivtest/regress-sv.list b/ivtest/regress-sv.list index df3af90a2..a3480f7a3 100644 --- a/ivtest/regress-sv.list +++ b/ivtest/regress-sv.list @@ -516,7 +516,6 @@ struct_packed_queue_fail CE,-g2009 ivltests struct_packed_sysfunct normal,-g2009 ivltests struct_packed_sysfunct2 normal,-g2009 ivltests struct_packed_uarray_fail CE,-g2009 ivltests -struct_packed_write_read2 normal,-g2009 ivltests struct_invalid_member CE,-g2009 ivltests gold=struct_invalid_member.gold struct_signed normal,-g2009 ivltests sv-constants normal,-g2005-sv ivltests diff --git a/ivtest/regress-synth.list b/ivtest/regress-synth.list index 7096ceab0..f925122d9 100644 --- a/ivtest/regress-synth.list +++ b/ivtest/regress-synth.list @@ -94,9 +94,6 @@ br_gh99x normal ivltests br_gh115 normal ivltests br_gh306a CE ivltests br_gh306b CE ivltests -case1 normal ivltests -case2 normal ivltests -case3 normal ivltests case4 normal ivltests case5 normal ivltests case5-syn-fail CE ivltests @@ -112,22 +109,10 @@ casesynth6 normal ivltests casesynth7 normal ivltests gold=casesynth7.gold casesynth8 CE ivltests casesynth9 normal ivltests -casex_synth normal ivltests condit1 normal ivltests conditsynth1 normal ivltests conditsynth2 normal ivltests conditsynth3 normal ivltests -dffsynth normal ivltests -dffsynth2 normal ivltests -dffsynth3 normal ivltests -dffsynth4 normal ivltests -dffsynth5 normal ivltests -dffsynth6 normal ivltests -dffsynth7 normal ivltests -dffsynth8 CE ivltests -dffsynth9 normal ivltests -dffsynth10 normal ivltests -dffsynth11 normal ivltests ff_dual_enable normal ivltests for_loop_synth normal ivltests for_loop_synth2 normal ivltests @@ -138,7 +123,6 @@ inside_synth2 normal ivltests inside_synth3 normal ivltests land5 normal ivltests lcatsynth normal ivltests -memsynth1 normal ivltests memsynth2 normal ivltests memsynth3 normal ivltests memsynth4 normal ivltests diff --git a/ivtest/regress-v10.list b/ivtest/regress-v10.list index c2221d5a1..524b2a293 100644 --- a/ivtest/regress-v10.list +++ b/ivtest/regress-v10.list @@ -289,8 +289,6 @@ br_gh115 CE,-S ivltests basiclatch normal ivltests blocksynth2 normal ivltests blocksynth3 normal ivltests -case1 normal ivltests -case2 normal ivltests case4 normal ivltests case5 normal ivltests case5-syn-fail normal ivltests @@ -299,20 +297,13 @@ casesynth1 normal ivltests casesynth2 normal ivltests casesynth3 normal ivltests casesynth7 NI -casex_synth normal ivltests condit1 normal ivltests conditsynth1 normal ivltests conditsynth2 normal ivltests conditsynth3 normal ivltests -dffsynth normal ivltests -dffsynth3 normal ivltests -dffsynth4 normal ivltests -dffsynth9 normal ivltests -dffsynth10 normal ivltests -dffsynth11 normal ivltests + inside_synth normal ivltests inside_synth3 normal ivltests -memsynth1 normal ivltests memsynth2 normal ivltests memsynth3 normal ivltests memsynth5 normal ivltests diff --git a/ivtest/regress-v11.list b/ivtest/regress-v11.list index 57fca1d72..259d48833 100644 --- a/ivtest/regress-v11.list +++ b/ivtest/regress-v11.list @@ -87,8 +87,6 @@ sv_default_port_value3 CE,-g2009 ivltests br_gh440 CE,-g2009 ivltests gold=br_gh440-v11.gold # v11 has incomplete synthesis support -dffsynth CE,-S ivltests -memsynth1 CE,-S ivltests memsynth2 CE,-S ivltests memsynth3 CE,-S ivltests memsynth5 CE,-S ivltests @@ -97,11 +95,6 @@ memsynth7 CE,-S ivltests memsynth9 CE,-S ivltests mix_reset CE,-S ivltests -# These tests pass, but synthesis is creating unnecessary latches. -case1 normal ivltests -case2 normal ivltests -casex_synth normal ivltests - # For V11 vvp does not fail for these tests automatic_error11 normal ivltests gold=automatic_error11.gold automatic_error12 normal ivltests gold=automatic_error12.gold diff --git a/ivtest/regress-v12.list b/ivtest/regress-v12.list index 8f09d06f0..766b2eeef 100644 --- a/ivtest/regress-v12.list +++ b/ivtest/regress-v12.list @@ -68,8 +68,6 @@ # # v11 has incomplete synthesis support -dffsynth CE,-S ivltests -memsynth1 CE,-S ivltests memsynth2 CE,-S ivltests memsynth3 CE,-S ivltests memsynth5 CE,-S ivltests @@ -77,8 +75,3 @@ memsynth6 CE,-S ivltests memsynth7 CE,-S ivltests memsynth9 CE,-S ivltests mix_reset CE,-S ivltests - -# These tests pass, but synthesis is creating unnecessary latches. -case1 normal ivltests -case2 normal ivltests -casex_synth normal ivltests diff --git a/ivtest/regress-v13.list b/ivtest/regress-v13.list index 3a773a9c3..d62d6b1af 100644 --- a/ivtest/regress-v13.list +++ b/ivtest/regress-v13.list @@ -68,8 +68,6 @@ # # v13 has incomplete synthesis support -dffsynth CE,-S ivltests -memsynth1 CE,-S ivltests memsynth2 CE,-S ivltests memsynth3 CE,-S ivltests memsynth5 CE,-S ivltests @@ -77,8 +75,3 @@ memsynth6 CE,-S ivltests memsynth7 CE,-S ivltests memsynth9 CE,-S ivltests mix_reset CE,-S ivltests - -# These tests pass, but synthesis is creating unnecessary latches. -case1 normal ivltests -case2 normal ivltests -casex_synth normal ivltests diff --git a/ivtest/regress-vvp.list b/ivtest/regress-vvp.list new file mode 100644 index 000000000..eb22d756a --- /dev/null +++ b/ivtest/regress-vvp.list @@ -0,0 +1,26 @@ + +# Test list files are a list of test names and the json that +# describes the test. + +array_packed_write_read vvp_tests/array_packed_write_read.json +case1 vvp_tests/case1.json +case2 vvp_tests/case2.json +case2-S vvp_tests/case2-S.json +case3 vvp_tests/case3.json +casex_synth vvp_tests/casex_synth.json +dffsynth vvp_tests/dffsynth.json +dffsynth-S vvp_tests/dffsynth-S.json +dffsynth2 vvp_tests/dffsynth2.json +dffsynth3 vvp_tests/dffsynth3.json +dffsynth4 vvp_tests/dffsynth4.json +dffsynth5 vvp_tests/dffsynth5.json +dffsynth6 vvp_tests/dffsynth6.json +dffsynth7 vvp_tests/dffsynth7.json +dffsynth8 vvp_tests/dffsynth8.json +dffsynth9 vvp_tests/dffsynth9.json +dffsynth10 vvp_tests/dffsynth10.json +dffsynth11 vvp_tests/dffsynth11.json +macro_str_esc vvp_tests/macro_str_esc.json +memsynth1 vvp_tests/memsynth1.json +struct_packed_write_read vvp_tests/struct_packed_write_read.json +struct_packed_write_read2 vvp_tests/struct_packed_write_read2.json diff --git a/ivtest/run_ivl.py b/ivtest/run_ivl.py new file mode 100644 index 000000000..6e5821bc4 --- /dev/null +++ b/ivtest/run_ivl.py @@ -0,0 +1,242 @@ +'''Functions for running Icarus Verilog + +''' + +import subprocess +import difflib +import os +import sys +import re + +def assemble_iverilog_cmd(source: str, it_dir: str, args: list) -> list: + res = ["iverilog", "-o", os.path.join("work", "a.out")] + res += ["-D__ICARUS_UNSIZED__"] + res += args + src = os.path.join(it_dir, source) + res += [src] + return res + + +def assemble_vvp_cmd(args: list = [], plusargs: list = []) -> list: + res = ["vvp", os.path.join("work", "a.out")] + return res + + +def get_ivl_version () -> list: + '''Figure out the version of the installed iverilog compler. + + The return value is a list of 2 numbers, the major and minor version + numbers, or None if the version string couldn't be found.''' + + # Get the output from the "iverilog -V" command for the version string. + text = subprocess.check_output(["iverilog", "-V"]) + match = re.search(b'Icarus Verilog version ([0-9]+)\.([0-9]+)', text) + if not match: + return None + + return [int(match[1]), int(match[2])] + +def build_runtime(it_key: str) -> None: + '''Check and prepare the runtime environment for a test + + This is called in front of tests to make sure that the directory + structure is correct, and common temp files that might linger from + a previous run are removed.''' + + try: + os.mkdir("log") + except FileExistsError: + pass + + try: + os.remove(os.path.join("log", it_key + ".log")) + except FileNotFoundError: + pass + + try: + os.mkdir("work") + except FileExistsError: + pass + + try: + os.remove(os.path.join("work", "a.out")) + except FileNotFoundError: + pass + +def log_results(key, title, res) -> None: + ''' Write results into log files. + + Generate a log file with the name of the key and title, and + put the stdout and stderr into separate files.''' + + with open(os.path.join("log", f"{key}-{title}-stdout.log"), 'wb') as fd: + fd.write(res.stdout) + + with open(os.path.join("log", f"{key}-{title}-stderr.log"), 'wb') as fd: + fd.write(res.stderr) + + +def compare_files(log_path, gold_path): + '''Compare the log file and the gold file + + The files are read it, line at a time, and the lines are compared. + If they differ, then write tou stdout a unified diff. In any case, + return True or False to indicate the results of the test.''' + + with open(log_path, 'rt') as fd: + a = fd.readlines() + with open(gold_path, 'rt') as fd: + b = fd.readlines() + + flag = a == b + if not flag: + print(f"{log_path} and {gold_path} differ:") + sys.stdout.writelines(difflib.unified_diff(a, b, log_path, gold_path)) + + return flag + + +def run_CE(options : dict) -> list: + ''' Run the compiler, and expect an error + + In this case, we assert that the command fails to run and reports + an error. This is to check that invalid input generates errors.''' + + it_key = options['key'] + it_dir = options['directory'] + it_args = options['iverilog_args'] + + build_runtime(it_key) + + cmd = assemble_iverilog_cmd(options['source'], it_dir, it_args) + res = subprocess.run(cmd, capture_output=True) + log_results(it_key, "iverilog", res) + + if res.returncode == 0: + return [1, "Failed - CE (no error reported)"] + elif res.returncode >= 256: + return [1, "Failed - CE (execution error)"] + else: + return [0, "Passed - CE"] + + +def do_run_normal(options : dict, expected_fail : bool) -> list: + '''Run the iverilog and vvp commands. + + In this case, run the compiler to generate a vvp output file, and + run the vvp command to actually execute the simulation. Collect + the results and look for a "PASSED" string.''' + + it_key = options['key'] + it_dir = options['directory'] + it_iverilog_args = options['iverilog_args'] + it_gold = options['gold'] + it_diff = options['diff'] + + build_runtime(it_key) + + # Run the iverilog command + ivl_cmd = assemble_iverilog_cmd(options['source'], it_dir, it_iverilog_args) + ivl_res = subprocess.run(ivl_cmd, capture_output=True) + + log_results(it_key, "iverilog", ivl_res) + if ivl_res.returncode != 0: + return [1, "Failed - Compile failed"] + + # run the vvp command + vvp_cmd = assemble_vvp_cmd() + vvp_res = subprocess.run(vvp_cmd, capture_output=True) + log_results(it_key, "vvp", vvp_res); + + if vvp_res.returncode != 0: + return [1, "Failed - Vvp execution failed"] + + it_stdout = vvp_res.stdout.decode('ascii') + + # If there is a gold file configured, the test result depends on + # the outputs matching the gold file. + if it_gold is not None: + + compared = True + + log_path = os.path.join("log", f"{it_key}-iverilog-stdout.log") + gold_path = os.path.join("gold", f"{it_gold}-iverilog-stdout.gold") + compared_ivl_stdout = compare_files(log_path, gold_path) + compared = compared and compared_ivl_stdout + + log_path = os.path.join("log", f"{it_key}-iverilog-stderr.log") + gold_path = os.path.join("gold", f"{it_gold}-iverilog-stderr.gold") + compared_ivl_stderr = compare_files(log_path, gold_path) + compared = compared and compared_ivl_stderr + + log_path = os.path.join("log", f"{it_key}-vvp-stdout.log") + gold_path = os.path.join("gold", f"{it_gold}-vvp-stdout.gold") + compared_vvp_stdout = compare_files(log_path, gold_path) + compared = compared and compared_vvp_stdout + + log_path = os.path.join("log", f"{it_key}-vvp-stderr.log") + gold_path = os.path.join("gold", f"{it_gold}-vvp-stderr.gold") + compared_vvp_stderr = compare_files(log_path, gold_path) + compared = compared and compared_vvp_stderr + + if expected_fail: + if compared: + return [1, "Failed - Passed, but expected failure"] + else: + return [0, "Passed - Expected fail"] + else: + if compared: + return [0, "Passed"] + else: + return [1, "Failed - Gold output doesn't match actual output."] + + + # If there is a diff description, then compare named files instead of + # the log and a gold file. + if it_diff is not None: + diff_name1 = it_diff[0] + diff_name2 = it_diff[1] + diff_skip = int(it_diff[2]) + + with open(diff_name1) as fd: + for idx in range(diff_skip): + fd.readline() + diff_data1 = fd.read() + + with open(diff_name2) as fd: + for idx in range(diff_skip): + fd.readline() + diff_data2 = fd.read() + + if expected_fail: + if diff_data1 == diff_data2: + return [1, "Failed - Passed, but expected failure"] + else: + return [0, "Passed"] + else: + if diff_data1 == diff_data2: + return [0, "Passed"] + else: + return [1, f"Failed - Files {diff_name1} and {diff_name2} differ."] + + # Otherwise, look for the PASSED output string in stdout. + for line in it_stdout.splitlines(): + if line == "PASSED": + if expected_fail: + return [1, "Failed - Passed, but expected failure"] + else: + return [0, "Passed"] + + # If there is no PASSED output, and nothing else to check, then + # assume a failure. + if expected_fail: + return [0, "Passed"] + else: + return [1, "Failed - No PASSED output, and no gold file"] + + +def run_normal(options : dict) -> list: + return do_run_normal(options, False) + +def run_EF(options : dict) -> list: + return do_run_normal(options, True) diff --git a/ivtest/sv_regress.list b/ivtest/sv_regress.list index dc834464a..88e5ffe0e 100644 --- a/ivtest/sv_regress.list +++ b/ivtest/sv_regress.list @@ -59,10 +59,8 @@ # gold or diff commands. # -struct_packed_write_read normal,-g2009 ivltests struct_packed_value_list normal,-g2009 ivltests -array_packed_write_read normal,-g2009 ivltests array_packed_value_list normal,-g2009 ivltests array_packed_sysfunct normal,-g2009 ivltests diff --git a/ivtest/test_lists.py b/ivtest/test_lists.py new file mode 100644 index 000000000..13c966fff --- /dev/null +++ b/ivtest/test_lists.py @@ -0,0 +1,76 @@ + +'''Functions for processing test list files + +The read_lists() function is the main export of this package. This function +takes a list of file names, reads all the test items in the files, and +puts the result into a sorted list. + +The tests list file is formatted like so: + + + +The is the name of the test. This is used to generate the source file +name for the test program. + +The is the name of a subdirectory were we search for the test. +So for example, if ==foo and ==bar, then the Verilog source +file will be inferred to be bar/foo.v. + +The is the test type. + +The field sets up how the tests will be checked. Things like gold +files and working directories are given here. +''' + +def read_list(fd) -> list: + '''Return a list of test items (each in list form) from the file. + + The input fd is the file opened in text mode. This function will read + the file, a line at a time, and make a list of lists, with each list + in the list a list of tokens for the line. This is used by the read_lists + function.''' + + build_list = list() + for line_raw in fd: + # Strip comments and leading/traling white space + idx = line_raw.find("#") + if idx < 0: + idx = len(line_raw) + line = line_raw[0:idx].strip() + + # Split into tokens + line_list = line.split() + if len(line_list) == 0: + continue + + build_list.append(line_list) + + return build_list + + +def read_lists(paths: list) -> list: + '''Read the paths in the list, and return the list of tests. + + The input is a list of list file names, and the result is a list + of all the tests, sorted, and with duplicates resolved. The order + of the test file lists is important, as is the order of tests + within each list file.''' + + tests_list = list() + for path in paths: + with open(path, "r") as fd: + tests_list += read_list(fd) + + # The loop above creates a tests_list to list all of the tests in the + # order that they appear. Now we go though the list in order and eliminate + # duplictes by test name. This allows that lists might override tests that + # are already declared. + tests_dict = dict() + for item in tests_list: + tests_dict[item[0]] = item; + + # Convert the result to a sorted list, and return that. + tests_list = list(tests_dict.values()) + tests_list.sort() + + return tests_list diff --git a/ivtest/vvp_reg.py b/ivtest/vvp_reg.py new file mode 100755 index 000000000..a8f47d365 --- /dev/null +++ b/ivtest/vvp_reg.py @@ -0,0 +1,110 @@ +#! python3 +''' +Usage: + vvp_reg + vvp_reg ... + + is a list of files in the current working directory that + each contain a list of tests. By convention, the file has the + suffix ".list". The files will be processed in order, so tests + can be overridden if listed twice. If no files are given, a + default list is used. +''' + +import sys +# It appears that docopt doesn't work on msys2 installations, so +# skip it completely on win32 platforms. +if sys.platform != 'win32': + from docopt import docopt +import test_lists +import json +import run_ivl + + +def process_test(item: list) -> str: + '''Process a single test + + This takes in the list of tokens from the tests list file, and converts + them (interprets them) to a collection of values.''' + + # This is the name of the test, and the name of the main sorce file + it_key = item[0] + test_path = item[1] + with open(test_path, 'rt') as fd: + it_dict = json.load(fd) + + # Get the test type from the json configuration. + it_type = it_dict['type'] + + # Wrap all of this into an options dictionary for ease of handling. + it_options = { + 'key' : it_key, + 'type' : it_type, + 'iverilog_args' : it_dict.get('iverilog-args', [ ]), + 'directory' : "ivltests", + 'source' : it_dict['source'], + 'modulename' : None, + 'gold' : it_dict.get('gold', None), + 'diff' : None + } + + if it_type == "NI": + res = [0, "Not Implemented."] + + elif it_type == "normal": + res = run_ivl.run_normal(it_options) + + elif it_type == "CE": + res = run_ivl.run_CE(it_options) + + elif it_type == "EF": + res = run_ivl.run_EF(it_options) + + else: + res = f"{it_key}: I don't understand the test type ({it_type})." + raise Exception(res) + + return res + + +if __name__ == "__main__": + print(f"Running tests on platform: {sys.platform}") + if sys.platform == 'win32': + args = { "" : [] } + else: + args = docopt(__doc__) + + # This returns [13, 0] or similar + ivl_version = run_ivl.get_ivl_version() + ivl_version_major = ivl_version[0] + print(f"Icarus Verilog version: {ivl_version_major}") + + # Select the lists to use. If any list paths are given on the command + # line, then use only those. Otherwise, use a default list. + list_paths = args[""] + if len(list_paths) == 0: + list_paths = list() + list_paths += ["regress-vvp.list"] + + print(f"Use lists: {list_paths}") + + # Read the list files, to get the tests. + tests_list = test_lists.read_lists(list_paths) + + # We need the width of the widest key so that we can figure out + # how to align the key:result columns. + width = 0 + for cur in tests_list: + if len(cur[0]) > width: + width = len(cur[0]) + + error_count = 0 + for cur in tests_list: + result = process_test(cur) + error_count += result[0] + print(f"{cur[0]:>{width}}: {result[1]}") + + print("===================================================") + print(f"Test results: Ran {len(tests_list)}, Failed {error_count}.") + exit(error_count) + diff --git a/ivtest/vvp_tests/README.txt b/ivtest/vvp_tests/README.txt new file mode 100644 index 000000000..9c760de9a --- /dev/null +++ b/ivtest/vvp_tests/README.txt @@ -0,0 +1,11 @@ + +This directory contains configurations for thet tests that test the iverilog +compiler with the vvp simulation engine. Eash test file is actually a JSON +file that calls out the test type, names the source file, the gold file, any +command argument flags. + +{ + "type" : "normal", + "source" : "macro_str_esc.v", + "gold" : "macro_str_esc" +} diff --git a/ivtest/vvp_tests/array_packed_write_read.json b/ivtest/vvp_tests/array_packed_write_read.json new file mode 100644 index 000000000..f83a222ff --- /dev/null +++ b/ivtest/vvp_tests/array_packed_write_read.json @@ -0,0 +1,5 @@ +{ + "type" : "normal", + "source" : "array_packed_write_read.v", + "iverilog-args" : [ "-g2009" ] +} diff --git a/ivtest/vvp_tests/case1.json b/ivtest/vvp_tests/case1.json new file mode 100644 index 000000000..0b64e7e35 --- /dev/null +++ b/ivtest/vvp_tests/case1.json @@ -0,0 +1,4 @@ +{ + "type" : "normal", + "source" : "case1.v" +} diff --git a/ivtest/vvp_tests/case2-S.json b/ivtest/vvp_tests/case2-S.json new file mode 100644 index 000000000..4bab5fe3a --- /dev/null +++ b/ivtest/vvp_tests/case2-S.json @@ -0,0 +1,5 @@ +{ + "type" : "normal", + "source" : "case2.v", + "iverilog-args" : [ "-S" ] +} diff --git a/ivtest/vvp_tests/case2.json b/ivtest/vvp_tests/case2.json new file mode 100644 index 000000000..f01a5f78c --- /dev/null +++ b/ivtest/vvp_tests/case2.json @@ -0,0 +1,4 @@ +{ + "type" : "normal", + "source" : "case2.v" +} diff --git a/ivtest/vvp_tests/case3.json b/ivtest/vvp_tests/case3.json new file mode 100644 index 000000000..6b2bad14a --- /dev/null +++ b/ivtest/vvp_tests/case3.json @@ -0,0 +1,4 @@ +{ + "type" : "normal", + "source" : "case3.v" +} diff --git a/ivtest/vvp_tests/casex_synth.json b/ivtest/vvp_tests/casex_synth.json new file mode 100644 index 000000000..77925bcab --- /dev/null +++ b/ivtest/vvp_tests/casex_synth.json @@ -0,0 +1,5 @@ +{ + "type" : "normal", + "source" : "casex_synth.v", + "iverilog-args" : [ "-S" ] +} diff --git a/ivtest/vvp_tests/dffsynth-S.json b/ivtest/vvp_tests/dffsynth-S.json new file mode 100644 index 000000000..be08bda34 --- /dev/null +++ b/ivtest/vvp_tests/dffsynth-S.json @@ -0,0 +1,5 @@ +{ + "type" : "NI", + "source" : "dffsynth.v", + "iverilog-args" : [ "-S" ] +} diff --git a/ivtest/vvp_tests/dffsynth.json b/ivtest/vvp_tests/dffsynth.json new file mode 100644 index 000000000..c25531c2e --- /dev/null +++ b/ivtest/vvp_tests/dffsynth.json @@ -0,0 +1,4 @@ +{ + "type" : "normal", + "source" : "dffsynth.v" +} diff --git a/ivtest/vvp_tests/dffsynth10.json b/ivtest/vvp_tests/dffsynth10.json new file mode 100644 index 000000000..817d7a37b --- /dev/null +++ b/ivtest/vvp_tests/dffsynth10.json @@ -0,0 +1,5 @@ +{ + "type" : "normal", + "source" : "dffsynth10.v", + "iverilog-args" : [ "-S" ] +} diff --git a/ivtest/vvp_tests/dffsynth11.json b/ivtest/vvp_tests/dffsynth11.json new file mode 100644 index 000000000..519dfdcae --- /dev/null +++ b/ivtest/vvp_tests/dffsynth11.json @@ -0,0 +1,5 @@ +{ + "type" : "normal", + "source" : "dffsynth11.v", + "iverilog-args" : [ "-S" ] +} diff --git a/ivtest/vvp_tests/dffsynth2.json b/ivtest/vvp_tests/dffsynth2.json new file mode 100644 index 000000000..61718a896 --- /dev/null +++ b/ivtest/vvp_tests/dffsynth2.json @@ -0,0 +1,5 @@ +{ + "type" : "normal", + "source" : "dffsynth2.v", + "iverilog-args" : [ "-S" ] +} diff --git a/ivtest/vvp_tests/dffsynth3.json b/ivtest/vvp_tests/dffsynth3.json new file mode 100644 index 000000000..295dba54d --- /dev/null +++ b/ivtest/vvp_tests/dffsynth3.json @@ -0,0 +1,5 @@ +{ + "type" : "normal", + "source" : "dffsynth3.v", + "iverilog-args" : [ "-S" ] +} diff --git a/ivtest/vvp_tests/dffsynth4.json b/ivtest/vvp_tests/dffsynth4.json new file mode 100644 index 000000000..460354bfd --- /dev/null +++ b/ivtest/vvp_tests/dffsynth4.json @@ -0,0 +1,5 @@ +{ + "type" : "normal", + "source" : "dffsynth4.v", + "iverilog-args" : [ "-S" ] +} diff --git a/ivtest/vvp_tests/dffsynth5.json b/ivtest/vvp_tests/dffsynth5.json new file mode 100644 index 000000000..8e3519af8 --- /dev/null +++ b/ivtest/vvp_tests/dffsynth5.json @@ -0,0 +1,5 @@ +{ + "type" : "normal", + "source" : "dffsynth5.v", + "iverilog-args" : [ "-S" ] +} diff --git a/ivtest/vvp_tests/dffsynth6.json b/ivtest/vvp_tests/dffsynth6.json new file mode 100644 index 000000000..a240aa2bf --- /dev/null +++ b/ivtest/vvp_tests/dffsynth6.json @@ -0,0 +1,5 @@ +{ + "type" : "normal", + "source" : "dffsynth6.v", + "iverilog-args" : [ "-S" ] +} diff --git a/ivtest/vvp_tests/dffsynth7.json b/ivtest/vvp_tests/dffsynth7.json new file mode 100644 index 000000000..a0b0c0e4d --- /dev/null +++ b/ivtest/vvp_tests/dffsynth7.json @@ -0,0 +1,5 @@ +{ + "type" : "normal", + "source" : "dffsynth7.v", + "iverilog-args" : [ "-S" ] +} diff --git a/ivtest/vvp_tests/dffsynth8.json b/ivtest/vvp_tests/dffsynth8.json new file mode 100644 index 000000000..f5bdcbc65 --- /dev/null +++ b/ivtest/vvp_tests/dffsynth8.json @@ -0,0 +1,5 @@ +{ + "type" : "NI", + "source" : "dffsynth8.v", + "iverilog-args" : [ "-S" ] +} diff --git a/ivtest/vvp_tests/dffsynth9.json b/ivtest/vvp_tests/dffsynth9.json new file mode 100644 index 000000000..cfc0c8a97 --- /dev/null +++ b/ivtest/vvp_tests/dffsynth9.json @@ -0,0 +1,5 @@ +{ + "type" : "normal", + "source" : "dffsynth9.v", + "iverilog-args" : [ "-S" ] +} diff --git a/ivtest/vvp_tests/macro_str_esc.json b/ivtest/vvp_tests/macro_str_esc.json new file mode 100644 index 000000000..09c7bd5e5 --- /dev/null +++ b/ivtest/vvp_tests/macro_str_esc.json @@ -0,0 +1,5 @@ +{ + "type" : "normal", + "source" : "macro_str_esc.v", + "gold" : "macro_str_esc" +} diff --git a/ivtest/vvp_tests/memsynth1.json b/ivtest/vvp_tests/memsynth1.json new file mode 100644 index 000000000..c25185d51 --- /dev/null +++ b/ivtest/vvp_tests/memsynth1.json @@ -0,0 +1,5 @@ +{ + "type" : "NI", + "source" : "memsynth1.v", + "iverilog-args" : [ "-S" ] +} diff --git a/ivtest/vvp_tests/struct_packed_write_read.json b/ivtest/vvp_tests/struct_packed_write_read.json new file mode 100644 index 000000000..766d7d0c0 --- /dev/null +++ b/ivtest/vvp_tests/struct_packed_write_read.json @@ -0,0 +1,5 @@ +{ + "type" : "normal", + "source" : "struct_packed_write_read.v", + "iverilog-args" : [ "-g2009" ] +} diff --git a/ivtest/vvp_tests/struct_packed_write_read2.json b/ivtest/vvp_tests/struct_packed_write_read2.json new file mode 100644 index 000000000..a7d60a6c6 --- /dev/null +++ b/ivtest/vvp_tests/struct_packed_write_read2.json @@ -0,0 +1,5 @@ +{ + "type" : "normal", + "source" : "struct_packed_write_read2.v", + "iverilog-args" : [ "-g2009" ] +}