luke
/
iverilog
mirror of https://github.com/steveicarus/iverilog.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #587 from steveicarus/ivtests-fail-exit 

Rework the regression tests to not use regression_report references
This commit is contained in:
Stephen Williams 2022-01-16 13:36:09 -08:00 committed by GitHub
parent f5f07d52b0 7c73ef8fb6
commit be06e293f6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 12 additions and 15718 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
.github/test.sh vendored
View File

@ -5,17 +5,9 @@ echo " pwd = $(pwd)"
cd ivtest
version=devel
status=0
if [ "$OS" = 'Windows_NT' ]; then
perl update_msys2_report.pl $version
perl vvp_reg.pl || status=1
diff --strip-trailing-cr regression_report-msys2-$version.txt regression_report.txt || status=1
else
perl vvp_reg.pl || status=1
diff regression_report-$version.txt regression_report.txt || status=1
fi
perl vvp_reg.pl || status=1
perl vpi_reg.pl || status=1

6
ivtest/regress-vams.list
View File

@ -67,8 +67,10 @@
# gold or diff commands.
#
analog1 normal,-gverilog-ams ivltests
analog2 normal,-gverilog-ams ivltests
#analog1 normal,-gverilog-ams ivltests
#analog2 normal,-gverilog-ams ivltests
analog1 NI ivltests
analog2 NI ivltests
br_gh99c normal,-gverilog-ams ivltests
cast_int_ams normal,-gverilog-ams ivltests
constfunc4_ams normal,-gverilog-ams ivltests

3
ivtest/regress-vhdl.list
View File

@ -118,7 +118,8 @@ vhdl_labeled_assign normal,-g2005-sv,ivltests/vhdl_labeled_assign.vhd ivltests
vhdl_lfcr normal,-g2005-sv,ivltests/vhdl_lfcr.vhd ivltests gold=vhdl_lfcr.gold
vhdl_logic normal,-g2005-sv,ivltests/vhdl_logic.vhd ivltests
vhdl_loop normal,-g2005-sv,ivltests/vhdl_loop.vhd ivltests
vhdl_multidim_array normal,-g2005-sv,ivltests/vhdl_multidim_array.vhd ivltests
#vhdl_multidim_array normal,-g2005-sv,ivltests/vhdl_multidim_array.vhd ivltests
vhdl_multidim_array NI ivltests
vhdl_nand104_stdlogic normal,-g2005-sv,ivltests/vhdl_nand104_stdlogic.vhd ivltests
vhdl_nand23_bit normal,-g2005-sv,ivltests/vhdl_nand23_bit.vhd ivltests
vhdl_nandg_bit normal,-g2005-sv,ivltests/vhdl_nandg_bit.vhd ivltests

2569
ivtest/regression_report-devel.txt
View File

File diff suppressed because it is too large Load Diff

2569
ivtest/regression_report-fsv.txt
View File

File diff suppressed because it is too large Load Diff

2566
ivtest/regression_report-strict.txt
View File

File diff suppressed because it is too large Load Diff

2482
ivtest/regression_report-v10.txt
View File

File diff suppressed because it is too large Load Diff

2560
ivtest/regression_report-v11.txt
View File

File diff suppressed because it is too large Load Diff

2569
ivtest/regression_report-vlog95.txt
View File

File diff suppressed because it is too large Load Diff

90
ivtest/update_msys2_report.pl
View File

@ -1,90 +0,0 @@
#!/usr/bin/env perl
#
# Script to automatically generate regression_report-msys2.txt
# from regression_report-devel.txt and regress-msys2.list.
#use strict;
use lib './perl-lib';
use RegressionList;
my $version = $ARGV[0] || 'devel';
read_regression_list("regress-msys2.list", "any", 0, "");
my $input_name = 'regression_report-' . $version . '.txt';
open(my $input, '<', $input_name)
or die "ERROR - can't open '$input_name'";
my $output_name = 'regression_report-msys2-' . $version . '.txt';
open(my $output, '>', $output_name)
or die "ERROR - can't open '$output_name'";
# Copy header.
my $line_count = 0;
while (my $line = <$input>) {
print $output $line;
last if ++$line_count == 2;
}
# Output results for MSYS2 test exceptions.
my $passed = 0;
my $failed = 0;
my $not_impl = 0;
my $exp_fail = 0;
my %skip_test;
foreach my $name (@testlist) {
seek($input, 0, 0);
while (my $line = <$input>) {
my ($prefix, $result) = split(':', $line);
my $test_name = $prefix =~ s/^\s+//r; # strip leading spaces
next if $test_name ne $name;
if ($testtype{$test_name} eq "NI") {
print $output "$prefix: Not Implemented.\n";
$not_impl++;
} elsif ($testtype{$test_name} eq "EF") {
print $output "$prefix: Passed - expected fail.\n";
$exp_fail++;
} elsif ($testtype{$test_name} eq "CO") {
print $output "$prefix: Passed - CO.\n";
$passed++;
} elsif ($testtype{$test_name} eq "CE") {
print $output "$prefix: Passed - CE.\n";
$passed++;
} elsif ($testtype{$test_name} eq "RE") {
print $output "$prefix: Passed - RE.\n";
$passed++;
} else {
print $output "$prefix: Passed.\n";
$passed++;
}
}
$skip_test{$name} = 1;
}
# Output remaining results.
seek($input, 0, 0);
while (my $line = <$input>) {
my ($prefix, $result) = split(':', $line);
next if !$result;
my $test_name = $prefix =~ s/^\s+//r; # strip leading spaces
next if $skip_test{$test_name};
if ($line =~ /Not Implemented/) {
$not_impl++;
} elsif ($line =~ /expected fail/) {
$exp_fail++;
} elsif ($line =~ /Failed/) {
$failed++;
} elsif ($line =~ /Passed/) {
$passed++;
} else {
next;
}
print $output $line;
}
my $total = $passed + $failed + $not_impl + $exp_fail;
print $output "=" x 76 . "\n";
print $output "Test results:\n Total=$total, Passed=$passed, Failed=$failed, Not Implemented=$not_impl, Expected Fail=$exp_fail\n";

299
ivtest/vhdl_regression_report-devel.txt
View File

@ -1,299 +0,0 @@
Running VHDL tests for Icarus Verilog version: 0.10.
----------------------------------------------------------------------
hello1: Passed.
mux2: Passed.
dff: Passed.
counter: Passed.
assign: Passed.
blocking: Passed.
constassign: Passed.
readout: Passed.
autof: Passed.
generics: Passed.
partpv: Passed - CO.
case1: Passed.
case2: Passed.
case3: Passed.
case4: Passed.
case5: Passed.
case6: Passed.
case7: Passed.
case3.8A: Passed.
case3.8B: Passed.
case3.8C: Passed.
case3.8D: Passed.
blocksynth1: Passed.
casesynth1: Passed.
casesynth2: Passed.
casesynth3: Passed.
casex_synth: Passed.
dffsynth: Passed.
dffsynth2: Passed.
dffsynth4: Passed.
inside_synth3: Passed.
memsynth1: Passed.
memsynth2: Passed.
memsynth3: Passed.
memsynth4: Passed.
memsynth5: Passed.
memsynth7: Passed.
memsynth8: Passed.
memsynth9: Passed.
multireg: Passed.
condit1: Passed.
conditsynth1: Passed.
conditsynth2: Passed.
conditsynth3: Passed.
basiclatch: Passed.
basicstate: Passed.
basicstate2: Passed.
ssetclr1: Passed.
ssetclr2: Passed.
ssetclr3: Passed.
always3.1.11A: Passed.
always3.1.11B: Passed.
always3.1.1C: Passed.
always3.1.1D: Passed.
always3.1.1E: Passed.
always3.1.1F: Passed.
always3.1.4A: Passed.
always3.1.4D: Passed.
always3.1.4E: Passed.
always3.1.4G: Passed.
always3.1.5A: Passed.
always3.1.5B: Passed.
always3.1.5C: Passed.
always3.1.5D: Passed.
always3.1.5E: Passed.
always3.1.5F: Passed.
always3.1.6D: Passed.
always3.1.7A: Passed.
always3.1.7B: Passed.
always3.1.7C: Passed.
always3.1.7D: Passed.
always3.1.8A: Passed.
function3.11B: Passed.
function3.11C: Passed.
function3.11D: Passed.
function3.11F: Passed.
module3.12A: Passed.
module3.12B: Passed.
muxtest: Passed.
ptest001: Passed.
ptest002: Passed.
ptest003: Passed.
ptest004: Passed.
ptest005: Passed.
ptest006: Passed.
ptest007: Passed.
ptest008: Passed.
ptest009: Passed.
ptest010: Passed.
qmark: Passed.
qmark1: Passed.
qmark3: Passed.
qmark5: Passed.
qmark6: Passed.
sdw_always1: Passed.
sdw_always2: Passed.
sdw_always3: Passed.
sdw_assign: Passed.
sdw_function1: Passed.
sdw_function2: Passed.
sdw_function3: Passed.
sdw_function4: Passed.
sdw_function5: Passed.
sdw_task1: Passed.
sdw_task2: Passed.
sdw_int: Passed.
sdw_lvalconcat2: Passed.
sdw_param1: Passed.
sdw_param2: Passed.
sdw_stmt002: Passed.
sdw_array: Passed.
sdw_instmod1: Passed.
sdw_instmod2: Passed.
sdw_lvalconcat: Passed.
task3.14A: Passed.
task3.14B: Passed.
task3.14C: Passed.
task3.14D: Passed.
task3.14E: Passed.
z1: Passed.
z2: Passed.
landor1: Passed.
land2: Passed.
land3: Passed.
contrib8.1: Passed.
contrib8.2: Passed.
contrib8.4: Passed.
dff1: Passed.
fifo: Passed.
gencrc: Passed.
idiv2: Passed.
event_list2: Passed.
timescale1: Passed.
integer1lt: Passed.
integer2le: Passed.
integer3gt: Passed.
integer4ge: Passed.
time1: Passed.
time3: Passed.
time8: Passed.
wireadd1: Passed.
wiresl: Passed.
wiresr: Passed.
wiresub1: Passed.
wirexor1: Passed.
wirele: Passed.
wirege: Passed.
wireeq: Passed.
andnot1: Passed.
constmult: Passed.
constadd: Passed.
constadd2: Passed.
constadd3: Passed.
consttern: Passed.
talu: Passed.
udp_bufg: Passed.
udp_bufg2: Passed.
unary_not: Passed.
unary_and: Passed.
unary_nand: Passed.
unary_nand2: Passed.
unary_or: Passed.
unary_nor: Passed.
unary_nor2: Passed.
unary_xor: Passed.
unary_xnor1: Passed.
unary_xnor2: Passed.
unary_minus: Passed.
unary_minus2: Passed.
unary_minus3: Passed.
unary_minus4: Passed.
pr2224949: Passed.
pr2281479: Passed.
pr2281519: Passed - CO.
pr2147135a: Passed - CO.
pr2147135b: Passed - CO.
pr2391405: Passed - CO.
pr2362426: Passed.
ga_and: Passed.
ga_or: Passed.
ga_xor: Passed.
ga_nand: Passed.
ga_nor: Passed.
ga_xnor: Passed.
binary_nand: Passed.
binary_nor: Passed.
rptconcat: Passed.
rptconcat2: Passed.
inout: Passed.
modparam: Passed.
port-test2: Passed.
scope2: Passed.
scope2b: Passed.
tri0: ==> Failed - output does not match gold file.
tri0b: ==> Failed - output does not match gold file.
tri1: ==> Failed - output does not match gold file.
posedge: Passed.
nblkorder: Passed.
task_inpad: Passed.
cond_band: Passed.
cond_wide: Passed.
cond_wide2: Passed.
wildsense: Passed.
wildsense2: Passed.
assign_mem1: Passed.
meminit: Passed.
meminit2: Passed.
task_noop: Passed.
task_bypath: Passed.
task_iotypes: Passed.
assign_nb1: Passed.
assign_nb2: Passed.
assign_delay: Passed.
define1: Passed.
delay2: Passed.
delay3: Passed.
delay4: Passed.
delay5: Passed.
delay_assign_nb: Passed.
delay_assign_nb2: Passed.
ldelay1: Passed.
ldelay2: Passed.
ldelay3: Passed.
ldelay5: Passed.
wireland: Passed.
param_concat: Passed.
param_select: Passed.
param_select2: Passed.
param_select3: Passed.
param_times: Passed.
function1: Passed.
function_exp: Passed.
addsr: Passed.
tern1: Passed.
tern4: Passed.
tern6: Passed.
tern7: Passed.
tern9: Passed.
tern10: Passed.
bnot: Passed.
stask_parm1: Passed.
task_omemw2: Passed.
lh_varindx2: Passed.
lh_varindx4: Passed.
lh_varindx5: Passed.
lh_catadd: Passed.
signed1: Passed.
signed2: Passed.
signed3: Passed.
signed4: Passed.
signed5: Passed.
signed6: Passed.
signed7: Passed.
signed9: Passed.
signed11: Passed.
repeat2: Passed.
decl_assign1: Passed.
shift2: Passed.
pr1903520: Passed.
pr142: Passed.
bitsel: Passed.
bitsel2: Passed.
bitsel3: Passed.
bitsel4: Passed.
bitsel5: Passed.
select: Passed.
uwire: Passed.
xnor_test: Passed.
pr2202846a: Passed.
pr2202846b: Passed.
pr2202846c: Passed.
pr2489116: Passed.
pr2489237: Passed - CO.
pr2516774: Passed.
pr2516774b: Passed - CO.
pr2527366: Passed - CO.
pr2529315: Passed - CO.
pr2529315b: Passed - CO.
pr2531370: Passed - CO.
pr2526768: Passed.
pr2536040: Passed - CO.
pr2541625: Passed.
pr2534491: Passed.
pr2554173: Passed - CO.
pr2555813: Passed - CO.
pr2555813b: Passed - CO.
pr2554029: Passed - CO.
pr2554124: Passed - CO.
simple_gen: Passed.
pr2911213: Passed.
reserved: Passed.
pr2555831: Passed.
pr2661101: Passed.
pr3397689: Passed.
======================================================================
Test results:
Total=294, Passed=291, Failed=3, Not Implemented=0, Expected Fail=0

5
ivtest/vvp_reg.pl
View File

@ -68,9 +68,10 @@ if ($#ARGV != -1) {
&read_regression_list("regress-synth.list", $ver, $force_sv, "-S");
}
}
&execute_regression($suffix, $strict, $with_valg);
$failed = &execute_regression($suffix, $strict, $with_valg);
&close_report_file;
exit $failed;
#
# execute_regression sequentially compiles and executes each test in
@ -262,4 +263,6 @@ sub execute_regression {
# Remove remaining temporary files
system("rm -f *.tmp ivltests/*.tmp");
return($failed)
}