Tests: Strictly test exit codes (no unexpected core dumps)

bin/verilator

@@ -258,8 +258,8 @@ sub run {
             warn "%Error: Command Failed $command\n";
         }
         exit $! if $!;  # errno
-        exit $? >> 8 if $? >> 8;  # child exit status
+        exit $? >> 8 if $? >> 8;  # pass along child exit code
-        exit 255;  # last resort
+        exit 128 + ($status & 127);  # last resort
     }
 }
 

test_regress/driver.py

@@ -1724,7 +1724,7 @@ class VlTest:
             check_finished=False,  # Check for All Finished
             entering=None,  # Print entering directory information
             expect_filename=None,  # Filename that should match logfile
-            fails=False,  # Command should fail
+            fails=False,  # True: normal 1 exit code, 'any': any exit code
             logfile=None,  # Filename to write putput to
             tee=True,
             verilator_run=False) -> str:  # Move gcov data to parallel area
@@ -1812,9 +1812,12 @@ class VlTest:
                 -8,  # SIGFPA
                 -11)):  # SIGSEGV
             self.error("Exec failed with core dump")
-            status = 10
+            status = 128 + (-rc)  # So is "normal" shell 0-255 status
+        elif rc >= 256:
+            # waitpid returns status << 8; subprocess otherwise; handle both
+            status = int(rc / 256)  # So is shell $?-like
         elif rc:
-            status = 10
+            status = rc
         else:
             status = 0
 
@@ -1829,8 +1832,21 @@ class VlTest:
             # Strip ANSI escape sequences
             firstline = re.sub(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])', '', firstline)
             self.error("Exec of " + self._error_cmd_simplify(cmd) + " failed: " + firstline)
+
         if fails and status:
-            print("(Exec expected to fail, and did.)")
+            if not verilator_run:
+                print("(Exec failed, matching expected fail)")
+            elif fails == 'any':
+                print("(Exec failed, matching expected 'any' exit code fail)")
+            elif fails is True:
+                if status == 1:
+                    print("(Exec failed, matching expected 'True' exit code 1 fail)")
+                else:
+                    self.error("Exec of " + self._error_cmd_simplify(cmd) +
+                               " failed with exit code " + str(status) +
+                               ", but expected 'True' exit code 1 fail")
+            else:  # Future: support numeric exit code?
+                self.error("fails=" + str(fails) + " is not legal value")
         if fails and not status:
             self.error("Exec of " + self._error_cmd_simplify(cmd) + " ok, but expected to fail")
         if self.errors or self._skips:

test_regress/t/t_assert_elab_bad.py

@@ -9,7 +9,7 @@
 
 import vltest_bootstrap
 
-test.scenarios('simulator')
+test.scenarios('simulator_st')
 test.top_filename = "t/t_assert_elab.v"
 
 test.unlink_ok(test.obj_dir + "/t_assert_elab_bad.log")
@@ -19,8 +19,6 @@ test.compile(v_flags2=[
 ],
              fails=True)
 
-test.execute(fails=test.vlt_all)
-
 test.file_grep(test.compile_log_filename,
                r'%Warning-USERFATAL: "Parameter   5 is invalid...string and constant both work"')
 

test_regress/t/t_debug_fatalsrc_bad.py

@@ -16,7 +16,7 @@ if 'VERILATOR_TEST_NO_GDB' in os.environ:
 if not test.have_gdb:
     test.skip("No gdb installed")
 
-test.lint(verilator_flags2=["--debug-fatalsrc"], fails=test.vlt_all)
+test.lint(verilator_flags2=["--debug-fatalsrc"], fails='any')
 
 test.file_grep(test.compile_log_filename, r'%Error: Internal Error: .*: --debug-fatal-src')
 test.file_grep(test.compile_log_filename, r'See the manual')

test_regress/t/t_debug_fatalsrc_bt_bad.py

@@ -16,7 +16,7 @@ if 'VERILATOR_TEST_NO_GDB' in os.environ:
 if not test.have_gdb:
     test.skip("No gdb installed")
 
-test.lint(verilator_flags2=["--lint-only --debug --gdbbt --debug-fatalsrc"], fails=True)
+test.lint(verilator_flags2=["--lint-only --debug --gdbbt --debug-fatalsrc"], fails='any')
 
 test.file_grep(test.compile_log_filename, r'%Error: Internal Error: .*: --debug-fatal-src')
 test.file_grep(test.compile_log_filename, r'See the manual')

test_regress/t/t_debug_sigsegv_bad.py

@@ -14,7 +14,7 @@ test.scenarios('vlt')
 test.setenv("ASAN_OPTIONS", "handle_segv=0")
 test.leak_check_disable()
 
-test.lint(v_flags=["--debug-sigsegv"], fails=True, sanitize=0)
+test.lint(v_flags=["--debug-sigsegv"], fails='any', sanitize=0)
 
 test.file_grep(test.compile_log_filename,
                r'%Error: Verilator internal fault, sorry. Suggest trying --debug --gdbbt')

test_regress/t/t_debug_sigsegv_bt_bad.py

@@ -18,7 +18,7 @@ if not test.have_gdb:
 
 test.lint(verilator_flags2=["--lint-only --debug --gdbbt --debug-sigsegv"],
           sanitize=0,
-          fails=test.vlt_all)
+          fails='any')
 
 test.file_grep(test.compile_log_filename, r'Program received signal SIGSEGV')
 test.file_grep(test.compile_log_filename, r'in V3Options::')

test_regress/t/t_flag_build_bad2.py

@@ -19,6 +19,6 @@ test.compile(
     ],
     # Recursive make breaks the golden compare
     #expect_filename = test.golden_filename
-    fails=True)
+    fails='any')  # make returns exit code 2
 
 test.passes()