From 07056b11f5e77d92672f87e531fab59b573fdeb3 Mon Sep 17 00:00:00 2001 From: Wilson Snyder Date: Sun, 23 Nov 2025 11:57:08 -0500 Subject: [PATCH] Tests: add driver.py test.priority settings (#6725) --- test_regress/driver.py | 100 +++++++++++++++--- test_regress/t/t_a1_first_cc.py | 1 + test_regress/t/t_a2_first_sc.py | 1 + test_regress/t/t_a3_selftest.py | 1 + test_regress/t/t_a3_selftest_thread.py | 1 + test_regress/t/t_a6_examples.py | 1 + test_regress/t/t_case_write1.py | 1 + test_regress/t/t_case_write2.py | 1 + ...nclude.py => t_dist_attributes_include.py} | 1 + ...ibutes_src.py => t_dist_attributes_src.py} | 1 + test_regress/t/t_gantt.py | 1 + test_regress/t/t_gantt_hier.py | 1 + test_regress/t/t_gantt_two.py | 1 + test_regress/t/t_hier_block.py | 1 + test_regress/t/t_hier_block_binary.py | 1 + test_regress/t/t_hier_block_chained.py | 2 + ...r_block_cmake.py => t_hier_block_cmake.py} | 1 + test_regress/t/t_hier_block_import.py | 1 + test_regress/t/t_hier_block_import_cmake.py | 2 + test_regress/t/t_hier_block_perf.py | 2 + test_regress/t/t_hier_block_prot_lib.py | 1 + .../t/t_hier_block_prot_lib_shared.py | 1 + test_regress/t/t_hier_block_sc.py | 1 + test_regress/t/t_hier_block_sc_trace_fst.py | 1 + test_regress/t/t_hier_block_sc_trace_vcd.py | 1 + test_regress/t/t_hier_block_trace_fst.py | 1 + test_regress/t/t_hier_block_trace_saif.py | 1 + test_regress/t/t_hier_block_trace_vcd.py | 1 + test_regress/t/t_unopt_array_csplit.py | 1 + .../t/t_uvm_hello_all_v2017_1_0_nodpi.py | 1 + .../t/t_uvm_hello_all_v2020_3_1_nodpi.py | 1 + 31 files changed, 121 insertions(+), 12 deletions(-) rename test_regress/t/{t_a5_attributes_include.py => t_dist_attributes_include.py} (98%) rename test_regress/t/{t_a5_attributes_src.py => t_dist_attributes_src.py} (99%) rename test_regress/t/{t_a7_hier_block_cmake.py => t_hier_block_cmake.py} (99%) diff --git a/test_regress/driver.py b/test_regress/driver.py index b6f440fea..5b1729c19 100755 --- a/test_regress/driver.py +++ b/test_regress/driver.py @@ -383,7 +383,13 @@ class Forker: class Runner: - def __init__(self, driver_log_filename, quiet, ok_cnt=0, fail1_cnt=0, skip_cnt=0): + def __init__(self, + driver_log_filename, + quiet, + ok_cnt=0, + pass_tests=None, + fail1_cnt=0, + skip_cnt=0): self.driver_log_filename = driver_log_filename self.quiet = quiet # Counts @@ -396,6 +402,10 @@ class Runner: self.skip_msgs = [] self.fail_msgs = [] self.fail_tests = [] + if pass_tests: + self.pass_tests = pass_tests + else: + self.pass_tests = [] self._last_proc_finish_time = 0 self._last_summary_time = 0 self._last_summary_left = 0 @@ -448,6 +458,7 @@ class Runner: test = VlTest(py_filename=process.name, scenario=process.scenario, running_id=process.running_id) + test.oprint("=" * 50) test._prep() if process.rerun_skipping: @@ -475,8 +486,10 @@ class Runner: running_id=process.running_id) test._quit = Quitting test._read_status() + if test.ok: self.ok_cnt += 1 + self.pass_tests.append(test) if Args.driver_clean: test.clean() elif test._quit: @@ -525,17 +538,22 @@ class Runner: def report(self, filename: str) -> None: if filename: with open(filename, "w", encoding="utf8") as fh: - self._report_fh(fh) + # Time report is only in logfile to reduce user confusion + self._report_fh(fh, True) else: - self._report_fh(sys.stdout) + self._report_fh(sys.stdout, False) - def _report_fh(self, fh) -> None: + def _report_fh(self, fh, show_times: bool) -> None: fh.write("\n") fh.write('=' * 70 + "\n") for f in sorted(self.fail_msgs): fh.write(f.strip() + "\n") for f in sorted(self.skip_msgs): fh.write(f.strip() + "\n") + + if show_times: + self._report_times(fh) + if self.fail_cnt: sumtxt = 'FAILED' elif self.skip_cnt: @@ -544,6 +562,30 @@ class Runner: sumtxt = 'PASSED' fh.write("==TESTS DONE, " + sumtxt + ": " + self.sprint_summary() + "\n") + def _report_times(self, fh) -> None: + if forker.is_any_left(): + return + if len(self.pass_tests) < 10: + return + # PY file may be used for both vlt and vltmt, so take max + py_times = {} + for ftest in self.pass_tests: + if (ftest.py_filename not in py_times) or (py_times[ftest.py_filename]._wall_time + < ftest._wall_time): + py_times[ftest.py_filename] = ftest + n = 0 + top_n = 6 + for py_filename in sorted(py_times, key=lambda key: py_times[key]._wall_time, + reverse=True): + if n > top_n: + break + n += 1 + ftest = py_times[py_filename] + if ftest._priority == 1: + fh.write(py_filename + + ": walltime=%0.3f is in top %d, add a test.priority() for it?\n" % + (ftest._wall_time, top_n)) + def print_summary(self, force=False): change = self._last_summary_left != self.left_cnt if (force or ((time.time() - self._last_summary_time) >= 15) @@ -648,9 +690,13 @@ class VlTest: self._have_solver_called = False self._inputs = {} self._ok = False + self._priority = 1 self._quit = False + self._scenario_parsed = False self._scenario_off = False # scenarios() didn't match running scenario self._skips = None + self._start_time = time.time() + self._wall_time = 0 match = re.match(r'^(.*/)?([^/]*)\.py', self.py_filename) self.name = match.group(2) # Name of this test @@ -904,11 +950,23 @@ class VlTest: self._skips = message raise VtSkipException + def priority(self, level: int) -> None: + """Called from tests as: priority() to + specify what priority order the test should run at. + Higher numbers run first; in numeric tiers very roughly + corresponding to the wall time runtimes in seconds. + One is default. Tests with same priories run in name-sorted order. + priority() must be on one line; line is parsed outside Python.""" + self._priority = level + if self._scenario_parsed: + self.error("priority() must be called before scenarios()") + def scenarios(self, *scenario_list) -> None: """Called from tests as: scenarios() to specify which scenarios this test runs under. Where ... is one cases listed in All_Scenarios. - All scenarios must be on one line; this is parsed outside Python.""" + All scenarios must be on one line; line is parsed outside Python.""" + self._scenario_parsed = True enabled_scenarios = {} for param in scenario_list: hit = False @@ -926,19 +984,24 @@ class VlTest: # self._exit() implied by skip's exception @staticmethod - def _prefilter_scenario(py_filename: str, scenario: str) -> bool: + def _prefilter_scenario(py_filename: str) -> dict: """Read a python file to see if scenarios require it to be run. - Much faster than parsing the file for a runtime check.""" + Much faster than parsing the file for a runtime check. + Return dict information on scenarios and priority.""" (py_filename, _) = Runner._py_filename_adjust(py_filename, ".") + result = {'priority': 1} # Also all secenarios enabled with open(py_filename, 'r', encoding="utf-8") as fh: for line in fh: + # Required that test.priority be earlier in file, + # the priority() function checks for this + m = re.search(r'^\s*test.priority\((.*?)\)', line) + if m: + result['priority'] = int(m.group(1)) m = re.search(r'^\s*test.scenarios\((.*?)\)', line) if m: for param in re.findall(r"""["']([^,]*)["']""", m.group(1)): - for allscarg in All_Scenarios[scenario]: - if param == allscarg: - return True - return False + result[param] = True + return result def _prep(self) -> None: VtOs.mkdir_ok(self.obj_dir) # Ok if already exists @@ -977,10 +1040,13 @@ class VlTest: def _write_status(self) -> None: with open(self._status_filename, "wb") as fh: + # Vtest/self values to propagate up to driver's Vtest object pass_to_driver = { '_ok': self._ok, + '_priority': self._priority, '_scenario_off': self._scenario_off, '_skips': self._skips, + '_wall_time': time.time() - self._start_time, 'errors': self.errors, } pickle.dump(pass_to_driver, fh) @@ -2820,10 +2886,19 @@ def run_them() -> None: timestart = time.strftime("%Y%m%d_%H%M%S") runner = Runner(driver_log_filename="obj_dist/driver_" + timestart + ".log", quiet=Args.quiet) + + test_data = {} for test_py in Arg_Tests: + test_data[test_py] = VlTest._prefilter_scenario(test_py) + for test_py in sorted(Arg_Tests, key=lambda key: test_data[key]['priority'], reverse=True): for scenario in sorted(set(Args.scenarios)): - if VlTest._prefilter_scenario(test_py, scenario): + run_it = False + for allscarg in All_Scenarios[scenario]: + if allscarg in test_data[test_py]: + run_it = True + if run_it: runner.one_test(py_filename=test_py, scenario=scenario) + runner.wait_and_report() if Args.rerun and runner.fail_cnt and not Quitting: @@ -2839,6 +2914,7 @@ def run_them() -> None: orig_runner = runner runner = Runner(driver_log_filename="obj_dist/driver_" + timestart + "_rerun.log", quiet=False, + pass_tests=orig_runner.pass_tests, fail1_cnt=orig_runner.fail_cnt, ok_cnt=orig_runner.ok_cnt, skip_cnt=orig_runner.skip_cnt) diff --git a/test_regress/t/t_a1_first_cc.py b/test_regress/t/t_a1_first_cc.py index d8942fe63..db89b3543 100755 --- a/test_regress/t/t_a1_first_cc.py +++ b/test_regress/t/t_a1_first_cc.py @@ -13,6 +13,7 @@ import vltest_bootstrap +test.priority(100) test.scenarios('vlt') test.leak_check_disable() diff --git a/test_regress/t/t_a2_first_sc.py b/test_regress/t/t_a2_first_sc.py index bbb5c7e23..b72c58858 100755 --- a/test_regress/t/t_a2_first_sc.py +++ b/test_regress/t/t_a2_first_sc.py @@ -13,6 +13,7 @@ import vltest_bootstrap +test.priority(100) test.scenarios('vlt') test.top_filename = "t/t_a1_first_cc.v" diff --git a/test_regress/t/t_a3_selftest.py b/test_regress/t/t_a3_selftest.py index dc4bcb78f..ff241c671 100755 --- a/test_regress/t/t_a3_selftest.py +++ b/test_regress/t/t_a3_selftest.py @@ -9,6 +9,7 @@ import vltest_bootstrap +test.priority(100) test.scenarios('vlt') test.top_filename = "t/t_EXAMPLE.v" diff --git a/test_regress/t/t_a3_selftest_thread.py b/test_regress/t/t_a3_selftest_thread.py index 6f8a00043..9ad2150da 100755 --- a/test_regress/t/t_a3_selftest_thread.py +++ b/test_regress/t/t_a3_selftest_thread.py @@ -9,6 +9,7 @@ import vltest_bootstrap +test.priority(100) test.scenarios('vltmt') test.top_filename = "t/t_EXAMPLE.v" diff --git a/test_regress/t/t_a6_examples.py b/test_regress/t/t_a6_examples.py index 34ba28900..c0c1dae26 100755 --- a/test_regress/t/t_a6_examples.py +++ b/test_regress/t/t_a6_examples.py @@ -9,6 +9,7 @@ import vltest_bootstrap +test.priority(180) test.scenarios('dist') test.clean_command = '/bin/rm -rf ../examples/*/build ../examples/*/obj*' diff --git a/test_regress/t/t_case_write1.py b/test_regress/t/t_case_write1.py index 37d0f7a42..68242656b 100755 --- a/test_regress/t/t_case_write1.py +++ b/test_regress/t/t_case_write1.py @@ -9,6 +9,7 @@ import vltest_bootstrap +test.priority(50) test.scenarios('simulator') test.compile(verilator_flags2=["--stats -O3 -x-assign fast"]) diff --git a/test_regress/t/t_case_write2.py b/test_regress/t/t_case_write2.py index 37d0f7a42..68242656b 100755 --- a/test_regress/t/t_case_write2.py +++ b/test_regress/t/t_case_write2.py @@ -9,6 +9,7 @@ import vltest_bootstrap +test.priority(50) test.scenarios('simulator') test.compile(verilator_flags2=["--stats -O3 -x-assign fast"]) diff --git a/test_regress/t/t_a5_attributes_include.py b/test_regress/t/t_dist_attributes_include.py similarity index 98% rename from test_regress/t/t_a5_attributes_include.py rename to test_regress/t/t_dist_attributes_include.py index aa7fb958a..5ec098959 100755 --- a/test_regress/t/t_a5_attributes_include.py +++ b/test_regress/t/t_dist_attributes_include.py @@ -9,6 +9,7 @@ import vltest_bootstrap +test.priority(30) test.scenarios('dist') test.rerunnable = False diff --git a/test_regress/t/t_a5_attributes_src.py b/test_regress/t/t_dist_attributes_src.py similarity index 99% rename from test_regress/t/t_a5_attributes_src.py rename to test_regress/t/t_dist_attributes_src.py index 28c719609..f4bf78ea1 100755 --- a/test_regress/t/t_a5_attributes_src.py +++ b/test_regress/t/t_dist_attributes_src.py @@ -9,6 +9,7 @@ import vltest_bootstrap +test.priority(30) test.scenarios('dist') test.rerunnable = False diff --git a/test_regress/t/t_gantt.py b/test_regress/t/t_gantt.py index 2ade0911a..4523e81e5 100755 --- a/test_regress/t/t_gantt.py +++ b/test_regress/t/t_gantt.py @@ -11,6 +11,7 @@ import vltest_bootstrap +test.priority(30) test.scenarios('vlt_all') test.top_filename = "t/t_gantt.v" test.pli_filename = "t/t_gantt_c.cpp" diff --git a/test_regress/t/t_gantt_hier.py b/test_regress/t/t_gantt_hier.py index ea5237f6b..86c6ce7bf 100755 --- a/test_regress/t/t_gantt_hier.py +++ b/test_regress/t/t_gantt_hier.py @@ -11,6 +11,7 @@ import vltest_bootstrap +test.priority(30) test.scenarios('vlt_all') test.top_filename = "t/t_gantt.v" test.pli_filename = "t/t_gantt_c.cpp" diff --git a/test_regress/t/t_gantt_two.py b/test_regress/t/t_gantt_two.py index 3ee322182..2b1a2125e 100755 --- a/test_regress/t/t_gantt_two.py +++ b/test_regress/t/t_gantt_two.py @@ -11,6 +11,7 @@ import vltest_bootstrap +test.priority(50) test.scenarios('vlt_all') test.top_filename = "t/t_gantt.v" test.pli_filename = "t/t_gantt_c.cpp" diff --git a/test_regress/t/t_hier_block.py b/test_regress/t/t_hier_block.py index 8c0a75780..ff13a4925 100755 --- a/test_regress/t/t_hier_block.py +++ b/test_regress/t/t_hier_block.py @@ -9,6 +9,7 @@ import vltest_bootstrap +test.priority(30) test.scenarios('vlt_all') # stats will be deleted but generation will be skipped if libs of hierarchical blocks exist. diff --git a/test_regress/t/t_hier_block_binary.py b/test_regress/t/t_hier_block_binary.py index 3e0199a2c..e0b1706b4 100755 --- a/test_regress/t/t_hier_block_binary.py +++ b/test_regress/t/t_hier_block_binary.py @@ -9,6 +9,7 @@ import vltest_bootstrap +test.priority(30) test.scenarios('vlt_all') test.top_filename = "t/t_hier_block.v" diff --git a/test_regress/t/t_hier_block_chained.py b/test_regress/t/t_hier_block_chained.py index 7cd6a2e24..af9546e7f 100755 --- a/test_regress/t/t_hier_block_chained.py +++ b/test_regress/t/t_hier_block_chained.py @@ -9,7 +9,9 @@ import vltest_bootstrap +test.priority(30) test.scenarios('vlt_all') + test.init_benchmarksim() test.cycles = (int(test.benchmark) if test.benchmark else 100000) test.sim_time = test.cycles * 10 + 1000 diff --git a/test_regress/t/t_a7_hier_block_cmake.py b/test_regress/t/t_hier_block_cmake.py similarity index 99% rename from test_regress/t/t_a7_hier_block_cmake.py rename to test_regress/t/t_hier_block_cmake.py index 535de75a3..80161d6c1 100755 --- a/test_regress/t/t_a7_hier_block_cmake.py +++ b/test_regress/t/t_hier_block_cmake.py @@ -13,6 +13,7 @@ import os # If a test fails, broken .cmake may disturb the next run test.clean_objs() +test.priority(30) test.scenarios('simulator') test.top_filename = "t/t_hier_block.v" diff --git a/test_regress/t/t_hier_block_import.py b/test_regress/t/t_hier_block_import.py index 890c1fee6..12bfe08f5 100755 --- a/test_regress/t/t_hier_block_import.py +++ b/test_regress/t/t_hier_block_import.py @@ -9,6 +9,7 @@ import vltest_bootstrap +test.priority(30) test.scenarios('vlt_all') # stats will be deleted but generation will be skipped if libs of hierarchical blocks exist. diff --git a/test_regress/t/t_hier_block_import_cmake.py b/test_regress/t/t_hier_block_import_cmake.py index 0188abdb2..0f246454d 100755 --- a/test_regress/t/t_hier_block_import_cmake.py +++ b/test_regress/t/t_hier_block_import_cmake.py @@ -9,7 +9,9 @@ import vltest_bootstrap +test.priority(30) test.scenarios('vlt_all') + # CMake build executes from a different directory than the Make one. test.top_filename = os.path.abspath("t/t_hier_block_import.v") diff --git a/test_regress/t/t_hier_block_perf.py b/test_regress/t/t_hier_block_perf.py index 5847c7588..c77172122 100755 --- a/test_regress/t/t_hier_block_perf.py +++ b/test_regress/t/t_hier_block_perf.py @@ -9,7 +9,9 @@ import vltest_bootstrap +test.priority(30) test.scenarios('vlt_all') + test.init_benchmarksim() test.cycles = (int(test.benchmark) if test.benchmark else 100000) test.sim_time = test.cycles * 10 + 1000 diff --git a/test_regress/t/t_hier_block_prot_lib.py b/test_regress/t/t_hier_block_prot_lib.py index d8f4103ee..b20c4eec4 100755 --- a/test_regress/t/t_hier_block_prot_lib.py +++ b/test_regress/t/t_hier_block_prot_lib.py @@ -9,6 +9,7 @@ import vltest_bootstrap +test.priority(30) test.scenarios('vlt_all', 'xsim') test.top_filename = "t/t_hier_block.v" diff --git a/test_regress/t/t_hier_block_prot_lib_shared.py b/test_regress/t/t_hier_block_prot_lib_shared.py index 1f073834c..09db88bb1 100755 --- a/test_regress/t/t_hier_block_prot_lib_shared.py +++ b/test_regress/t/t_hier_block_prot_lib_shared.py @@ -9,6 +9,7 @@ import vltest_bootstrap +test.priority(30) test.scenarios('vlt_all', 'xsim') test.top_filename = "t/t_hier_block.v" diff --git a/test_regress/t/t_hier_block_sc.py b/test_regress/t/t_hier_block_sc.py index ee03f4602..3660f8311 100755 --- a/test_regress/t/t_hier_block_sc.py +++ b/test_regress/t/t_hier_block_sc.py @@ -9,6 +9,7 @@ import vltest_bootstrap +test.priority(30) test.scenarios('vlt_all') test.top_filename = "t/t_hier_block.v" diff --git a/test_regress/t/t_hier_block_sc_trace_fst.py b/test_regress/t/t_hier_block_sc_trace_fst.py index 5dbe51557..b3ca507cc 100755 --- a/test_regress/t/t_hier_block_sc_trace_fst.py +++ b/test_regress/t/t_hier_block_sc_trace_fst.py @@ -9,6 +9,7 @@ import vltest_bootstrap +test.priority(30) test.scenarios('vlt_all') test.top_filename = "t/t_hier_block.v" diff --git a/test_regress/t/t_hier_block_sc_trace_vcd.py b/test_regress/t/t_hier_block_sc_trace_vcd.py index dff5601e7..0fbd7237a 100755 --- a/test_regress/t/t_hier_block_sc_trace_vcd.py +++ b/test_regress/t/t_hier_block_sc_trace_vcd.py @@ -9,6 +9,7 @@ import vltest_bootstrap +test.priority(30) test.scenarios('vlt_all') test.top_filename = "t/t_hier_block.v" diff --git a/test_regress/t/t_hier_block_trace_fst.py b/test_regress/t/t_hier_block_trace_fst.py index f6b26d0f5..b3d4c0343 100755 --- a/test_regress/t/t_hier_block_trace_fst.py +++ b/test_regress/t/t_hier_block_trace_fst.py @@ -9,6 +9,7 @@ import vltest_bootstrap +test.priority(30) test.scenarios('vlt_all') test.top_filename = "t/t_hier_block.v" diff --git a/test_regress/t/t_hier_block_trace_saif.py b/test_regress/t/t_hier_block_trace_saif.py index 44a437fd5..e4a50a5d1 100755 --- a/test_regress/t/t_hier_block_trace_saif.py +++ b/test_regress/t/t_hier_block_trace_saif.py @@ -9,6 +9,7 @@ import vltest_bootstrap +test.priority(30) test.scenarios('vlt_all') test.top_filename = "t/t_hier_block.v" test.golden_filename = "t/t_hier_block_trace_saif.out" diff --git a/test_regress/t/t_hier_block_trace_vcd.py b/test_regress/t/t_hier_block_trace_vcd.py index 7bffe147f..6e06e809d 100755 --- a/test_regress/t/t_hier_block_trace_vcd.py +++ b/test_regress/t/t_hier_block_trace_vcd.py @@ -9,6 +9,7 @@ import vltest_bootstrap +test.priority(30) test.scenarios('vlt_all') test.top_filename = "t/t_hier_block.v" diff --git a/test_regress/t/t_unopt_array_csplit.py b/test_regress/t/t_unopt_array_csplit.py index ec16097b5..3d22d22a4 100755 --- a/test_regress/t/t_unopt_array_csplit.py +++ b/test_regress/t/t_unopt_array_csplit.py @@ -9,6 +9,7 @@ import vltest_bootstrap +test.priority(30) test.scenarios('vlt_all') test.top_filename = "t/t_unopt_array.v" diff --git a/test_regress/t/t_uvm_hello_all_v2017_1_0_nodpi.py b/test_regress/t/t_uvm_hello_all_v2017_1_0_nodpi.py index 9ab431c0f..ec2d6922c 100755 --- a/test_regress/t/t_uvm_hello_all_v2017_1_0_nodpi.py +++ b/test_regress/t/t_uvm_hello_all_v2017_1_0_nodpi.py @@ -9,6 +9,7 @@ import vltest_bootstrap +test.priority(50) test.scenarios('vlt') test.top_filename = 't/t_uvm_hello.v' diff --git a/test_regress/t/t_uvm_hello_all_v2020_3_1_nodpi.py b/test_regress/t/t_uvm_hello_all_v2020_3_1_nodpi.py index 206f5c729..19e0b3d6f 100755 --- a/test_regress/t/t_uvm_hello_all_v2020_3_1_nodpi.py +++ b/test_regress/t/t_uvm_hello_all_v2020_3_1_nodpi.py @@ -9,6 +9,7 @@ import vltest_bootstrap +test.priority(50) test.scenarios('vlt') test.top_filename = 't/t_uvm_hello.v'