From 7e5c5d677fca871ea4bca7dae2ef65e1d369b278 Mon Sep 17 00:00:00 2001 From: Wilson Snyder Date: Tue, 16 Dec 2025 21:17:27 -0500 Subject: [PATCH] Tests: Remove old benchmarksim, should use rtlmeter instead --- docs/internals.rst | 5 --- test_regress/driver.py | 45 -------------------------- test_regress/t/t_benchmark_sim.py | 33 +------------------ test_regress/t/t_hier_block_chained.py | 2 -- test_regress/t/t_hier_block_perf.py | 2 -- test_regress/t/t_pgo_threads_hier.py | 5 ++- 6 files changed, 3 insertions(+), 89 deletions(-) diff --git a/docs/internals.rst b/docs/internals.rst index 5b050170d..cf5325f4b 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -2630,11 +2630,6 @@ verilator_flags / verilator_flags2 Verilator. If a flag is a standard flag, ``+incdir`` for example, pass it with ``v_flags2`` instead. -benchmarksim - Output the number of model evaluations and execution time of a test to - ``test_output_dir>/_benchmarksim.csv``. Multiple invocations - of the same test file will append to to the same .csv file. - xsim_flags / xsim_flags2 / xsim_run_flags The equivalent of ``v_flags``, ``v_flags2`` and ``all_run_flags``, but only for use with the Xilinx XSim simulator. diff --git a/test_regress/driver.py b/test_regress/driver.py index 235d37973..ac61baac2 100755 --- a/test_regress/driver.py +++ b/test_regress/driver.py @@ -702,7 +702,6 @@ class VlTest: self.name = match.group(2) # Name of this test self.benchmark = Args.benchmark - self.benchmarksim = False self.clean_command = None self.context_threads = 0 # Number of threads to allocate in the context self.errors = None @@ -891,16 +890,6 @@ class VlTest: def _define_opt_calc(self) -> str: return "--define " if self.xsim else "+define+" - def init_benchmarksim(self) -> None: - # Simulations with benchmarksim enabled append to the same file between runs. - # Test files must ensure a clean benchmark data file before executing tests. - filename = self.benchmarksim_filename - with open(filename, 'w', encoding="utf8") as fh: - fh.write("# Verilator simulation benchmark data\n") - fh.write("# Test name: " + self.name + "\n") - fh.write("# Top file: " + self.top_filename + "\n") - fh.write("evals, time[s]\n") - def soprint(self, message: str) -> str: message = message.rstrip() + "\n" message = self.scenario + "/" + self.name + ": " + message @@ -1138,9 +1127,6 @@ class VlTest: else: self.trace_format = 'vcd-c' # pylint: disable=attribute-defined-outside-init - if param.get('benchmarksim', None): - self.benchmarksim = True # pylint: disable=attribute-defined-outside-init - verilator_flags = [*param.get('verilator_flags', "")] if Args.gdb: verilator_flags += ["--gdb"] @@ -1687,10 +1673,6 @@ class VlTest: VlTest._cached_aslr_off = "" return VlTest._cached_aslr_off - @property - def benchmarksim_filename(self) -> str: - return self.obj_dir + "/" + self.name + "_benchmarksim.csv" - @property def driver_verilator_flags(self) -> list: return Args.passdown_verilator_flags @@ -2038,10 +2020,6 @@ class VlTest: str(int(round(self.main_time_multiplier, 0))) + "\n") fh.write("#include \n") - if self.benchmarksim: - fh.write("#include \n") - fh.write("#include \n") - fh.write("#include \n") fh.write("// OS header\n") fh.write('#include "verilatedos.h"' + "\n") @@ -2139,11 +2117,6 @@ class VlTest: fh.write(" topp->eval();\n") setp = "topp->" - if self.benchmarksim: - fh.write(" std::chrono::time_point starttime;\n") - fh.write(" bool warm = false;\n") - fh.write(" uint64_t n_evals = 0;\n") - if self.trace: fh.write("\n") fh.write("#if VM_TRACE\n") @@ -2249,27 +2222,9 @@ class VlTest: fh.write(" return 0;\n") fh.write(" }\n") self._print_advance_time(fh, self.sc_time_resolution_multiplier, action) - if self.benchmarksim: - fh.write(" if (VL_UNLIKELY(!warm)) {\n") - fh.write(" starttime = std::chrono::steady_clock::now();\n") - fh.write(" warm = true;\n") - fh.write(" } else {\n") - fh.write(" ++n_evals;\n") - fh.write(" }\n") fh.write(" }\n") - if self.benchmarksim: - fh.write(" {\n") - fh.write(" const std::chrono::duration exec_s" - " = std::chrono::steady_clock::now() - starttime;\n") - fh.write(" std::ofstream benchfile(\"" + self.benchmarksim_filename + - "\", std::ofstream::out | std::ofstream::app);\n") - fh.write(" benchfile << std::fixed << std::setprecision(9)" - " << n_evals << \",\" << exec_s.count() << std::endl;\n") - fh.write(" benchfile.close();\n") - fh.write(" }\n") - fh.write(" if (!contextp->gotFinish()) {\n") fh.write(' vl_fatal(__FILE__, __LINE__, "main",' + ' "%Error: Timeout; never got a $finish");' + "\n") diff --git a/test_regress/t/t_benchmark_sim.py b/test_regress/t/t_benchmark_sim.py index 501a8b0fd..4b23385b0 100755 --- a/test_regress/t/t_benchmark_sim.py +++ b/test_regress/t/t_benchmark_sim.py @@ -12,43 +12,12 @@ import vltest_bootstrap test.scenarios('vlt') test.top_filename = "t/t_gen_alw.v" # Use any top file -test.init_benchmarksim() - # As an example, compile and simulate the top file with varying optimization level l_opts = ['-O0', '-O1', '-O2', '-O3'] for l_opt in l_opts: - test.compile(benchmarksim=1, v_flags2=[l_opt]) + test.compile(v_flags2=[l_opt]) test.execute() -filename = test.benchmarksim_filename -gotn = 0 -with open(filename, 'r', encoding="utf8") as fh: - lineno = 0 - headered = False - for line in fh: - lineno += 1 - if re.match(r'^#', line): - continue - if not headered: - headered = True - if not re.search(r'evals, ', line): - test.error(filename + ":" + str(lineno) + ": Expected header but found: " + line) - else: - m = re.search(r'(\d+\.?\d*),(\d+\.?\d*)', line) - if not m: - test.error(filename + ":" + str(lineno) + ": Expected 2 tokens on line: " + line) - continue - cycles = float(m.group(1)) - time = float(m.group(2)) - if cycles <= 0.0 or time <= 0.0: - test.error(filename + ":" + str(lineno) + ": Invalid data on line: " + line) - continue - gotn += 1 - -n_lines_expected = len(l_opts) -if gotn != int(n_lines_expected): - test.error("Expected " + str(n_lines_expected) + " lines but found " + str(gotn)) - test.passes() diff --git a/test_regress/t/t_hier_block_chained.py b/test_regress/t/t_hier_block_chained.py index af9546e7f..6a6f5a8aa 100755 --- a/test_regress/t/t_hier_block_chained.py +++ b/test_regress/t/t_hier_block_chained.py @@ -12,7 +12,6 @@ import vltest_bootstrap test.priority(30) test.scenarios('vlt_all') -test.init_benchmarksim() test.cycles = (int(test.benchmark) if test.benchmark else 100000) test.sim_time = test.cycles * 10 + 1000 @@ -23,7 +22,6 @@ HIER_THREADS = 4 config_file = test.t_dir + "/" + test.name + ".vlt" test.compile( - benchmarksim=1, v_flags2=[ config_file, "+define+SIM_CYCLES=" + str(test.cycles), "--hierarchical", "--stats", (f"-DWORKERS={HIER_BLOCK_THREADS}" if test.vltmt and HIER_BLOCK_THREADS > 1 else ""), diff --git a/test_regress/t/t_hier_block_perf.py b/test_regress/t/t_hier_block_perf.py index c77172122..694cbc7d4 100755 --- a/test_regress/t/t_hier_block_perf.py +++ b/test_regress/t/t_hier_block_perf.py @@ -12,7 +12,6 @@ import vltest_bootstrap test.priority(30) test.scenarios('vlt_all') -test.init_benchmarksim() test.cycles = (int(test.benchmark) if test.benchmark else 100000) test.sim_time = test.cycles * 10 + 1000 @@ -23,7 +22,6 @@ HIER_THREADS = 4 config_file = test.t_dir + "/" + test.name + ".vlt" test.compile( - benchmarksim=1, v_flags2=[ config_file, "+define+SIM_CYCLES=" + str(test.cycles), "--prof-exec", "--hierarchical", "--stats", "-Wno-UNOPTFLAT", diff --git a/test_regress/t/t_pgo_threads_hier.py b/test_regress/t/t_pgo_threads_hier.py index 0c0449825..404deb791 100755 --- a/test_regress/t/t_pgo_threads_hier.py +++ b/test_regress/t/t_pgo_threads_hier.py @@ -18,7 +18,7 @@ threads = 2 config_file = test.t_dir + "/" + test.name + ".vlt" flags = [config_file, "--hierarchical", "-Wno-UNOPTFLAT", "-DSIM_CYCLES=" + str(cycles)] -test.compile(benchmarksim=1, v_flags2=["--prof-pgo"] + flags, threads=threads) +test.compile(v_flags2=["--prof-pgo"] + flags, threads=threads) test.execute(all_run_flags=[ "+verilator+prof+exec+start+0", @@ -33,10 +33,9 @@ test.file_grep(test.obj_dir + "/profile.vlt", r'profile_data -model "V' + test.n # Check for cost rollovers test.file_grep_not(test.obj_dir + "/profile.vlt", r'.*cost 64\'d\d{18}.*') -# Differentiate benchmarksim results +# Differentiate results test.name = test.name + "_optimized" test.compile( - benchmarksim=1, # Intentionally no --prof-pgo here to make sure profile data can be read in # without it (that is: --prof-pgo has no effect on profile_data hash names) v_flags2=[test.obj_dir + "/profile.vlt"] + flags,