From 65bd9df5f7846015313734d08a5a6367df79453c Mon Sep 17 00:00:00 2001
From: Drew <cannada@google.com>
Date: Thu, 27 Aug 2026 08:50:21 -0400
Subject: [PATCH] Chunk vertices in BFS parallel iteration for dynamic load
 balancing

Dispatch fixed-size chunks (8 vertices per task) instead of one
contiguous slice per thread, so threads that finish early pick up more
work. Chunk size vs. runtime is U-shaped; 8 is the smallest size at the
bottom of the curve.

report_checks on a 32-core machine (min of 5 runs):
- gcd_sky130hd, 32 threads: 1845 us -> 1255 us (-32%)
- gcd_sky130hd,  8 threads: 1254 us -> 1117 us (-11%)
- aes_nangate45 (~17k inst): neutral; single-threaded path unchanged.

To keep upstream merges clean, the fork logic lives in a new
BfsIterator::visitLevelChunked() wrapped in "OpenROAD fork: BFS chunked
dispatch" markers, called from one marked line in visitParallel's
existing else branch. All other upstream lines are byte-identical. This
is a stop-gap until the upstream BFS rework lands; on merge conflict
take upstream and drop it.

Signed-off-by: Drew <cannada@google.com>
---
 include/sta/Bfs.hh |  9 ++++++++
 search/Bfs.cc      | 55 ++++++++++++++++++++++++++++++----------------
 2 files changed, 45 insertions(+), 19 deletions(-)

diff --git a/include/sta/Bfs.hh b/include/sta/Bfs.hh
index c0c24f1e..af5f687e 100644
--- a/include/sta/Bfs.hh
+++ b/include/sta/Bfs.hh
@@ -109,6 +109,15 @@ protected:
                   Level level);
   void findNext(Level to_level);
 
+  // ---- OpenROAD fork: BFS chunked dispatch (begin) ----
+  // Dispatch level_vertices to the worker pool in fixed-size chunks so
+  // idle threads pick up remaining work. Stop-gap until upstream BFS
+  // rework lands; on merge conflict take upstream and drop this.
+  void visitLevelChunked(VertexSeq &level_vertices,
+                         Level level,
+                         std::vector<VertexVisitor *> &visitors);
+  // ---- OpenROAD fork: BFS chunked dispatch (end) ----
+
   BfsIndex bfs_index_;
   Level level_min_;
   Level level_max_;
diff --git a/search/Bfs.cc b/search/Bfs.cc
index 08a37999..c9e90ec2 100644
--- a/search/Bfs.cc
+++ b/search/Bfs.cc
@@ -182,25 +182,9 @@ BfsIterator::visitParallel(Level to_level,
             }
           }
           else {
-            size_t from = 0;
-            size_t chunk_size = vertex_count / thread_count;
-            BfsIndex bfs_index = bfs_index_;
-            for (size_t k = 0; k < thread_count; k++) {
-              // Last thread gets the left overs.
-              size_t to = (k == thread_count - 1) ? vertex_count : from + chunk_size;
-              dispatch_queue_->dispatch([=, this](size_t) {
-                for (size_t i = from; i < to; i++) {
-                  Vertex *vertex = level_vertices[i];
-                  if (vertex) {
-                    checkLevel(vertex, level);
-                    vertex->setBfsInQueue(bfs_index, false);
-                    visitors[k]->visit(vertex);
-                  }
-                }
-              });
-              from = to;
-            }
-            dispatch_queue_->finishTasks();
+            // ---- OpenROAD fork: BFS chunked dispatch (begin) ----
+            visitLevelChunked(level_vertices, level, visitors);
+            // ---- OpenROAD fork: BFS chunked dispatch (end) ----
           }
           level_vertices.clear();
           visit_count += vertex_count;
@@ -336,6 +320,39 @@ BfsIterator::findNext(Level to_level)
   }
 }
 
+// ---- OpenROAD fork: BFS chunked dispatch (begin) ----
+void
+BfsIterator::visitLevelChunked(VertexSeq &level_vertices,
+                               Level level,
+                               std::vector<VertexVisitor *> &visitors)
+{
+  // Tasks read level_vertices in place and unlocked. This relies on
+  // visitors never enqueuing at the current level (they only enqueue
+  // fanout/fanin, which levelize places at a higher/lower level).
+  // Chunk size vs. runtime is U-shaped; 8 is the smallest size at the
+  // bottom of the curve.
+  constexpr size_t chunk_size = 8;
+  size_t vertex_count = level_vertices.size();
+  BfsIndex bfs_index = bfs_index_;
+  for (size_t from = 0; from < vertex_count; from += chunk_size) {
+    size_t to = (from + chunk_size < vertex_count) ? from + chunk_size : vertex_count;
+    dispatch_queue_->dispatch([this, &level_vertices, from, to, level,
+                               bfs_index, &visitors](int thread_id) {
+      VertexVisitor *thread_visitor = visitors[thread_id];
+      for (size_t i = from; i < to; i++) {
+        Vertex *vertex = level_vertices[i];
+        if (vertex) {
+          checkLevel(vertex, level);
+          vertex->setBfsInQueue(bfs_index, false);
+          thread_visitor->visit(vertex);
+        }
+      }
+    });
+  }
+  dispatch_queue_->finishTasks();
+}
+// ---- OpenROAD fork: BFS chunked dispatch (end) ----
+
 ////////////////////////////////////////////////////////////////
 
 BfsFwdIterator::BfsFwdIterator(BfsIndex bfs_index,