Merge pull request #405 from calewis/chunking-in-bfs

Chunk vertices in BFS parallel iteration for dynamic load balancing

include/sta/Bfs.hh

@@ -109,6 +109,15 @@ protected:
                   Level level);
   void findNext(Level to_level);
 
+  // ---- OpenROAD fork: BFS chunked dispatch (begin) ----
+  // Dispatch level_vertices to the worker pool in fixed-size chunks so
+  // idle threads pick up remaining work. Stop-gap until upstream BFS
+  // rework lands; on merge conflict take upstream and drop this.
+  void visitLevelChunked(VertexSeq &level_vertices,
+                         Level level,
+                         std::vector<VertexVisitor *> &visitors);
+  // ---- OpenROAD fork: BFS chunked dispatch (end) ----
+
   BfsIndex bfs_index_;
   Level level_min_;
   Level level_max_;

search/Bfs.cc

@@ -182,25 +182,9 @@ BfsIterator::visitParallel(Level to_level,
             }
           }
           else {
-            size_t from = 0;
-            size_t chunk_size = vertex_count / thread_count;
-            BfsIndex bfs_index = bfs_index_;
-            for (size_t k = 0; k < thread_count; k++) {
-              // Last thread gets the left overs.
-              size_t to = (k == thread_count - 1) ? vertex_count : from + chunk_size;
-              dispatch_queue_->dispatch([=, this](size_t) {
-                for (size_t i = from; i < to; i++) {
-                  Vertex *vertex = level_vertices[i];
-                  if (vertex) {
-                    checkLevel(vertex, level);
-                    vertex->setBfsInQueue(bfs_index, false);
-                    visitors[k]->visit(vertex);
-                  }
-                }
-              });
-              from = to;
-            }
-            dispatch_queue_->finishTasks();
+            // ---- OpenROAD fork: BFS chunked dispatch (begin) ----
+            visitLevelChunked(level_vertices, level, visitors);
+            // ---- OpenROAD fork: BFS chunked dispatch (end) ----
           }
           level_vertices.clear();
           visit_count += vertex_count;
@@ -336,6 +320,39 @@ BfsIterator::findNext(Level to_level)
   }
 }
 
+// ---- OpenROAD fork: BFS chunked dispatch (begin) ----
+void
+BfsIterator::visitLevelChunked(VertexSeq &level_vertices,
+                               Level level,
+                               std::vector<VertexVisitor *> &visitors)
+{
+  // Tasks read level_vertices in place and unlocked. This relies on
+  // visitors never enqueuing at the current level (they only enqueue
+  // fanout/fanin, which levelize places at a higher/lower level).
+  // Chunk size vs. runtime is U-shaped; 8 is the smallest size at the
+  // bottom of the curve.
+  constexpr size_t chunk_size = 8;
+  size_t vertex_count = level_vertices.size();
+  BfsIndex bfs_index = bfs_index_;
+  for (size_t from = 0; from < vertex_count; from += chunk_size) {
+    size_t to = (from + chunk_size < vertex_count) ? from + chunk_size : vertex_count;
+    dispatch_queue_->dispatch([this, &level_vertices, from, to, level,
+                               bfs_index, &visitors](int thread_id) {
+      VertexVisitor *thread_visitor = visitors[thread_id];
+      for (size_t i = from; i < to; i++) {
+        Vertex *vertex = level_vertices[i];
+        if (vertex) {
+          checkLevel(vertex, level);
+          vertex->setBfsInQueue(bfs_index, false);
+          thread_visitor->visit(vertex);
+        }
+      }
+    });
+  }
+  dispatch_queue_->finishTasks();
+}
+// ---- OpenROAD fork: BFS chunked dispatch (end) ----
+
 ////////////////////////////////////////////////////////////////
 
 BfsFwdIterator::BfsFwdIterator(BfsIndex bfs_index,