luke/verilator
1
0
Fork 0
mirror of https://github.com/verilator/verilator.git synced 2026-10-06 10:03:44 +02:00
Code Issues Packages Projects Releases Wiki Activity

Improve MTask coarsening in multi-threaded scheduling (#8120)

Browse Source 
This is a large refactor of the MTask graph and coarsening algorithm, in
prep for fixing the bug described in #7913, it can also improve the
resulting multi-threaded schedule.

Two major changes:

OrderMTaskGraph now maintains the critical paths of the MTasks through
mutation. There are 2 ways to mutate the graph, which are done via
methods on the graph itself: adding an edge (used during construction,
and will be used later during fixing data hazards), or merging an MTask
into another (used during contraction). All critical path measures are
automatically updated and propagated on any mutation, so no external
algorithm needs to maintain them explicitly.

The merge candidate scoreboard used during contraction is simplified to
remove deferral of updated scores. This simplifies the code and results
in a greedily more optimal schedule. (The previous tranched rescore was
an optimization to work around the previous std::set based scoreboard,
however since the algorithm now uses an efficient PairingHeap,
verilation time is not impacted by the more accurate scoring, while
yielding better results).

Combining these two into a single patch as the code is highly
interdependent and any one change without the other would be just a
noisy transit point with unclear performance implications. Together it
should be a clear improvement.

Also added a stronger validation step run with '--debug-partition',
which checks all invariants throughout the algorithms.
This commit is contained in:
Geza Lore
2026-08-17 23:14:52 +02:00
committed by GitHub
parent ace84ef2b3
commit 7dcd4e0b65
10 changed files with 834 additions and 1144 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/V3OrderMTaskGraph.cpp
+288 -25
View File
@@ -21,28 +21,16 @@
#include "V3Global.h"
#include "V3InstrCount.h"
#include <algorithm>
#include <memory>
#include <unordered_set>
VL_DEFINE_DEBUG_FUNCTIONS;
//######################################################################
// OrderMTaskGraph
OrderMTaskGraph::OrderMTaskGraph(OrderMoveGraph& moveGraph)
: m_moveGraph{moveGraph}
, m_entryp{new LogicMTask{*this, nullptr}}
, m_exitp{new LogicMTask{*this, nullptr}}
, m_forwardPropagator{v3Global.opt.debugPartition()}
, m_reversePropagator{v3Global.opt.debugPartition()} {}
uint64_t OrderMTaskGraph::totalCost() const {
uint64_t cost = 0;
for (const V3GraphVertex& vtx : vertices()) cost += static_cast<const LogicMTask&>(vtx).cost();
return cost;
}
//######################################################################
// LogicMTask
uint32_t LogicMTask::s_nextId = 1; // Start at 1, so that 0 indicates no mtask.
uint32_t LogicMTask::s_nextId = 1; // Start at 1, for historic reasons
LogicMTask::LogicMTask(OrderMTaskGraph& graph, OrderMoveVertex* mVtxp)
: V3GraphVertex{&graph} {
@@ -54,6 +42,281 @@ LogicMTask::LogicMTask(OrderMTaskGraph& graph, OrderMoveVertex* mVtxp)
}
}
//######################################################################
// OrderMTaskGraph
OrderMTaskGraph::OrderMTaskGraph(OrderMoveGraph& moveGraph)
: m_moveGraph{moveGraph}
, m_entryp{new LogicMTask{*this, nullptr}}
, m_exitp{new LogicMTask{*this, nullptr}}
, m_slowAsserts{v3Global.opt.debugPartition()} {}
bool OrderMTaskGraph::pathExistsImpl(LogicMTask* fromp, LogicMTask* top,
const MTaskEdge* excludedEdgep) {
UDEBUGONLY(UASSERT_OBJ(fromp->m_generation != m_currentGeneration, fromp,
"Should not visit an MTask twice in the same search"););
// Mark visited.
fromp->m_generation = m_currentGeneration;
// Base case: we found a path.
if (fromp == top) return true;
// Base case: fromp is too late, cannot possibly be a prereq for top.
if (fromp->cpExclusive<GraphWay::REVERSE>() < top->cpInclusive<GraphWay::REVERSE>()) {
return false;
}
if (fromp->cpInclusive<GraphWay::FORWARD>() > top->cpExclusive<GraphWay::FORWARD>()) {
return false;
}
// Recursively look for a path
for (const V3GraphEdge& follow : fromp->outEdges()) {
if (&follow == excludedEdgep) continue;
LogicMTask* const nextp = static_cast<LogicMTask*>(follow.top());
// Don't visit the same MTask twice in the same search.
if (nextp->m_generation == m_currentGeneration) continue;
if (pathExistsImpl(nextp, top, nullptr)) return true;
}
return false;
}
template <GraphWay::en N_Way>
void OrderMTaskGraph::propagatePush(LogicMTask* mtaskp) {
constexpr GraphWay way{N_Way};
constexpr GraphWay inv{way.invert()};
const uint64_t inclusiveCp = mtaskp->cpInclusive<way>();
for (V3GraphEdge& graphEdge : mtaskp->edges<way>()) {
MTaskEdge& edge = static_cast<MTaskEdge&>(graphEdge);
LogicMTask* const relativep = edge.furtherMTaskp<N_Way>();
EdgeHeap::Node& edgeHeapNode = edge.m_edgeHeapNode[inv];
if (inclusiveCp > edgeHeapNode.key().m_cp) {
relativep->m_edgeHeap[inv].increaseKey(&edgeHeapNode, inclusiveCp);
}
const uint64_t relativeCp = relativep->cpExclusive<way>();
if (relativeCp >= inclusiveCp) continue;
// relativep's critical path is out of step with its longest !wayward edge.
// Schedule that to be resolved.
const uint64_t increment = inclusiveCp - relativeCp;
PropagatePendingHeap::Node*& pendingNodepRef = relativep->m_propagateHeapNodep;
if (PropagatePendingHeap::Node* const nodep = pendingNodepRef) {
// Already in heap. Increase the increment if needed.
if (increment > nodep->key().m_increment) {
m_pendingHeap.increaseKey(nodep, increment);
}
continue;
}
// Add to heap
PropagatePendingHeap::Node* const nodep = m_pendingNodePool.alloc();
pendingNodepRef = nodep;
m_pendingHeap.insert(nodep, {increment, relativep->id(), relativep});
}
}
template <GraphWay::en N_Way>
void OrderMTaskGraph::propagateResolve() {
constexpr GraphWay way{N_Way};
constexpr GraphWay inv{way.invert()};
// Each pending MTask is keyed on how much its critical path will grow by. Resolving them in
// decreasing order of that growth means each MTask needs resolving only once: the growth of a
// wayward MTask is never larger than the growth of the MTask it was pushed from, so once an
// MTask has been resolved, no larger growth can be pushed onto it later.
while (!m_pendingHeap.empty()) {
// Pop max element from heap
PropagatePendingHeap::Node* const maxp = m_pendingHeap.max();
m_pendingHeap.remove(maxp);
// Pick up values
LogicMTask* const mtaskp = maxp->key().m_mtaskp;
const uint64_t cpGrowBy = maxp->key().m_increment;
// Confirm that we only set each node's CP once. That's an important property of this
// algorithm, which allows it to be far faster than a recursive one.
UASSERT_OBJ(mtaskp->m_generation != m_currentGeneration, mtaskp, "Set CP on node twice");
mtaskp->m_generation = m_currentGeneration;
// Free the heap node, we are done with it
m_pendingNodePool.free(maxp);
mtaskp->m_propagateHeapNodep = nullptr;
// Update the critical path of mtaskp, that was out-of-date with respect to its edges
uint64_t& cpRef = mtaskp->m_cpExclusive[way];
const uint64_t newCp = cpRef + cpGrowBy;
// Check that CP matches that of the longest edge wayward of mtaskp.
if (VL_UNLIKELY(m_slowAsserts)) {
const uint64_t edgeCp = mtaskp->m_edgeHeap[inv].max()->key().m_cp;
UASSERT_OBJ(edgeCp == newCp, mtaskp, "CP doesn't match longest wayward edge");
}
cpRef = newCp;
propagatePush<N_Way>(mtaskp);
}
}
uint64_t OrderMTaskGraph::totalCost() const {
uint64_t cost = 0;
for (const V3GraphVertex& vtx : vertices()) cost += static_cast<const LogicMTask&>(vtx).cost();
return cost;
}
void OrderMTaskGraph::addEdge(LogicMTask* fromp, LogicMTask* top) {
UASSERT_OBJ(fromp != top, fromp, "Should not create self-edges");
UDEBUGONLY(UASSERT_OBJ(!fromp->hasEdgeTo(top), fromp, "Should not create redundant edges"););
// Create the edge. This inserts it into the edge heap of both endpoints with the correct
// critical path keys, as the critical paths of the endpoints are still unchanged here.
new MTaskEdge{this, fromp, top};
// The path through the new edge might be longer than the current critical path of its
// endpoints, in which case the critical paths need updating. Note each endpoint is the seed of
// one propagation, and is updated by the other: the inclusive critical paths of the endpoints
// themselves did not change (a new out-edge cannot lengthen a path into 'fromp', nor a new
// in-edge a path out of 'top'), so it is the new relative of each seed whose critical path
// might need to grow. That is, 'top' is updated wayward of 'fromp' below, and vice versa,
// together with the relatives of each, transitively.
//
// The guards below are an asymptotic optimization. The graph is consistent apart from the new
// edge, so the new relative is the only relative of either seed that can have a stale critical
// path, and if it does not need updating the propagation does nothing. It would however still
// walk all edges of the seed to discover that, which is expensive for a high degree seed.
if (fromp->cpInclusive<GraphWay::FORWARD>() > top->cpExclusive<GraphWay::FORWARD>()) {
propagate<GraphWay::FORWARD>(fromp);
}
if (top->cpInclusive<GraphWay::REVERSE>() > fromp->cpExclusive<GraphWay::REVERSE>()) {
propagate<GraphWay::REVERSE>(top);
}
}
void OrderMTaskGraph::mergeMTasks(LogicMTask* recipientp, LogicMTask* donorp) {
UASSERT_OBJ(recipientp != donorp, recipientp, "Should not merge an MTask with itself");
// Note we redirect the edges before updating the cost and critical paths of the recipient,
// which means the redirected edges are inserted into the edge heaps of the relatives using the
// pre-merge values of the recipient. The critical path propagation below then brings all of
// them up to date. This works because the keys in the edge heaps only ever need increasing:
// the inclusive critical path of the merged MTask is at least the inclusive critical path of
// either of the two MTasks it is made of, in both directions.
// Process outgoing edges of donor
while (MTaskEdge* const edgep = static_cast<MTaskEdge*>(donorp->outEdges().frontp())) {
LogicMTask* const relativep = edgep->toMTaskp();
relativep->removeRelativeEdge<GraphWay::REVERSE>(edgep);
if (relativep == recipientp || recipientp->hasEdgeTo(relativep)) {
// This is either the edge connecting the two MTasks, which becomes internal to the
// merged MTask, or is parallel with an existing edge of the recipient. Drop it.
VL_DO_DANGLING(edgep->unlinkDelete(), edgep);
} else {
// No existing edge between recipient and relative of donor.
// Redirect the edge from donor -> relative to recipient -> relative.
edgep->relinkFromp(recipientp);
recipientp->addDependent(relativep);
recipientp->stealRelativeEdge<GraphWay::FORWARD>(edgep);
relativep->addRelativeEdge<GraphWay::REVERSE>(edgep);
}
}
// Process incoming edges of donor
while (MTaskEdge* const edgep = static_cast<MTaskEdge*>(donorp->inEdges().frontp())) {
LogicMTask* const relativep = edgep->fromMTaskp();
relativep->removeDependent(donorp);
relativep->removeRelativeEdge<GraphWay::FORWARD>(edgep);
if (relativep == recipientp || relativep->hasEdgeTo(recipientp)) {
// This is either the edge connecting the two MTasks, which becomes internal to the
// merged MTask, or is parallel with an existing edge of the recipient. Drop it.
VL_DO_DANGLING(edgep->unlinkDelete(), edgep);
} else {
// No existing edge between recipient and relative of donor.
// Redirect the edge from relative -> donor to relative -> recipient.
edgep->relinkTop(recipientp);
relativep->addDependent(recipientp);
relativep->addRelativeEdge<GraphWay::FORWARD>(edgep);
recipientp->stealRelativeEdge<GraphWay::REVERSE>(edgep);
}
}
// Move the contents of the donor into the recipient, update its cost
recipientp->m_mVertices.splice(recipientp->m_mVertices.end(), donorp->m_mVertices);
recipientp->m_cost += donorp->m_cost;
// The recipient now holds all edges of the merged MTask, and the critical paths of all its
// relatives are still up to date, so the critical paths implied by its edges are the critical
// paths of the merged MTask.
const uint64_t newCpFwd = recipientp->cpExclusiveFromEdges<GraphWay::FORWARD>();
const uint64_t newCpRev = recipientp->cpExclusiveFromEdges<GraphWay::REVERSE>();
// Set the new critical paths, then propagate the increases to the relatives. Note this also
// brings the keys of all edges of the merged MTask up to date in the relatives' edge heaps.
recipientp->cpExclusive<GraphWay::FORWARD>(newCpFwd);
propagate<GraphWay::FORWARD>(recipientp);
recipientp->cpExclusive<GraphWay::REVERSE>(newCpRev);
propagate<GraphWay::REVERSE>(recipientp);
// Remove the donor from the graph
VL_DO_DANGLING(donorp->unlinkDelete(this), donorp);
}
// Check the critical paths in the given direction, and the critical paths cached in the edge heaps
// in the opposite direction, against those implied by the edges. Note this deliberately iterates
// the edge lists, rather than consulting the edge heaps, so the heaps are validated, not trusted.
template <GraphWay::en N_Way>
void OrderMTaskGraph::validateWay() const {
constexpr GraphWay way{N_Way};
constexpr GraphWay inv = way.invert();
for (const V3GraphVertex& vtx : vertices()) {
const LogicMTask& mtask = *vtx.as<LogicMTask>();
uint64_t cpCost = 0;
std::unordered_set<const V3GraphVertex*> relatives;
for (const V3GraphEdge& graphEdge : mtask.edges<inv>()) {
const MTaskEdge& edge = *graphEdge.as<MTaskEdge>();
const LogicMTask& relative = *(edge.furtherp<inv>()->template as<LogicMTask>());
// Run a few asserts on the graph, while we are iterating through...
UASSERT_OBJ(edge.weight() != 0, &mtask, "Should be no cut edges in MTask graph");
UASSERT_OBJ(&relative != &mtask, &mtask, "Should be no self edges in MTask graph");
const bool first = relatives.insert(&relative).second;
UASSERT_OBJ(first, &mtask, "Should be no redundant edges in MTask graph");
const uint64_t inclusiveCp = relative.cpInclusive<way>();
// The critical path cached in the edge heap must match that of the relative
UASSERT_OBJ(edge.cachedCp(inv) == inclusiveCp, &mtask,
"Cached critical path does not match the relative");
// As must the ID it is keyed on, which breaks ties between equal critical paths
UASSERT_OBJ(edge.cachedId(inv) == relative.id(), &mtask,
"Cached ID does not match the relative");
cpCost = std::max(cpCost, inclusiveCp);
}
const uint64_t cp = mtask.cpExclusive<way>();
UASSERT_OBJ(cp == cpCost, &mtask, "Critical path does not match the edges");
// The edge heap must yield the same, that is: it must return the largest of its keys
UASSERT_OBJ(mtask.cpExclusiveFromEdges<N_Way>() == cpCost, &mtask,
"Edge heap maximum does not match the edges");
}
}
void OrderMTaskGraph::validate() const {
if (!m_slowAsserts) return;
validateWay<GraphWay::FORWARD>();
validateWay<GraphWay::REVERSE>();
// Check the dependents set of each MTask agrees with its out-edges
for (const V3GraphVertex& vtx : vertices()) {
const LogicMTask& mtask = *vtx.as<LogicMTask>();
size_t nDependents = 0;
for (const V3GraphEdge& graphEdge : mtask.outEdges()) {
LogicMTask* const top = graphEdge.as<MTaskEdge>()->toMTaskp();
UASSERT_OBJ(mtask.hasEdgeTo(top), &mtask, "Dependent missing from the dependents set");
++nDependents;
}
UASSERT_OBJ(mtask.m_dependents.size() == nDependents, &mtask,
"Stale entry in the dependents set");
}
}
//######################################################################
// OrderMTaskGraphBuilder
@@ -103,10 +366,9 @@ class OrderMTaskGraphBuilder final {
}
// Add an edge to the graph, if there is not already an edge between the two vertices.
void addEdge(LogicMTask& src, LogicMTask& dst) {
UASSERT_OBJ(&src != &dst, &src, "Should not create self-edges");
if (src.hasRelativeMTask(&dst)) return; // Don't create redundant edges.
new MTaskEdge{&m_mtaskGraph, &src, &dst, 1};
void addEdge(LogicMTask* srcp, LogicMTask* dstp) {
if (srcp->hasEdgeTo(dstp)) return; // Don't create redundant edges.
m_mtaskGraph.addEdge(srcp, dstp);
}
// CONSTRUCTORS
@@ -145,7 +407,7 @@ class OrderMTaskGraphBuilder final {
// If the opposite end of the edge is not a bypassed vertex, add direct dependency
if (LogicMTask* const otherp = static_cast<LogicMTask*>(top->userp())) {
addEdge(mtask, *otherp);
addEdge(&mtask, otherp);
continue;
}
@@ -155,7 +417,7 @@ class OrderMTaskGraphBuilder final {
// The Move graph is bipartite (logic <-> var), and logic is never
// bypassed, hence 'transp' must be non-nullptr.
UASSERT_OBJ(transp, mVtxp, "This cannot be a bypassed vertex");
addEdge(mtask, *transp);
addEdge(&mtask, transp);
}
}
}
@@ -166,8 +428,8 @@ class OrderMTaskGraphBuilder final {
LogicMTask& mtask = static_cast<LogicMTask&>(vtx);
if (VL_UNLIKELY((&mtask == &entry) || (&mtask == &exit))) continue;
// Add the entry/exit edges if not otherwise connected
if (mtask.inEmpty()) addEdge(entry, mtask);
if (mtask.outEmpty()) addEdge(mtask, exit);
if (mtask.inEmpty()) addEdge(&entry, &mtask);
if (mtask.outEmpty()) addEdge(&mtask, &exit);
}
}
~OrderMTaskGraphBuilder() = default;
@@ -181,5 +443,6 @@ public:
std::unique_ptr<OrderMTaskGraph> OrderMTaskGraph::build(OrderMoveGraph& moveGraph) {
std::unique_ptr<OrderMTaskGraph> resp{new OrderMTaskGraph{moveGraph}};
OrderMTaskGraphBuilder::apply(*resp);
resp->validate();
return resp;
}