From 7dcd4e0b656ba1b4ca632b55f94b421e2f3bafe9 Mon Sep 17 00:00:00 2001 From: Geza Lore Date: Mon, 17 Aug 2026 23:14:52 +0200 Subject: [PATCH] Improve MTask coarsening in multi-threaded scheduling (#8120) This is a large refactor of the MTask graph and coarsening algorithm, in prep for fixing the bug described in #7913, it can also improve the resulting multi-threaded schedule. Two major changes: OrderMTaskGraph now maintains the critical paths of the MTasks through mutation. There are 2 ways to mutate the graph, which are done via methods on the graph itself: adding an edge (used during construction, and will be used later during fixing data hazards), or merging an MTask into another (used during contraction). All critical path measures are automatically updated and propagated on any mutation, so no external algorithm needs to maintain them explicitly. The merge candidate scoreboard used during contraction is simplified to remove deferral of updated scores. This simplifies the code and results in a greedily more optimal schedule. (The previous tranched rescore was an optimization to work around the previous std::set based scoreboard, however since the algorithm now uses an efficient PairingHeap, verilation time is not impacted by the more accurate scoring, while yielding better results). Combining these two into a single patch as the code is highly interdependent and any one change without the other would be just a noisy transit point with unclear performance implications. Together it should be a clear improvement. Also added a stronger validation step run with '--debug-partition', which checks all invariants throughout the algorithms. --- src/CMakeLists.txt | 1 + src/V3Graph.cpp | 25 - src/V3Graph.h | 4 - src/V3OrderMTaskContraction.cpp | 940 +++++++++----------------------- src/V3OrderMTaskFixHazards.cpp | 67 +-- src/V3OrderMTaskGraph.cpp | 313 ++++++++++- src/V3OrderMTaskGraph.h | 536 +++++++----------- src/V3OrderParallel.cpp | 3 + src/V3PairingHeap.h | 4 +- src/V3PoolAllocator.h | 85 +++ 10 files changed, 834 insertions(+), 1144 deletions(-) create mode 100644 src/V3PoolAllocator.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ef11e60ec..e54602de4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -157,6 +157,7 @@ set(HEADERS V3ParseImp.h V3PchAstMT.h V3PchAstNoMT.h + V3PoolAllocator.h V3PreExpr.h V3PreLex.h V3PreProc.h diff --git a/src/V3Graph.cpp b/src/V3Graph.cpp index 2ce4a0f26..96772bf02 100644 --- a/src/V3Graph.cpp +++ b/src/V3Graph.cpp @@ -79,31 +79,6 @@ void V3GraphVertex::rerouteEdges(V3Graph* graphp) { unlinkEdges(graphp); } -template -V3GraphEdge* V3GraphVertex::findConnectingEdgep(V3GraphVertex* waywardp) { - // O(edges) linear search. Searches search both nodes' edge lists in - // parallel. The lists probably aren't _both_ huge, so this is - // unlikely to blow up even on fairly nasty graphs. - constexpr GraphWay way{N_Way}; - constexpr GraphWay inv = way.invert(); - auto& aEdges = this->edges(); - auto aIt = aEdges.begin(); - auto aEnd = aEdges.end(); - auto& bEdges = waywardp->edges(); - auto bIt = bEdges.begin(); - auto bEnd = bEdges.end(); - while (aIt != aEnd && bIt != bEnd) { - V3GraphEdge& aedge = *aIt++; - if (aedge.furtherp() == waywardp) return &aedge; - V3GraphEdge& bedge = *bIt++; - if (bedge.furtherp() == this) return &bedge; - } - return nullptr; -} - -template V3GraphEdge* V3GraphVertex::findConnectingEdgep(V3GraphVertex*); -template V3GraphEdge* V3GraphVertex::findConnectingEdgep(V3GraphVertex*); - // cppcheck-has-bug-suppress constParameter void V3GraphVertex::v3errorEnd(const std::ostringstream& str) const // LCOV_EXCL_START VL_RELEASE(V3Error::s().m_mutex) { diff --git a/src/V3Graph.h b/src/V3Graph.h index a640a3df8..a954d43b1 100644 --- a/src/V3Graph.h +++ b/src/V3Graph.h @@ -311,10 +311,6 @@ public: VL_RELEASE(V3Error::s().m_mutex) VL_MT_DISABLED; /// Edges are routed around this vertex to point from "from" directly to "to" void rerouteEdges(V3Graph* graphp) VL_MT_DISABLED; - // Find the edge connecting this vertex to the given vertex. - // If edge is not found returns nullptr. O(edges) performance. - template - V3GraphEdge* findConnectingEdgep(V3GraphVertex* otherp) VL_MT_DISABLED; }; std::ostream& operator<<(std::ostream& os, V3GraphVertex* vertexp) VL_MT_DISABLED; diff --git a/src/V3OrderMTaskContraction.cpp b/src/V3OrderMTaskContraction.cpp index 6d4ea3c8a..af49acd72 100644 --- a/src/V3OrderMTaskContraction.cpp +++ b/src/V3OrderMTaskContraction.cpp @@ -14,11 +14,9 @@ // //************************************************************************* // -// Coarsens the fine-grained MTask graph produced by the partitioner by -// repeatedly contracting MTasks (merging along an edge, or merging two -// "sibling" MTasks) until a critical-path score limit is reached. Driven by -// the partitioner in V3OrderParallel.cpp via OrderMTaskGraph::contract, -// declared in V3OrderMTaskGraph.h. +// Coarsens the fine-grained MTask graph by repeatedly contracting MTasks, +// merging along an edge, or merging two "sibling" MTasks until a +// critical-path limit is reached. // //************************************************************************* @@ -26,9 +24,9 @@ #include "V3Global.h" #include "V3Graph.h" -#include "V3GraphStream.h" #include "V3OrderMTaskGraph.h" #include "V3PairingHeap.h" +#include "V3PoolAllocator.h" #include #include @@ -44,66 +42,38 @@ class MergeCandidate; class SiblingMC; class EdgeMC; -// ###################################################################### -// Partitioner tunable settings: -// -// Before describing these settings, a bit of background: -// -// Early during the development of the partitioner, V3Split was failing to -// split large always blocks (with ~100K assignments) so we had to handle -// very large vertices with ~100K incoming and outgoing edges. -// -// The partitioner attempts to deal with such densely connected -// graphs. Some of the tuning parameters below reference "huge vertices", -// that's what they're talking about, vertices with tens of thousands of -// edges in and out. Whereas most graphs have only tens of edges in and out -// of most vertices. -// -// V3Split has since been fixed to more reliably split large always -// blocks. It's kind of an open question whether the partitioner must -// handle huge nodes gracefully. Maybe not! But it still can, given -// appropriate tuning. +//###################################################################### +// Tunable settings -// PART_SIBLING_EDGE_LIMIT (integer) -// // Arbitrarily limit the number of edges on a single vertex that will be // considered when enumerating siblings, to the given value. This protects -// the partitioner runtime in the presence of huge vertices. +// the runtime in the presence of huge vertices. // -// The sibling-merge is less important than the edge merge. (You can -// totally disable the sibling merge and get halfway decent partitions; you -// can't disable edge merges, those are fundamental to the process.) So, -// skipping the enumeration of some siblings on a few vertices does not -// have a large impact on the result of the partitioner. +// The sibling-merge is less important than the edge merge. Sibling merges +// can be disabled and result in halfway decent coarsening. Edge merges +// cannot be disabled, those are fundamental to the process. So, skipping +// the enumeration of some siblings on a few vertices does not have a large +// impact on the result of the partitioner. // -// If your vertices are small, the limit (at 26) approaches a no-op. Hence -// there's basically no cost to applying this limit even when we don't -// expect huge vertices. +// If vertices are small, the limit (at 26) approaches a no-op. Hence +// there's basically no cost to applying this limit even when no huge +// vertices are expected. // -// If you don't care about partitioner runtime and you want the most -// aggressive partition, set the limit very high. If you have huge -// vertices, leave this as is. +// If runtime is not a concern and the most precise result is desired, +// set the limit very high. constexpr unsigned PART_SIBLING_EDGE_LIMIT = 26; -// Don't produce more than a certain maximum number of MTasks. This helps -// the TSP variable sort not to blow up (a concern for some of the tests) -// and we probably don't want a huge number of MTasks in practice anyway -// (50 to 100 is typical.) -// // If the user doesn't give one with '--threads-max-mtasks', we'll set the -// maximum # of MTasks to -// (# of threads * PART_DEFAULT_MAX_MTASKS_PER_THREAD) +// maximum # of MTasks to (# of threads * PART_DEFAULT_MAX_MTASKS_PER_THREAD) constexpr unsigned PART_DEFAULT_MAX_MTASKS_PER_THREAD = 50; -// end tunables. - //###################################################################### // MTask utility classes struct MergeCandidateKey final { // Note: Structure layout chosen to minimize padding in PairingHeap<*>::Node - uint64_t m_id; // Unique ID part of edge score - uint64_t m_score; // Score part of ID + uint64_t m_id; // Unique ID part of the key + uint64_t m_score; // Score part of the key bool operator<(const MergeCandidateKey& other) const { // First by Score then by ID, but notice that we want minimums using a max-heap, so reverse return m_score > other.m_score || (m_score == other.m_score && m_id > other.m_id); @@ -120,11 +90,10 @@ class MergeCandidate VL_NOT_FINAL : public MergeCandidateHeapNode { friend class SiblingMC; friend class EdgeMC; - // This structure is extremely hot. To save 8 bytes we pack - // one bit indicating removedFromSb with the id. To save another - // 8 bytes by not having a virtual function table, we implement the - // few polymorphic methods over the two known subclasses explicitly, - // using another bit of the id to denote the actual subtype. + // This structure is extremely hot. To save 8 bytes by not having a virtual + // function table, we implement the few polymorphic methods over the two + // known subclasses explicitly, using a bit of the id to denote the actual + // subtype. // By using the bottom bits for flags, we can still use < to compare IDs without masking. // <63:1> Serial number for ordering, <0> subtype (SiblingMC) @@ -145,10 +114,14 @@ public: // METHODS SiblingMC* toSiblingMC(); // Instead of cast<>/as<> EdgeMC* toEdgeMC(); // Instead of cast<>/as<> - bool mergeWouldCreateCycle() const; // Instead of virtual method + bool mergeWouldCreateCycle(OrderMTaskGraph& graph); - inline void rescore(); + // The current score of this candidate, which changes as the graph is contracted + inline uint64_t currentScore(); + // The score this candidate was last given, which is its key in the scoreboard heap uint64_t score() const { return m_key.m_score; } + // Set the score of this candidate to its current value. Only valid while it is not in a heap. + void updateScore() { m_key.m_score = currentScore(); } static MergeCandidate* heapNodeToElem(MergeCandidateHeapNode* nodep) { return static_cast(nodep); @@ -185,7 +158,6 @@ public: LogicMTask* ap() const { return m_ap; } LogicMTask* bp() const { return m_bp; } - bool mergeWouldCreateCycle() const; }; static_assert(!std::is_polymorphic::value, "Should not have a vtable"); @@ -203,7 +175,6 @@ public: // METHODS MTaskEdge* edgep() const { return m_edgep; } - bool mergeWouldCreateCycle() const; }; static_assert(!std::is_polymorphic::value, "Should not have a vtable"); @@ -236,44 +207,46 @@ SiblingMC* MergeCandidate::toSiblingMC() { EdgeMC* MergeCandidate::toEdgeMC() { return isSiblingMC() ? nullptr : static_cast(this); } -// Normally this would be a virtual function, but we save space by not having a vtable, -// and we know we only have 2 possible subclasses. -bool MergeCandidate::mergeWouldCreateCycle() const { - return isSiblingMC() ? static_cast(this)->mergeWouldCreateCycle() - : static_cast(this)->mergeWouldCreateCycle(); -} - -static uint64_t siblingScore(const SiblingMC* sibsp) { - const LogicMTask* const ap = sibsp->ap(); - const LogicMTask* const bp = sibsp->bp(); - const uint64_t mergedCpCostFwd - = std::max(ap->critPathCost(GraphWay::FORWARD), bp->critPathCost(GraphWay::FORWARD)); - const uint64_t mergedCpCostRev - = std::max(ap->critPathCost(GraphWay::REVERSE), bp->critPathCost(GraphWay::REVERSE)); - return mergedCpCostRev + mergedCpCostFwd + ap->cost() + bp->cost(); -} - -static uint64_t edgeScore(const MTaskEdge* edgep) { - // Score this edge. Lower is better. The score is the new local CP - // length if we merge these MTasks. ("Local" means the longest - // critical path running through the merged node.) - const LogicMTask* const top = edgep->toMTaskp(); - const LogicMTask* const fromp = edgep->fromMTaskp(); - const uint64_t mergedCpCostFwd = std::max(fromp->critPathCost(GraphWay::FORWARD), - top->critPathCostWithout(edgep)); - const uint64_t mergedCpCostRev = std::max(fromp->critPathCostWithout(edgep), - top->critPathCost(GraphWay::REVERSE)); - return mergedCpCostRev + mergedCpCostFwd + fromp->cost() + top->cost(); -} - -void MergeCandidate::rescore() { +bool MergeCandidate::mergeWouldCreateCycle(OrderMTaskGraph& graph) { + // Sibling merge: merging creates a cycle if either sibling is reachable from the other if (const SiblingMC* const sibp = toSiblingMC()) { - m_key.m_score = siblingScore(sibp); - } else { + return graph.pathExists(sibp->ap(), sibp->bp(), nullptr) + || graph.pathExists(sibp->bp(), sibp->ap(), nullptr); + } + + // Edge merge: merging creates a cycle if there is another path between the two MTasks + MTaskEdge* const edgep = toEdgeMC()->edgep(); + return graph.pathExists(edgep->fromMTaskp(), edgep->toMTaskp(), edgep); +} + +uint64_t MergeCandidate::currentScore() { + // Score this candidate. The score is the new local CP length if we merge this candidate. + // ("Local" means the longest critical path running through the merged node.) + + // Sibling merge + if (const SiblingMC* const sibp = toSiblingMC()) { + const LogicMTask* const ap = sibp->ap(); + const LogicMTask* const bp = sibp->bp(); + const uint64_t mergedCpFwd + = std::max(ap->cpExclusive(), bp->cpExclusive()); + const uint64_t mergedCpRev + = std::max(ap->cpExclusive(), bp->cpExclusive()); + return mergedCpRev + mergedCpFwd + ap->cost() + bp->cost(); + } + + // Edge merge + { + MTaskEdge* const edgep = toEdgeMC()->edgep(); + const LogicMTask* const fromp = edgep->fromMTaskp(); + const LogicMTask* const top = edgep->toMTaskp(); + const uint64_t mergedCpFwd = std::max(fromp->cpExclusive(), + top->cpExclusiveWithout(edgep)); + const uint64_t mergedCpRev = std::max(fromp->cpExclusiveWithout(edgep), + top->cpExclusive()); // Give a slight preference to sibling merges by increasing the cost of edge merges. // This biases towards sibling merges in case they are equal score with edge merges. // This avoid a central node growing while many leaves remain due to edge merges. - m_key.m_score = 1 + edgeScore(static_cast(this)->edgep()); + return 1 + mergedCpRev + mergedCpFwd + fromp->cost() + top->cost(); } } @@ -296,59 +269,43 @@ void SiblingMC::unlinkA() { void SiblingMC::unlinkB() { mtaskData(m_bp).bSiblingMCs.unlink(this); } -// cppcheck-suppress duplInheritedMember -bool SiblingMC::mergeWouldCreateCycle() const { - return (LogicMTask::pathExistsFrom(m_ap, m_bp, nullptr) - || LogicMTask::pathExistsFrom(m_bp, m_ap, nullptr)); -} - -// cppcheck-suppress duplInheritedMember -bool EdgeMC::mergeWouldCreateCycle() const { - return LogicMTask::pathExistsFrom(m_edgep->fromMTaskp(), m_edgep->toMTaskp(), m_edgep); -} - -// Scoreboard of MTask merge candidates. Owns the lifetime of the merge candidate objects: callers -// add/remove candidates via the methods below and never allocate or free them directly. For edges -// this maintains the invariant that an MTaskEdge has an associated EdgeMC (held in its userp()), -// if and only if it is currently on the scoreboard. +// Scoreboard of MTask merge candidates. // -// This is essentially a heap that can be hinted that some elements have changed keys, at which -// point those elements are deferred as 'unknown' until the next 'rescore' call. We use the -// generic PairingHeap, relying on its internal structure. For efficiency, the merge candidates are -// themselves the heap nodes (MergeCandidate derives from PairingHeap::Node), so -// a candidate can be on at most one scoreboard. +// This is a heap, sorted by the local critical path that would result from merging the candidate, +// that is: the longest critical path running through the merged MTask. Merges proceed by picking +// the candidate yielding the lowest such critical path, which is the merge that does the least +// damage: a merge can only ever lengthen paths, and one whose local critical path is no longer +// than the current global critical path cannot lengthen that at all. +// +// A candidate's score changes as the graph is contracted, so the score a candidate is in the heap +// with is only the score it was last given, which is a lower bound on its current score. This +// makes the top of the heap a lower bound on the score of every candidate, so the caller can pick +// the best candidate by checking whether the top still has the score it was given, and +// rescoring it if not (see the contraction loop). +// +// The scoreboard owns the lifetime of the merge candidate objects: callers add/remove candidates +// via the methods below and never allocate or free them directly. For edges this maintains the +// invariant that an MTaskEdge has an associated EdgeMC (held in its userp()), if and only if it is +// currently on the scoreboard. class MergeCandidateScoreboard final { // TYPES using Heap = PairingHeap; - using Node = Heap::Node; - using Link = Heap::Link; // MEMBERS - Heap m_known; // The heap of candidates with known scores - Link m_unknown; // List of candidates with unknown scores + Heap m_heap; // The heap of candidates, keyed on their score (see class comment above) + PoolAllocator m_edgeMCPool; // Allocator for the edge merge candidates + PoolAllocator m_siblingMCPool; // Allocator for the sibling merge candidates // METHODS - void addUnknown(MergeCandidate* nodep) { - // Just prepend it to the list of unknown entries - nodep->m_next.link(m_unknown.unlink()); - m_unknown.linkNonNull(nodep); - // We mark nodes on the unknown list by making their child pointer point to themselves - nodep->m_kids.m_ptr = nodep; + // Set the score of a candidate and add it to the heap. The score is computed from the critical + // paths of the MTasks, which are up to date while the graph is being contracted. + void insert(MergeCandidate* nodep) { + nodep->updateScore(); + m_heap.insert(nodep); } - // Add a freshly created candidate. Not returned by 'best' before the next 'rescore' call. - void add(MergeCandidate* nodep) { addUnknown(nodep); } - // Remove a candidate from the scoreboard. - void remove(MergeCandidate* nodep) { - if (nodep->m_kids.m_ptr == nodep) { - // Node is on the unknown list, replace with next - nodep->replaceWith(nodep->m_next.unlink()); - return; - } - // Node is in the known heap, remove it - m_known.remove(nodep); - } + void remove(MergeCandidate* nodep) { m_heap.remove(nodep); } public: // CONSTRUCTORS @@ -356,475 +313,94 @@ public: ~MergeCandidateScoreboard() = default; VL_UNCOPYABLE(MergeCandidateScoreboard); - // The candidate with the best (lowest) known score, or nullptr if none have a known score. - // This does not automatically 'rescore'; the caller must 'rescore' to reflect all candidates. - MergeCandidate* best() const { return MergeCandidate::heapNodeToElem(m_known.max()); } + // The candidate with the lowest score it was given, or nullptr if the scoreboard is empty. + // Note this is only a lower bound on the best current score, see the class comment above. + MergeCandidate* best() const { return MergeCandidate::heapNodeToElem(m_heap.max()); } - // Tell the scoreboard a candidate's score may have changed. Its score becomes 'unknown' and it - // will not be returned by 'best' until the next 'rescore'. - void hintScoreChanged(MergeCandidate* nodep) { - // If it's already in the unknown list, then nothing to do - if (nodep->m_kids.m_ptr == nodep) return; - // Otherwise it was in the heap, remove it - m_known.remove(nodep); - // Prepend it to the unknown list - addUnknown(nodep); + // Update the score of a candidate to its current value, and reposition it in the heap + void rescore(MergeCandidate* nodep) { + remove(nodep); + insert(nodep); } - // True if there are candidates with an unknown score - bool needsRescore() const { return m_unknown; } - // True if the given candidate's score is unknown - static bool needsRescore(const MergeCandidate* nodep) { return nodep->m_kids.m_ptr == nodep; } - - // For each candidate whose score is unknown, recompute the score and add to the known heap - void rescore() { - for (Node *nodep = m_unknown.unlink(), *nextp; nodep; nodep = nextp) { - // Pick up next - nextp = nodep->m_next.ptr(); - // Reset pointers - nodep->m_next.m_ptr = nullptr; - nodep->m_kids.m_ptr = nullptr; - nodep->m_ownerpp = nullptr; - // Re-compute the score of the candidate - MergeCandidate::heapNodeToElem(nodep)->rescore(); - // Re-insert into the heap - m_known.insert(nodep); - } - } - - // Create the merge candidate for 'edgep' and add it to the scoreboard (out-of-line below) - void addEdge(MTaskEdge* edgep) { + // Create a merge candidate for 'edgep' and add it to the scoreboard + void addEdgeMC(MTaskEdge* edgep) { UDEBUGONLY(UASSERT(!edgep->userp(), "Edge already has a merge candidate");); - EdgeMC* const edgeMCp = new EdgeMC{edgep}; + EdgeMC* const edgeMCp = m_edgeMCPool.alloc(edgep); edgep->userp(edgeMCp); - add(edgeMCp); + insert(edgeMCp); } - // Remove 'edgep's merge candidate from the scoreboard and delete it (out-of-line below) - void removeEdge(MTaskEdge* edgep) { + // Remove 'edgep's merge candidate from the scoreboard and release it + void removeEdgeMC(MTaskEdge* edgep) { EdgeMC* const edgeMCp = edgeMC(edgep); UDEBUGONLY(UASSERT(edgeMCp, "Edge has no merge candidate");); edgep->userp(nullptr); remove(edgeMCp); - VL_DO_DANGLING(delete edgeMCp, edgeMCp); + VL_DO_DANGLING(m_edgeMCPool.free(edgeMCp), edgeMCp); } // Create a sibling merge candidate for 'ap' and 'bp' and add it to the scoreboard - void addSibling(LogicMTask* ap, LogicMTask* bp) { add(new SiblingMC{ap, bp}); } - // Remove sibling merge candidate 'smcp' from the scoreboard and delete it - void removeSibling(SiblingMC* smcp) { + void addSiblingMC(LogicMTask* ap, LogicMTask* bp) { insert(m_siblingMCPool.alloc(ap, bp)); } + // Remove sibling merge candidate 'smcp' from the scoreboard and release it + void removeSiblingMC(SiblingMC* smcp) { remove(smcp); smcp->unlinkA(); smcp->unlinkB(); - VL_DO_DANGLING(delete smcp, smcp); + VL_DO_DANGLING(m_siblingMCPool.free(smcp), smcp); + } + + // Remove the given merge candidate, whichever kind it is, from the scoreboard and release it + void removeMC(MergeCandidate* nodep) { + if (SiblingMC* const smcp = nodep->toSiblingMC()) { + removeSiblingMC(smcp); + } else { + removeEdgeMC(nodep->toEdgeMC()->edgep()); + } } }; //###################################################################### - -// Look at vertex costs (in one way) to form critical paths for each -// vertex. -template -static void partInitHalfCriticalPaths(V3Graph& mTaskGraph, bool checkOnly) { - constexpr GraphWay way{N_Way}; - constexpr GraphWay rev = way.invert(); - GraphStreamUnordered order{&mTaskGraph, way}; - for (const V3GraphVertex* vertexp; (vertexp = order.nextp());) { - const LogicMTask* const mtaskcp = static_cast(vertexp); - LogicMTask* const mtaskp = const_cast(mtaskcp); - uint64_t cpCost = 0; -#if VL_DEBUG - std::unordered_set relatives; -#endif - for (const V3GraphEdge& edge : vertexp->edges()) { -#if VL_DEBUG - // Run a few asserts on the initial mtask graph, - // while we're iterating through... - UASSERT_OBJ(edge.weight() != 0, mtaskp, "Should be no cut edges in MTask graph"); - UASSERT_OBJ(relatives.find(edge.furtherp()) == relatives.end(), mtaskp, - "Should be no redundant edges in MTask graph"); - relatives.insert(edge.furtherp()); -#endif - const LogicMTask* const relativep = static_cast(edge.furtherp()); - cpCost = std::max(cpCost, (relativep->critPathCost(way) + relativep->cost())); - } - if (checkOnly) { - UASSERT(mtaskp->critPathCost(way) == cpCost, "Calculation error in scoring"); - } else { - mtaskp->setCritPathCost(way, cpCost); - } - } -} - -// Look at vertex costs to form critical paths for each vertex. -static void partInitCriticalPaths(V3Graph& mTaskGraph) { - partInitHalfCriticalPaths(mTaskGraph, false); - partInitHalfCriticalPaths(mTaskGraph, false); - - // Reset all MTaskEdges so that 'm_edges' will show correct CP numbers. - // They would have been all zeroes on initial creation of the MTaskEdges. - for (V3GraphVertex& vtx : mTaskGraph.vertices()) { - for (V3GraphEdge& edge : vtx.outEdges()) edge.as()->resetCriticalPaths(); - } -} - -// Do an EXPENSIVE check to make sure that all incremental CP updates have -// gone correctly. -static void partCheckCriticalPaths(V3Graph& mTaskGraph) { - partInitHalfCriticalPaths(mTaskGraph, true); - partInitHalfCriticalPaths(mTaskGraph, true); - for (const V3GraphVertex& vtx : mTaskGraph.vertices()) { - const LogicMTask& mtask = static_cast(vtx); - mtask.checkRelativesCp(); - mtask.checkRelativesCp(); - } -} - -//###################################################################### -// Contraction - -// Perform edge or sibling contraction on the partition graph +// Contraction - Perform greedy edge or sibling merges on the MTask graph class Contraction final { - // TYPES - // New CP information for mtaskp reflecting an upcoming merge - struct NewCp final { - uint64_t cp; - uint64_t propagateCp; - bool propagate; - }; - // MEMBERS OrderMTaskGraph& m_mTaskGraph; // The Mtask graph - uint64_t m_scoreLimit; // Sloppy score allowed when picking merges - // Next score rescore at - uint64_t m_scoreLimitBeforeRescore = std::numeric_limits::max(); - unsigned m_mergesSinceRescore = 0; // Merges since last rescore - const bool m_slowAsserts{v3Global.opt.debugPartition()}; // Take extra time to validate steps + uint64_t m_scoreLimit; // Critical path limit for merges MergeCandidateScoreboard m_sb; // Scoreboard // Auxiliary per-MTask data (the SiblingMC lists) attached to each MTask via its user pointer. - // Owned here for the lifetime of this Contraction. A single array, as the number of MTasks is - // fixed for that lifetime: merging only ever deletes vertices, never creates them. + // Owned here for the lifetime of this Contraction. A single array, as the number of MTasks + // can only decrease during contraction. std::unique_ptr m_mtaskDatap; - // Singular source vertex of the OrderMTaskGraph - LogicMTask* const m_entryMTaskp = m_mTaskGraph.entryp(); - // Singular sink vertex of the dependency graph - LogicMTask* const m_exitMTaskp = m_mTaskGraph.exitp(); - - // Merge edges from a LogicMtask, keeping the merge candidate scoreboard in sync. - static void partRedirectEdgesFrom(V3Graph& graph, LogicMTask* recipientp, LogicMTask* donorp, - MergeCandidateScoreboard& sb) { - // This code removes adjacent edges. When this occurs, mark it in need - // of a rescore, in case its score has fallen and we need to move it up - // toward the front of the scoreboard. - // - // Wait, what? Shouldn't the scores only increase as we merge nodes? Well - // that's almost true. But there is one exception. - // - // Suppose we have A->B, B->C, and A->C. - // - // The A->C edge is a "transitive" edge. It's ineligible to be merged, as - // the merge would create a cycle. We score it on the scoreboard like any - // other edge. - // - // However, our "score" estimate for A->C is bogus, because the forward - // critical path to C and the reverse critical path to A both contain the - // same node (B) so we overestimate the score of A->C. At first this - // doesn't matter, since transitive edges aren't eligible to merge anyway. - // - // Later, suppose the edge contractor decides to merge the B->C edge, with - // B donating all its incoming edges into C, say. (So we reach this - // function.) - // - // With B going away, the A->C edge will no longer be transitive and it - // will become eligible to merge. But if we don't mark it for rescore, - // it'll stay in the scoreboard with its old (overestimate) score. We'll - // merge it too late due to the bogus score. When we finally merge it, we - // fail the assert in the main edge contraction loop which checks that the - // actual score did not fall below the scoreboard's score. - // - // Another way of stating this: this code ensures that scores of - // non-transitive edges only ever increase. - - // Process outgoing edges - while (MTaskEdge* const edgep = static_cast(donorp->outEdges().frontp())) { - LogicMTask* const relativep = edgep->toMTaskp(); - - relativep->removeRelativeEdge(edgep); - - if (recipientp->hasRelativeMTask(relativep)) { - // An edge already exists between recipient and relative of donor. - // Mark it in need of a rescore - // The donor edge is going away, so remove it from the scoreboard - if (edgep->userp()) sb.removeEdge(edgep); - MTaskEdge* const existMTaskEdgep = static_cast( - recipientp->findConnectingEdgep(relativep)); - UDEBUGONLY(UASSERT(existMTaskEdgep, "findConnectingEdge didn't find edge");); - // The existing edge is no longer transitive, so may need a rescore - if (EdgeMC* const existEdgeMCp = edgeMC(existMTaskEdgep)) { - sb.hintScoreChanged(existEdgeMCp); - } - VL_DO_DANGLING(edgep->unlinkDelete(), edgep); - } else { - // No existing edge between recipient and relative of donor. - // Redirect the edge from donor<->relative to recipient<->relative. - edgep->relinkFromp(recipientp); - recipientp->addRelativeMTask(relativep); - recipientp->stealRelativeEdge(edgep); - relativep->addRelativeEdge(edgep); - // The redirected edge is a merge candidate again - if (EdgeMC* const edgeMCp = edgeMC(edgep)) { - sb.hintScoreChanged(edgeMCp); - } else { - sb.addEdge(edgep); - } - } - } - - // Process incoming edges - while (MTaskEdge* const edgep = static_cast(donorp->inEdges().frontp())) { - LogicMTask* const relativep = edgep->fromMTaskp(); - - relativep->removeRelativeMTask(donorp); - relativep->removeRelativeEdge(edgep); - - if (relativep->hasRelativeMTask(recipientp)) { - // An edge already exists between recipient and relative of donor. - // Mark it in need of a rescore - // The donor edge is going away, so remove it from the scoreboard - if (edgep->userp()) sb.removeEdge(edgep); - MTaskEdge* const existMTaskEdgep = static_cast( - recipientp->findConnectingEdgep(relativep)); - UDEBUGONLY(UASSERT(existMTaskEdgep, "findConnectingEdge didn't find edge");); - // The existing edge is no longer transitive, so may need a rescore - if (EdgeMC* const existEdgeMCp = edgeMC(existMTaskEdgep)) { - sb.hintScoreChanged(existEdgeMCp); - } - VL_DO_DANGLING(edgep->unlinkDelete(), edgep); - } else { - // No existing edge between recipient and relative of donor. - // Redirect the edge from donor<->relative to recipient<->relative. - edgep->relinkTop(recipientp); - relativep->addRelativeMTask(recipientp); - relativep->addRelativeEdge(edgep); - recipientp->stealRelativeEdge(edgep); - // The redirected edge is a merge candidate again - if (EdgeMC* const edgeMCp = edgeMC(edgep)) { - sb.hintScoreChanged(edgeMCp); - } else { - sb.addEdge(edgep); - } - } - } - - // Remove donorp from the graph - VL_DO_DANGLING(donorp->unlinkDelete(&graph), donorp); + // Add merge candidates for all edges of 'mtaskp' to the scoreboard + void addEdgeMCs(LogicMTask* mtaskp) { + for (V3GraphEdge& e : mtaskp->outEdges()) m_sb.addEdgeMC(static_cast(&e)); + for (V3GraphEdge& e : mtaskp->inEdges()) m_sb.addEdgeMC(static_cast(&e)); } - template - NewCp newCp(const LogicMTask* mtaskp, const LogicMTask* otherp, const MTaskEdge* mergeEdgep) { - constexpr GraphWay way{N_Way}; - // Return new wayward-CP for mtaskp reflecting its upcoming merge - // with otherp. Set 'result.propagate' if mtaskp's wayward - // relatives will see a new wayward CP from this merge. - uint64_t newCp; - if (mergeEdgep) { - if (mtaskp == mergeEdgep->furtherp()) { - newCp = std::max(otherp->critPathCost(way), - mtaskp->critPathCostWithout(mergeEdgep)); - } else { - newCp = std::max(mtaskp->critPathCost(way), - otherp->critPathCostWithout(mergeEdgep)); - } - } else { - newCp = std::max(otherp->critPathCost(way), mtaskp->critPathCost(way)); + // Remove the merge candidates of all edges of 'mtaskp' from the scoreboard. + void removeEdgeMCs(LogicMTask* mtaskp) { + // Note not all edges have a merge candidate: + // those rejected by the main contraction loop had theirs removed there. + for (V3GraphEdge& edge : mtaskp->outEdges()) { + MTaskEdge* const edgep = static_cast(&edge); + if (edgeMC(edgep)) m_sb.removeEdgeMC(edgep); } - - const uint64_t oldRelativesCp = mtaskp->critPathCost(way) + mtaskp->cost(); - const uint64_t newRelativesCp = newCp + mtaskp->cost() + otherp->cost(); - - NewCp result; - result.cp = newCp; - result.propagate = (newRelativesCp > oldRelativesCp); - result.propagateCp = newRelativesCp; - return result; - } - - void removeSiblingMCsWith(LogicMTask* mtaskp) { - // Note: 'removeSibling' unlinks the candidate from both of its MTasks' lists, so taking - // the front element repeatedly does terminate. It also erases the candidate from the - // owning (higher id) MTask's sibling set as it goes, so both the sets and the lists are - // left consistent, whichever side of the candidate 'mtaskp' happens to be on. - while (SiblingMC* const smcp = mtaskData(mtaskp).aSiblingMCs.frontp()) { - m_sb.removeSibling(smcp); + for (V3GraphEdge& edge : mtaskp->inEdges()) { + MTaskEdge* const edgep = static_cast(&edge); + if (edgeMC(edgep)) m_sb.removeEdgeMC(edgep); } - while (SiblingMC* const smcp = mtaskData(mtaskp).bSiblingMCs.frontp()) { - m_sb.removeSibling(smcp); - } - } - - void removeSiblingMCs(LogicMTask* recipientp, LogicMTask* donorp) { - // These two can share a SiblingMC (an edge between them does not preclude one). That is - // fine: 'removeSiblingMCsWith' unlinks each candidate from both sides, so the shared one - // is gone by the time we get to the donor. - // - // This also leaves both sibling sets empty, so they need no separate clearing: each entry - // in an MTask's sibling set is added by 'makeSiblingMC' together with a SiblingMC on that - // same MTask's 'aSiblingMCs' list, and draining that list erases the matching entry (see - // 'SiblingMC::unlinkA'). The slow assert in 'makeSiblingMC' catches it if that ever - // diverges, as a stale set entry there suppresses creating the SiblingMC it stands for. - removeSiblingMCsWith(recipientp); - removeSiblingMCsWith(donorp); - } - - void contract(MergeCandidate* mergeCanp) { - LogicMTask* top = nullptr; - LogicMTask* fromp = nullptr; - EdgeMC* const mergeEdgeMCp = mergeCanp->toEdgeMC(); - MTaskEdge* const mergeEdgep = mergeEdgeMCp ? mergeEdgeMCp->edgep() : nullptr; - SiblingMC* const mergeSibsp = mergeCanp->toSiblingMC(); - if (mergeEdgep) { - top = mergeEdgep->toMTaskp(); - fromp = mergeEdgep->fromMTaskp(); - } else { - top = mergeSibsp->ap(); - fromp = mergeSibsp->bp(); - } - - // Merge the smaller mtask into the larger mtask. If one of them - // is much larger, this will save time in partRedirectEdgesFrom(). - // Assume the more costly mtask has more edges. - // - // [TODO: now that we have edge maps, we could count the edges - // exactly without a linear search.] - LogicMTask* recipientp; - LogicMTask* donorp; - if (fromp->cost() > top->cost()) { - recipientp = fromp; - donorp = top; - } else { - donorp = fromp; - recipientp = top; - } - VL_DANGLING(fromp); - VL_DANGLING(top); // Use donorp and recipientp now instead - - // Recursively update forward and reverse CP numbers. - // - // Doing this before merging the MTasks lets us often avoid - // recursing through either incoming or outgoing edges on one or - // both MTasks. - // - // These 'NewCp' objects carry a bit indicating whether we must - // propagate CP for each of the four cases: - const NewCp recipientNewCpFwd = newCp(recipientp, donorp, mergeEdgep); - const NewCp donorNewCpFwd = newCp(donorp, recipientp, mergeEdgep); - const NewCp recipientNewCpRev = newCp(recipientp, donorp, mergeEdgep); - const NewCp donorNewCpRev = newCp(donorp, recipientp, mergeEdgep); - - if (mergeEdgep) { - // Remove and free the connecting edge. Must do this before propagating CP's below. - m_sb.removeEdge(mergeEdgep); - mergeEdgep->fromMTaskp()->removeRelativeMTask(mergeEdgep->toMTaskp()); - mergeEdgep->fromMTaskp()->removeRelativeEdge(mergeEdgep); - mergeEdgep->toMTaskp()->removeRelativeEdge(mergeEdgep); - VL_DO_DANGLING(mergeEdgep->unlinkDelete(), mergeEdgep); - } else { - // Remove the siblingMC - m_sb.removeSibling(mergeSibsp); - } - - // This also updates cost on recipientp - recipientp->moveAllVerticesFrom(donorp); - - UINFO(9, "recipient = " << recipientp->id() << ", donor = " << donorp->id() - << ", mergeEdgep = " << mergeEdgep << "\n" - << "recipientNewCpFwd = " << recipientNewCpFwd.cp - << (recipientNewCpFwd.propagate ? " true " : " false ") - << recipientNewCpFwd.propagateCp << "\n" - << "donorNewCpFwd = " << donorNewCpFwd.cp - << (donorNewCpFwd.propagate ? " true " : " false ") - << donorNewCpFwd.propagateCp); - - recipientp->setCritPathCost(GraphWay::FORWARD, recipientNewCpFwd.cp); - if (recipientNewCpFwd.propagate) { - m_mTaskGraph.forwardPropagator().cpHasIncreased(recipientp, - recipientNewCpFwd.propagateCp); - } - recipientp->setCritPathCost(GraphWay::REVERSE, recipientNewCpRev.cp); - if (recipientNewCpRev.propagate) { - m_mTaskGraph.reversePropagator().cpHasIncreased(recipientp, - recipientNewCpRev.propagateCp); - } - if (donorNewCpFwd.propagate) { - m_mTaskGraph.forwardPropagator().cpHasIncreased(donorp, donorNewCpFwd.propagateCp); - } - if (donorNewCpRev.propagate) { - m_mTaskGraph.reversePropagator().cpHasIncreased(donorp, donorNewCpRev.propagateCp); - } - m_mTaskGraph.forwardPropagator().go(); - m_mTaskGraph.reversePropagator().go(); - - // Remove all other SiblingMCs that include recipientp or donorp. We remove all siblingMCs - // of recipientp so we do not get huge numbers of SiblingMCs. We'll recreate them below, up - // to a bounded number. - removeSiblingMCs(recipientp, donorp); - - // Redirect all edges, delete donorp - partRedirectEdgesFrom(m_mTaskGraph, recipientp, donorp, m_sb); - - ++m_mergesSinceRescore; - - // Do an expensive check, confirm we haven't botched the CP - // updates. - if (m_slowAsserts) partCheckCriticalPaths(m_mTaskGraph); - - // Finally, make new sibling pairs as needed: - // - prereqs and postreqs of recipientp - // - prereqs of recipientp's postreqs - // - postreqs of recipientp's prereqs - // Note that this depends on the updated critical paths (above). - siblingPairFromRelatives(recipientp); - siblingPairFromRelatives(recipientp); - unsigned edges = 0; - for (V3GraphEdge& edge : recipientp->outEdges()) { - LogicMTask* const postreqp = static_cast(edge.top()); - siblingPairFromRelatives(postreqp); - ++edges; - if (edges >= PART_SIBLING_EDGE_LIMIT) break; - } - edges = 0; - for (V3GraphEdge& edge : recipientp->inEdges()) { - LogicMTask* const prereqp = static_cast(edge.fromp()); - siblingPairFromRelatives(prereqp); - ++edges; - if (edges >= PART_SIBLING_EDGE_LIMIT) break; - } - } - - void doRescore() { - // During rescore, we know that graph isn't changing, so allow - // the critPathCost*Without() routines to cache some data in - // each LogicMTask. This is just an optimization, things should - // behave identically without the caching (just slower) - - m_sb.rescore(); - UINFO(6, "Did rescore. Merges since previous = " << m_mergesSinceRescore); - - m_mergesSinceRescore = 0; - m_scoreLimitBeforeRescore - = std::numeric_limits::max(); } void makeSiblingMC(LogicMTask* ap, LogicMTask* bp) { if (ap->id() < bp->id()) std::swap(ap, bp); // The higher id vertex owns the association set - const auto first = mtaskData(ap).siblings.insert(bp).second; + const bool first = mtaskData(ap).siblings.insert(bp).second; if (first) { - m_sb.addSibling(ap, bp); + m_sb.addSiblingMC(ap, bp); return; } - if (VL_UNLIKELY(m_slowAsserts)) { + if (VL_UNLIKELY(m_mTaskGraph.slowAsserts())) { // It's fine if we already have this SiblingMC, we may have // created it earlier. Just confirm that we have associated data. bool found = false; @@ -837,7 +413,7 @@ class Contraction final { } template - void siblingPairFromRelatives(V3GraphVertex* mtaskp) { + void addSiblingMCsFromRelatives(LogicMTask* mtaskp) { constexpr GraphWay way{N_Way}; // Need at least 2 edges auto& edges = mtaskp->edges(); @@ -872,7 +448,7 @@ class Contraction final { LogicMTask* const otherp = static_cast(edge.furtherp()); neighbors[n] = otherp; sortRecs[n].m_id = otherp->id(); - sortRecs[n].m_cp = otherp->critPathCost(way) + otherp->cost(); + sortRecs[n].m_cp = otherp->cpInclusive(); sortRecs[n].m_idx = n; ++n; // Prevent nodes with huge numbers of edges from massively slowing down us down @@ -898,25 +474,97 @@ class Contraction final { } } - // CONSTRUCTORS - Contraction(OrderMTaskGraph& mTaskGraph, uint64_t scoreLimit) - : m_mTaskGraph{mTaskGraph} - , m_scoreLimit{scoreLimit} { + void removeSiblingMCs(LogicMTask* mtaskp) { + // Note: 'removeSiblingMC' unlinks the candidate from both of its MTasks' lists, so taking + // the front element repeatedly does terminate. It also erases the candidate from the + // owning (higher id) MTask's sibling set as it goes, so both the sets and the lists are + // left consistent, whichever side of the candidate 'mtaskp' happens to be on. + while (SiblingMC* const smcp = mtaskData(mtaskp).aSiblingMCs.frontp()) { + m_sb.removeSiblingMC(smcp); + } + while (SiblingMC* const smcp = mtaskData(mtaskp).bSiblingMCs.frontp()) { + m_sb.removeSiblingMC(smcp); + } + } - if (m_slowAsserts) { - // Check there are no redundant edges - for (V3GraphVertex& vtx : m_mTaskGraph.vertices()) { - std::unordered_set neighbors; - for (V3GraphEdge& edge : vtx.outEdges()) { - const bool first = neighbors.insert(edge.top()).second; - UASSERT_OBJ(first, &vtx, "Redundant edge found in input to Contraction()"); - } - } + // Merge the two MTasks of 'mergeCanp' + void contract(MergeCandidate* mergeCanp) { + // The two MTasks to merge. Note the order of the two decides which of them becomes the + // recipient below when their costs are equal, so keep it stable. + LogicMTask* fromp; + LogicMTask* top; + if (const EdgeMC* const edgeMCp = mergeCanp->toEdgeMC()) { + fromp = edgeMCp->edgep()->fromMTaskp(); + top = edgeMCp->edgep()->toMTaskp(); + } else { + const SiblingMC* const sibMCp = mergeCanp->toSiblingMC(); + fromp = sibMCp->bp(); + top = sibMCp->ap(); } - // Set up the critical path into and out of each node, then coarsen the graph. - partInitCriticalPaths(mTaskGraph); + // Merge the smaller mtask into the larger mtask. + LogicMTask* recipientp; + LogicMTask* donorp; + if (fromp->cost() > top->cost()) { + recipientp = fromp; + donorp = top; + } else { + donorp = fromp; + recipientp = top; + } + VL_DANGLING(fromp); + VL_DANGLING(top); // Use donorp and recipientp now instead + // Remove all SiblingMCs that include either MTask + removeSiblingMCs(recipientp); + removeSiblingMCs(donorp); + + // Remove the EdgeMCs of both MTasks + removeEdgeMCs(recipientp); + removeEdgeMCs(donorp); + + // Merge the MTasks. This redirects all edges, updates critical paths, and deletes donorp + m_mTaskGraph.mergeMTasks(recipientp, donorp); + VL_DANGLING(donorp); + + // Confirm we haven't botched the CP updates. + m_mTaskGraph.validate(); + + // Add the EdgeMCs of the merged MTask + addEdgeMCs(recipientp); + + // Finally, make new sibling pairs as needed: + // - prereqs and postreqs of recipientp + // - prereqs of recipientp's postreqs + // - postreqs of recipientp's prereqs + // Note that this depends on the updated critical paths (above). + addSiblingMCsFromRelatives(recipientp); + addSiblingMCsFromRelatives(recipientp); + unsigned edges = 0; + for (V3GraphEdge& edge : recipientp->outEdges()) { + LogicMTask* const postreqp = static_cast(edge.top()); + addSiblingMCsFromRelatives(postreqp); + ++edges; + if (edges >= PART_SIBLING_EDGE_LIMIT) break; + } + edges = 0; + for (V3GraphEdge& edge : recipientp->inEdges()) { + LogicMTask* const prereqp = static_cast(edge.fromp()); + addSiblingMCsFromRelatives(prereqp); + ++edges; + if (edges >= PART_SIBLING_EDGE_LIMIT) break; + } + } + + // CONSTRUCTORS + Contraction(OrderMTaskGraph& mTaskGraph, uint64_t cpLimit) + : m_mTaskGraph{mTaskGraph} + , m_scoreLimit{cpLimit} { + + // Check the graph we were given is consistent. + m_mTaskGraph.validate(); + + // Figure out maximum number of MTasks const uint32_t maxMTasks = []() -> uint32_t { // If specified, use the given value const int given = v3Global.opt.threadsMaxMTasks(); @@ -925,15 +573,6 @@ class Contraction final { return PART_DEFAULT_MAX_MTASKS_PER_THREAD * v3Global.opt.threads(); }(); - // OPTIMIZATION PASS: Edge contraction and sibling contraction. - // - Score pairs of LogicMTask which are a candidate to merge. - // * Each edge defines such a candidate pair - // * Two LogicMTask that are prereqs or postreqs of a common third - // vertex are "siblings", these are also a candidate pair. - // - Build a list of MergeCandidates, sorted by score. - // - Merge the best pair. - // - Incrementally recompute critical paths near the merged mtask. - // Allocate and assign the auxiliary data for every LogicMTask. { const size_t nMTasks = m_mTaskGraph.vertices().size(); @@ -945,56 +584,33 @@ class Contraction final { // Add initial candidates for (V3GraphVertex& vtx : m_mTaskGraph.vertices()) { - for (V3GraphEdge& edge : vtx.outEdges()) m_sb.addEdge(static_cast(&edge)); - siblingPairFromRelatives(&vtx); - siblingPairFromRelatives(&vtx); + for (V3GraphEdge& edge : vtx.outEdges()) + m_sb.addEdgeMC(static_cast(&edge)); + LogicMTask* const mtaskp = static_cast(&vtx); + addSiblingMCsFromRelatives(mtaskp); + addSiblingMCsFromRelatives(mtaskp); } - // Set initial scores in scoreboard - doRescore(); - while (true) { - // This is the best edge to merge, with the lowest score (shortest local critical path) + // Pick the candidate yielding the lowest local critical path. MergeCandidate* const mergeCanp = m_sb.best(); - if (!mergeCanp) { - if (!m_sb.needsRescore()) break; // No more eligible candidates - // Rescore the scoreboard and try again - doRescore(); + if (!mergeCanp) break; // No more candidates + + // If the score has changed since it was inserted into the scoreboard, rescore it and + // pick again. (The real scores can differ from the one used to insert it into the + // scoreboard, due to merges between insertion and retrieval.) + const uint64_t score = mergeCanp->currentScore(); + if (score != mergeCanp->score()) { + m_sb.rescore(mergeCanp); continue; } - UASSERT(!m_sb.needsRescore(mergeCanp), - "Need-rescore items should not be returned by bestp"); + // Check if the critical path limit is reached. + if (score > m_scoreLimit) { - const uint64_t cachedScore = mergeCanp->score(); - mergeCanp->rescore(); - const uint64_t actualScore = mergeCanp->score(); - - // If cached score is out-of-date, mark this elem as in need of a rescore and continue. - // cppcheck-suppress knownConditionTrueFalse // they are in fact different - if (actualScore > cachedScore) { - m_sb.hintScoreChanged(mergeCanp); - continue; - } - - // ... we'll also confirm that actualScore hasn't shrunk relative - // to cached score, after the mergeWouldCreateCycle() check. - - if (actualScore > m_scoreLimit) { - // Our best option isn't good enough - if (m_sb.needsRescore()) { - // Some pairs need a rescore, maybe those will be - // eligible to merge afterward. - doRescore(); - continue; - } - - // We've exhausted everything below m_scoreLimit; stop. - - // Except, if we have too many LogicMTasks, raise the score limit and keep going... + // If there are still too many MTasks, raise the limit and keep going const unsigned mtaskCount = m_mTaskGraph.vertices().size(); if (mtaskCount > maxMTasks) { - const uint64_t oldLimit = m_scoreLimit; m_scoreLimit = (m_scoreLimit * 120) / 100; FileLine* const flp = v3Global.rootp()->fileline(); if (!flp->warnIsOff(V3ErrorCode::UNOPTTHREADS)) { @@ -1003,80 +619,38 @@ class Contraction final { "parallelism; suggest asking for fewer threads."); flp->modifyWarnOff(V3ErrorCode::UNOPTTHREADS, true); } - UINFO(6, "Critical path limit was=" << oldLimit << " now=" << m_scoreLimit); continue; } - // Really stop + // MTasks limit and CP limit reached. Stop. break; } - // If time to rescore, that will result in a higher scoreLimitBeforeRescore, and - // possibly lower-scoring elements returned from bestp(). - if (actualScore > m_scoreLimitBeforeRescore) { - doRescore(); - continue; - } - // Avoid merging the entry/exit nodes. This would create serialization, by forcing the // merged MTask to run before/after everything else. Empirically this helps performance // in a modest way by allowing other MTasks to start earlier. if (EdgeMC* const edgeMCp = mergeCanp->toEdgeMC()) { MTaskEdge* const edgep = edgeMCp->edgep(); - if (edgep->fromp() == m_entryMTaskp || edgep->top() == m_exitMTaskp) { - m_sb.removeEdge(edgep); + if (edgep->fromp() == m_mTaskGraph.entryp() + || edgep->top() == m_mTaskGraph.exitp()) { + m_sb.removeEdgeMC(edgep); continue; } } - // Avoid merging any edge that would create a cycle. - // - // For example suppose we begin with vertices A, B, C and edges - // A->B, B->C, A->C. - // - // Suppose we want to merge A->C into a single vertex. - // New edges would be AC->B and B->AC which is not a DAG. - // Do not allow this. - if (mergeCanp->mergeWouldCreateCycle()) { - // Remove this candidate from scoreboard so we don't keep - // reconsidering it on every loop. - if (SiblingMC* const smcp = mergeCanp->toSiblingMC()) { - m_sb.removeSibling(smcp); - } else { - m_sb.removeEdge(mergeCanp->toEdgeMC()->edgep()); - } + // Avoid merging any edge that would create a cycle. For example suppose we begin with + // vertices A, B, C and edges A->B, B->C, A->C. Merging A->C would create a cycle. + if (mergeCanp->mergeWouldCreateCycle(m_mTaskGraph)) { + m_sb.removeMC(mergeCanp); continue; } - UASSERT(cachedScore == actualScore, "Calculation error in scoring"); - - // Finally there's no cycle risk, no need to rescore, we're - // within m_scoreLimit and m_scoreLimitBeforeRescore. - // This is the edge to merge. - - // Bookkeeping: if this is the first edge we'll merge since - // the last rescore, compute the new m_scoreLimitBeforeRescore - // to be somewhat higher than this edge's score. - if (!m_mergesSinceRescore) m_scoreLimitBeforeRescore = actualScore; - - // Finally merge this candidate. + // Merge this candidate contract(mergeCanp); } - // Free all remaining merge candidates. As an EdgeMC exists exactly while its edge is on - // the scoreboard, draining the scoreboard here frees every remaining EdgeMC; edges removed - // from the scoreboard earlier already had theirs freed. Note 'best' only ever returns - // candidates with a known score, so this only drains the scoreboard completely if nothing - // is left with an unknown score. Every 'break' out of the loop above is guarded on that, - // but assert it here, as otherwise we would leak candidates. - UASSERT(!m_sb.needsRescore(), "Should have no unknown score candidates at this point"); - while (MergeCandidate* const mergeCanp = m_sb.best()) { - if (SiblingMC* const smcp = mergeCanp->toSiblingMC()) { - m_sb.removeSibling(smcp); - } else { - m_sb.removeEdge(mergeCanp->toEdgeMC()->edgep()); - } - } + // Free all remaining merge candidates. + while (MergeCandidate* const mergeCanp = m_sb.best()) m_sb.removeMC(mergeCanp); } public: diff --git a/src/V3OrderMTaskFixHazards.cpp b/src/V3OrderMTaskFixHazards.cpp index ecd8ab468..17563ea95 100644 --- a/src/V3OrderMTaskFixHazards.cpp +++ b/src/V3OrderMTaskFixHazards.cpp @@ -162,46 +162,6 @@ class FixDataHazards final { // METHODS - // Redirect all edges of 'donorp' onto 'recipientp' - static void redirectEdgesFrom(LogicMTask* recipientp, LogicMTask* donorp) { - // Process outgoing edges - while (MTaskEdge* const edgep = static_cast(donorp->outEdges().frontp())) { - LogicMTask* const top = edgep->toMTaskp(); - top->removeRelativeEdge(edgep); - - // If an edge already exists between recipient and sink of donor, drop the duplicate. - if (recipientp->hasRelativeMTask(top)) { - VL_DO_DANGLING(edgep->unlinkDelete(), edgep); - continue; - } - - // Otherwise redirect the edge from donorp->top to recipientp->top. - edgep->relinkFromp(recipientp); - recipientp->addRelativeMTask(top); - recipientp->stealRelativeEdge(edgep); - top->addRelativeEdge(edgep); - } - - // Process incoming edges - while (MTaskEdge* const edgep = static_cast(donorp->inEdges().frontp())) { - LogicMTask* const fromp = edgep->fromMTaskp(); - fromp->removeRelativeMTask(donorp); - fromp->removeRelativeEdge(edgep); - - // If an edge already exists between recipient and source of donor, drop the duplicate. - if (fromp->hasRelativeMTask(recipientp)) { - VL_DO_DANGLING(edgep->unlinkDelete(), edgep); - continue; - } - - // Otherwise redirect the edge from fromp->donorp to fromp->recipientp. - edgep->relinkTop(recipientp); - fromp->addRelativeMTask(recipientp); - fromp->addRelativeEdge(edgep); - recipientp->stealRelativeEdge(edgep); - } - } - void findAdjacentTasks(const OrderVarStdVertex* varVtxp, TasksByRank& tasksByRank) { // Find all writer tasks for this variable, group by rank. for (const V3GraphEdge& edge : varVtxp->inEdges()) { @@ -219,8 +179,8 @@ class FixDataHazards final { LogicMTask* lastRecipientp = nullptr; for (const auto& pair : tasksByRank) { // Find the largest node at this rank, merge into it. (If we - // happen to find a huge node, this saves time in - // redirectEdgesFrom() versus merging into an arbitrary node.) + // happen to find a huge node, this saves time in the merge + // versus merging into an arbitrary node.) LogicMTask* recipientp = nullptr; for (LogicMTask* const mtaskp : pair.second) { if (!recipientp || (recipientp->cost() < mtaskp->cost())) recipientp = mtaskp; @@ -231,20 +191,18 @@ class FixDataHazards final { for (LogicMTask* const donorp : pair.second) { // Merge donor into recipient. if (donorp == recipientp) continue; - // Fix up the map, so donor's OLVs map to recipientp + // Fix up the map, so donor's OLVs map to recipientp. Must do this while the donor + // still holds them. for (const OrderMoveVertex& vtx : donorp->vertexList()) { vtx.logicp()->userp(recipientp); } - // Move all vertices from donorp to recipientp - recipientp->moveAllVerticesFrom(donorp); - // Redirect edges from donorp to recipientp - redirectEdgesFrom(recipientp, donorp); - // Remove donorp from the graph - VL_DO_DANGLING(donorp->unlinkDelete(&m_mTaskGraph), donorp); + // Merge donorp into recipientp, which also deletes donorp + m_mTaskGraph.mergeMTasks(recipientp, donorp); + VL_DANGLING(donorp); } - if (lastRecipientp && !lastRecipientp->hasRelativeMTask(recipientp)) { - new MTaskEdge{&m_mTaskGraph, lastRecipientp, recipientp, 1}; + if (lastRecipientp && !lastRecipientp->hasEdgeTo(recipientp)) { + m_mTaskGraph.addEdge(lastRecipientp, recipientp); } lastRecipientp = recipientp; } @@ -319,9 +277,8 @@ class FixDataHazards final { // given OVV.) Create edges across these remaining MTasks to ensure // they run in serial order (going along with the existing ranks.) // - // NOTE: we don't update the CP's stored in the LogicMTasks to - // reflect the changes we make to the graph. That's OK, as we - // haven't yet initialized CPs when we call this routine. + // NOTE: all graph mutations below go through OrderMTaskGraph (adding an edge, or merging + // two MTasks), so the CP's stored in the LogicMTasks are kept up to date throughout. for (const OrderVarStdVertex* const varVtxp : regularVars) { // Build a set of MTasks, per rank, which access this var. // Within a rank, sort by MTaskID to avoid nondeterminism. @@ -387,4 +344,6 @@ public: void OrderMTaskGraph::fixDataHazards(OrderMTaskGraph& mtaskGraph) { FixDataHazards::apply(mtaskGraph); + // The critical paths are maintained as the graph is mutated, check them + mtaskGraph.validate(); } diff --git a/src/V3OrderMTaskGraph.cpp b/src/V3OrderMTaskGraph.cpp index e47b61fb4..50d3a175c 100644 --- a/src/V3OrderMTaskGraph.cpp +++ b/src/V3OrderMTaskGraph.cpp @@ -21,28 +21,16 @@ #include "V3Global.h" #include "V3InstrCount.h" +#include +#include +#include + VL_DEFINE_DEBUG_FUNCTIONS; -//###################################################################### -// OrderMTaskGraph - -OrderMTaskGraph::OrderMTaskGraph(OrderMoveGraph& moveGraph) - : m_moveGraph{moveGraph} - , m_entryp{new LogicMTask{*this, nullptr}} - , m_exitp{new LogicMTask{*this, nullptr}} - , m_forwardPropagator{v3Global.opt.debugPartition()} - , m_reversePropagator{v3Global.opt.debugPartition()} {} - -uint64_t OrderMTaskGraph::totalCost() const { - uint64_t cost = 0; - for (const V3GraphVertex& vtx : vertices()) cost += static_cast(vtx).cost(); - return cost; -} - //###################################################################### // LogicMTask -uint32_t LogicMTask::s_nextId = 1; // Start at 1, so that 0 indicates no mtask. +uint32_t LogicMTask::s_nextId = 1; // Start at 1, for historic reasons LogicMTask::LogicMTask(OrderMTaskGraph& graph, OrderMoveVertex* mVtxp) : V3GraphVertex{&graph} { @@ -54,6 +42,281 @@ LogicMTask::LogicMTask(OrderMTaskGraph& graph, OrderMoveVertex* mVtxp) } } +//###################################################################### +// OrderMTaskGraph + +OrderMTaskGraph::OrderMTaskGraph(OrderMoveGraph& moveGraph) + : m_moveGraph{moveGraph} + , m_entryp{new LogicMTask{*this, nullptr}} + , m_exitp{new LogicMTask{*this, nullptr}} + , m_slowAsserts{v3Global.opt.debugPartition()} {} + +bool OrderMTaskGraph::pathExistsImpl(LogicMTask* fromp, LogicMTask* top, + const MTaskEdge* excludedEdgep) { + UDEBUGONLY(UASSERT_OBJ(fromp->m_generation != m_currentGeneration, fromp, + "Should not visit an MTask twice in the same search");); + // Mark visited. + fromp->m_generation = m_currentGeneration; + + // Base case: we found a path. + if (fromp == top) return true; + + // Base case: fromp is too late, cannot possibly be a prereq for top. + if (fromp->cpExclusive() < top->cpInclusive()) { + return false; + } + if (fromp->cpInclusive() > top->cpExclusive()) { + return false; + } + + // Recursively look for a path + for (const V3GraphEdge& follow : fromp->outEdges()) { + if (&follow == excludedEdgep) continue; + LogicMTask* const nextp = static_cast(follow.top()); + // Don't visit the same MTask twice in the same search. + if (nextp->m_generation == m_currentGeneration) continue; + if (pathExistsImpl(nextp, top, nullptr)) return true; + } + return false; +} + +template +void OrderMTaskGraph::propagatePush(LogicMTask* mtaskp) { + constexpr GraphWay way{N_Way}; + constexpr GraphWay inv{way.invert()}; + const uint64_t inclusiveCp = mtaskp->cpInclusive(); + + for (V3GraphEdge& graphEdge : mtaskp->edges()) { + MTaskEdge& edge = static_cast(graphEdge); + + LogicMTask* const relativep = edge.furtherMTaskp(); + EdgeHeap::Node& edgeHeapNode = edge.m_edgeHeapNode[inv]; + if (inclusiveCp > edgeHeapNode.key().m_cp) { + relativep->m_edgeHeap[inv].increaseKey(&edgeHeapNode, inclusiveCp); + } + + const uint64_t relativeCp = relativep->cpExclusive(); + + if (relativeCp >= inclusiveCp) continue; + + // relativep's critical path is out of step with its longest !wayward edge. + // Schedule that to be resolved. + const uint64_t increment = inclusiveCp - relativeCp; + + PropagatePendingHeap::Node*& pendingNodepRef = relativep->m_propagateHeapNodep; + if (PropagatePendingHeap::Node* const nodep = pendingNodepRef) { + // Already in heap. Increase the increment if needed. + if (increment > nodep->key().m_increment) { + m_pendingHeap.increaseKey(nodep, increment); + } + continue; + } + + // Add to heap + PropagatePendingHeap::Node* const nodep = m_pendingNodePool.alloc(); + pendingNodepRef = nodep; + m_pendingHeap.insert(nodep, {increment, relativep->id(), relativep}); + } +} + +template +void OrderMTaskGraph::propagateResolve() { + constexpr GraphWay way{N_Way}; + constexpr GraphWay inv{way.invert()}; + + // Each pending MTask is keyed on how much its critical path will grow by. Resolving them in + // decreasing order of that growth means each MTask needs resolving only once: the growth of a + // wayward MTask is never larger than the growth of the MTask it was pushed from, so once an + // MTask has been resolved, no larger growth can be pushed onto it later. + while (!m_pendingHeap.empty()) { + // Pop max element from heap + PropagatePendingHeap::Node* const maxp = m_pendingHeap.max(); + m_pendingHeap.remove(maxp); + // Pick up values + LogicMTask* const mtaskp = maxp->key().m_mtaskp; + const uint64_t cpGrowBy = maxp->key().m_increment; + // Confirm that we only set each node's CP once. That's an important property of this + // algorithm, which allows it to be far faster than a recursive one. + UASSERT_OBJ(mtaskp->m_generation != m_currentGeneration, mtaskp, "Set CP on node twice"); + mtaskp->m_generation = m_currentGeneration; + // Free the heap node, we are done with it + m_pendingNodePool.free(maxp); + mtaskp->m_propagateHeapNodep = nullptr; + // Update the critical path of mtaskp, that was out-of-date with respect to its edges + uint64_t& cpRef = mtaskp->m_cpExclusive[way]; + const uint64_t newCp = cpRef + cpGrowBy; + // Check that CP matches that of the longest edge wayward of mtaskp. + if (VL_UNLIKELY(m_slowAsserts)) { + const uint64_t edgeCp = mtaskp->m_edgeHeap[inv].max()->key().m_cp; + UASSERT_OBJ(edgeCp == newCp, mtaskp, "CP doesn't match longest wayward edge"); + } + cpRef = newCp; + propagatePush(mtaskp); + } +} + +uint64_t OrderMTaskGraph::totalCost() const { + uint64_t cost = 0; + for (const V3GraphVertex& vtx : vertices()) cost += static_cast(vtx).cost(); + return cost; +} + +void OrderMTaskGraph::addEdge(LogicMTask* fromp, LogicMTask* top) { + UASSERT_OBJ(fromp != top, fromp, "Should not create self-edges"); + UDEBUGONLY(UASSERT_OBJ(!fromp->hasEdgeTo(top), fromp, "Should not create redundant edges");); + + // Create the edge. This inserts it into the edge heap of both endpoints with the correct + // critical path keys, as the critical paths of the endpoints are still unchanged here. + new MTaskEdge{this, fromp, top}; + + // The path through the new edge might be longer than the current critical path of its + // endpoints, in which case the critical paths need updating. Note each endpoint is the seed of + // one propagation, and is updated by the other: the inclusive critical paths of the endpoints + // themselves did not change (a new out-edge cannot lengthen a path into 'fromp', nor a new + // in-edge a path out of 'top'), so it is the new relative of each seed whose critical path + // might need to grow. That is, 'top' is updated wayward of 'fromp' below, and vice versa, + // together with the relatives of each, transitively. + // + // The guards below are an asymptotic optimization. The graph is consistent apart from the new + // edge, so the new relative is the only relative of either seed that can have a stale critical + // path, and if it does not need updating the propagation does nothing. It would however still + // walk all edges of the seed to discover that, which is expensive for a high degree seed. + if (fromp->cpInclusive() > top->cpExclusive()) { + propagate(fromp); + } + if (top->cpInclusive() > fromp->cpExclusive()) { + propagate(top); + } +} + +void OrderMTaskGraph::mergeMTasks(LogicMTask* recipientp, LogicMTask* donorp) { + UASSERT_OBJ(recipientp != donorp, recipientp, "Should not merge an MTask with itself"); + + // Note we redirect the edges before updating the cost and critical paths of the recipient, + // which means the redirected edges are inserted into the edge heaps of the relatives using the + // pre-merge values of the recipient. The critical path propagation below then brings all of + // them up to date. This works because the keys in the edge heaps only ever need increasing: + // the inclusive critical path of the merged MTask is at least the inclusive critical path of + // either of the two MTasks it is made of, in both directions. + + // Process outgoing edges of donor + while (MTaskEdge* const edgep = static_cast(donorp->outEdges().frontp())) { + LogicMTask* const relativep = edgep->toMTaskp(); + + relativep->removeRelativeEdge(edgep); + + if (relativep == recipientp || recipientp->hasEdgeTo(relativep)) { + // This is either the edge connecting the two MTasks, which becomes internal to the + // merged MTask, or is parallel with an existing edge of the recipient. Drop it. + VL_DO_DANGLING(edgep->unlinkDelete(), edgep); + } else { + // No existing edge between recipient and relative of donor. + // Redirect the edge from donor -> relative to recipient -> relative. + edgep->relinkFromp(recipientp); + recipientp->addDependent(relativep); + recipientp->stealRelativeEdge(edgep); + relativep->addRelativeEdge(edgep); + } + } + + // Process incoming edges of donor + while (MTaskEdge* const edgep = static_cast(donorp->inEdges().frontp())) { + LogicMTask* const relativep = edgep->fromMTaskp(); + + relativep->removeDependent(donorp); + relativep->removeRelativeEdge(edgep); + + if (relativep == recipientp || relativep->hasEdgeTo(recipientp)) { + // This is either the edge connecting the two MTasks, which becomes internal to the + // merged MTask, or is parallel with an existing edge of the recipient. Drop it. + VL_DO_DANGLING(edgep->unlinkDelete(), edgep); + } else { + // No existing edge between recipient and relative of donor. + // Redirect the edge from relative -> donor to relative -> recipient. + edgep->relinkTop(recipientp); + relativep->addDependent(recipientp); + relativep->addRelativeEdge(edgep); + recipientp->stealRelativeEdge(edgep); + } + } + + // Move the contents of the donor into the recipient, update its cost + recipientp->m_mVertices.splice(recipientp->m_mVertices.end(), donorp->m_mVertices); + recipientp->m_cost += donorp->m_cost; + + // The recipient now holds all edges of the merged MTask, and the critical paths of all its + // relatives are still up to date, so the critical paths implied by its edges are the critical + // paths of the merged MTask. + const uint64_t newCpFwd = recipientp->cpExclusiveFromEdges(); + const uint64_t newCpRev = recipientp->cpExclusiveFromEdges(); + + // Set the new critical paths, then propagate the increases to the relatives. Note this also + // brings the keys of all edges of the merged MTask up to date in the relatives' edge heaps. + recipientp->cpExclusive(newCpFwd); + propagate(recipientp); + recipientp->cpExclusive(newCpRev); + propagate(recipientp); + + // Remove the donor from the graph + VL_DO_DANGLING(donorp->unlinkDelete(this), donorp); +} + +// Check the critical paths in the given direction, and the critical paths cached in the edge heaps +// in the opposite direction, against those implied by the edges. Note this deliberately iterates +// the edge lists, rather than consulting the edge heaps, so the heaps are validated, not trusted. +template +void OrderMTaskGraph::validateWay() const { + constexpr GraphWay way{N_Way}; + constexpr GraphWay inv = way.invert(); + for (const V3GraphVertex& vtx : vertices()) { + const LogicMTask& mtask = *vtx.as(); + uint64_t cpCost = 0; + std::unordered_set relatives; + for (const V3GraphEdge& graphEdge : mtask.edges()) { + const MTaskEdge& edge = *graphEdge.as(); + const LogicMTask& relative = *(edge.furtherp()->template as()); + // Run a few asserts on the graph, while we are iterating through... + UASSERT_OBJ(edge.weight() != 0, &mtask, "Should be no cut edges in MTask graph"); + UASSERT_OBJ(&relative != &mtask, &mtask, "Should be no self edges in MTask graph"); + const bool first = relatives.insert(&relative).second; + UASSERT_OBJ(first, &mtask, "Should be no redundant edges in MTask graph"); + const uint64_t inclusiveCp = relative.cpInclusive(); + // The critical path cached in the edge heap must match that of the relative + UASSERT_OBJ(edge.cachedCp(inv) == inclusiveCp, &mtask, + "Cached critical path does not match the relative"); + // As must the ID it is keyed on, which breaks ties between equal critical paths + UASSERT_OBJ(edge.cachedId(inv) == relative.id(), &mtask, + "Cached ID does not match the relative"); + cpCost = std::max(cpCost, inclusiveCp); + } + const uint64_t cp = mtask.cpExclusive(); + UASSERT_OBJ(cp == cpCost, &mtask, "Critical path does not match the edges"); + // The edge heap must yield the same, that is: it must return the largest of its keys + UASSERT_OBJ(mtask.cpExclusiveFromEdges() == cpCost, &mtask, + "Edge heap maximum does not match the edges"); + } +} + +void OrderMTaskGraph::validate() const { + if (!m_slowAsserts) return; + + validateWay(); + validateWay(); + + // Check the dependents set of each MTask agrees with its out-edges + for (const V3GraphVertex& vtx : vertices()) { + const LogicMTask& mtask = *vtx.as(); + size_t nDependents = 0; + for (const V3GraphEdge& graphEdge : mtask.outEdges()) { + LogicMTask* const top = graphEdge.as()->toMTaskp(); + UASSERT_OBJ(mtask.hasEdgeTo(top), &mtask, "Dependent missing from the dependents set"); + ++nDependents; + } + UASSERT_OBJ(mtask.m_dependents.size() == nDependents, &mtask, + "Stale entry in the dependents set"); + } +} + //###################################################################### // OrderMTaskGraphBuilder @@ -103,10 +366,9 @@ class OrderMTaskGraphBuilder final { } // Add an edge to the graph, if there is not already an edge between the two vertices. - void addEdge(LogicMTask& src, LogicMTask& dst) { - UASSERT_OBJ(&src != &dst, &src, "Should not create self-edges"); - if (src.hasRelativeMTask(&dst)) return; // Don't create redundant edges. - new MTaskEdge{&m_mtaskGraph, &src, &dst, 1}; + void addEdge(LogicMTask* srcp, LogicMTask* dstp) { + if (srcp->hasEdgeTo(dstp)) return; // Don't create redundant edges. + m_mtaskGraph.addEdge(srcp, dstp); } // CONSTRUCTORS @@ -145,7 +407,7 @@ class OrderMTaskGraphBuilder final { // If the opposite end of the edge is not a bypassed vertex, add direct dependency if (LogicMTask* const otherp = static_cast(top->userp())) { - addEdge(mtask, *otherp); + addEdge(&mtask, otherp); continue; } @@ -155,7 +417,7 @@ class OrderMTaskGraphBuilder final { // The Move graph is bipartite (logic <-> var), and logic is never // bypassed, hence 'transp' must be non-nullptr. UASSERT_OBJ(transp, mVtxp, "This cannot be a bypassed vertex"); - addEdge(mtask, *transp); + addEdge(&mtask, transp); } } } @@ -166,8 +428,8 @@ class OrderMTaskGraphBuilder final { LogicMTask& mtask = static_cast(vtx); if (VL_UNLIKELY((&mtask == &entry) || (&mtask == &exit))) continue; // Add the entry/exit edges if not otherwise connected - if (mtask.inEmpty()) addEdge(entry, mtask); - if (mtask.outEmpty()) addEdge(mtask, exit); + if (mtask.inEmpty()) addEdge(&entry, &mtask); + if (mtask.outEmpty()) addEdge(&mtask, &exit); } } ~OrderMTaskGraphBuilder() = default; @@ -181,5 +443,6 @@ public: std::unique_ptr OrderMTaskGraph::build(OrderMoveGraph& moveGraph) { std::unique_ptr resp{new OrderMTaskGraph{moveGraph}}; OrderMTaskGraphBuilder::apply(*resp); + resp->validate(); return resp; } diff --git a/src/V3OrderMTaskGraph.h b/src/V3OrderMTaskGraph.h index 8643d0787..39dd7e7b1 100644 --- a/src/V3OrderMTaskGraph.h +++ b/src/V3OrderMTaskGraph.h @@ -20,9 +20,9 @@ // candidate machinery: any auxiliary data the algorithms need is attached // externally via the vertex/edge user pointers. // -// PropagateCp propagates increasing critical path costs through the graph. -// OrderMTaskGraph owns one instance for each direction, which the algorithms -// operating on the graph use to keep the critical paths up to date. +// OrderMTaskGraph maintains the critical paths of the MTasks, and the ones +// cached in the edge heaps, as the graph is mutated via 'addEdge' and +// 'mergeMTasks'. // //************************************************************************* @@ -35,38 +35,56 @@ #include "V3Graph.h" #include "V3OrderMoveGraph.h" #include "V3PairingHeap.h" +#include "V3PoolAllocator.h" #include -#include #include #include #include -#include class LogicMTask; class OrderMTaskGraph; -template -class PropagateCp; //============================================================================= -// We keep MTaskEdge graph edges in a PairingHeap, sorted by score and id +// MTaskEdge graph edges are stored in a PairingHeap in each LogicMTask they +// connect to, sorted by critical path through that edge (and id for stability). struct EdgeKey final { - uint64_t m_score; // Score part of edge key - uint64_t m_id; // Unique ID part of edge key - void increase(uint64_t score) { - UDEBUGONLY(UASSERT(score >= m_score, "Must increase");); - m_score = score; + uint64_t m_cp; // The inclusive critical path of the further MTask of the edge + uint32_t m_id; // The ID of the further MTask, for stable comparison + void increase(uint64_t cp) { + UDEBUGONLY(UASSERT(cp >= m_cp, "Must increase");); + m_cp = cp; } - // Sort first by Score then by ID + // Sort first by critical path, then by ID bool operator<(const EdgeKey& other) const { - if (m_score != other.m_score) return m_score < other.m_score; + if (m_cp != other.m_cp) return m_cp < other.m_cp; return m_id < other.m_id; } }; using EdgeHeap = PairingHeap; +//============================================================================= +// LogicMTasks are stored in a PairingHeap during critical path update propagation. + +struct PropagatePendingKey final { + uint64_t m_increment; // The amount the critical path of the MTask will grow by + uint32_t m_id; // The ID of the MTask, for stable comparison + LogicMTask* m_mtaskp; // The MTask the heap entry corresponds to + void increase(uint64_t increment) { + UDEBUGONLY(UASSERT(increment >= m_increment, "Must increase");); + m_increment = increment; + } + // Sort first by increment, then by ID + bool operator<(const PropagatePendingKey& other) const { + if (m_increment != other.m_increment) return m_increment < other.m_increment; + return m_id < other.m_id; + } +}; + +using PropagatePendingHeap = PairingHeap; + //============================================================================= // GraphEdge for the MTask graph @@ -74,22 +92,19 @@ class MTaskEdge final : public V3GraphEdge { VL_RTTI_IMPL(MTaskEdge, V3GraphEdge) friend class LogicMTask; - template - friend class PropagateCp; + friend class OrderMTaskGraph; // MEMBERS // This edge can be in 2 EdgeHeaps, one forward and one reverse. We allocate the heap nodes // directly within the edge as they are always required and this makes association cheap. std::array m_edgeHeapNode; - // Note: The edge's contraction merge candidate (if any) is held in the inherited user pointer - // (V3GraphEdge::userp), managed entirely by the partitioner; see edgeMC() and - // MergeCandidateScoreboard. Kept out of MTaskEdge so it does not depend on the MergeCandidate - // hierarchy. + // CONSTRUCTORS + // Private, so edges can only be created via OrderMTaskGraph, which also updates the critical + // paths on graph mutation. + inline MTaskEdge(OrderMTaskGraph* graphp, LogicMTask* fromp, LogicMTask* top); public: - // CONSTRUCTORS - inline MTaskEdge(OrderMTaskGraph* graphp, LogicMTask* fromp, LogicMTask* top, int weight); VL_UNCOPYABLE(MTaskEdge); VL_UNMOVABLE(MTaskEdge); @@ -99,11 +114,8 @@ public: inline LogicMTask* fromMTaskp() const; inline LogicMTask* toMTaskp() const; - // Following initial assignment of critical paths, clear this MTaskEdge - // out of the edge-map for each node and reinsert at a new location - // with updated critical path. - inline void resetCriticalPaths(); - uint64_t cachedCp(GraphWay way) const { return m_edgeHeapNode[way].key().m_score; } + uint64_t cachedCp(GraphWay way) const { return m_edgeHeapNode[way].key().m_cp; } + uint32_t cachedId(GraphWay way) const { return m_edgeHeapNode[way].key().m_id; } // Convert from the address of the m_edgeHeapNode[way] in an MTaskEdge back to the MTaskEdge static const MTaskEdge* toMTaskEdge(GraphWay way, const EdgeHeap::Node* nodep) { const size_t offset = VL_OFFSETOF(MTaskEdge, m_edgeHeapNode[way]); @@ -117,8 +129,8 @@ public: class LogicMTask final : public V3GraphVertex { VL_RTTI_IMPL(LogicMTask, V3GraphVertex) - template - friend class PropagateCp; + friend class MTaskEdge; + friend class OrderMTaskGraph; // MEMBERS @@ -126,18 +138,23 @@ class LogicMTask final : public V3GraphVertex { // OrderMoveVertex objects, we merely keep them in a list here. OrderMoveVertex::List m_mVertices; + static uint32_t s_nextId; // Next ID number to use + const uint32_t m_id = s_nextId++; // Unique LogicMTask ID number for stable comparison + // Cost estimate for this LogicMTask, derived from V3InstrCount, in abstract time units. // Cost estimates and critical path lengths are bounded by number of AstNodes * constant, // will run out of host memory storing the Ast way before they can overflow. uint64_t m_cost = 0; - // Cost of critical paths going FORWARD from graph-start to the start - // of this vertex, and also going REVERSE from the end of the graph to - // the end of the vertex. Same units as m_cost. - std::array m_critPathCost = {}; + // Critical path in each direction: going FORWARD from graph-start to the start of this vertex, + // and going REVERSE from graph-exit to the end of this vertex. Exclusive of the cost of this + // vertex itself, see cpInclusive() for the value including it. + std::array m_cpExclusive = {0, 0}; - static uint32_t s_nextId; // Next ID number to use - const uint32_t m_id = s_nextId++; // Unique LogicMTask ID number for stable comparison + // The MTasks this MTask has an out-edge to, so checking for an existing edge is O(1) + std::unordered_set m_dependents; + // Store the out/in edges in a heaps sorted by the critical path length through each edge + std::array m_edgeHeap; // Count "generations" which are just operations that scan through the // graph. We'll mark each node with the last generation that scanned @@ -145,15 +162,9 @@ class LogicMTask final : public V3GraphVertex { // while searching for a path. uint64_t m_generation = 0; - // Store a set of forward relatives so we can quickly check if we have a given child - std::unordered_set m_edgeSet; - // Store the outgoing and incoming edges in a heap sorted by the critical path length - std::array m_edgeHeap; - - // Scratch pointer used only by PropagateCp: this MTask's node in the pending heap, or nullptr - // if this MTask is not pending. Type erased, as the heap node type is private to PropagateCp, - // and differs between its two instantiations (which never run concurrently). - void* m_propagateHeapNodep = nullptr; + // Scratch pointer used only by the critical path propagation in OrderMTaskGraph: this MTask's + // node in the pending heap, or nullptr if this MTask is not pending. + PropagatePendingHeap::Node* m_propagateHeapNodep = nullptr; public: // CONSTRUCTORS @@ -161,20 +172,65 @@ public: VL_UNCOPYABLE(LogicMTask); VL_UNMOVABLE(LogicMTask); - // ACCESSORS - OrderMoveVertex::List& vertexList() { return m_mVertices; } - const OrderMoveVertex::List& vertexList() const { return m_mVertices; } - uint32_t id() const { return m_id; } - uint64_t cost() const VL_MT_SAFE { return m_cost; } - uint64_t critPathCost(GraphWay way) const { return m_critPathCost[way]; } - void setCritPathCost(GraphWay way, uint64_t cost) { m_critPathCost[way] = cost; } - // METHODS + OrderMoveVertex::List& vertexList() { return m_mVertices; } + uint32_t id() const { return m_id; } bool operator<(const LogicMTask& rhs) const { return id() < rhs.id(); } - void moveAllVerticesFrom(LogicMTask* otherp) { - m_mVertices.splice(m_mVertices.end(), otherp->vertexList()); - m_cost += otherp->m_cost; + uint64_t cost() const VL_MT_SAFE { return m_cost; } + template + uint64_t cpExclusive() const { + return m_cpExclusive[N_Way]; + } + template + uint64_t cpInclusive() const { + return m_cpExclusive[N_Way] + m_cost; + } + // The critical path of this MTask without considering the given edge. + template + uint64_t cpExclusiveWithout(const V3GraphEdge* edgep) const { + const GraphWay way{N_Way}; + const GraphWay inv = way.invert(); + UDEBUGONLY(UASSERT(edgep->furtherp() == this, + "In cpExclusiveWithout(), 'edgep' must further to 'this'");); + // At most two edges need to be considered: the critical path, if that is not via 'edgep', + // or the second-worst path, if the critical path is via 'edgep'. + const EdgeHeap& edgeHeap = m_edgeHeap[inv]; + // Pick up the critical path edge + const EdgeHeap::Node* const maxp = edgeHeap.max(); + UDEBUGONLY(UASSERT(maxp, "Edge not in heap");); + // If 'edgep' is not the critical path edge, return its critical path + if (MTaskEdge::toMTaskEdge(inv, maxp) != edgep) return maxp->key().m_cp; + // Otherwise return the second-worst path, if there is one + const EdgeHeap::Node* const secp = edgeHeap.secondMax(); + if (!secp) return 0; + return secp->key().m_cp; + } + + bool hasEdgeTo(LogicMTask* dependentp) const { return m_dependents.count(dependentp); } + + // For Graphviz dumps only + std::string name() const override VL_MT_STABLE { + std::ostringstream out; + out << "mt" << m_id // + << " | cpFwd " << m_cpExclusive[GraphWay::FORWARD] // + << " | cost " << cost() // + << " | cpRev " << m_cpExclusive[GraphWay::REVERSE]; + return out.str(); + } + +private: + // Following only used by OrderMTaskGraph, which maintains cached CPs and graph invariants. + + template + uint64_t cpExclusiveFromEdges() const { + constexpr GraphWay inv = GraphWay{N_Way}.invert(); + const EdgeHeap::Node* const maxp = m_edgeHeap[inv].max(); + return maxp ? maxp->key().m_cp : 0; + } + template + void cpExclusive(uint64_t cp) { + m_cpExclusive[N_Way] = cp; } template @@ -183,8 +239,8 @@ public: constexpr GraphWay inv = way.invert(); // Add to the edge heap LogicMTask* const relativep = edgep->furtherMTaskp(); - // Value is !way cp to this edge - const uint64_t cp = relativep->cost() + relativep->critPathCost(inv); + // Value is the !way inclusive cp of the relative + const uint64_t cp = relativep->cpInclusive(); m_edgeHeap[way].insert(&edgep->m_edgeHeapNode[way], {cp, relativep->id()}); } template @@ -202,283 +258,14 @@ public: m_edgeHeap[way].remove(&edgep->m_edgeHeapNode[way]); } - void addRelativeMTask(LogicMTask* relativep) { - // Add the relative to connecting edge map - const bool exits = !m_edgeSet.emplace(relativep).second; - UDEBUGONLY(UASSERT(!exits, "Adding existing relative");); + void addDependent(LogicMTask* dependentp) { + const bool exists = !m_dependents.emplace(dependentp).second; + UDEBUGONLY(UASSERT(!exists, "Adding existing dependent");); } - void removeRelativeMTask(LogicMTask* relativep) { - const size_t removed = m_edgeSet.erase(relativep); - UDEBUGONLY(UASSERT(removed, "Relative should have been in set");); + void removeDependent(LogicMTask* dependentp) { + const size_t removed = m_dependents.erase(dependentp); + UDEBUGONLY(UASSERT(removed, "Dependent should have been in set");); } - bool hasRelativeMTask(LogicMTask* relativep) const { return m_edgeSet.count(relativep); } - - template - void checkRelativesCp() const { - constexpr GraphWay way{N_Way}; - for (const V3GraphEdge& edge : edges()) { - const LogicMTask* const relativep - = static_cast(edge.furtherp()); - const uint64_t cachedCp = static_cast(edge).cachedCp(way); - const uint64_t cp = relativep->critPathCost(way.invert()) + relativep->cost(); - UASSERT(cachedCp == cp, "Calculation error in scoring"); - } - } - - template - uint64_t critPathCostWithout(const V3GraphEdge* withoutp) const { - const GraphWay way{N_Way}; - const GraphWay inv = way.invert(); - // Compute the critical path cost wayward to this node, without considering edge - // 'withoutp'. We need to look at two edges at most, the critical path if that is not via - // 'withoutp', or the second-worst path, if the critical path is via 'withoutp'. - UDEBUGONLY(UASSERT(withoutp->furtherp() == this, - "In critPathCostWithout(), edge 'withoutp' must further to 'this'");); - const EdgeHeap& edgeHeap = m_edgeHeap[inv]; - const EdgeHeap::Node* const maxp = edgeHeap.max(); - if (!maxp) return 0; - if (MTaskEdge::toMTaskEdge(inv, maxp) != withoutp) return maxp->key().m_score; - const EdgeHeap::Node* const secp = edgeHeap.secondMax(); - if (!secp) return 0; - return secp->key().m_score; - } - -private: - // This takes LogicMTask instead of generic V3GraphVertex. We will use the critical - // paths known to LogicMTask to prune the recursion for speed. Also store 'generation' in - // LogicMTask::m_generation so we can prune the search and avoid recursing through the same - // node more than once in a single search. - static bool pathExistsFromInternal(LogicMTask* fromp, LogicMTask* top, - const MTaskEdge* excludedEdgep, uint64_t generation) { - - // If already looked at this node in the current search, since we're back again, - // we must not have found a path on the first go. - if (fromp->m_generation == generation) return false; - - // Mark visited - fromp->m_generation = generation; - - // Base case: we found a path. - if (fromp == top) return true; - - // Base case: fromp is too late, cannot possibly be a prereq for top. - if (fromp->critPathCost(GraphWay::REVERSE) - < (top->critPathCost(GraphWay::REVERSE) + top->cost())) { - return false; - } - if ((fromp->critPathCost(GraphWay::FORWARD) + fromp->cost()) - > top->critPathCost(GraphWay::FORWARD)) { - return false; - } - - // Recursively look for a path - for (const V3GraphEdge& follow : fromp->outEdges()) { - if (&follow == excludedEdgep) continue; - LogicMTask* const nextp = static_cast(follow.top()); - if (pathExistsFromInternal(nextp, top, nullptr, generation)) return true; - } - return false; - } - -public: - // True if there's a path from 'fromp' to 'top' excluding 'excludedEdgep', false otherwise. - // 'excludedEdgep' may be nullptr in which case no edge is excluded. If 'excludedEdgep' is - // non-nullptr it must connect fromp and top. - static bool pathExistsFrom(LogicMTask* fromp, LogicMTask* top, - const MTaskEdge* excludedEdgep) { - static uint64_t s_generation = 0; - return pathExistsFromInternal(fromp, top, excludedEdgep, ++s_generation); - } - - // For Graphviz dumps only - std::string name() const override VL_MT_STABLE { - std::ostringstream out; - out << "mt" << m_id // - << " | fwdCP " << m_critPathCost[GraphWay::FORWARD] // - << " | revCP " << m_critPathCost[GraphWay::REVERSE] // - << " | cost " << cost(); - return out.str(); - } -}; - -//============================================================================= -// PropagateCp - -template -class PropagateCp final { - // Propagate increasing critical path (CP) costs through a graph. - // - // Usage: - // * Client increases the cost and/or CP at a node or small set of nodes - // (often a pair in practice, eg. edge contraction.) - // * Client calls PropagateCp::cpHasIncreased() one or more times. - // Each call indicates that the inclusive CP of some "seed" vertex - // has increased to a given value. - // * NOTE: PropagateCp will neither read nor modify the cost - // or CPs at the seed vertices, it only accesses and modifies - // vertices wayward from the seeds. - // * Client calls PropagateCp::go(). Internally, this iteratively - // propagates the new CPs wayward through the graph. - // - - // TYPES - - // We keep pending vertices in a heap during critical path propagation - struct PendingKey final { - LogicMTask* m_mtaskp; // The vertex in the heap - uint64_t m_score; // The score of this entry - void increase(uint64_t score) { - UDEBUGONLY(UASSERT(score >= m_score, "Must increase");); - m_score = score; - } - bool operator<(const PendingKey& other) const { - if (m_score != other.m_score) return m_score < other.m_score; - return *m_mtaskp < *other.m_mtaskp; - } - }; - - using PendingHeap = PairingHeap; - using PendingHeapNode = typename PendingHeap::Node; - - // MEMBERS - PendingHeap m_pendingHeap; // Heap of pending rescores - - // We allocate this many heap nodes at once - static constexpr size_t ALLOC_CHUNK_SIZE = 128; - PendingHeapNode* m_freep = nullptr; // List of free heap nodes - std::vector> m_allocated; // Allocated heap nodes - - const bool m_slowAsserts; // Enable nontrivial asserts - // Used only with slow asserts to check MTasks visited only once - std::unordered_set m_seen; - -public: - // CONSTRUCTORS - explicit PropagateCp(bool slowAsserts) - : m_slowAsserts{slowAsserts} {} - - // METHODS -private: - // Allocate a HeapNode for the given element - PendingHeapNode* allocNode() { - // If no free nodes available, then make some - if (!m_freep) { - // Allocate in chunks for efficiency - m_allocated.emplace_back(new PendingHeapNode[ALLOC_CHUNK_SIZE]); - // Set up free list pointer - m_freep = m_allocated.back().get(); - // Set up free list chain - for (size_t i = 1; i < ALLOC_CHUNK_SIZE; ++i) { - m_freep[i - 1].m_next.m_ptr = &m_freep[i]; - } - // Clear the next pointer of the last entry - m_freep[ALLOC_CHUNK_SIZE - 1].m_next.m_ptr = nullptr; - } - // Free nodes are available, pick up the first one - PendingHeapNode* const resultp = m_freep; - m_freep = resultp->m_next.m_ptr; - resultp->m_next.m_ptr = nullptr; - return resultp; - } - - // Release a heap node (make it available for future allocation) - void freeNode(PendingHeapNode* nodep) { - // Re-use the existing link pointers and simply prepend it to the free list - nodep->m_next.m_ptr = m_freep; - m_freep = nodep; - } - -public: - void cpHasIncreased(LogicMTask* vxp, uint64_t newInclusiveCp) { - constexpr GraphWay way{N_Way}; - constexpr GraphWay inv{way.invert()}; - - // For *vxp, whose CP-inclusive has just increased to - // newInclusiveCp, iterate to all wayward nodes, update the edges - // of each, and add each to m_pending if its overall CP has grown. - for (V3GraphEdge& graphEdge : vxp->edges()) { - MTaskEdge& edge = static_cast(graphEdge); - - LogicMTask* const relativep = edge.furtherMTaskp(); - EdgeHeap::Node& edgeHeapNode = edge.m_edgeHeapNode[inv]; - if (newInclusiveCp > edgeHeapNode.key().m_score) { - relativep->m_edgeHeap[inv].increaseKey(&edgeHeapNode, newInclusiveCp); - } - - const uint64_t critPathCost = relativep->critPathCost(way); - - if (critPathCost >= newInclusiveCp) continue; - - // relativep's critPathCost() is out of step with its longest !wayward edge. - // Schedule that to be resolved. - const uint64_t newVal = newInclusiveCp - critPathCost; - - void*& pendingNodepRef = relativep->m_propagateHeapNodep; - if (PendingHeapNode* const nodep = static_cast(pendingNodepRef)) { - // Already in heap. Increase score if needed. - if (newVal > nodep->key().m_score) m_pendingHeap.increaseKey(nodep, newVal); - continue; - } - - // Add to heap - PendingHeapNode* const nodep = allocNode(); - pendingNodepRef = nodep; - m_pendingHeap.insert(nodep, {relativep, newVal}); - } - } - - void go() { - constexpr GraphWay way{N_Way}; - constexpr GraphWay inv{way.invert()}; - - // m_pending maps each pending vertex to the amount that it wayward - // CP will grow. - // - // We can iterate over the pending set in reverse order, always - // choosing the nodes with the largest pending CP-growth. - // - // The intuition is: if the original seed node had its CP grow by - // 50, the most any wayward node can possibly grow is also 50. So - // for anything pending to grow by 50, we know we can process it - // once and we won't have to grow its CP again on the current pass. - // After we're done with all the grow-by-50s, nothing else will - // grow by 50 again on the current pass, and we can process the - // grow-by-49s and we know we'll only have to process each one - // once. And so on. - // - // This generalizes to multiple seed nodes also. - while (!m_pendingHeap.empty()) { - // Pop max element from heap - PendingHeapNode* const maxp = m_pendingHeap.max(); - m_pendingHeap.remove(maxp); - // Pick up values - LogicMTask* const mtaskp = maxp->key().m_mtaskp; - const uint64_t cpGrowBy = maxp->key().m_score; - // Free the heap node, we are done with it - freeNode(maxp); - mtaskp->m_propagateHeapNodep = nullptr; - // Update the critPathCost of mtaskp, that was out-of-date with respect to its edges - const uint64_t startCp = mtaskp->critPathCost(way); - const uint64_t newCp = startCp + cpGrowBy; - if (VL_UNLIKELY(m_slowAsserts)) { - // Check that CP matches that of the longest edge wayward of vxp. - const uint64_t edgeCp = mtaskp->m_edgeHeap[inv].max()->key().m_score; - UASSERT_OBJ(edgeCp == newCp, mtaskp, "CP doesn't match longest wayward edge"); - // Confirm that we only set each node's CP once. That's an - // important property of PropagateCp which allows it to be far - // faster than a recursive algorithm on some graphs. - const bool first = m_seen.insert(mtaskp).second; - UASSERT_OBJ(first, mtaskp, "Set CP on node twice"); - } - mtaskp->setCritPathCost(way, newCp); - cpHasIncreased(mtaskp, newCp + mtaskp->cost()); - } - - if (VL_UNLIKELY(m_slowAsserts)) m_seen.clear(); - } - -private: - VL_UNCOPYABLE(PropagateCp); }; //============================================================================= @@ -486,31 +273,87 @@ private: // The graph of LogicMTask vertices and MTaskEdge edges, used during multi-threaded scheduling. class OrderMTaskGraph final : public V3Graph { + // MEMBERS OrderMoveGraph& m_moveGraph; // The OrderMoveGraph this graph is built from LogicMTask* const m_entryp; // The singular entry point vertex LogicMTask* const m_exitp; // The singular exit point vertex - // The critical path propagators, one for each direction. Owned here so the algorithms - // operating on this graph (contraction, hazard fixing) share them. - PropagateCp m_forwardPropagator; // Forward propagator - PropagateCp m_reversePropagator; // Reverse propagator + const bool m_slowAsserts; // Take extra time to validate the graph ('--debug-partition') + + // Critical path propagation state. Scratch only: the heap is empty, and no MTask is pending, + // between calls to 'propagate'. The node pool persists to recycle the heap nodes. + PropagatePendingHeap m_pendingHeap; // Heap of MTasks pending a critical path update + PoolAllocator m_pendingNodePool; // Allocator for the heap nodes + + // Generation counter, e.g. for marking the MTasks visited by algorithms + uint64_t m_currentGeneration = 0; // CONSTRUCTOR explicit OrderMTaskGraph(OrderMoveGraph& moveGraph); // Used by build(), hence private VL_UNCOPYABLE(OrderMTaskGraph); VL_UNMOVABLE(OrderMTaskGraph); + // METHODS + + bool pathExistsImpl(LogicMTask* fromp, LogicMTask* top, const MTaskEdge* excludedEdgep); + + // Bring the critical paths of all MTasks wayward of 'mtaskp' in direction N_Way, and those + // cached in the edge heaps on the way, up to date. Call after mutating the graph such that + // only MTasks wayward of 'mtaskp' can have a stale critical path, and the critical path of + // 'mtaskp' itself is already correct. 'mtaskp' is read, but never modified. + // + // Note critical paths can only ever grow: those cached in the edge heaps are heap keys, and a + // heap key can be increased in place, but not decreased. + template + void propagate(LogicMTask* mtaskp) { + ++m_currentGeneration; + propagatePush(mtaskp); + propagateResolve(); + } + // Push the inclusive critical path of 'mtaskp' onto each of its wayward relatives, and add any + // relative left with a stale critical path to the pending heap (out of line below) + template + void propagatePush(LogicMTask* mtaskp); + // Resolve all pending critical path increases (out of line below) + template + void propagateResolve(); + // Part of 'validate' + template + void validateWay() const; + public: // ACCESSORS OrderMoveGraph& moveGraph() const { return m_moveGraph; } LogicMTask* entryp() const { return m_entryp; } LogicMTask* exitp() const { return m_exitp; } - PropagateCp& forwardPropagator() { return m_forwardPropagator; } - PropagateCp& reversePropagator() { return m_reversePropagator; } + bool slowAsserts() const { return m_slowAsserts; } // METHODS uint64_t totalCost() const; // O(V), called once + // True if there's a path from 'fromp' to 'top' excluding 'excludedEdgep', false otherwise. + // 'excludedEdgep' may be nullptr in which case no edge is excluded. If 'excludedEdgep' is + // non-nullptr it must connect fromp and top. + bool pathExists(LogicMTask* fromp, LogicMTask* top, const MTaskEdge* excludedEdgep) { + ++m_currentGeneration; + return pathExistsImpl(fromp, top, excludedEdgep); + } + + // Add an edge to the graph, update impacted critical paths + void addEdge(LogicMTask* fromp, LogicMTask* top); + + // Merge 'donorp' into 'recipientp': move the contents and all edges of 'donorp' onto + // 'recipientp', update impacted critical paths, then delete 'donorp'. The edge connecting the + // two (if any) becomes internal to the merged MTask and is deleted, as is one of each pair of + // edges the two have to a common relative. Note this deletes edges, so the caller must have + // released any auxiliary data it attached to them via their user pointer. + void mergeMTasks(LogicMTask* recipientp, LogicMTask* donorp); + + // Do an EXPENSIVE check that the maintained critical paths, including the ones cached in the + // edge heaps, match those implied by the current edges of the graph, and that the graph itself + // is consistent. Does nothing unless 'slowAsserts', so it is safe to call unconditionally. + void validate() const; + // STATIC METHODS // Build an MTask graph from 'moveGraph' static std::unique_ptr build(OrderMoveGraph& moveGraph) VL_MT_DISABLED; @@ -523,9 +366,9 @@ public: //============================================================================= // MTaskEdge method definitions (need the full definition of LogicMTask) -MTaskEdge::MTaskEdge(OrderMTaskGraph* graphp, LogicMTask* fromp, LogicMTask* top, int weight) - : V3GraphEdge{graphp, fromp, top, weight} { - fromp->addRelativeMTask(top); +MTaskEdge::MTaskEdge(OrderMTaskGraph* graphp, LogicMTask* fromp, LogicMTask* top) + : V3GraphEdge{graphp, fromp, top, 1} { + fromp->addDependent(top); fromp->addRelativeEdge(this); top->addRelativeEdge(this); } @@ -537,13 +380,4 @@ LogicMTask* MTaskEdge::furtherMTaskp() const { LogicMTask* MTaskEdge::fromMTaskp() const { return static_cast(fromp()); } LogicMTask* MTaskEdge::toMTaskp() const { return static_cast(top()); } -void MTaskEdge::resetCriticalPaths() { - LogicMTask* const fromp = fromMTaskp(); - LogicMTask* const top = toMTaskp(); - fromp->removeRelativeEdge(this); - top->removeRelativeEdge(this); - fromp->addRelativeEdge(this); - top->addRelativeEdge(this); -} - #endif // Guard diff --git a/src/V3OrderParallel.cpp b/src/V3OrderParallel.cpp index 0f20eb11d..a68ceed97 100644 --- a/src/V3OrderParallel.cpp +++ b/src/V3OrderParallel.cpp @@ -71,6 +71,9 @@ static std::unique_ptr partition(OrderMoveGraph& moveGraph) { mTaskGraphp->hashGraphDebug("MTask graph after contract()"); } + // Note the graph is only mutated by generic V3Graph algorithms from here on. These neither + // maintain the critical paths of the MTasks, nor create MTaskEdges when rerouting, so the + // critical paths are stale below, and the graph must not be handed back to OrderMTaskGraph. mTaskGraphp->removeTransitiveEdges(); mTaskGraphp->hashGraphDebug("MTask graph after removeTransitiveEdges()"); diff --git a/src/V3PairingHeap.h b/src/V3PairingHeap.h index f435ac75f..7beb1ee41 100644 --- a/src/V3PairingHeap.h +++ b/src/V3PairingHeap.h @@ -25,8 +25,8 @@ //============================================================================= // Pairing heap (max-heap) with increase key and delete. // -// While this is written as a generic data structure, it's interface and -// implementation is finely tuned for use by V3Partition, and is critical +// While this is written as a generic data structure, its interface and +// implementation is finely tuned for use by parallel scheduling, and is critical // to Verilation performance, so be very careful changing anything or adding any // new operations that would impact either memory usage, or performance of the // existing operations. This data structure is fully deterministic, meaning diff --git a/src/V3PoolAllocator.h b/src/V3PoolAllocator.h new file mode 100644 index 000000000..436ffa308 --- /dev/null +++ b/src/V3PoolAllocator.h @@ -0,0 +1,85 @@ +// -*- mode: C++; c-file-style: "cc-mode" -*- +//************************************************************************* +// DESCRIPTION: Verilator: Chunked pool allocator +// +// Code available from: https://verilator.org +// +//************************************************************************* +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of either the GNU Lesser General Public License Version 3 +// or the Perl Artistic License Version 2.0. +// SPDX-FileCopyrightText: 2003-2026 Wilson Snyder +// SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0 +// +//************************************************************************* + +#ifndef VERILATOR_V3POOLALLOCATOR_H_ +#define VERILATOR_V3POOLALLOCATOR_H_ + +#include "config_build.h" +#include "verilatedos.h" + +#include +#include +#include +#include +#include + +// Hands out elements of a single type, allocated 'N_ChunkSize' at a time for +// efficiency. Released elements are recycled via a free list stored in the +// same storage. All memory is released when the pool is destroyed, so the +// elements it handed out must not be used beyond its lifetime, and must all +// have been released by then. +template +class PoolAllocator final { + static_assert(N_ChunkSize > 0, "Chunk size must be non-zero"); + + // A slot of storage, holding either a live element, or a link in the free list. + union Slot final { + T_Elem m_elem; // Storage for the allocated element + Slot* m_nextFreep; // Link to the next free slot + Slot() {} + ~Slot() {} + }; + + // MEMBERS + Slot* m_freep = nullptr; // Head of the free list + std::vector> m_allocated; // The allocated chunks + +public: + // CONSTRUCTORS + PoolAllocator() = default; + VL_UNCOPYABLE(PoolAllocator); + VL_UNMOVABLE(PoolAllocator); + + // METHODS + // Allocate an element, constructed with the given arguments + template + T_Elem* alloc(Args&&... args) { + // If no free slots available, then make some + if (!m_freep) { + // Allocate in chunks for efficiency + m_allocated.emplace_back(new Slot[N_ChunkSize]); + // Chain the new slots into the free list + Slot* const chunkp = m_allocated.back().get(); + for (size_t i = 1; i < N_ChunkSize; ++i) chunkp[i - 1].m_nextFreep = &chunkp[i]; + chunkp[N_ChunkSize - 1].m_nextFreep = nullptr; + m_freep = chunkp; + } + // Free slots are available, pick up the first one + Slot* const slotp = m_freep; + m_freep = slotp->m_nextFreep; + return new (&slotp->m_elem) T_Elem{std::forward(args)...}; + } + + // Destroy an element, and return its slot for future allocation + void free(T_Elem* elemp) { + elemp->~T_Elem(); + Slot* const slotp = reinterpret_cast(elemp); + slotp->m_nextFreep = m_freep; + m_freep = slotp; + } +}; + +#endif // Guard