diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ef11e60ec..e54602de4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -157,6 +157,7 @@ set(HEADERS V3ParseImp.h V3PchAstMT.h V3PchAstNoMT.h + V3PoolAllocator.h V3PreExpr.h V3PreLex.h V3PreProc.h diff --git a/src/V3Graph.cpp b/src/V3Graph.cpp index 2ce4a0f26..96772bf02 100644 --- a/src/V3Graph.cpp +++ b/src/V3Graph.cpp @@ -79,31 +79,6 @@ void V3GraphVertex::rerouteEdges(V3Graph* graphp) { unlinkEdges(graphp); } -template -V3GraphEdge* V3GraphVertex::findConnectingEdgep(V3GraphVertex* waywardp) { - // O(edges) linear search. Searches search both nodes' edge lists in - // parallel. The lists probably aren't _both_ huge, so this is - // unlikely to blow up even on fairly nasty graphs. - constexpr GraphWay way{N_Way}; - constexpr GraphWay inv = way.invert(); - auto& aEdges = this->edges(); - auto aIt = aEdges.begin(); - auto aEnd = aEdges.end(); - auto& bEdges = waywardp->edges(); - auto bIt = bEdges.begin(); - auto bEnd = bEdges.end(); - while (aIt != aEnd && bIt != bEnd) { - V3GraphEdge& aedge = *aIt++; - if (aedge.furtherp() == waywardp) return &aedge; - V3GraphEdge& bedge = *bIt++; - if (bedge.furtherp() == this) return &bedge; - } - return nullptr; -} - -template V3GraphEdge* V3GraphVertex::findConnectingEdgep(V3GraphVertex*); -template V3GraphEdge* V3GraphVertex::findConnectingEdgep(V3GraphVertex*); - // cppcheck-has-bug-suppress constParameter void V3GraphVertex::v3errorEnd(const std::ostringstream& str) const // LCOV_EXCL_START VL_RELEASE(V3Error::s().m_mutex) { diff --git a/src/V3Graph.h b/src/V3Graph.h index a640a3df8..a954d43b1 100644 --- a/src/V3Graph.h +++ b/src/V3Graph.h @@ -311,10 +311,6 @@ public: VL_RELEASE(V3Error::s().m_mutex) VL_MT_DISABLED; /// Edges are routed around this vertex to point from "from" directly to "to" void rerouteEdges(V3Graph* graphp) VL_MT_DISABLED; - // Find the edge connecting this vertex to the given vertex. - // If edge is not found returns nullptr. O(edges) performance. - template - V3GraphEdge* findConnectingEdgep(V3GraphVertex* otherp) VL_MT_DISABLED; }; std::ostream& operator<<(std::ostream& os, V3GraphVertex* vertexp) VL_MT_DISABLED; diff --git a/src/V3OrderMTaskContraction.cpp b/src/V3OrderMTaskContraction.cpp index 6d4ea3c8a..af49acd72 100644 --- a/src/V3OrderMTaskContraction.cpp +++ b/src/V3OrderMTaskContraction.cpp @@ -14,11 +14,9 @@ // //************************************************************************* // -// Coarsens the fine-grained MTask graph produced by the partitioner by -// repeatedly contracting MTasks (merging along an edge, or merging two -// "sibling" MTasks) until a critical-path score limit is reached. Driven by -// the partitioner in V3OrderParallel.cpp via OrderMTaskGraph::contract, -// declared in V3OrderMTaskGraph.h. +// Coarsens the fine-grained MTask graph by repeatedly contracting MTasks, +// merging along an edge, or merging two "sibling" MTasks until a +// critical-path limit is reached. // //************************************************************************* @@ -26,9 +24,9 @@ #include "V3Global.h" #include "V3Graph.h" -#include "V3GraphStream.h" #include "V3OrderMTaskGraph.h" #include "V3PairingHeap.h" +#include "V3PoolAllocator.h" #include #include @@ -44,66 +42,38 @@ class MergeCandidate; class SiblingMC; class EdgeMC; -// ###################################################################### -// Partitioner tunable settings: -// -// Before describing these settings, a bit of background: -// -// Early during the development of the partitioner, V3Split was failing to -// split large always blocks (with ~100K assignments) so we had to handle -// very large vertices with ~100K incoming and outgoing edges. -// -// The partitioner attempts to deal with such densely connected -// graphs. Some of the tuning parameters below reference "huge vertices", -// that's what they're talking about, vertices with tens of thousands of -// edges in and out. Whereas most graphs have only tens of edges in and out -// of most vertices. -// -// V3Split has since been fixed to more reliably split large always -// blocks. It's kind of an open question whether the partitioner must -// handle huge nodes gracefully. Maybe not! But it still can, given -// appropriate tuning. +//###################################################################### +// Tunable settings -// PART_SIBLING_EDGE_LIMIT (integer) -// // Arbitrarily limit the number of edges on a single vertex that will be // considered when enumerating siblings, to the given value. This protects -// the partitioner runtime in the presence of huge vertices. +// the runtime in the presence of huge vertices. // -// The sibling-merge is less important than the edge merge. (You can -// totally disable the sibling merge and get halfway decent partitions; you -// can't disable edge merges, those are fundamental to the process.) So, -// skipping the enumeration of some siblings on a few vertices does not -// have a large impact on the result of the partitioner. +// The sibling-merge is less important than the edge merge. Sibling merges +// can be disabled and result in halfway decent coarsening. Edge merges +// cannot be disabled, those are fundamental to the process. So, skipping +// the enumeration of some siblings on a few vertices does not have a large +// impact on the result of the partitioner. // -// If your vertices are small, the limit (at 26) approaches a no-op. Hence -// there's basically no cost to applying this limit even when we don't -// expect huge vertices. +// If vertices are small, the limit (at 26) approaches a no-op. Hence +// there's basically no cost to applying this limit even when no huge +// vertices are expected. // -// If you don't care about partitioner runtime and you want the most -// aggressive partition, set the limit very high. If you have huge -// vertices, leave this as is. +// If runtime is not a concern and the most precise result is desired, +// set the limit very high. constexpr unsigned PART_SIBLING_EDGE_LIMIT = 26; -// Don't produce more than a certain maximum number of MTasks. This helps -// the TSP variable sort not to blow up (a concern for some of the tests) -// and we probably don't want a huge number of MTasks in practice anyway -// (50 to 100 is typical.) -// // If the user doesn't give one with '--threads-max-mtasks', we'll set the -// maximum # of MTasks to -// (# of threads * PART_DEFAULT_MAX_MTASKS_PER_THREAD) +// maximum # of MTasks to (# of threads * PART_DEFAULT_MAX_MTASKS_PER_THREAD) constexpr unsigned PART_DEFAULT_MAX_MTASKS_PER_THREAD = 50; -// end tunables. - //###################################################################### // MTask utility classes struct MergeCandidateKey final { // Note: Structure layout chosen to minimize padding in PairingHeap<*>::Node - uint64_t m_id; // Unique ID part of edge score - uint64_t m_score; // Score part of ID + uint64_t m_id; // Unique ID part of the key + uint64_t m_score; // Score part of the key bool operator<(const MergeCandidateKey& other) const { // First by Score then by ID, but notice that we want minimums using a max-heap, so reverse return m_score > other.m_score || (m_score == other.m_score && m_id > other.m_id); @@ -120,11 +90,10 @@ class MergeCandidate VL_NOT_FINAL : public MergeCandidateHeapNode { friend class SiblingMC; friend class EdgeMC; - // This structure is extremely hot. To save 8 bytes we pack - // one bit indicating removedFromSb with the id. To save another - // 8 bytes by not having a virtual function table, we implement the - // few polymorphic methods over the two known subclasses explicitly, - // using another bit of the id to denote the actual subtype. + // This structure is extremely hot. To save 8 bytes by not having a virtual + // function table, we implement the few polymorphic methods over the two + // known subclasses explicitly, using a bit of the id to denote the actual + // subtype. // By using the bottom bits for flags, we can still use < to compare IDs without masking. // <63:1> Serial number for ordering, <0> subtype (SiblingMC) @@ -145,10 +114,14 @@ public: // METHODS SiblingMC* toSiblingMC(); // Instead of cast<>/as<> EdgeMC* toEdgeMC(); // Instead of cast<>/as<> - bool mergeWouldCreateCycle() const; // Instead of virtual method + bool mergeWouldCreateCycle(OrderMTaskGraph& graph); - inline void rescore(); + // The current score of this candidate, which changes as the graph is contracted + inline uint64_t currentScore(); + // The score this candidate was last given, which is its key in the scoreboard heap uint64_t score() const { return m_key.m_score; } + // Set the score of this candidate to its current value. Only valid while it is not in a heap. + void updateScore() { m_key.m_score = currentScore(); } static MergeCandidate* heapNodeToElem(MergeCandidateHeapNode* nodep) { return static_cast(nodep); @@ -185,7 +158,6 @@ public: LogicMTask* ap() const { return m_ap; } LogicMTask* bp() const { return m_bp; } - bool mergeWouldCreateCycle() const; }; static_assert(!std::is_polymorphic::value, "Should not have a vtable"); @@ -203,7 +175,6 @@ public: // METHODS MTaskEdge* edgep() const { return m_edgep; } - bool mergeWouldCreateCycle() const; }; static_assert(!std::is_polymorphic::value, "Should not have a vtable"); @@ -236,44 +207,46 @@ SiblingMC* MergeCandidate::toSiblingMC() { EdgeMC* MergeCandidate::toEdgeMC() { return isSiblingMC() ? nullptr : static_cast(this); } -// Normally this would be a virtual function, but we save space by not having a vtable, -// and we know we only have 2 possible subclasses. -bool MergeCandidate::mergeWouldCreateCycle() const { - return isSiblingMC() ? static_cast(this)->mergeWouldCreateCycle() - : static_cast(this)->mergeWouldCreateCycle(); -} - -static uint64_t siblingScore(const SiblingMC* sibsp) { - const LogicMTask* const ap = sibsp->ap(); - const LogicMTask* const bp = sibsp->bp(); - const uint64_t mergedCpCostFwd - = std::max(ap->critPathCost(GraphWay::FORWARD), bp->critPathCost(GraphWay::FORWARD)); - const uint64_t mergedCpCostRev - = std::max(ap->critPathCost(GraphWay::REVERSE), bp->critPathCost(GraphWay::REVERSE)); - return mergedCpCostRev + mergedCpCostFwd + ap->cost() + bp->cost(); -} - -static uint64_t edgeScore(const MTaskEdge* edgep) { - // Score this edge. Lower is better. The score is the new local CP - // length if we merge these MTasks. ("Local" means the longest - // critical path running through the merged node.) - const LogicMTask* const top = edgep->toMTaskp(); - const LogicMTask* const fromp = edgep->fromMTaskp(); - const uint64_t mergedCpCostFwd = std::max(fromp->critPathCost(GraphWay::FORWARD), - top->critPathCostWithout(edgep)); - const uint64_t mergedCpCostRev = std::max(fromp->critPathCostWithout(edgep), - top->critPathCost(GraphWay::REVERSE)); - return mergedCpCostRev + mergedCpCostFwd + fromp->cost() + top->cost(); -} - -void MergeCandidate::rescore() { +bool MergeCandidate::mergeWouldCreateCycle(OrderMTaskGraph& graph) { + // Sibling merge: merging creates a cycle if either sibling is reachable from the other if (const SiblingMC* const sibp = toSiblingMC()) { - m_key.m_score = siblingScore(sibp); - } else { + return graph.pathExists(sibp->ap(), sibp->bp(), nullptr) + || graph.pathExists(sibp->bp(), sibp->ap(), nullptr); + } + + // Edge merge: merging creates a cycle if there is another path between the two MTasks + MTaskEdge* const edgep = toEdgeMC()->edgep(); + return graph.pathExists(edgep->fromMTaskp(), edgep->toMTaskp(), edgep); +} + +uint64_t MergeCandidate::currentScore() { + // Score this candidate. The score is the new local CP length if we merge this candidate. + // ("Local" means the longest critical path running through the merged node.) + + // Sibling merge + if (const SiblingMC* const sibp = toSiblingMC()) { + const LogicMTask* const ap = sibp->ap(); + const LogicMTask* const bp = sibp->bp(); + const uint64_t mergedCpFwd + = std::max(ap->cpExclusive(), bp->cpExclusive()); + const uint64_t mergedCpRev + = std::max(ap->cpExclusive(), bp->cpExclusive()); + return mergedCpRev + mergedCpFwd + ap->cost() + bp->cost(); + } + + // Edge merge + { + MTaskEdge* const edgep = toEdgeMC()->edgep(); + const LogicMTask* const fromp = edgep->fromMTaskp(); + const LogicMTask* const top = edgep->toMTaskp(); + const uint64_t mergedCpFwd = std::max(fromp->cpExclusive(), + top->cpExclusiveWithout(edgep)); + const uint64_t mergedCpRev = std::max(fromp->cpExclusiveWithout(edgep), + top->cpExclusive()); // Give a slight preference to sibling merges by increasing the cost of edge merges. // This biases towards sibling merges in case they are equal score with edge merges. // This avoid a central node growing while many leaves remain due to edge merges. - m_key.m_score = 1 + edgeScore(static_cast(this)->edgep()); + return 1 + mergedCpRev + mergedCpFwd + fromp->cost() + top->cost(); } } @@ -296,59 +269,43 @@ void SiblingMC::unlinkA() { void SiblingMC::unlinkB() { mtaskData(m_bp).bSiblingMCs.unlink(this); } -// cppcheck-suppress duplInheritedMember -bool SiblingMC::mergeWouldCreateCycle() const { - return (LogicMTask::pathExistsFrom(m_ap, m_bp, nullptr) - || LogicMTask::pathExistsFrom(m_bp, m_ap, nullptr)); -} - -// cppcheck-suppress duplInheritedMember -bool EdgeMC::mergeWouldCreateCycle() const { - return LogicMTask::pathExistsFrom(m_edgep->fromMTaskp(), m_edgep->toMTaskp(), m_edgep); -} - -// Scoreboard of MTask merge candidates. Owns the lifetime of the merge candidate objects: callers -// add/remove candidates via the methods below and never allocate or free them directly. For edges -// this maintains the invariant that an MTaskEdge has an associated EdgeMC (held in its userp()), -// if and only if it is currently on the scoreboard. +// Scoreboard of MTask merge candidates. // -// This is essentially a heap that can be hinted that some elements have changed keys, at which -// point those elements are deferred as 'unknown' until the next 'rescore' call. We use the -// generic PairingHeap, relying on its internal structure. For efficiency, the merge candidates are -// themselves the heap nodes (MergeCandidate derives from PairingHeap::Node), so -// a candidate can be on at most one scoreboard. +// This is a heap, sorted by the local critical path that would result from merging the candidate, +// that is: the longest critical path running through the merged MTask. Merges proceed by picking +// the candidate yielding the lowest such critical path, which is the merge that does the least +// damage: a merge can only ever lengthen paths, and one whose local critical path is no longer +// than the current global critical path cannot lengthen that at all. +// +// A candidate's score changes as the graph is contracted, so the score a candidate is in the heap +// with is only the score it was last given, which is a lower bound on its current score. This +// makes the top of the heap a lower bound on the score of every candidate, so the caller can pick +// the best candidate by checking whether the top still has the score it was given, and +// rescoring it if not (see the contraction loop). +// +// The scoreboard owns the lifetime of the merge candidate objects: callers add/remove candidates +// via the methods below and never allocate or free them directly. For edges this maintains the +// invariant that an MTaskEdge has an associated EdgeMC (held in its userp()), if and only if it is +// currently on the scoreboard. class MergeCandidateScoreboard final { // TYPES using Heap = PairingHeap; - using Node = Heap::Node; - using Link = Heap::Link; // MEMBERS - Heap m_known; // The heap of candidates with known scores - Link m_unknown; // List of candidates with unknown scores + Heap m_heap; // The heap of candidates, keyed on their score (see class comment above) + PoolAllocator m_edgeMCPool; // Allocator for the edge merge candidates + PoolAllocator m_siblingMCPool; // Allocator for the sibling merge candidates // METHODS - void addUnknown(MergeCandidate* nodep) { - // Just prepend it to the list of unknown entries - nodep->m_next.link(m_unknown.unlink()); - m_unknown.linkNonNull(nodep); - // We mark nodes on the unknown list by making their child pointer point to themselves - nodep->m_kids.m_ptr = nodep; + // Set the score of a candidate and add it to the heap. The score is computed from the critical + // paths of the MTasks, which are up to date while the graph is being contracted. + void insert(MergeCandidate* nodep) { + nodep->updateScore(); + m_heap.insert(nodep); } - // Add a freshly created candidate. Not returned by 'best' before the next 'rescore' call. - void add(MergeCandidate* nodep) { addUnknown(nodep); } - // Remove a candidate from the scoreboard. - void remove(MergeCandidate* nodep) { - if (nodep->m_kids.m_ptr == nodep) { - // Node is on the unknown list, replace with next - nodep->replaceWith(nodep->m_next.unlink()); - return; - } - // Node is in the known heap, remove it - m_known.remove(nodep); - } + void remove(MergeCandidate* nodep) { m_heap.remove(nodep); } public: // CONSTRUCTORS @@ -356,475 +313,94 @@ public: ~MergeCandidateScoreboard() = default; VL_UNCOPYABLE(MergeCandidateScoreboard); - // The candidate with the best (lowest) known score, or nullptr if none have a known score. - // This does not automatically 'rescore'; the caller must 'rescore' to reflect all candidates. - MergeCandidate* best() const { return MergeCandidate::heapNodeToElem(m_known.max()); } + // The candidate with the lowest score it was given, or nullptr if the scoreboard is empty. + // Note this is only a lower bound on the best current score, see the class comment above. + MergeCandidate* best() const { return MergeCandidate::heapNodeToElem(m_heap.max()); } - // Tell the scoreboard a candidate's score may have changed. Its score becomes 'unknown' and it - // will not be returned by 'best' until the next 'rescore'. - void hintScoreChanged(MergeCandidate* nodep) { - // If it's already in the unknown list, then nothing to do - if (nodep->m_kids.m_ptr == nodep) return; - // Otherwise it was in the heap, remove it - m_known.remove(nodep); - // Prepend it to the unknown list - addUnknown(nodep); + // Update the score of a candidate to its current value, and reposition it in the heap + void rescore(MergeCandidate* nodep) { + remove(nodep); + insert(nodep); } - // True if there are candidates with an unknown score - bool needsRescore() const { return m_unknown; } - // True if the given candidate's score is unknown - static bool needsRescore(const MergeCandidate* nodep) { return nodep->m_kids.m_ptr == nodep; } - - // For each candidate whose score is unknown, recompute the score and add to the known heap - void rescore() { - for (Node *nodep = m_unknown.unlink(), *nextp; nodep; nodep = nextp) { - // Pick up next - nextp = nodep->m_next.ptr(); - // Reset pointers - nodep->m_next.m_ptr = nullptr; - nodep->m_kids.m_ptr = nullptr; - nodep->m_ownerpp = nullptr; - // Re-compute the score of the candidate - MergeCandidate::heapNodeToElem(nodep)->rescore(); - // Re-insert into the heap - m_known.insert(nodep); - } - } - - // Create the merge candidate for 'edgep' and add it to the scoreboard (out-of-line below) - void addEdge(MTaskEdge* edgep) { + // Create a merge candidate for 'edgep' and add it to the scoreboard + void addEdgeMC(MTaskEdge* edgep) { UDEBUGONLY(UASSERT(!edgep->userp(), "Edge already has a merge candidate");); - EdgeMC* const edgeMCp = new EdgeMC{edgep}; + EdgeMC* const edgeMCp = m_edgeMCPool.alloc(edgep); edgep->userp(edgeMCp); - add(edgeMCp); + insert(edgeMCp); } - // Remove 'edgep's merge candidate from the scoreboard and delete it (out-of-line below) - void removeEdge(MTaskEdge* edgep) { + // Remove 'edgep's merge candidate from the scoreboard and release it + void removeEdgeMC(MTaskEdge* edgep) { EdgeMC* const edgeMCp = edgeMC(edgep); UDEBUGONLY(UASSERT(edgeMCp, "Edge has no merge candidate");); edgep->userp(nullptr); remove(edgeMCp); - VL_DO_DANGLING(delete edgeMCp, edgeMCp); + VL_DO_DANGLING(m_edgeMCPool.free(edgeMCp), edgeMCp); } // Create a sibling merge candidate for 'ap' and 'bp' and add it to the scoreboard - void addSibling(LogicMTask* ap, LogicMTask* bp) { add(new SiblingMC{ap, bp}); } - // Remove sibling merge candidate 'smcp' from the scoreboard and delete it - void removeSibling(SiblingMC* smcp) { + void addSiblingMC(LogicMTask* ap, LogicMTask* bp) { insert(m_siblingMCPool.alloc(ap, bp)); } + // Remove sibling merge candidate 'smcp' from the scoreboard and release it + void removeSiblingMC(SiblingMC* smcp) { remove(smcp); smcp->unlinkA(); smcp->unlinkB(); - VL_DO_DANGLING(delete smcp, smcp); + VL_DO_DANGLING(m_siblingMCPool.free(smcp), smcp); + } + + // Remove the given merge candidate, whichever kind it is, from the scoreboard and release it + void removeMC(MergeCandidate* nodep) { + if (SiblingMC* const smcp = nodep->toSiblingMC()) { + removeSiblingMC(smcp); + } else { + removeEdgeMC(nodep->toEdgeMC()->edgep()); + } } }; //###################################################################### - -// Look at vertex costs (in one way) to form critical paths for each -// vertex. -template -static void partInitHalfCriticalPaths(V3Graph& mTaskGraph, bool checkOnly) { - constexpr GraphWay way{N_Way}; - constexpr GraphWay rev = way.invert(); - GraphStreamUnordered order{&mTaskGraph, way}; - for (const V3GraphVertex* vertexp; (vertexp = order.nextp());) { - const LogicMTask* const mtaskcp = static_cast(vertexp); - LogicMTask* const mtaskp = const_cast(mtaskcp); - uint64_t cpCost = 0; -#if VL_DEBUG - std::unordered_set relatives; -#endif - for (const V3GraphEdge& edge : vertexp->edges()) { -#if VL_DEBUG - // Run a few asserts on the initial mtask graph, - // while we're iterating through... - UASSERT_OBJ(edge.weight() != 0, mtaskp, "Should be no cut edges in MTask graph"); - UASSERT_OBJ(relatives.find(edge.furtherp()) == relatives.end(), mtaskp, - "Should be no redundant edges in MTask graph"); - relatives.insert(edge.furtherp()); -#endif - const LogicMTask* const relativep = static_cast(edge.furtherp()); - cpCost = std::max(cpCost, (relativep->critPathCost(way) + relativep->cost())); - } - if (checkOnly) { - UASSERT(mtaskp->critPathCost(way) == cpCost, "Calculation error in scoring"); - } else { - mtaskp->setCritPathCost(way, cpCost); - } - } -} - -// Look at vertex costs to form critical paths for each vertex. -static void partInitCriticalPaths(V3Graph& mTaskGraph) { - partInitHalfCriticalPaths(mTaskGraph, false); - partInitHalfCriticalPaths(mTaskGraph, false); - - // Reset all MTaskEdges so that 'm_edges' will show correct CP numbers. - // They would have been all zeroes on initial creation of the MTaskEdges. - for (V3GraphVertex& vtx : mTaskGraph.vertices()) { - for (V3GraphEdge& edge : vtx.outEdges()) edge.as()->resetCriticalPaths(); - } -} - -// Do an EXPENSIVE check to make sure that all incremental CP updates have -// gone correctly. -static void partCheckCriticalPaths(V3Graph& mTaskGraph) { - partInitHalfCriticalPaths(mTaskGraph, true); - partInitHalfCriticalPaths(mTaskGraph, true); - for (const V3GraphVertex& vtx : mTaskGraph.vertices()) { - const LogicMTask& mtask = static_cast(vtx); - mtask.checkRelativesCp(); - mtask.checkRelativesCp(); - } -} - -//###################################################################### -// Contraction - -// Perform edge or sibling contraction on the partition graph +// Contraction - Perform greedy edge or sibling merges on the MTask graph class Contraction final { - // TYPES - // New CP information for mtaskp reflecting an upcoming merge - struct NewCp final { - uint64_t cp; - uint64_t propagateCp; - bool propagate; - }; - // MEMBERS OrderMTaskGraph& m_mTaskGraph; // The Mtask graph - uint64_t m_scoreLimit; // Sloppy score allowed when picking merges - // Next score rescore at - uint64_t m_scoreLimitBeforeRescore = std::numeric_limits::max(); - unsigned m_mergesSinceRescore = 0; // Merges since last rescore - const bool m_slowAsserts{v3Global.opt.debugPartition()}; // Take extra time to validate steps + uint64_t m_scoreLimit; // Critical path limit for merges MergeCandidateScoreboard m_sb; // Scoreboard // Auxiliary per-MTask data (the SiblingMC lists) attached to each MTask via its user pointer. - // Owned here for the lifetime of this Contraction. A single array, as the number of MTasks is - // fixed for that lifetime: merging only ever deletes vertices, never creates them. + // Owned here for the lifetime of this Contraction. A single array, as the number of MTasks + // can only decrease during contraction. std::unique_ptr m_mtaskDatap; - // Singular source vertex of the OrderMTaskGraph - LogicMTask* const m_entryMTaskp = m_mTaskGraph.entryp(); - // Singular sink vertex of the dependency graph - LogicMTask* const m_exitMTaskp = m_mTaskGraph.exitp(); - - // Merge edges from a LogicMtask, keeping the merge candidate scoreboard in sync. - static void partRedirectEdgesFrom(V3Graph& graph, LogicMTask* recipientp, LogicMTask* donorp, - MergeCandidateScoreboard& sb) { - // This code removes adjacent edges. When this occurs, mark it in need - // of a rescore, in case its score has fallen and we need to move it up - // toward the front of the scoreboard. - // - // Wait, what? Shouldn't the scores only increase as we merge nodes? Well - // that's almost true. But there is one exception. - // - // Suppose we have A->B, B->C, and A->C. - // - // The A->C edge is a "transitive" edge. It's ineligible to be merged, as - // the merge would create a cycle. We score it on the scoreboard like any - // other edge. - // - // However, our "score" estimate for A->C is bogus, because the forward - // critical path to C and the reverse critical path to A both contain the - // same node (B) so we overestimate the score of A->C. At first this - // doesn't matter, since transitive edges aren't eligible to merge anyway. - // - // Later, suppose the edge contractor decides to merge the B->C edge, with - // B donating all its incoming edges into C, say. (So we reach this - // function.) - // - // With B going away, the A->C edge will no longer be transitive and it - // will become eligible to merge. But if we don't mark it for rescore, - // it'll stay in the scoreboard with its old (overestimate) score. We'll - // merge it too late due to the bogus score. When we finally merge it, we - // fail the assert in the main edge contraction loop which checks that the - // actual score did not fall below the scoreboard's score. - // - // Another way of stating this: this code ensures that scores of - // non-transitive edges only ever increase. - - // Process outgoing edges - while (MTaskEdge* const edgep = static_cast(donorp->outEdges().frontp())) { - LogicMTask* const relativep = edgep->toMTaskp(); - - relativep->removeRelativeEdge(edgep); - - if (recipientp->hasRelativeMTask(relativep)) { - // An edge already exists between recipient and relative of donor. - // Mark it in need of a rescore - // The donor edge is going away, so remove it from the scoreboard - if (edgep->userp()) sb.removeEdge(edgep); - MTaskEdge* const existMTaskEdgep = static_cast( - recipientp->findConnectingEdgep(relativep)); - UDEBUGONLY(UASSERT(existMTaskEdgep, "findConnectingEdge didn't find edge");); - // The existing edge is no longer transitive, so may need a rescore - if (EdgeMC* const existEdgeMCp = edgeMC(existMTaskEdgep)) { - sb.hintScoreChanged(existEdgeMCp); - } - VL_DO_DANGLING(edgep->unlinkDelete(), edgep); - } else { - // No existing edge between recipient and relative of donor. - // Redirect the edge from donor<->relative to recipient<->relative. - edgep->relinkFromp(recipientp); - recipientp->addRelativeMTask(relativep); - recipientp->stealRelativeEdge(edgep); - relativep->addRelativeEdge(edgep); - // The redirected edge is a merge candidate again - if (EdgeMC* const edgeMCp = edgeMC(edgep)) { - sb.hintScoreChanged(edgeMCp); - } else { - sb.addEdge(edgep); - } - } - } - - // Process incoming edges - while (MTaskEdge* const edgep = static_cast(donorp->inEdges().frontp())) { - LogicMTask* const relativep = edgep->fromMTaskp(); - - relativep->removeRelativeMTask(donorp); - relativep->removeRelativeEdge(edgep); - - if (relativep->hasRelativeMTask(recipientp)) { - // An edge already exists between recipient and relative of donor. - // Mark it in need of a rescore - // The donor edge is going away, so remove it from the scoreboard - if (edgep->userp()) sb.removeEdge(edgep); - MTaskEdge* const existMTaskEdgep = static_cast( - recipientp->findConnectingEdgep(relativep)); - UDEBUGONLY(UASSERT(existMTaskEdgep, "findConnectingEdge didn't find edge");); - // The existing edge is no longer transitive, so may need a rescore - if (EdgeMC* const existEdgeMCp = edgeMC(existMTaskEdgep)) { - sb.hintScoreChanged(existEdgeMCp); - } - VL_DO_DANGLING(edgep->unlinkDelete(), edgep); - } else { - // No existing edge between recipient and relative of donor. - // Redirect the edge from donor<->relative to recipient<->relative. - edgep->relinkTop(recipientp); - relativep->addRelativeMTask(recipientp); - relativep->addRelativeEdge(edgep); - recipientp->stealRelativeEdge(edgep); - // The redirected edge is a merge candidate again - if (EdgeMC* const edgeMCp = edgeMC(edgep)) { - sb.hintScoreChanged(edgeMCp); - } else { - sb.addEdge(edgep); - } - } - } - - // Remove donorp from the graph - VL_DO_DANGLING(donorp->unlinkDelete(&graph), donorp); + // Add merge candidates for all edges of 'mtaskp' to the scoreboard + void addEdgeMCs(LogicMTask* mtaskp) { + for (V3GraphEdge& e : mtaskp->outEdges()) m_sb.addEdgeMC(static_cast(&e)); + for (V3GraphEdge& e : mtaskp->inEdges()) m_sb.addEdgeMC(static_cast(&e)); } - template - NewCp newCp(const LogicMTask* mtaskp, const LogicMTask* otherp, const MTaskEdge* mergeEdgep) { - constexpr GraphWay way{N_Way}; - // Return new wayward-CP for mtaskp reflecting its upcoming merge - // with otherp. Set 'result.propagate' if mtaskp's wayward - // relatives will see a new wayward CP from this merge. - uint64_t newCp; - if (mergeEdgep) { - if (mtaskp == mergeEdgep->furtherp()) { - newCp = std::max(otherp->critPathCost(way), - mtaskp->critPathCostWithout(mergeEdgep)); - } else { - newCp = std::max(mtaskp->critPathCost(way), - otherp->critPathCostWithout(mergeEdgep)); - } - } else { - newCp = std::max(otherp->critPathCost(way), mtaskp->critPathCost(way)); + // Remove the merge candidates of all edges of 'mtaskp' from the scoreboard. + void removeEdgeMCs(LogicMTask* mtaskp) { + // Note not all edges have a merge candidate: + // those rejected by the main contraction loop had theirs removed there. + for (V3GraphEdge& edge : mtaskp->outEdges()) { + MTaskEdge* const edgep = static_cast(&edge); + if (edgeMC(edgep)) m_sb.removeEdgeMC(edgep); } - - const uint64_t oldRelativesCp = mtaskp->critPathCost(way) + mtaskp->cost(); - const uint64_t newRelativesCp = newCp + mtaskp->cost() + otherp->cost(); - - NewCp result; - result.cp = newCp; - result.propagate = (newRelativesCp > oldRelativesCp); - result.propagateCp = newRelativesCp; - return result; - } - - void removeSiblingMCsWith(LogicMTask* mtaskp) { - // Note: 'removeSibling' unlinks the candidate from both of its MTasks' lists, so taking - // the front element repeatedly does terminate. It also erases the candidate from the - // owning (higher id) MTask's sibling set as it goes, so both the sets and the lists are - // left consistent, whichever side of the candidate 'mtaskp' happens to be on. - while (SiblingMC* const smcp = mtaskData(mtaskp).aSiblingMCs.frontp()) { - m_sb.removeSibling(smcp); + for (V3GraphEdge& edge : mtaskp->inEdges()) { + MTaskEdge* const edgep = static_cast(&edge); + if (edgeMC(edgep)) m_sb.removeEdgeMC(edgep); } - while (SiblingMC* const smcp = mtaskData(mtaskp).bSiblingMCs.frontp()) { - m_sb.removeSibling(smcp); - } - } - - void removeSiblingMCs(LogicMTask* recipientp, LogicMTask* donorp) { - // These two can share a SiblingMC (an edge between them does not preclude one). That is - // fine: 'removeSiblingMCsWith' unlinks each candidate from both sides, so the shared one - // is gone by the time we get to the donor. - // - // This also leaves both sibling sets empty, so they need no separate clearing: each entry - // in an MTask's sibling set is added by 'makeSiblingMC' together with a SiblingMC on that - // same MTask's 'aSiblingMCs' list, and draining that list erases the matching entry (see - // 'SiblingMC::unlinkA'). The slow assert in 'makeSiblingMC' catches it if that ever - // diverges, as a stale set entry there suppresses creating the SiblingMC it stands for. - removeSiblingMCsWith(recipientp); - removeSiblingMCsWith(donorp); - } - - void contract(MergeCandidate* mergeCanp) { - LogicMTask* top = nullptr; - LogicMTask* fromp = nullptr; - EdgeMC* const mergeEdgeMCp = mergeCanp->toEdgeMC(); - MTaskEdge* const mergeEdgep = mergeEdgeMCp ? mergeEdgeMCp->edgep() : nullptr; - SiblingMC* const mergeSibsp = mergeCanp->toSiblingMC(); - if (mergeEdgep) { - top = mergeEdgep->toMTaskp(); - fromp = mergeEdgep->fromMTaskp(); - } else { - top = mergeSibsp->ap(); - fromp = mergeSibsp->bp(); - } - - // Merge the smaller mtask into the larger mtask. If one of them - // is much larger, this will save time in partRedirectEdgesFrom(). - // Assume the more costly mtask has more edges. - // - // [TODO: now that we have edge maps, we could count the edges - // exactly without a linear search.] - LogicMTask* recipientp; - LogicMTask* donorp; - if (fromp->cost() > top->cost()) { - recipientp = fromp; - donorp = top; - } else { - donorp = fromp; - recipientp = top; - } - VL_DANGLING(fromp); - VL_DANGLING(top); // Use donorp and recipientp now instead - - // Recursively update forward and reverse CP numbers. - // - // Doing this before merging the MTasks lets us often avoid - // recursing through either incoming or outgoing edges on one or - // both MTasks. - // - // These 'NewCp' objects carry a bit indicating whether we must - // propagate CP for each of the four cases: - const NewCp recipientNewCpFwd = newCp(recipientp, donorp, mergeEdgep); - const NewCp donorNewCpFwd = newCp(donorp, recipientp, mergeEdgep); - const NewCp recipientNewCpRev = newCp(recipientp, donorp, mergeEdgep); - const NewCp donorNewCpRev = newCp(donorp, recipientp, mergeEdgep); - - if (mergeEdgep) { - // Remove and free the connecting edge. Must do this before propagating CP's below. - m_sb.removeEdge(mergeEdgep); - mergeEdgep->fromMTaskp()->removeRelativeMTask(mergeEdgep->toMTaskp()); - mergeEdgep->fromMTaskp()->removeRelativeEdge(mergeEdgep); - mergeEdgep->toMTaskp()->removeRelativeEdge(mergeEdgep); - VL_DO_DANGLING(mergeEdgep->unlinkDelete(), mergeEdgep); - } else { - // Remove the siblingMC - m_sb.removeSibling(mergeSibsp); - } - - // This also updates cost on recipientp - recipientp->moveAllVerticesFrom(donorp); - - UINFO(9, "recipient = " << recipientp->id() << ", donor = " << donorp->id() - << ", mergeEdgep = " << mergeEdgep << "\n" - << "recipientNewCpFwd = " << recipientNewCpFwd.cp - << (recipientNewCpFwd.propagate ? " true " : " false ") - << recipientNewCpFwd.propagateCp << "\n" - << "donorNewCpFwd = " << donorNewCpFwd.cp - << (donorNewCpFwd.propagate ? " true " : " false ") - << donorNewCpFwd.propagateCp); - - recipientp->setCritPathCost(GraphWay::FORWARD, recipientNewCpFwd.cp); - if (recipientNewCpFwd.propagate) { - m_mTaskGraph.forwardPropagator().cpHasIncreased(recipientp, - recipientNewCpFwd.propagateCp); - } - recipientp->setCritPathCost(GraphWay::REVERSE, recipientNewCpRev.cp); - if (recipientNewCpRev.propagate) { - m_mTaskGraph.reversePropagator().cpHasIncreased(recipientp, - recipientNewCpRev.propagateCp); - } - if (donorNewCpFwd.propagate) { - m_mTaskGraph.forwardPropagator().cpHasIncreased(donorp, donorNewCpFwd.propagateCp); - } - if (donorNewCpRev.propagate) { - m_mTaskGraph.reversePropagator().cpHasIncreased(donorp, donorNewCpRev.propagateCp); - } - m_mTaskGraph.forwardPropagator().go(); - m_mTaskGraph.reversePropagator().go(); - - // Remove all other SiblingMCs that include recipientp or donorp. We remove all siblingMCs - // of recipientp so we do not get huge numbers of SiblingMCs. We'll recreate them below, up - // to a bounded number. - removeSiblingMCs(recipientp, donorp); - - // Redirect all edges, delete donorp - partRedirectEdgesFrom(m_mTaskGraph, recipientp, donorp, m_sb); - - ++m_mergesSinceRescore; - - // Do an expensive check, confirm we haven't botched the CP - // updates. - if (m_slowAsserts) partCheckCriticalPaths(m_mTaskGraph); - - // Finally, make new sibling pairs as needed: - // - prereqs and postreqs of recipientp - // - prereqs of recipientp's postreqs - // - postreqs of recipientp's prereqs - // Note that this depends on the updated critical paths (above). - siblingPairFromRelatives(recipientp); - siblingPairFromRelatives(recipientp); - unsigned edges = 0; - for (V3GraphEdge& edge : recipientp->outEdges()) { - LogicMTask* const postreqp = static_cast(edge.top()); - siblingPairFromRelatives(postreqp); - ++edges; - if (edges >= PART_SIBLING_EDGE_LIMIT) break; - } - edges = 0; - for (V3GraphEdge& edge : recipientp->inEdges()) { - LogicMTask* const prereqp = static_cast(edge.fromp()); - siblingPairFromRelatives(prereqp); - ++edges; - if (edges >= PART_SIBLING_EDGE_LIMIT) break; - } - } - - void doRescore() { - // During rescore, we know that graph isn't changing, so allow - // the critPathCost*Without() routines to cache some data in - // each LogicMTask. This is just an optimization, things should - // behave identically without the caching (just slower) - - m_sb.rescore(); - UINFO(6, "Did rescore. Merges since previous = " << m_mergesSinceRescore); - - m_mergesSinceRescore = 0; - m_scoreLimitBeforeRescore - = std::numeric_limits::max(); } void makeSiblingMC(LogicMTask* ap, LogicMTask* bp) { if (ap->id() < bp->id()) std::swap(ap, bp); // The higher id vertex owns the association set - const auto first = mtaskData(ap).siblings.insert(bp).second; + const bool first = mtaskData(ap).siblings.insert(bp).second; if (first) { - m_sb.addSibling(ap, bp); + m_sb.addSiblingMC(ap, bp); return; } - if (VL_UNLIKELY(m_slowAsserts)) { + if (VL_UNLIKELY(m_mTaskGraph.slowAsserts())) { // It's fine if we already have this SiblingMC, we may have // created it earlier. Just confirm that we have associated data. bool found = false; @@ -837,7 +413,7 @@ class Contraction final { } template - void siblingPairFromRelatives(V3GraphVertex* mtaskp) { + void addSiblingMCsFromRelatives(LogicMTask* mtaskp) { constexpr GraphWay way{N_Way}; // Need at least 2 edges auto& edges = mtaskp->edges(); @@ -872,7 +448,7 @@ class Contraction final { LogicMTask* const otherp = static_cast(edge.furtherp()); neighbors[n] = otherp; sortRecs[n].m_id = otherp->id(); - sortRecs[n].m_cp = otherp->critPathCost(way) + otherp->cost(); + sortRecs[n].m_cp = otherp->cpInclusive(); sortRecs[n].m_idx = n; ++n; // Prevent nodes with huge numbers of edges from massively slowing down us down @@ -898,25 +474,97 @@ class Contraction final { } } - // CONSTRUCTORS - Contraction(OrderMTaskGraph& mTaskGraph, uint64_t scoreLimit) - : m_mTaskGraph{mTaskGraph} - , m_scoreLimit{scoreLimit} { + void removeSiblingMCs(LogicMTask* mtaskp) { + // Note: 'removeSiblingMC' unlinks the candidate from both of its MTasks' lists, so taking + // the front element repeatedly does terminate. It also erases the candidate from the + // owning (higher id) MTask's sibling set as it goes, so both the sets and the lists are + // left consistent, whichever side of the candidate 'mtaskp' happens to be on. + while (SiblingMC* const smcp = mtaskData(mtaskp).aSiblingMCs.frontp()) { + m_sb.removeSiblingMC(smcp); + } + while (SiblingMC* const smcp = mtaskData(mtaskp).bSiblingMCs.frontp()) { + m_sb.removeSiblingMC(smcp); + } + } - if (m_slowAsserts) { - // Check there are no redundant edges - for (V3GraphVertex& vtx : m_mTaskGraph.vertices()) { - std::unordered_set neighbors; - for (V3GraphEdge& edge : vtx.outEdges()) { - const bool first = neighbors.insert(edge.top()).second; - UASSERT_OBJ(first, &vtx, "Redundant edge found in input to Contraction()"); - } - } + // Merge the two MTasks of 'mergeCanp' + void contract(MergeCandidate* mergeCanp) { + // The two MTasks to merge. Note the order of the two decides which of them becomes the + // recipient below when their costs are equal, so keep it stable. + LogicMTask* fromp; + LogicMTask* top; + if (const EdgeMC* const edgeMCp = mergeCanp->toEdgeMC()) { + fromp = edgeMCp->edgep()->fromMTaskp(); + top = edgeMCp->edgep()->toMTaskp(); + } else { + const SiblingMC* const sibMCp = mergeCanp->toSiblingMC(); + fromp = sibMCp->bp(); + top = sibMCp->ap(); } - // Set up the critical path into and out of each node, then coarsen the graph. - partInitCriticalPaths(mTaskGraph); + // Merge the smaller mtask into the larger mtask. + LogicMTask* recipientp; + LogicMTask* donorp; + if (fromp->cost() > top->cost()) { + recipientp = fromp; + donorp = top; + } else { + donorp = fromp; + recipientp = top; + } + VL_DANGLING(fromp); + VL_DANGLING(top); // Use donorp and recipientp now instead + // Remove all SiblingMCs that include either MTask + removeSiblingMCs(recipientp); + removeSiblingMCs(donorp); + + // Remove the EdgeMCs of both MTasks + removeEdgeMCs(recipientp); + removeEdgeMCs(donorp); + + // Merge the MTasks. This redirects all edges, updates critical paths, and deletes donorp + m_mTaskGraph.mergeMTasks(recipientp, donorp); + VL_DANGLING(donorp); + + // Confirm we haven't botched the CP updates. + m_mTaskGraph.validate(); + + // Add the EdgeMCs of the merged MTask + addEdgeMCs(recipientp); + + // Finally, make new sibling pairs as needed: + // - prereqs and postreqs of recipientp + // - prereqs of recipientp's postreqs + // - postreqs of recipientp's prereqs + // Note that this depends on the updated critical paths (above). + addSiblingMCsFromRelatives(recipientp); + addSiblingMCsFromRelatives(recipientp); + unsigned edges = 0; + for (V3GraphEdge& edge : recipientp->outEdges()) { + LogicMTask* const postreqp = static_cast(edge.top()); + addSiblingMCsFromRelatives(postreqp); + ++edges; + if (edges >= PART_SIBLING_EDGE_LIMIT) break; + } + edges = 0; + for (V3GraphEdge& edge : recipientp->inEdges()) { + LogicMTask* const prereqp = static_cast(edge.fromp()); + addSiblingMCsFromRelatives(prereqp); + ++edges; + if (edges >= PART_SIBLING_EDGE_LIMIT) break; + } + } + + // CONSTRUCTORS + Contraction(OrderMTaskGraph& mTaskGraph, uint64_t cpLimit) + : m_mTaskGraph{mTaskGraph} + , m_scoreLimit{cpLimit} { + + // Check the graph we were given is consistent. + m_mTaskGraph.validate(); + + // Figure out maximum number of MTasks const uint32_t maxMTasks = []() -> uint32_t { // If specified, use the given value const int given = v3Global.opt.threadsMaxMTasks(); @@ -925,15 +573,6 @@ class Contraction final { return PART_DEFAULT_MAX_MTASKS_PER_THREAD * v3Global.opt.threads(); }(); - // OPTIMIZATION PASS: Edge contraction and sibling contraction. - // - Score pairs of LogicMTask which are a candidate to merge. - // * Each edge defines such a candidate pair - // * Two LogicMTask that are prereqs or postreqs of a common third - // vertex are "siblings", these are also a candidate pair. - // - Build a list of MergeCandidates, sorted by score. - // - Merge the best pair. - // - Incrementally recompute critical paths near the merged mtask. - // Allocate and assign the auxiliary data for every LogicMTask. { const size_t nMTasks = m_mTaskGraph.vertices().size(); @@ -945,56 +584,33 @@ class Contraction final { // Add initial candidates for (V3GraphVertex& vtx : m_mTaskGraph.vertices()) { - for (V3GraphEdge& edge : vtx.outEdges()) m_sb.addEdge(static_cast(&edge)); - siblingPairFromRelatives(&vtx); - siblingPairFromRelatives(&vtx); + for (V3GraphEdge& edge : vtx.outEdges()) + m_sb.addEdgeMC(static_cast(&edge)); + LogicMTask* const mtaskp = static_cast(&vtx); + addSiblingMCsFromRelatives(mtaskp); + addSiblingMCsFromRelatives(mtaskp); } - // Set initial scores in scoreboard - doRescore(); - while (true) { - // This is the best edge to merge, with the lowest score (shortest local critical path) + // Pick the candidate yielding the lowest local critical path. MergeCandidate* const mergeCanp = m_sb.best(); - if (!mergeCanp) { - if (!m_sb.needsRescore()) break; // No more eligible candidates - // Rescore the scoreboard and try again - doRescore(); + if (!mergeCanp) break; // No more candidates + + // If the score has changed since it was inserted into the scoreboard, rescore it and + // pick again. (The real scores can differ from the one used to insert it into the + // scoreboard, due to merges between insertion and retrieval.) + const uint64_t score = mergeCanp->currentScore(); + if (score != mergeCanp->score()) { + m_sb.rescore(mergeCanp); continue; } - UASSERT(!m_sb.needsRescore(mergeCanp), - "Need-rescore items should not be returned by bestp"); + // Check if the critical path limit is reached. + if (score > m_scoreLimit) { - const uint64_t cachedScore = mergeCanp->score(); - mergeCanp->rescore(); - const uint64_t actualScore = mergeCanp->score(); - - // If cached score is out-of-date, mark this elem as in need of a rescore and continue. - // cppcheck-suppress knownConditionTrueFalse // they are in fact different - if (actualScore > cachedScore) { - m_sb.hintScoreChanged(mergeCanp); - continue; - } - - // ... we'll also confirm that actualScore hasn't shrunk relative - // to cached score, after the mergeWouldCreateCycle() check. - - if (actualScore > m_scoreLimit) { - // Our best option isn't good enough - if (m_sb.needsRescore()) { - // Some pairs need a rescore, maybe those will be - // eligible to merge afterward. - doRescore(); - continue; - } - - // We've exhausted everything below m_scoreLimit; stop. - - // Except, if we have too many LogicMTasks, raise the score limit and keep going... + // If there are still too many MTasks, raise the limit and keep going const unsigned mtaskCount = m_mTaskGraph.vertices().size(); if (mtaskCount > maxMTasks) { - const uint64_t oldLimit = m_scoreLimit; m_scoreLimit = (m_scoreLimit * 120) / 100; FileLine* const flp = v3Global.rootp()->fileline(); if (!flp->warnIsOff(V3ErrorCode::UNOPTTHREADS)) { @@ -1003,80 +619,38 @@ class Contraction final { "parallelism; suggest asking for fewer threads."); flp->modifyWarnOff(V3ErrorCode::UNOPTTHREADS, true); } - UINFO(6, "Critical path limit was=" << oldLimit << " now=" << m_scoreLimit); continue; } - // Really stop + // MTasks limit and CP limit reached. Stop. break; } - // If time to rescore, that will result in a higher scoreLimitBeforeRescore, and - // possibly lower-scoring elements returned from bestp(). - if (actualScore > m_scoreLimitBeforeRescore) { - doRescore(); - continue; - } - // Avoid merging the entry/exit nodes. This would create serialization, by forcing the // merged MTask to run before/after everything else. Empirically this helps performance // in a modest way by allowing other MTasks to start earlier. if (EdgeMC* const edgeMCp = mergeCanp->toEdgeMC()) { MTaskEdge* const edgep = edgeMCp->edgep(); - if (edgep->fromp() == m_entryMTaskp || edgep->top() == m_exitMTaskp) { - m_sb.removeEdge(edgep); + if (edgep->fromp() == m_mTaskGraph.entryp() + || edgep->top() == m_mTaskGraph.exitp()) { + m_sb.removeEdgeMC(edgep); continue; } } - // Avoid merging any edge that would create a cycle. - // - // For example suppose we begin with vertices A, B, C and edges - // A->B, B->C, A->C. - // - // Suppose we want to merge A->C into a single vertex. - // New edges would be AC->B and B->AC which is not a DAG. - // Do not allow this. - if (mergeCanp->mergeWouldCreateCycle()) { - // Remove this candidate from scoreboard so we don't keep - // reconsidering it on every loop. - if (SiblingMC* const smcp = mergeCanp->toSiblingMC()) { - m_sb.removeSibling(smcp); - } else { - m_sb.removeEdge(mergeCanp->toEdgeMC()->edgep()); - } + // Avoid merging any edge that would create a cycle. For example suppose we begin with + // vertices A, B, C and edges A->B, B->C, A->C. Merging A->C would create a cycle. + if (mergeCanp->mergeWouldCreateCycle(m_mTaskGraph)) { + m_sb.removeMC(mergeCanp); continue; } - UASSERT(cachedScore == actualScore, "Calculation error in scoring"); - - // Finally there's no cycle risk, no need to rescore, we're - // within m_scoreLimit and m_scoreLimitBeforeRescore. - // This is the edge to merge. - - // Bookkeeping: if this is the first edge we'll merge since - // the last rescore, compute the new m_scoreLimitBeforeRescore - // to be somewhat higher than this edge's score. - if (!m_mergesSinceRescore) m_scoreLimitBeforeRescore = actualScore; - - // Finally merge this candidate. + // Merge this candidate contract(mergeCanp); } - // Free all remaining merge candidates. As an EdgeMC exists exactly while its edge is on - // the scoreboard, draining the scoreboard here frees every remaining EdgeMC; edges removed - // from the scoreboard earlier already had theirs freed. Note 'best' only ever returns - // candidates with a known score, so this only drains the scoreboard completely if nothing - // is left with an unknown score. Every 'break' out of the loop above is guarded on that, - // but assert it here, as otherwise we would leak candidates. - UASSERT(!m_sb.needsRescore(), "Should have no unknown score candidates at this point"); - while (MergeCandidate* const mergeCanp = m_sb.best()) { - if (SiblingMC* const smcp = mergeCanp->toSiblingMC()) { - m_sb.removeSibling(smcp); - } else { - m_sb.removeEdge(mergeCanp->toEdgeMC()->edgep()); - } - } + // Free all remaining merge candidates. + while (MergeCandidate* const mergeCanp = m_sb.best()) m_sb.removeMC(mergeCanp); } public: diff --git a/src/V3OrderMTaskFixHazards.cpp b/src/V3OrderMTaskFixHazards.cpp index ecd8ab468..17563ea95 100644 --- a/src/V3OrderMTaskFixHazards.cpp +++ b/src/V3OrderMTaskFixHazards.cpp @@ -162,46 +162,6 @@ class FixDataHazards final { // METHODS - // Redirect all edges of 'donorp' onto 'recipientp' - static void redirectEdgesFrom(LogicMTask* recipientp, LogicMTask* donorp) { - // Process outgoing edges - while (MTaskEdge* const edgep = static_cast(donorp->outEdges().frontp())) { - LogicMTask* const top = edgep->toMTaskp(); - top->removeRelativeEdge(edgep); - - // If an edge already exists between recipient and sink of donor, drop the duplicate. - if (recipientp->hasRelativeMTask(top)) { - VL_DO_DANGLING(edgep->unlinkDelete(), edgep); - continue; - } - - // Otherwise redirect the edge from donorp->top to recipientp->top. - edgep->relinkFromp(recipientp); - recipientp->addRelativeMTask(top); - recipientp->stealRelativeEdge(edgep); - top->addRelativeEdge(edgep); - } - - // Process incoming edges - while (MTaskEdge* const edgep = static_cast(donorp->inEdges().frontp())) { - LogicMTask* const fromp = edgep->fromMTaskp(); - fromp->removeRelativeMTask(donorp); - fromp->removeRelativeEdge(edgep); - - // If an edge already exists between recipient and source of donor, drop the duplicate. - if (fromp->hasRelativeMTask(recipientp)) { - VL_DO_DANGLING(edgep->unlinkDelete(), edgep); - continue; - } - - // Otherwise redirect the edge from fromp->donorp to fromp->recipientp. - edgep->relinkTop(recipientp); - fromp->addRelativeMTask(recipientp); - fromp->addRelativeEdge(edgep); - recipientp->stealRelativeEdge(edgep); - } - } - void findAdjacentTasks(const OrderVarStdVertex* varVtxp, TasksByRank& tasksByRank) { // Find all writer tasks for this variable, group by rank. for (const V3GraphEdge& edge : varVtxp->inEdges()) { @@ -219,8 +179,8 @@ class FixDataHazards final { LogicMTask* lastRecipientp = nullptr; for (const auto& pair : tasksByRank) { // Find the largest node at this rank, merge into it. (If we - // happen to find a huge node, this saves time in - // redirectEdgesFrom() versus merging into an arbitrary node.) + // happen to find a huge node, this saves time in the merge + // versus merging into an arbitrary node.) LogicMTask* recipientp = nullptr; for (LogicMTask* const mtaskp : pair.second) { if (!recipientp || (recipientp->cost() < mtaskp->cost())) recipientp = mtaskp; @@ -231,20 +191,18 @@ class FixDataHazards final { for (LogicMTask* const donorp : pair.second) { // Merge donor into recipient. if (donorp == recipientp) continue; - // Fix up the map, so donor's OLVs map to recipientp + // Fix up the map, so donor's OLVs map to recipientp. Must do this while the donor + // still holds them. for (const OrderMoveVertex& vtx : donorp->vertexList()) { vtx.logicp()->userp(recipientp); } - // Move all vertices from donorp to recipientp - recipientp->moveAllVerticesFrom(donorp); - // Redirect edges from donorp to recipientp - redirectEdgesFrom(recipientp, donorp); - // Remove donorp from the graph - VL_DO_DANGLING(donorp->unlinkDelete(&m_mTaskGraph), donorp); + // Merge donorp into recipientp, which also deletes donorp + m_mTaskGraph.mergeMTasks(recipientp, donorp); + VL_DANGLING(donorp); } - if (lastRecipientp && !lastRecipientp->hasRelativeMTask(recipientp)) { - new MTaskEdge{&m_mTaskGraph, lastRecipientp, recipientp, 1}; + if (lastRecipientp && !lastRecipientp->hasEdgeTo(recipientp)) { + m_mTaskGraph.addEdge(lastRecipientp, recipientp); } lastRecipientp = recipientp; } @@ -319,9 +277,8 @@ class FixDataHazards final { // given OVV.) Create edges across these remaining MTasks to ensure // they run in serial order (going along with the existing ranks.) // - // NOTE: we don't update the CP's stored in the LogicMTasks to - // reflect the changes we make to the graph. That's OK, as we - // haven't yet initialized CPs when we call this routine. + // NOTE: all graph mutations below go through OrderMTaskGraph (adding an edge, or merging + // two MTasks), so the CP's stored in the LogicMTasks are kept up to date throughout. for (const OrderVarStdVertex* const varVtxp : regularVars) { // Build a set of MTasks, per rank, which access this var. // Within a rank, sort by MTaskID to avoid nondeterminism. @@ -387,4 +344,6 @@ public: void OrderMTaskGraph::fixDataHazards(OrderMTaskGraph& mtaskGraph) { FixDataHazards::apply(mtaskGraph); + // The critical paths are maintained as the graph is mutated, check them + mtaskGraph.validate(); } diff --git a/src/V3OrderMTaskGraph.cpp b/src/V3OrderMTaskGraph.cpp index e47b61fb4..50d3a175c 100644 --- a/src/V3OrderMTaskGraph.cpp +++ b/src/V3OrderMTaskGraph.cpp @@ -21,28 +21,16 @@ #include "V3Global.h" #include "V3InstrCount.h" +#include +#include +#include + VL_DEFINE_DEBUG_FUNCTIONS; -//###################################################################### -// OrderMTaskGraph - -OrderMTaskGraph::OrderMTaskGraph(OrderMoveGraph& moveGraph) - : m_moveGraph{moveGraph} - , m_entryp{new LogicMTask{*this, nullptr}} - , m_exitp{new LogicMTask{*this, nullptr}} - , m_forwardPropagator{v3Global.opt.debugPartition()} - , m_reversePropagator{v3Global.opt.debugPartition()} {} - -uint64_t OrderMTaskGraph::totalCost() const { - uint64_t cost = 0; - for (const V3GraphVertex& vtx : vertices()) cost += static_cast(vtx).cost(); - return cost; -} - //###################################################################### // LogicMTask -uint32_t LogicMTask::s_nextId = 1; // Start at 1, so that 0 indicates no mtask. +uint32_t LogicMTask::s_nextId = 1; // Start at 1, for historic reasons LogicMTask::LogicMTask(OrderMTaskGraph& graph, OrderMoveVertex* mVtxp) : V3GraphVertex{&graph} { @@ -54,6 +42,281 @@ LogicMTask::LogicMTask(OrderMTaskGraph& graph, OrderMoveVertex* mVtxp) } } +//###################################################################### +// OrderMTaskGraph + +OrderMTaskGraph::OrderMTaskGraph(OrderMoveGraph& moveGraph) + : m_moveGraph{moveGraph} + , m_entryp{new LogicMTask{*this, nullptr}} + , m_exitp{new LogicMTask{*this, nullptr}} + , m_slowAsserts{v3Global.opt.debugPartition()} {} + +bool OrderMTaskGraph::pathExistsImpl(LogicMTask* fromp, LogicMTask* top, + const MTaskEdge* excludedEdgep) { + UDEBUGONLY(UASSERT_OBJ(fromp->m_generation != m_currentGeneration, fromp, + "Should not visit an MTask twice in the same search");); + // Mark visited. + fromp->m_generation = m_currentGeneration; + + // Base case: we found a path. + if (fromp == top) return true; + + // Base case: fromp is too late, cannot possibly be a prereq for top. + if (fromp->cpExclusive() < top->cpInclusive()) { + return false; + } + if (fromp->cpInclusive() > top->cpExclusive()) { + return false; + } + + // Recursively look for a path + for (const V3GraphEdge& follow : fromp->outEdges()) { + if (&follow == excludedEdgep) continue; + LogicMTask* const nextp = static_cast(follow.top()); + // Don't visit the same MTask twice in the same search. + if (nextp->m_generation == m_currentGeneration) continue; + if (pathExistsImpl(nextp, top, nullptr)) return true; + } + return false; +} + +template +void OrderMTaskGraph::propagatePush(LogicMTask* mtaskp) { + constexpr GraphWay way{N_Way}; + constexpr GraphWay inv{way.invert()}; + const uint64_t inclusiveCp = mtaskp->cpInclusive(); + + for (V3GraphEdge& graphEdge : mtaskp->edges()) { + MTaskEdge& edge = static_cast(graphEdge); + + LogicMTask* const relativep = edge.furtherMTaskp(); + EdgeHeap::Node& edgeHeapNode = edge.m_edgeHeapNode[inv]; + if (inclusiveCp > edgeHeapNode.key().m_cp) { + relativep->m_edgeHeap[inv].increaseKey(&edgeHeapNode, inclusiveCp); + } + + const uint64_t relativeCp = relativep->cpExclusive(); + + if (relativeCp >= inclusiveCp) continue; + + // relativep's critical path is out of step with its longest !wayward edge. + // Schedule that to be resolved. + const uint64_t increment = inclusiveCp - relativeCp; + + PropagatePendingHeap::Node*& pendingNodepRef = relativep->m_propagateHeapNodep; + if (PropagatePendingHeap::Node* const nodep = pendingNodepRef) { + // Already in heap. Increase the increment if needed. + if (increment > nodep->key().m_increment) { + m_pendingHeap.increaseKey(nodep, increment); + } + continue; + } + + // Add to heap + PropagatePendingHeap::Node* const nodep = m_pendingNodePool.alloc(); + pendingNodepRef = nodep; + m_pendingHeap.insert(nodep, {increment, relativep->id(), relativep}); + } +} + +template +void OrderMTaskGraph::propagateResolve() { + constexpr GraphWay way{N_Way}; + constexpr GraphWay inv{way.invert()}; + + // Each pending MTask is keyed on how much its critical path will grow by. Resolving them in + // decreasing order of that growth means each MTask needs resolving only once: the growth of a + // wayward MTask is never larger than the growth of the MTask it was pushed from, so once an + // MTask has been resolved, no larger growth can be pushed onto it later. + while (!m_pendingHeap.empty()) { + // Pop max element from heap + PropagatePendingHeap::Node* const maxp = m_pendingHeap.max(); + m_pendingHeap.remove(maxp); + // Pick up values + LogicMTask* const mtaskp = maxp->key().m_mtaskp; + const uint64_t cpGrowBy = maxp->key().m_increment; + // Confirm that we only set each node's CP once. That's an important property of this + // algorithm, which allows it to be far faster than a recursive one. + UASSERT_OBJ(mtaskp->m_generation != m_currentGeneration, mtaskp, "Set CP on node twice"); + mtaskp->m_generation = m_currentGeneration; + // Free the heap node, we are done with it + m_pendingNodePool.free(maxp); + mtaskp->m_propagateHeapNodep = nullptr; + // Update the critical path of mtaskp, that was out-of-date with respect to its edges + uint64_t& cpRef = mtaskp->m_cpExclusive[way]; + const uint64_t newCp = cpRef + cpGrowBy; + // Check that CP matches that of the longest edge wayward of mtaskp. + if (VL_UNLIKELY(m_slowAsserts)) { + const uint64_t edgeCp = mtaskp->m_edgeHeap[inv].max()->key().m_cp; + UASSERT_OBJ(edgeCp == newCp, mtaskp, "CP doesn't match longest wayward edge"); + } + cpRef = newCp; + propagatePush(mtaskp); + } +} + +uint64_t OrderMTaskGraph::totalCost() const { + uint64_t cost = 0; + for (const V3GraphVertex& vtx : vertices()) cost += static_cast(vtx).cost(); + return cost; +} + +void OrderMTaskGraph::addEdge(LogicMTask* fromp, LogicMTask* top) { + UASSERT_OBJ(fromp != top, fromp, "Should not create self-edges"); + UDEBUGONLY(UASSERT_OBJ(!fromp->hasEdgeTo(top), fromp, "Should not create redundant edges");); + + // Create the edge. This inserts it into the edge heap of both endpoints with the correct + // critical path keys, as the critical paths of the endpoints are still unchanged here. + new MTaskEdge{this, fromp, top}; + + // The path through the new edge might be longer than the current critical path of its + // endpoints, in which case the critical paths need updating. Note each endpoint is the seed of + // one propagation, and is updated by the other: the inclusive critical paths of the endpoints + // themselves did not change (a new out-edge cannot lengthen a path into 'fromp', nor a new + // in-edge a path out of 'top'), so it is the new relative of each seed whose critical path + // might need to grow. That is, 'top' is updated wayward of 'fromp' below, and vice versa, + // together with the relatives of each, transitively. + // + // The guards below are an asymptotic optimization. The graph is consistent apart from the new + // edge, so the new relative is the only relative of either seed that can have a stale critical + // path, and if it does not need updating the propagation does nothing. It would however still + // walk all edges of the seed to discover that, which is expensive for a high degree seed. + if (fromp->cpInclusive() > top->cpExclusive()) { + propagate(fromp); + } + if (top->cpInclusive() > fromp->cpExclusive()) { + propagate(top); + } +} + +void OrderMTaskGraph::mergeMTasks(LogicMTask* recipientp, LogicMTask* donorp) { + UASSERT_OBJ(recipientp != donorp, recipientp, "Should not merge an MTask with itself"); + + // Note we redirect the edges before updating the cost and critical paths of the recipient, + // which means the redirected edges are inserted into the edge heaps of the relatives using the + // pre-merge values of the recipient. The critical path propagation below then brings all of + // them up to date. This works because the keys in the edge heaps only ever need increasing: + // the inclusive critical path of the merged MTask is at least the inclusive critical path of + // either of the two MTasks it is made of, in both directions. + + // Process outgoing edges of donor + while (MTaskEdge* const edgep = static_cast(donorp->outEdges().frontp())) { + LogicMTask* const relativep = edgep->toMTaskp(); + + relativep->removeRelativeEdge(edgep); + + if (relativep == recipientp || recipientp->hasEdgeTo(relativep)) { + // This is either the edge connecting the two MTasks, which becomes internal to the + // merged MTask, or is parallel with an existing edge of the recipient. Drop it. + VL_DO_DANGLING(edgep->unlinkDelete(), edgep); + } else { + // No existing edge between recipient and relative of donor. + // Redirect the edge from donor -> relative to recipient -> relative. + edgep->relinkFromp(recipientp); + recipientp->addDependent(relativep); + recipientp->stealRelativeEdge(edgep); + relativep->addRelativeEdge(edgep); + } + } + + // Process incoming edges of donor + while (MTaskEdge* const edgep = static_cast(donorp->inEdges().frontp())) { + LogicMTask* const relativep = edgep->fromMTaskp(); + + relativep->removeDependent(donorp); + relativep->removeRelativeEdge(edgep); + + if (relativep == recipientp || relativep->hasEdgeTo(recipientp)) { + // This is either the edge connecting the two MTasks, which becomes internal to the + // merged MTask, or is parallel with an existing edge of the recipient. Drop it. + VL_DO_DANGLING(edgep->unlinkDelete(), edgep); + } else { + // No existing edge between recipient and relative of donor. + // Redirect the edge from relative -> donor to relative -> recipient. + edgep->relinkTop(recipientp); + relativep->addDependent(recipientp); + relativep->addRelativeEdge(edgep); + recipientp->stealRelativeEdge(edgep); + } + } + + // Move the contents of the donor into the recipient, update its cost + recipientp->m_mVertices.splice(recipientp->m_mVertices.end(), donorp->m_mVertices); + recipientp->m_cost += donorp->m_cost; + + // The recipient now holds all edges of the merged MTask, and the critical paths of all its + // relatives are still up to date, so the critical paths implied by its edges are the critical + // paths of the merged MTask. + const uint64_t newCpFwd = recipientp->cpExclusiveFromEdges(); + const uint64_t newCpRev = recipientp->cpExclusiveFromEdges(); + + // Set the new critical paths, then propagate the increases to the relatives. Note this also + // brings the keys of all edges of the merged MTask up to date in the relatives' edge heaps. + recipientp->cpExclusive(newCpFwd); + propagate(recipientp); + recipientp->cpExclusive(newCpRev); + propagate(recipientp); + + // Remove the donor from the graph + VL_DO_DANGLING(donorp->unlinkDelete(this), donorp); +} + +// Check the critical paths in the given direction, and the critical paths cached in the edge heaps +// in the opposite direction, against those implied by the edges. Note this deliberately iterates +// the edge lists, rather than consulting the edge heaps, so the heaps are validated, not trusted. +template +void OrderMTaskGraph::validateWay() const { + constexpr GraphWay way{N_Way}; + constexpr GraphWay inv = way.invert(); + for (const V3GraphVertex& vtx : vertices()) { + const LogicMTask& mtask = *vtx.as(); + uint64_t cpCost = 0; + std::unordered_set relatives; + for (const V3GraphEdge& graphEdge : mtask.edges()) { + const MTaskEdge& edge = *graphEdge.as(); + const LogicMTask& relative = *(edge.furtherp()->template as()); + // Run a few asserts on the graph, while we are iterating through... + UASSERT_OBJ(edge.weight() != 0, &mtask, "Should be no cut edges in MTask graph"); + UASSERT_OBJ(&relative != &mtask, &mtask, "Should be no self edges in MTask graph"); + const bool first = relatives.insert(&relative).second; + UASSERT_OBJ(first, &mtask, "Should be no redundant edges in MTask graph"); + const uint64_t inclusiveCp = relative.cpInclusive(); + // The critical path cached in the edge heap must match that of the relative + UASSERT_OBJ(edge.cachedCp(inv) == inclusiveCp, &mtask, + "Cached critical path does not match the relative"); + // As must the ID it is keyed on, which breaks ties between equal critical paths + UASSERT_OBJ(edge.cachedId(inv) == relative.id(), &mtask, + "Cached ID does not match the relative"); + cpCost = std::max(cpCost, inclusiveCp); + } + const uint64_t cp = mtask.cpExclusive(); + UASSERT_OBJ(cp == cpCost, &mtask, "Critical path does not match the edges"); + // The edge heap must yield the same, that is: it must return the largest of its keys + UASSERT_OBJ(mtask.cpExclusiveFromEdges() == cpCost, &mtask, + "Edge heap maximum does not match the edges"); + } +} + +void OrderMTaskGraph::validate() const { + if (!m_slowAsserts) return; + + validateWay(); + validateWay(); + + // Check the dependents set of each MTask agrees with its out-edges + for (const V3GraphVertex& vtx : vertices()) { + const LogicMTask& mtask = *vtx.as(); + size_t nDependents = 0; + for (const V3GraphEdge& graphEdge : mtask.outEdges()) { + LogicMTask* const top = graphEdge.as()->toMTaskp(); + UASSERT_OBJ(mtask.hasEdgeTo(top), &mtask, "Dependent missing from the dependents set"); + ++nDependents; + } + UASSERT_OBJ(mtask.m_dependents.size() == nDependents, &mtask, + "Stale entry in the dependents set"); + } +} + //###################################################################### // OrderMTaskGraphBuilder @@ -103,10 +366,9 @@ class OrderMTaskGraphBuilder final { } // Add an edge to the graph, if there is not already an edge between the two vertices. - void addEdge(LogicMTask& src, LogicMTask& dst) { - UASSERT_OBJ(&src != &dst, &src, "Should not create self-edges"); - if (src.hasRelativeMTask(&dst)) return; // Don't create redundant edges. - new MTaskEdge{&m_mtaskGraph, &src, &dst, 1}; + void addEdge(LogicMTask* srcp, LogicMTask* dstp) { + if (srcp->hasEdgeTo(dstp)) return; // Don't create redundant edges. + m_mtaskGraph.addEdge(srcp, dstp); } // CONSTRUCTORS @@ -145,7 +407,7 @@ class OrderMTaskGraphBuilder final { // If the opposite end of the edge is not a bypassed vertex, add direct dependency if (LogicMTask* const otherp = static_cast(top->userp())) { - addEdge(mtask, *otherp); + addEdge(&mtask, otherp); continue; } @@ -155,7 +417,7 @@ class OrderMTaskGraphBuilder final { // The Move graph is bipartite (logic <-> var), and logic is never // bypassed, hence 'transp' must be non-nullptr. UASSERT_OBJ(transp, mVtxp, "This cannot be a bypassed vertex"); - addEdge(mtask, *transp); + addEdge(&mtask, transp); } } } @@ -166,8 +428,8 @@ class OrderMTaskGraphBuilder final { LogicMTask& mtask = static_cast(vtx); if (VL_UNLIKELY((&mtask == &entry) || (&mtask == &exit))) continue; // Add the entry/exit edges if not otherwise connected - if (mtask.inEmpty()) addEdge(entry, mtask); - if (mtask.outEmpty()) addEdge(mtask, exit); + if (mtask.inEmpty()) addEdge(&entry, &mtask); + if (mtask.outEmpty()) addEdge(&mtask, &exit); } } ~OrderMTaskGraphBuilder() = default; @@ -181,5 +443,6 @@ public: std::unique_ptr OrderMTaskGraph::build(OrderMoveGraph& moveGraph) { std::unique_ptr resp{new OrderMTaskGraph{moveGraph}}; OrderMTaskGraphBuilder::apply(*resp); + resp->validate(); return resp; } diff --git a/src/V3OrderMTaskGraph.h b/src/V3OrderMTaskGraph.h index 8643d0787..39dd7e7b1 100644 --- a/src/V3OrderMTaskGraph.h +++ b/src/V3OrderMTaskGraph.h @@ -20,9 +20,9 @@ // candidate machinery: any auxiliary data the algorithms need is attached // externally via the vertex/edge user pointers. // -// PropagateCp propagates increasing critical path costs through the graph. -// OrderMTaskGraph owns one instance for each direction, which the algorithms -// operating on the graph use to keep the critical paths up to date. +// OrderMTaskGraph maintains the critical paths of the MTasks, and the ones +// cached in the edge heaps, as the graph is mutated via 'addEdge' and +// 'mergeMTasks'. // //************************************************************************* @@ -35,38 +35,56 @@ #include "V3Graph.h" #include "V3OrderMoveGraph.h" #include "V3PairingHeap.h" +#include "V3PoolAllocator.h" #include -#include #include #include #include -#include class LogicMTask; class OrderMTaskGraph; -template -class PropagateCp; //============================================================================= -// We keep MTaskEdge graph edges in a PairingHeap, sorted by score and id +// MTaskEdge graph edges are stored in a PairingHeap in each LogicMTask they +// connect to, sorted by critical path through that edge (and id for stability). struct EdgeKey final { - uint64_t m_score; // Score part of edge key - uint64_t m_id; // Unique ID part of edge key - void increase(uint64_t score) { - UDEBUGONLY(UASSERT(score >= m_score, "Must increase");); - m_score = score; + uint64_t m_cp; // The inclusive critical path of the further MTask of the edge + uint32_t m_id; // The ID of the further MTask, for stable comparison + void increase(uint64_t cp) { + UDEBUGONLY(UASSERT(cp >= m_cp, "Must increase");); + m_cp = cp; } - // Sort first by Score then by ID + // Sort first by critical path, then by ID bool operator<(const EdgeKey& other) const { - if (m_score != other.m_score) return m_score < other.m_score; + if (m_cp != other.m_cp) return m_cp < other.m_cp; return m_id < other.m_id; } }; using EdgeHeap = PairingHeap; +//============================================================================= +// LogicMTasks are stored in a PairingHeap during critical path update propagation. + +struct PropagatePendingKey final { + uint64_t m_increment; // The amount the critical path of the MTask will grow by + uint32_t m_id; // The ID of the MTask, for stable comparison + LogicMTask* m_mtaskp; // The MTask the heap entry corresponds to + void increase(uint64_t increment) { + UDEBUGONLY(UASSERT(increment >= m_increment, "Must increase");); + m_increment = increment; + } + // Sort first by increment, then by ID + bool operator<(const PropagatePendingKey& other) const { + if (m_increment != other.m_increment) return m_increment < other.m_increment; + return m_id < other.m_id; + } +}; + +using PropagatePendingHeap = PairingHeap; + //============================================================================= // GraphEdge for the MTask graph @@ -74,22 +92,19 @@ class MTaskEdge final : public V3GraphEdge { VL_RTTI_IMPL(MTaskEdge, V3GraphEdge) friend class LogicMTask; - template - friend class PropagateCp; + friend class OrderMTaskGraph; // MEMBERS // This edge can be in 2 EdgeHeaps, one forward and one reverse. We allocate the heap nodes // directly within the edge as they are always required and this makes association cheap. std::array m_edgeHeapNode; - // Note: The edge's contraction merge candidate (if any) is held in the inherited user pointer - // (V3GraphEdge::userp), managed entirely by the partitioner; see edgeMC() and - // MergeCandidateScoreboard. Kept out of MTaskEdge so it does not depend on the MergeCandidate - // hierarchy. + // CONSTRUCTORS + // Private, so edges can only be created via OrderMTaskGraph, which also updates the critical + // paths on graph mutation. + inline MTaskEdge(OrderMTaskGraph* graphp, LogicMTask* fromp, LogicMTask* top); public: - // CONSTRUCTORS - inline MTaskEdge(OrderMTaskGraph* graphp, LogicMTask* fromp, LogicMTask* top, int weight); VL_UNCOPYABLE(MTaskEdge); VL_UNMOVABLE(MTaskEdge); @@ -99,11 +114,8 @@ public: inline LogicMTask* fromMTaskp() const; inline LogicMTask* toMTaskp() const; - // Following initial assignment of critical paths, clear this MTaskEdge - // out of the edge-map for each node and reinsert at a new location - // with updated critical path. - inline void resetCriticalPaths(); - uint64_t cachedCp(GraphWay way) const { return m_edgeHeapNode[way].key().m_score; } + uint64_t cachedCp(GraphWay way) const { return m_edgeHeapNode[way].key().m_cp; } + uint32_t cachedId(GraphWay way) const { return m_edgeHeapNode[way].key().m_id; } // Convert from the address of the m_edgeHeapNode[way] in an MTaskEdge back to the MTaskEdge static const MTaskEdge* toMTaskEdge(GraphWay way, const EdgeHeap::Node* nodep) { const size_t offset = VL_OFFSETOF(MTaskEdge, m_edgeHeapNode[way]); @@ -117,8 +129,8 @@ public: class LogicMTask final : public V3GraphVertex { VL_RTTI_IMPL(LogicMTask, V3GraphVertex) - template - friend class PropagateCp; + friend class MTaskEdge; + friend class OrderMTaskGraph; // MEMBERS @@ -126,18 +138,23 @@ class LogicMTask final : public V3GraphVertex { // OrderMoveVertex objects, we merely keep them in a list here. OrderMoveVertex::List m_mVertices; + static uint32_t s_nextId; // Next ID number to use + const uint32_t m_id = s_nextId++; // Unique LogicMTask ID number for stable comparison + // Cost estimate for this LogicMTask, derived from V3InstrCount, in abstract time units. // Cost estimates and critical path lengths are bounded by number of AstNodes * constant, // will run out of host memory storing the Ast way before they can overflow. uint64_t m_cost = 0; - // Cost of critical paths going FORWARD from graph-start to the start - // of this vertex, and also going REVERSE from the end of the graph to - // the end of the vertex. Same units as m_cost. - std::array m_critPathCost = {}; + // Critical path in each direction: going FORWARD from graph-start to the start of this vertex, + // and going REVERSE from graph-exit to the end of this vertex. Exclusive of the cost of this + // vertex itself, see cpInclusive() for the value including it. + std::array m_cpExclusive = {0, 0}; - static uint32_t s_nextId; // Next ID number to use - const uint32_t m_id = s_nextId++; // Unique LogicMTask ID number for stable comparison + // The MTasks this MTask has an out-edge to, so checking for an existing edge is O(1) + std::unordered_set m_dependents; + // Store the out/in edges in a heaps sorted by the critical path length through each edge + std::array m_edgeHeap; // Count "generations" which are just operations that scan through the // graph. We'll mark each node with the last generation that scanned @@ -145,15 +162,9 @@ class LogicMTask final : public V3GraphVertex { // while searching for a path. uint64_t m_generation = 0; - // Store a set of forward relatives so we can quickly check if we have a given child - std::unordered_set m_edgeSet; - // Store the outgoing and incoming edges in a heap sorted by the critical path length - std::array m_edgeHeap; - - // Scratch pointer used only by PropagateCp: this MTask's node in the pending heap, or nullptr - // if this MTask is not pending. Type erased, as the heap node type is private to PropagateCp, - // and differs between its two instantiations (which never run concurrently). - void* m_propagateHeapNodep = nullptr; + // Scratch pointer used only by the critical path propagation in OrderMTaskGraph: this MTask's + // node in the pending heap, or nullptr if this MTask is not pending. + PropagatePendingHeap::Node* m_propagateHeapNodep = nullptr; public: // CONSTRUCTORS @@ -161,20 +172,65 @@ public: VL_UNCOPYABLE(LogicMTask); VL_UNMOVABLE(LogicMTask); - // ACCESSORS - OrderMoveVertex::List& vertexList() { return m_mVertices; } - const OrderMoveVertex::List& vertexList() const { return m_mVertices; } - uint32_t id() const { return m_id; } - uint64_t cost() const VL_MT_SAFE { return m_cost; } - uint64_t critPathCost(GraphWay way) const { return m_critPathCost[way]; } - void setCritPathCost(GraphWay way, uint64_t cost) { m_critPathCost[way] = cost; } - // METHODS + OrderMoveVertex::List& vertexList() { return m_mVertices; } + uint32_t id() const { return m_id; } bool operator<(const LogicMTask& rhs) const { return id() < rhs.id(); } - void moveAllVerticesFrom(LogicMTask* otherp) { - m_mVertices.splice(m_mVertices.end(), otherp->vertexList()); - m_cost += otherp->m_cost; + uint64_t cost() const VL_MT_SAFE { return m_cost; } + template + uint64_t cpExclusive() const { + return m_cpExclusive[N_Way]; + } + template + uint64_t cpInclusive() const { + return m_cpExclusive[N_Way] + m_cost; + } + // The critical path of this MTask without considering the given edge. + template + uint64_t cpExclusiveWithout(const V3GraphEdge* edgep) const { + const GraphWay way{N_Way}; + const GraphWay inv = way.invert(); + UDEBUGONLY(UASSERT(edgep->furtherp() == this, + "In cpExclusiveWithout(), 'edgep' must further to 'this'");); + // At most two edges need to be considered: the critical path, if that is not via 'edgep', + // or the second-worst path, if the critical path is via 'edgep'. + const EdgeHeap& edgeHeap = m_edgeHeap[inv]; + // Pick up the critical path edge + const EdgeHeap::Node* const maxp = edgeHeap.max(); + UDEBUGONLY(UASSERT(maxp, "Edge not in heap");); + // If 'edgep' is not the critical path edge, return its critical path + if (MTaskEdge::toMTaskEdge(inv, maxp) != edgep) return maxp->key().m_cp; + // Otherwise return the second-worst path, if there is one + const EdgeHeap::Node* const secp = edgeHeap.secondMax(); + if (!secp) return 0; + return secp->key().m_cp; + } + + bool hasEdgeTo(LogicMTask* dependentp) const { return m_dependents.count(dependentp); } + + // For Graphviz dumps only + std::string name() const override VL_MT_STABLE { + std::ostringstream out; + out << "mt" << m_id // + << " | cpFwd " << m_cpExclusive[GraphWay::FORWARD] // + << " | cost " << cost() // + << " | cpRev " << m_cpExclusive[GraphWay::REVERSE]; + return out.str(); + } + +private: + // Following only used by OrderMTaskGraph, which maintains cached CPs and graph invariants. + + template + uint64_t cpExclusiveFromEdges() const { + constexpr GraphWay inv = GraphWay{N_Way}.invert(); + const EdgeHeap::Node* const maxp = m_edgeHeap[inv].max(); + return maxp ? maxp->key().m_cp : 0; + } + template + void cpExclusive(uint64_t cp) { + m_cpExclusive[N_Way] = cp; } template @@ -183,8 +239,8 @@ public: constexpr GraphWay inv = way.invert(); // Add to the edge heap LogicMTask* const relativep = edgep->furtherMTaskp(); - // Value is !way cp to this edge - const uint64_t cp = relativep->cost() + relativep->critPathCost(inv); + // Value is the !way inclusive cp of the relative + const uint64_t cp = relativep->cpInclusive(); m_edgeHeap[way].insert(&edgep->m_edgeHeapNode[way], {cp, relativep->id()}); } template @@ -202,283 +258,14 @@ public: m_edgeHeap[way].remove(&edgep->m_edgeHeapNode[way]); } - void addRelativeMTask(LogicMTask* relativep) { - // Add the relative to connecting edge map - const bool exits = !m_edgeSet.emplace(relativep).second; - UDEBUGONLY(UASSERT(!exits, "Adding existing relative");); + void addDependent(LogicMTask* dependentp) { + const bool exists = !m_dependents.emplace(dependentp).second; + UDEBUGONLY(UASSERT(!exists, "Adding existing dependent");); } - void removeRelativeMTask(LogicMTask* relativep) { - const size_t removed = m_edgeSet.erase(relativep); - UDEBUGONLY(UASSERT(removed, "Relative should have been in set");); + void removeDependent(LogicMTask* dependentp) { + const size_t removed = m_dependents.erase(dependentp); + UDEBUGONLY(UASSERT(removed, "Dependent should have been in set");); } - bool hasRelativeMTask(LogicMTask* relativep) const { return m_edgeSet.count(relativep); } - - template - void checkRelativesCp() const { - constexpr GraphWay way{N_Way}; - for (const V3GraphEdge& edge : edges()) { - const LogicMTask* const relativep - = static_cast(edge.furtherp()); - const uint64_t cachedCp = static_cast(edge).cachedCp(way); - const uint64_t cp = relativep->critPathCost(way.invert()) + relativep->cost(); - UASSERT(cachedCp == cp, "Calculation error in scoring"); - } - } - - template - uint64_t critPathCostWithout(const V3GraphEdge* withoutp) const { - const GraphWay way{N_Way}; - const GraphWay inv = way.invert(); - // Compute the critical path cost wayward to this node, without considering edge - // 'withoutp'. We need to look at two edges at most, the critical path if that is not via - // 'withoutp', or the second-worst path, if the critical path is via 'withoutp'. - UDEBUGONLY(UASSERT(withoutp->furtherp() == this, - "In critPathCostWithout(), edge 'withoutp' must further to 'this'");); - const EdgeHeap& edgeHeap = m_edgeHeap[inv]; - const EdgeHeap::Node* const maxp = edgeHeap.max(); - if (!maxp) return 0; - if (MTaskEdge::toMTaskEdge(inv, maxp) != withoutp) return maxp->key().m_score; - const EdgeHeap::Node* const secp = edgeHeap.secondMax(); - if (!secp) return 0; - return secp->key().m_score; - } - -private: - // This takes LogicMTask instead of generic V3GraphVertex. We will use the critical - // paths known to LogicMTask to prune the recursion for speed. Also store 'generation' in - // LogicMTask::m_generation so we can prune the search and avoid recursing through the same - // node more than once in a single search. - static bool pathExistsFromInternal(LogicMTask* fromp, LogicMTask* top, - const MTaskEdge* excludedEdgep, uint64_t generation) { - - // If already looked at this node in the current search, since we're back again, - // we must not have found a path on the first go. - if (fromp->m_generation == generation) return false; - - // Mark visited - fromp->m_generation = generation; - - // Base case: we found a path. - if (fromp == top) return true; - - // Base case: fromp is too late, cannot possibly be a prereq for top. - if (fromp->critPathCost(GraphWay::REVERSE) - < (top->critPathCost(GraphWay::REVERSE) + top->cost())) { - return false; - } - if ((fromp->critPathCost(GraphWay::FORWARD) + fromp->cost()) - > top->critPathCost(GraphWay::FORWARD)) { - return false; - } - - // Recursively look for a path - for (const V3GraphEdge& follow : fromp->outEdges()) { - if (&follow == excludedEdgep) continue; - LogicMTask* const nextp = static_cast(follow.top()); - if (pathExistsFromInternal(nextp, top, nullptr, generation)) return true; - } - return false; - } - -public: - // True if there's a path from 'fromp' to 'top' excluding 'excludedEdgep', false otherwise. - // 'excludedEdgep' may be nullptr in which case no edge is excluded. If 'excludedEdgep' is - // non-nullptr it must connect fromp and top. - static bool pathExistsFrom(LogicMTask* fromp, LogicMTask* top, - const MTaskEdge* excludedEdgep) { - static uint64_t s_generation = 0; - return pathExistsFromInternal(fromp, top, excludedEdgep, ++s_generation); - } - - // For Graphviz dumps only - std::string name() const override VL_MT_STABLE { - std::ostringstream out; - out << "mt" << m_id // - << " | fwdCP " << m_critPathCost[GraphWay::FORWARD] // - << " | revCP " << m_critPathCost[GraphWay::REVERSE] // - << " | cost " << cost(); - return out.str(); - } -}; - -//============================================================================= -// PropagateCp - -template -class PropagateCp final { - // Propagate increasing critical path (CP) costs through a graph. - // - // Usage: - // * Client increases the cost and/or CP at a node or small set of nodes - // (often a pair in practice, eg. edge contraction.) - // * Client calls PropagateCp::cpHasIncreased() one or more times. - // Each call indicates that the inclusive CP of some "seed" vertex - // has increased to a given value. - // * NOTE: PropagateCp will neither read nor modify the cost - // or CPs at the seed vertices, it only accesses and modifies - // vertices wayward from the seeds. - // * Client calls PropagateCp::go(). Internally, this iteratively - // propagates the new CPs wayward through the graph. - // - - // TYPES - - // We keep pending vertices in a heap during critical path propagation - struct PendingKey final { - LogicMTask* m_mtaskp; // The vertex in the heap - uint64_t m_score; // The score of this entry - void increase(uint64_t score) { - UDEBUGONLY(UASSERT(score >= m_score, "Must increase");); - m_score = score; - } - bool operator<(const PendingKey& other) const { - if (m_score != other.m_score) return m_score < other.m_score; - return *m_mtaskp < *other.m_mtaskp; - } - }; - - using PendingHeap = PairingHeap; - using PendingHeapNode = typename PendingHeap::Node; - - // MEMBERS - PendingHeap m_pendingHeap; // Heap of pending rescores - - // We allocate this many heap nodes at once - static constexpr size_t ALLOC_CHUNK_SIZE = 128; - PendingHeapNode* m_freep = nullptr; // List of free heap nodes - std::vector> m_allocated; // Allocated heap nodes - - const bool m_slowAsserts; // Enable nontrivial asserts - // Used only with slow asserts to check MTasks visited only once - std::unordered_set m_seen; - -public: - // CONSTRUCTORS - explicit PropagateCp(bool slowAsserts) - : m_slowAsserts{slowAsserts} {} - - // METHODS -private: - // Allocate a HeapNode for the given element - PendingHeapNode* allocNode() { - // If no free nodes available, then make some - if (!m_freep) { - // Allocate in chunks for efficiency - m_allocated.emplace_back(new PendingHeapNode[ALLOC_CHUNK_SIZE]); - // Set up free list pointer - m_freep = m_allocated.back().get(); - // Set up free list chain - for (size_t i = 1; i < ALLOC_CHUNK_SIZE; ++i) { - m_freep[i - 1].m_next.m_ptr = &m_freep[i]; - } - // Clear the next pointer of the last entry - m_freep[ALLOC_CHUNK_SIZE - 1].m_next.m_ptr = nullptr; - } - // Free nodes are available, pick up the first one - PendingHeapNode* const resultp = m_freep; - m_freep = resultp->m_next.m_ptr; - resultp->m_next.m_ptr = nullptr; - return resultp; - } - - // Release a heap node (make it available for future allocation) - void freeNode(PendingHeapNode* nodep) { - // Re-use the existing link pointers and simply prepend it to the free list - nodep->m_next.m_ptr = m_freep; - m_freep = nodep; - } - -public: - void cpHasIncreased(LogicMTask* vxp, uint64_t newInclusiveCp) { - constexpr GraphWay way{N_Way}; - constexpr GraphWay inv{way.invert()}; - - // For *vxp, whose CP-inclusive has just increased to - // newInclusiveCp, iterate to all wayward nodes, update the edges - // of each, and add each to m_pending if its overall CP has grown. - for (V3GraphEdge& graphEdge : vxp->edges()) { - MTaskEdge& edge = static_cast(graphEdge); - - LogicMTask* const relativep = edge.furtherMTaskp(); - EdgeHeap::Node& edgeHeapNode = edge.m_edgeHeapNode[inv]; - if (newInclusiveCp > edgeHeapNode.key().m_score) { - relativep->m_edgeHeap[inv].increaseKey(&edgeHeapNode, newInclusiveCp); - } - - const uint64_t critPathCost = relativep->critPathCost(way); - - if (critPathCost >= newInclusiveCp) continue; - - // relativep's critPathCost() is out of step with its longest !wayward edge. - // Schedule that to be resolved. - const uint64_t newVal = newInclusiveCp - critPathCost; - - void*& pendingNodepRef = relativep->m_propagateHeapNodep; - if (PendingHeapNode* const nodep = static_cast(pendingNodepRef)) { - // Already in heap. Increase score if needed. - if (newVal > nodep->key().m_score) m_pendingHeap.increaseKey(nodep, newVal); - continue; - } - - // Add to heap - PendingHeapNode* const nodep = allocNode(); - pendingNodepRef = nodep; - m_pendingHeap.insert(nodep, {relativep, newVal}); - } - } - - void go() { - constexpr GraphWay way{N_Way}; - constexpr GraphWay inv{way.invert()}; - - // m_pending maps each pending vertex to the amount that it wayward - // CP will grow. - // - // We can iterate over the pending set in reverse order, always - // choosing the nodes with the largest pending CP-growth. - // - // The intuition is: if the original seed node had its CP grow by - // 50, the most any wayward node can possibly grow is also 50. So - // for anything pending to grow by 50, we know we can process it - // once and we won't have to grow its CP again on the current pass. - // After we're done with all the grow-by-50s, nothing else will - // grow by 50 again on the current pass, and we can process the - // grow-by-49s and we know we'll only have to process each one - // once. And so on. - // - // This generalizes to multiple seed nodes also. - while (!m_pendingHeap.empty()) { - // Pop max element from heap - PendingHeapNode* const maxp = m_pendingHeap.max(); - m_pendingHeap.remove(maxp); - // Pick up values - LogicMTask* const mtaskp = maxp->key().m_mtaskp; - const uint64_t cpGrowBy = maxp->key().m_score; - // Free the heap node, we are done with it - freeNode(maxp); - mtaskp->m_propagateHeapNodep = nullptr; - // Update the critPathCost of mtaskp, that was out-of-date with respect to its edges - const uint64_t startCp = mtaskp->critPathCost(way); - const uint64_t newCp = startCp + cpGrowBy; - if (VL_UNLIKELY(m_slowAsserts)) { - // Check that CP matches that of the longest edge wayward of vxp. - const uint64_t edgeCp = mtaskp->m_edgeHeap[inv].max()->key().m_score; - UASSERT_OBJ(edgeCp == newCp, mtaskp, "CP doesn't match longest wayward edge"); - // Confirm that we only set each node's CP once. That's an - // important property of PropagateCp which allows it to be far - // faster than a recursive algorithm on some graphs. - const bool first = m_seen.insert(mtaskp).second; - UASSERT_OBJ(first, mtaskp, "Set CP on node twice"); - } - mtaskp->setCritPathCost(way, newCp); - cpHasIncreased(mtaskp, newCp + mtaskp->cost()); - } - - if (VL_UNLIKELY(m_slowAsserts)) m_seen.clear(); - } - -private: - VL_UNCOPYABLE(PropagateCp); }; //============================================================================= @@ -486,31 +273,87 @@ private: // The graph of LogicMTask vertices and MTaskEdge edges, used during multi-threaded scheduling. class OrderMTaskGraph final : public V3Graph { + // MEMBERS OrderMoveGraph& m_moveGraph; // The OrderMoveGraph this graph is built from LogicMTask* const m_entryp; // The singular entry point vertex LogicMTask* const m_exitp; // The singular exit point vertex - // The critical path propagators, one for each direction. Owned here so the algorithms - // operating on this graph (contraction, hazard fixing) share them. - PropagateCp m_forwardPropagator; // Forward propagator - PropagateCp m_reversePropagator; // Reverse propagator + const bool m_slowAsserts; // Take extra time to validate the graph ('--debug-partition') + + // Critical path propagation state. Scratch only: the heap is empty, and no MTask is pending, + // between calls to 'propagate'. The node pool persists to recycle the heap nodes. + PropagatePendingHeap m_pendingHeap; // Heap of MTasks pending a critical path update + PoolAllocator m_pendingNodePool; // Allocator for the heap nodes + + // Generation counter, e.g. for marking the MTasks visited by algorithms + uint64_t m_currentGeneration = 0; // CONSTRUCTOR explicit OrderMTaskGraph(OrderMoveGraph& moveGraph); // Used by build(), hence private VL_UNCOPYABLE(OrderMTaskGraph); VL_UNMOVABLE(OrderMTaskGraph); + // METHODS + + bool pathExistsImpl(LogicMTask* fromp, LogicMTask* top, const MTaskEdge* excludedEdgep); + + // Bring the critical paths of all MTasks wayward of 'mtaskp' in direction N_Way, and those + // cached in the edge heaps on the way, up to date. Call after mutating the graph such that + // only MTasks wayward of 'mtaskp' can have a stale critical path, and the critical path of + // 'mtaskp' itself is already correct. 'mtaskp' is read, but never modified. + // + // Note critical paths can only ever grow: those cached in the edge heaps are heap keys, and a + // heap key can be increased in place, but not decreased. + template + void propagate(LogicMTask* mtaskp) { + ++m_currentGeneration; + propagatePush(mtaskp); + propagateResolve(); + } + // Push the inclusive critical path of 'mtaskp' onto each of its wayward relatives, and add any + // relative left with a stale critical path to the pending heap (out of line below) + template + void propagatePush(LogicMTask* mtaskp); + // Resolve all pending critical path increases (out of line below) + template + void propagateResolve(); + // Part of 'validate' + template + void validateWay() const; + public: // ACCESSORS OrderMoveGraph& moveGraph() const { return m_moveGraph; } LogicMTask* entryp() const { return m_entryp; } LogicMTask* exitp() const { return m_exitp; } - PropagateCp& forwardPropagator() { return m_forwardPropagator; } - PropagateCp& reversePropagator() { return m_reversePropagator; } + bool slowAsserts() const { return m_slowAsserts; } // METHODS uint64_t totalCost() const; // O(V), called once + // True if there's a path from 'fromp' to 'top' excluding 'excludedEdgep', false otherwise. + // 'excludedEdgep' may be nullptr in which case no edge is excluded. If 'excludedEdgep' is + // non-nullptr it must connect fromp and top. + bool pathExists(LogicMTask* fromp, LogicMTask* top, const MTaskEdge* excludedEdgep) { + ++m_currentGeneration; + return pathExistsImpl(fromp, top, excludedEdgep); + } + + // Add an edge to the graph, update impacted critical paths + void addEdge(LogicMTask* fromp, LogicMTask* top); + + // Merge 'donorp' into 'recipientp': move the contents and all edges of 'donorp' onto + // 'recipientp', update impacted critical paths, then delete 'donorp'. The edge connecting the + // two (if any) becomes internal to the merged MTask and is deleted, as is one of each pair of + // edges the two have to a common relative. Note this deletes edges, so the caller must have + // released any auxiliary data it attached to them via their user pointer. + void mergeMTasks(LogicMTask* recipientp, LogicMTask* donorp); + + // Do an EXPENSIVE check that the maintained critical paths, including the ones cached in the + // edge heaps, match those implied by the current edges of the graph, and that the graph itself + // is consistent. Does nothing unless 'slowAsserts', so it is safe to call unconditionally. + void validate() const; + // STATIC METHODS // Build an MTask graph from 'moveGraph' static std::unique_ptr build(OrderMoveGraph& moveGraph) VL_MT_DISABLED; @@ -523,9 +366,9 @@ public: //============================================================================= // MTaskEdge method definitions (need the full definition of LogicMTask) -MTaskEdge::MTaskEdge(OrderMTaskGraph* graphp, LogicMTask* fromp, LogicMTask* top, int weight) - : V3GraphEdge{graphp, fromp, top, weight} { - fromp->addRelativeMTask(top); +MTaskEdge::MTaskEdge(OrderMTaskGraph* graphp, LogicMTask* fromp, LogicMTask* top) + : V3GraphEdge{graphp, fromp, top, 1} { + fromp->addDependent(top); fromp->addRelativeEdge(this); top->addRelativeEdge(this); } @@ -537,13 +380,4 @@ LogicMTask* MTaskEdge::furtherMTaskp() const { LogicMTask* MTaskEdge::fromMTaskp() const { return static_cast(fromp()); } LogicMTask* MTaskEdge::toMTaskp() const { return static_cast(top()); } -void MTaskEdge::resetCriticalPaths() { - LogicMTask* const fromp = fromMTaskp(); - LogicMTask* const top = toMTaskp(); - fromp->removeRelativeEdge(this); - top->removeRelativeEdge(this); - fromp->addRelativeEdge(this); - top->addRelativeEdge(this); -} - #endif // Guard diff --git a/src/V3OrderParallel.cpp b/src/V3OrderParallel.cpp index 0f20eb11d..a68ceed97 100644 --- a/src/V3OrderParallel.cpp +++ b/src/V3OrderParallel.cpp @@ -71,6 +71,9 @@ static std::unique_ptr partition(OrderMoveGraph& moveGraph) { mTaskGraphp->hashGraphDebug("MTask graph after contract()"); } + // Note the graph is only mutated by generic V3Graph algorithms from here on. These neither + // maintain the critical paths of the MTasks, nor create MTaskEdges when rerouting, so the + // critical paths are stale below, and the graph must not be handed back to OrderMTaskGraph. mTaskGraphp->removeTransitiveEdges(); mTaskGraphp->hashGraphDebug("MTask graph after removeTransitiveEdges()"); diff --git a/src/V3PairingHeap.h b/src/V3PairingHeap.h index f435ac75f..7beb1ee41 100644 --- a/src/V3PairingHeap.h +++ b/src/V3PairingHeap.h @@ -25,8 +25,8 @@ //============================================================================= // Pairing heap (max-heap) with increase key and delete. // -// While this is written as a generic data structure, it's interface and -// implementation is finely tuned for use by V3Partition, and is critical +// While this is written as a generic data structure, its interface and +// implementation is finely tuned for use by parallel scheduling, and is critical // to Verilation performance, so be very careful changing anything or adding any // new operations that would impact either memory usage, or performance of the // existing operations. This data structure is fully deterministic, meaning diff --git a/src/V3PoolAllocator.h b/src/V3PoolAllocator.h new file mode 100644 index 000000000..436ffa308 --- /dev/null +++ b/src/V3PoolAllocator.h @@ -0,0 +1,85 @@ +// -*- mode: C++; c-file-style: "cc-mode" -*- +//************************************************************************* +// DESCRIPTION: Verilator: Chunked pool allocator +// +// Code available from: https://verilator.org +// +//************************************************************************* +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of either the GNU Lesser General Public License Version 3 +// or the Perl Artistic License Version 2.0. +// SPDX-FileCopyrightText: 2003-2026 Wilson Snyder +// SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0 +// +//************************************************************************* + +#ifndef VERILATOR_V3POOLALLOCATOR_H_ +#define VERILATOR_V3POOLALLOCATOR_H_ + +#include "config_build.h" +#include "verilatedos.h" + +#include +#include +#include +#include +#include + +// Hands out elements of a single type, allocated 'N_ChunkSize' at a time for +// efficiency. Released elements are recycled via a free list stored in the +// same storage. All memory is released when the pool is destroyed, so the +// elements it handed out must not be used beyond its lifetime, and must all +// have been released by then. +template +class PoolAllocator final { + static_assert(N_ChunkSize > 0, "Chunk size must be non-zero"); + + // A slot of storage, holding either a live element, or a link in the free list. + union Slot final { + T_Elem m_elem; // Storage for the allocated element + Slot* m_nextFreep; // Link to the next free slot + Slot() {} + ~Slot() {} + }; + + // MEMBERS + Slot* m_freep = nullptr; // Head of the free list + std::vector> m_allocated; // The allocated chunks + +public: + // CONSTRUCTORS + PoolAllocator() = default; + VL_UNCOPYABLE(PoolAllocator); + VL_UNMOVABLE(PoolAllocator); + + // METHODS + // Allocate an element, constructed with the given arguments + template + T_Elem* alloc(Args&&... args) { + // If no free slots available, then make some + if (!m_freep) { + // Allocate in chunks for efficiency + m_allocated.emplace_back(new Slot[N_ChunkSize]); + // Chain the new slots into the free list + Slot* const chunkp = m_allocated.back().get(); + for (size_t i = 1; i < N_ChunkSize; ++i) chunkp[i - 1].m_nextFreep = &chunkp[i]; + chunkp[N_ChunkSize - 1].m_nextFreep = nullptr; + m_freep = chunkp; + } + // Free slots are available, pick up the first one + Slot* const slotp = m_freep; + m_freep = slotp->m_nextFreep; + return new (&slotp->m_elem) T_Elem{std::forward(args)...}; + } + + // Destroy an element, and return its slot for future allocation + void free(T_Elem* elemp) { + elemp->~T_Elem(); + Slot* const slotp = reinterpret_cast(elemp); + slotp->m_nextFreep = m_freep; + m_freep = slotp; + } +}; + +#endif // Guard