diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 90a0d3ab4..ef11e60ec 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -149,6 +149,7 @@ set(HEADERS V3OrderGraph.h V3OrderInternal.h V3OrderMoveGraph.h + V3OrderMTaskGraph.h V3Os.h V3PairingHeap.h V3Param.h @@ -170,7 +171,6 @@ set(HEADERS V3Sampled.h V3Sched.h V3Scope.h - V3Scoreboard.h V3SenExprBuilder.h V3SenTree.h V3Simulate.h @@ -319,6 +319,9 @@ set(COMMON_SOURCES V3Order.cpp V3OrderGraphBuilder.cpp V3OrderMoveGraph.cpp + V3OrderMTaskContraction.cpp + V3OrderMTaskFixHazards.cpp + V3OrderMTaskGraph.cpp V3OrderParallel.cpp V3OrderProcessDomains.cpp V3OrderSerial.cpp @@ -346,7 +349,6 @@ set(COMMON_SOURCES V3SchedUtil.cpp V3SchedVirtIface.cpp V3Scope.cpp - V3Scoreboard.cpp V3Slice.cpp V3Split.cpp V3SplitVar.cpp diff --git a/src/Makefile_obj.in b/src/Makefile_obj.in index f5a728063..1c2b08a73 100644 --- a/src/Makefile_obj.in +++ b/src/Makefile_obj.in @@ -309,6 +309,9 @@ RAW_OBJS_PCH_ASTNOMT = \ V3Order.o \ V3OrderGraphBuilder.o \ V3OrderMoveGraph.o \ + V3OrderMTaskContraction.o \ + V3OrderMTaskFixHazards.o \ + V3OrderMTaskGraph.o \ V3OrderParallel.o \ V3OrderProcessDomains.o \ V3OrderSerial.o \ @@ -330,7 +333,6 @@ RAW_OBJS_PCH_ASTNOMT = \ V3SchedUtil.o \ V3SchedVirtIface.o \ V3Scope.o \ - V3Scoreboard.o \ V3Slice.o \ V3Split.o \ V3SplitVar.o \ diff --git a/src/V3ExecGraph.cpp b/src/V3ExecGraph.cpp index edafaa962..1129360a0 100644 --- a/src/V3ExecGraph.cpp +++ b/src/V3ExecGraph.cpp @@ -879,8 +879,9 @@ void finalizeCosts(V3Graph* execMTaskGraphp) { execMTaskGraphp->removeTransitiveEdges(); // Record summary stats for final m_tasks graph. - const auto report = execMTaskGraphp->parallelismReport( - [](const V3GraphVertex* vtxp) { return vtxp->as()->cost(); }); + const auto report = execMTaskGraphp->parallelismReport([](const V3GraphVertex* vtxp) { // + return vtxp->as()->cost(); + }); V3Stats::addStat("MTask graph, final, critical path cost", report.criticalPathCost()); V3Stats::addStat("MTask graph, final, total graph cost", report.totalGraphCost()); V3Stats::addStat("MTask graph, final, mtask count", report.vertexCount()); diff --git a/src/V3Graph.cpp b/src/V3Graph.cpp index bdb72f6d5..2ce4a0f26 100644 --- a/src/V3Graph.cpp +++ b/src/V3Graph.cpp @@ -388,3 +388,18 @@ void V3Graph::dumpDotFile(const string& filename, bool colorAsSubgraph) const { cout << "dot -Tpdf -o ~/a.pdf " << filename << "\n"; } + +void V3Graph::hashGraphDebug(const char* debugName) const { + // Disabled when there are no nondeterminism issues in flight. + if (!v3Global.opt.debugNondeterminism()) return; + + // Assign a unique ID to each vertex for pointer stability, then hash + uint32_t id = 1; + std::unordered_map vx2Id; + for (const V3GraphVertex& vtx : vertices()) vx2Id[&vtx] = ++id; + V3Hash hash; + for (const V3GraphVertex& vtx : vertices()) { + for (const V3GraphEdge& edge : vtx.outEdges()) hash += vx2Id[edge.top()]; + } + UINFO(0, "Hash of shape (not contents) of " << debugName << " = " << cvtToHex(hash.value())); +} diff --git a/src/V3Graph.h b/src/V3Graph.h index 7ecb09418..a640a3df8 100644 --- a/src/V3Graph.h +++ b/src/V3Graph.h @@ -450,6 +450,10 @@ public: void dumpDotFilePrefixedAlways(const string& nameComment, bool colorAsSubgraph = false) const VL_MT_DISABLED; void dumpEdges(std::ostream& os, const V3GraphVertex& vertex) const VL_MT_DISABLED; + // Print a hash of the shape of graphp. When debugging nondeterminism, this can help + // pinpoint where it's coming from. + void hashGraphDebug(const char* debugName) const VL_MT_DISABLED; + static void selfTest() VL_MT_DISABLED; class ParallelismReport final { diff --git a/src/V3Order.cpp b/src/V3Order.cpp index 54991d0c5..54cb7fba3 100644 --- a/src/V3Order.cpp +++ b/src/V3Order.cpp @@ -121,7 +121,7 @@ AstCFunc* V3Order::order(AstNetlist* netlistp, // AstNodeStmt* stmtsp = nullptr; if (!moveGraphp->empty()) { if (parallel) { - stmtsp = createParallel(*graph, *moveGraphp, tag, slow); + stmtsp = createParallel(*moveGraphp, tag, slow); } else { stmtsp = createSerial(*moveGraphp, tag, slow); } diff --git a/src/V3Order.h b/src/V3Order.h index 7ff61a7b0..34d644e6e 100644 --- a/src/V3Order.h +++ b/src/V3Order.h @@ -51,8 +51,6 @@ AstCFunc* order(AstNetlist* netlistp, // bool slow, // const ExternalDomainsProvider& externalDomains) VL_MT_DISABLED; -void selfTestParallel(); - }; // namespace V3Order #endif // Guard diff --git a/src/V3OrderInternal.h b/src/V3OrderInternal.h index 86a508204..789a6e6d8 100644 --- a/src/V3OrderInternal.h +++ b/src/V3OrderInternal.h @@ -51,14 +51,9 @@ void processDomains(AstNetlist* netlistp, // const std::string& tag, // const ExternalDomainsProvider& externalDomains); -AstNodeStmt* createSerial(OrderMoveGraph& moveGraph, // - const std::string& tag, // - bool slow); +AstNodeStmt* createSerial(OrderMoveGraph& moveGraph, const std::string& tag, bool slow); -AstNodeStmt* createParallel(const OrderGraph& orderGraph, // - OrderMoveGraph& moveGraph, // - const std::string& tag, // - bool slow); +AstNodeStmt* createParallel(OrderMoveGraph& moveGraph, const std::string& tag, bool slow); }; // namespace V3Order diff --git a/src/V3OrderMTaskContraction.cpp b/src/V3OrderMTaskContraction.cpp new file mode 100644 index 000000000..94220a4ce --- /dev/null +++ b/src/V3OrderMTaskContraction.cpp @@ -0,0 +1,1277 @@ +// -*- mode: C++; c-file-style: "cc-mode" -*- +//************************************************************************* +// DESCRIPTION: Verilator: Multi-threaded MTask graph contraction (coarsening) +// +// Code available from: https://verilator.org +// +//************************************************************************* +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of either the GNU Lesser General Public License Version 3 +// or the Perl Artistic License Version 2.0. +// SPDX-FileCopyrightText: 2003-2026 Wilson Snyder +// SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0 +// +//************************************************************************* +// +// Coarsens the fine-grained MTask graph produced by the partitioner by +// repeatedly contracting MTasks (merging along an edge, or merging two +// "sibling" MTasks) until a critical-path score limit is reached. Driven by +// the partitioner in V3OrderParallel.cpp via OrderMTaskGraph::contract, +// declared in V3OrderMTaskGraph.h. +// +//************************************************************************* + +#include "V3PchAstNoMT.h" // VL_MT_DISABLED_CODE_UNIT + +#include "V3Global.h" +#include "V3Graph.h" +#include "V3GraphStream.h" +#include "V3OrderMTaskGraph.h" +#include "V3PairingHeap.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +VL_DEFINE_DEBUG_FUNCTIONS; + +class MergeCandidate; +class SiblingMC; +class EdgeMC; + +// ###################################################################### +// Partitioner tunable settings: +// +// Before describing these settings, a bit of background: +// +// Early during the development of the partitioner, V3Split was failing to +// split large always blocks (with ~100K assignments) so we had to handle +// very large vertices with ~100K incoming and outgoing edges. +// +// The partitioner attempts to deal with such densely connected +// graphs. Some of the tuning parameters below reference "huge vertices", +// that's what they're talking about, vertices with tens of thousands of +// edges in and out. Whereas most graphs have only tens of edges in and out +// of most vertices. +// +// V3Split has since been fixed to more reliably split large always +// blocks. It's kind of an open question whether the partitioner must +// handle huge nodes gracefully. Maybe not! But it still can, given +// appropriate tuning. + +// PART_SIBLING_EDGE_LIMIT (integer) +// +// Arbitrarily limit the number of edges on a single vertex that will be +// considered when enumerating siblings, to the given value. This protects +// the partitioner runtime in the presence of huge vertices. +// +// The sibling-merge is less important than the edge merge. (You can +// totally disable the sibling merge and get halfway decent partitions; you +// can't disable edge merges, those are fundamental to the process.) So, +// skipping the enumeration of some siblings on a few vertices does not +// have a large impact on the result of the partitioner. +// +// If your vertices are small, the limit (at 26) approaches a no-op. Hence +// there's basically no cost to applying this limit even when we don't +// expect huge vertices. +// +// If you don't care about partitioner runtime and you want the most +// aggressive partition, set the limit very high. If you have huge +// vertices, leave this as is. +constexpr unsigned PART_SIBLING_EDGE_LIMIT = 26; + +// Don't produce more than a certain maximum number of MTasks. This helps +// the TSP variable sort not to blow up (a concern for some of the tests) +// and we probably don't want a huge number of MTasks in practice anyway +// (50 to 100 is typical.) +// +// If the user doesn't give one with '--threads-max-mtasks', we'll set the +// maximum # of MTasks to +// (# of threads * PART_DEFAULT_MAX_MTASKS_PER_THREAD) +constexpr unsigned PART_DEFAULT_MAX_MTASKS_PER_THREAD = 50; + +// end tunables. + +//###################################################################### +// MTask utility classes + +struct MergeCandidateKey final { + // Note: Structure layout chosen to minimize padding in PairingHeap<*>::Node + uint64_t m_id; // Unique ID part of edge score + uint64_t m_score; // Score part of ID + bool operator<(const MergeCandidateKey& other) const { + // First by Score then by ID, but notice that we want minimums using a max-heap, so reverse + return m_score > other.m_score || (m_score == other.m_score && m_id > other.m_id); + } +}; + +// For efficiency, MergeCandidateScoreboard elements must derive from +// PairingHeap::Node +using MergeCandidateHeapNode = PairingHeap::Node; + +// Information associated with scoreboarding a merge candidate +class MergeCandidate VL_NOT_FINAL : public MergeCandidateHeapNode { + // Only the known subclasses can create or delete one of these + friend class SiblingMC; + friend class EdgeMC; + + // This structure is extremely hot. To save 8 bytes we pack + // one bit indicating removedFromSb with the id. To save another + // 8 bytes by not having a virtual function table, we implement the + // few polymorphic methods over the two known subclasses explicitly, + // using another bit of the id to denote the actual subtype. + + // By using the bottom bits for flags, we can still use < to compare IDs without masking. + // <63:1> Serial number for ordering, <0> subtype (SiblingMC) + static constexpr uint64_t IS_SIBLING_MASK = 1ULL << 0; + static constexpr uint64_t ID_INCREMENT = 1ULL << 1; + + bool isSiblingMC() const { return m_key.m_id & IS_SIBLING_MASK; } + + // CONSTRUCTORS + explicit MergeCandidate(bool isSiblingMC) { + static uint64_t s_serial = 0; + s_serial += ID_INCREMENT; // +ID_INCREMENT so doesn't set the special bottom bits + m_key.m_id = s_serial | (isSiblingMC * IS_SIBLING_MASK); + } + ~MergeCandidate() = default; + +public: + // METHODS + SiblingMC* toSiblingMC(); // Instead of cast<>/as<> + EdgeMC* toEdgeMC(); // Instead of cast<>/as<> + bool mergeWouldCreateCycle() const; // Instead of virtual method + + inline void rescore(); + uint64_t score() const { return m_key.m_score; } + + static MergeCandidate* heapNodeToElem(MergeCandidateHeapNode* nodep) { + return static_cast(nodep); + } +}; + +static_assert(sizeof(MergeCandidate) == sizeof(MergeCandidateHeapNode), + "Should not have a vtable"); + +// A pair of associated LogicMTask's that are merge candidates for sibling +// contraction +class SiblingMC final : public MergeCandidate { + LogicMTask* const m_ap; // The higher ID MTask + LogicMTask* const m_bp; // The lower ID MTask + + V3ListLinks m_aLinks; // List links to store instances of this class + V3ListLinks m_bLinks; // List links to store instances of this class + + V3ListLinks& aLinks() { return m_aLinks; } + V3ListLinks& bLinks() { return m_bLinks; } + +public: + // List type to store instances of this class + using AList = V3List; + using BList = V3List; + + // CONSTRUCTORS + SiblingMC(LogicMTask* ap, LogicMTask* bp); + ~SiblingMC() = default; + + // METHODS + void unlinkA(); + void unlinkB(); + + LogicMTask* ap() const { return m_ap; } + LogicMTask* bp() const { return m_bp; } + bool mergeWouldCreateCycle() const; +}; + +static_assert(!std::is_polymorphic::value, "Should not have a vtable"); + +// A merge candidate associated with an MTaskEdge (edge contraction candidate) +class EdgeMC final : public MergeCandidate { + MTaskEdge* const m_edgep; // The associated edge + +public: + // CONSTRUCTORS + explicit EdgeMC(MTaskEdge* edgep) + : MergeCandidate{/* isSiblingMC: */ false} + , m_edgep{edgep} {} + ~EdgeMC() = default; + + // METHODS + MTaskEdge* edgep() const { return m_edgep; } + bool mergeWouldCreateCycle() const; +}; + +static_assert(!std::is_polymorphic::value, "Should not have a vtable"); + +// Auxiliary data associated with each LogicMTask during Contraction, attached via +// LogicMTask::userp(). Kept out of LogicMTask itself so that LogicMTask does not depend on the +// MergeCandidate hierarchy (which the SiblingMC lists reference). +struct MTaskContractionData final { + // MTasks for which a SiblingMC exists with the owning MTask as the higher ID MTask (m_ap) + std::unordered_set siblings; + // SiblingMCs for which the owning MTask is the higher ID MTask (m_ap in SiblingMC) + SiblingMC::AList aSiblingMCs; + // SiblingMCs for which the owning MTask is the lower ID MTask (m_bp in SiblingMC) + SiblingMC::BList bSiblingMCs; +}; + +// The MTaskContractionData attached to 'mtaskp' (see LogicMTask::userp) +static MTaskContractionData& mtaskData(const LogicMTask* mtaskp) { + return *static_cast(mtaskp->userp()); +} + +// The EdgeMC associated with 'edgep' while it is on the scoreboard, or nullptr otherwise. Held in +// the edge's user pointer (see MTaskEdge), kept here so MTaskEdge does not depend on EdgeMC. +static EdgeMC* edgeMC(const MTaskEdge* edgep) { return static_cast(edgep->userp()); } + +// Instead of dynamic cast +SiblingMC* MergeCandidate::toSiblingMC() { + return isSiblingMC() ? static_cast(this) : nullptr; +} + +EdgeMC* MergeCandidate::toEdgeMC() { return isSiblingMC() ? nullptr : static_cast(this); } + +// Normally this would be a virtual function, but we save space by not having a vtable, +// and we know we only have 2 possible subclasses. +bool MergeCandidate::mergeWouldCreateCycle() const { + return isSiblingMC() ? static_cast(this)->mergeWouldCreateCycle() + : static_cast(this)->mergeWouldCreateCycle(); +} + +static uint64_t siblingScore(const SiblingMC* sibsp) { + const LogicMTask* const ap = sibsp->ap(); + const LogicMTask* const bp = sibsp->bp(); + const uint64_t mergedCpCostFwd + = std::max(ap->critPathCost(GraphWay::FORWARD), bp->critPathCost(GraphWay::FORWARD)); + const uint64_t mergedCpCostRev + = std::max(ap->critPathCost(GraphWay::REVERSE), bp->critPathCost(GraphWay::REVERSE)); + return mergedCpCostRev + mergedCpCostFwd + LogicMTask::stepCost(ap->cost() + bp->cost()); +} + +static uint64_t edgeScore(const MTaskEdge* edgep) { + // Score this edge. Lower is better. The score is the new local CP + // length if we merge these MTasks. ("Local" means the longest + // critical path running through the merged node.) + const LogicMTask* const top = edgep->toMTaskp(); + const LogicMTask* const fromp = edgep->fromMTaskp(); + const uint64_t mergedCpCostFwd = std::max(fromp->critPathCost(GraphWay::FORWARD), + top->critPathCostWithout(edgep)); + const uint64_t mergedCpCostRev = std::max(fromp->critPathCostWithout(edgep), + top->critPathCost(GraphWay::REVERSE)); + return mergedCpCostRev + mergedCpCostFwd + LogicMTask::stepCost(fromp->cost() + top->cost()); +} + +void MergeCandidate::rescore() { + if (const SiblingMC* const sibp = toSiblingMC()) { + m_key.m_score = siblingScore(sibp); + } else { + // Give a slight preference to sibling merges by increasing the cost of edge merges. + // This biases towards sibling merges in case they are equal score with edge merges. + // This avoid a central node growing while many leaves remain due to edge merges. + m_key.m_score = 1 + edgeScore(static_cast(this)->edgep()); + } +} + +SiblingMC::SiblingMC(LogicMTask* ap, LogicMTask* bp) + : MergeCandidate{/* isSiblingMC: */ true} + , m_ap{ap} + , m_bp{bp} { + // Storage management depends on this + UASSERT(ap->id() > bp->id(), "Should be ordered"); + UDEBUGONLY(UASSERT(mtaskData(ap).siblings.count(bp), "Should be in sibling map");); + mtaskData(m_ap).aSiblingMCs.linkBack(this); + mtaskData(m_bp).bSiblingMCs.linkBack(this); +} + +void SiblingMC::unlinkA() { + VL_ATTR_UNUSED const size_t removed = mtaskData(m_ap).siblings.erase(m_bp); + UDEBUGONLY(UASSERT(removed == 1, "Should have been in sibling set");); + mtaskData(m_ap).aSiblingMCs.unlink(this); +} + +void SiblingMC::unlinkB() { mtaskData(m_bp).bSiblingMCs.unlink(this); } + +// cppcheck-suppress duplInheritedMember +bool SiblingMC::mergeWouldCreateCycle() const { + return (LogicMTask::pathExistsFrom(m_ap, m_bp, nullptr) + || LogicMTask::pathExistsFrom(m_bp, m_ap, nullptr)); +} + +// cppcheck-suppress duplInheritedMember +bool EdgeMC::mergeWouldCreateCycle() const { + return LogicMTask::pathExistsFrom(m_edgep->fromMTaskp(), m_edgep->toMTaskp(), m_edgep); +} + +// Scoreboard of MTask merge candidates. Owns the lifetime of the merge candidate objects: callers +// add/remove candidates via the methods below and never allocate or free them directly. For edges +// this maintains the invariant that an MTaskEdge has an associated EdgeMC (held in its userp()), +// if and only if it is currently on the scoreboard. +// +// This is essentially a heap that can be hinted that some elements have changed keys, at which +// point those elements are deferred as 'unknown' until the next 'rescore' call. We use the +// generic PairingHeap, relying on its internal structure. For efficiency, the merge candidates are +// themselves the heap nodes (MergeCandidate derives from PairingHeap::Node), so +// a candidate can be on at most one scoreboard. +class MergeCandidateScoreboard final { + // TYPES + using Heap = PairingHeap; + using Node = Heap::Node; + using Link = Heap::Link; + + // MEMBERS + Heap m_known; // The heap of candidates with known scores + Link m_unknown; // List of candidates with unknown scores + + // METHODS + void addUnknown(MergeCandidate* nodep) { + // Just prepend it to the list of unknown entries + nodep->m_next.link(m_unknown.unlink()); + m_unknown.linkNonNull(nodep); + // We mark nodes on the unknown list by making their child pointer point to themselves + nodep->m_kids.m_ptr = nodep; + } + + // Add a freshly created candidate. Not returned by 'best' before the next 'rescore' call. + void add(MergeCandidate* nodep) { addUnknown(nodep); } + + // Remove a candidate from the scoreboard. + void remove(MergeCandidate* nodep) { + if (nodep->m_kids.m_ptr == nodep) { + // Node is on the unknown list, replace with next + nodep->replaceWith(nodep->m_next.unlink()); + return; + } + // Node is in the known heap, remove it + m_known.remove(nodep); + } + +public: + // CONSTRUCTORS + MergeCandidateScoreboard() = default; + ~MergeCandidateScoreboard() = default; + VL_UNCOPYABLE(MergeCandidateScoreboard); + + // The candidate with the best (lowest) known score, or nullptr if none have a known score. + // This does not automatically 'rescore'; the caller must 'rescore' to reflect all candidates. + MergeCandidate* best() const { return MergeCandidate::heapNodeToElem(m_known.max()); } + + // Tell the scoreboard a candidate's score may have changed. Its score becomes 'unknown' and it + // will not be returned by 'best' until the next 'rescore'. + void hintScoreChanged(MergeCandidate* nodep) { + // If it's already in the unknown list, then nothing to do + if (nodep->m_kids.m_ptr == nodep) return; + // Otherwise it was in the heap, remove it + m_known.remove(nodep); + // Prepend it to the unknown list + addUnknown(nodep); + } + + // True if there are candidates with an unknown score + bool needsRescore() const { return m_unknown; } + // True if the given candidate's score is unknown + static bool needsRescore(const MergeCandidate* nodep) { return nodep->m_kids.m_ptr == nodep; } + + // For each candidate whose score is unknown, recompute the score and add to the known heap + void rescore() { + for (Node *nodep = m_unknown.unlink(), *nextp; nodep; nodep = nextp) { + // Pick up next + nextp = nodep->m_next.ptr(); + // Reset pointers + nodep->m_next.m_ptr = nullptr; + nodep->m_kids.m_ptr = nullptr; + nodep->m_ownerpp = nullptr; + // Re-compute the score of the candidate + MergeCandidate::heapNodeToElem(nodep)->rescore(); + // Re-insert into the heap + m_known.insert(nodep); + } + } + + // Create the merge candidate for 'edgep' and add it to the scoreboard (out-of-line below) + void addEdge(MTaskEdge* edgep) { + UDEBUGONLY(UASSERT(!edgep->userp(), "Edge already has a merge candidate");); + EdgeMC* const edgeMCp = new EdgeMC{edgep}; + edgep->userp(edgeMCp); + add(edgeMCp); + } + // Remove 'edgep's merge candidate from the scoreboard and delete it (out-of-line below) + void removeEdge(MTaskEdge* edgep) { + EdgeMC* const edgeMCp = edgeMC(edgep); + UDEBUGONLY(UASSERT(edgeMCp, "Edge has no merge candidate");); + edgep->userp(nullptr); + remove(edgeMCp); + VL_DO_DANGLING(delete edgeMCp, edgeMCp); + } + + // Create a sibling merge candidate for 'ap' and 'bp' and add it to the scoreboard + void addSibling(LogicMTask* ap, LogicMTask* bp) { add(new SiblingMC{ap, bp}); } + // Remove sibling merge candidate 'smcp' from the scoreboard and delete it + void removeSibling(SiblingMC* smcp) { + remove(smcp); + smcp->unlinkA(); + smcp->unlinkB(); + VL_DO_DANGLING(delete smcp, smcp); + } +}; + +//###################################################################### + +// Look at vertex costs (in one way) to form critical paths for each +// vertex. +template +static void partInitHalfCriticalPaths(V3Graph& mTaskGraph, bool checkOnly) { + constexpr GraphWay way{N_Way}; + constexpr GraphWay rev = way.invert(); + GraphStreamUnordered order{&mTaskGraph, way}; + for (const V3GraphVertex* vertexp; (vertexp = order.nextp());) { + const LogicMTask* const mtaskcp = static_cast(vertexp); + LogicMTask* const mtaskp = const_cast(mtaskcp); + uint64_t cpCost = 0; +#if VL_DEBUG + std::unordered_set relatives; +#endif + for (const V3GraphEdge& edge : vertexp->edges()) { +#if VL_DEBUG + // Run a few asserts on the initial mtask graph, + // while we're iterating through... + UASSERT_OBJ(edge.weight() != 0, mtaskp, "Should be no cut edges in MTask graph"); + UASSERT_OBJ(relatives.find(edge.furtherp()) == relatives.end(), mtaskp, + "Should be no redundant edges in MTask graph"); + relatives.insert(edge.furtherp()); +#endif + const LogicMTask* const relativep = static_cast(edge.furtherp()); + cpCost = std::max(cpCost, (relativep->critPathCost(way) + + static_cast(relativep->stepCost()))); + } + if (checkOnly) { + partCheckCachedScoreVsActual(mtaskp->critPathCost(way), cpCost); + } else { + mtaskp->setCritPathCost(way, cpCost); + } + } +} + +// Look at vertex costs to form critical paths for each vertex. +static void partInitCriticalPaths(V3Graph& mTaskGraph) { + partInitHalfCriticalPaths(mTaskGraph, false); + partInitHalfCriticalPaths(mTaskGraph, false); + + // Reset all MTaskEdges so that 'm_edges' will show correct CP numbers. + // They would have been all zeroes on initial creation of the MTaskEdges. + for (V3GraphVertex& vtx : mTaskGraph.vertices()) { + for (V3GraphEdge& edge : vtx.outEdges()) edge.as()->resetCriticalPaths(); + } +} + +// Do an EXPENSIVE check to make sure that all incremental CP updates have +// gone correctly. +static void partCheckCriticalPaths(V3Graph& mTaskGraph) { + partInitHalfCriticalPaths(mTaskGraph, true); + partInitHalfCriticalPaths(mTaskGraph, true); + for (const V3GraphVertex& vtx : mTaskGraph.vertices()) { + const LogicMTask& mtask = static_cast(vtx); + mtask.checkRelativesCp(); + mtask.checkRelativesCp(); + } +} + +// ###################################################################### +// PropagateCp + +template +class PropagateCp final { + // Propagate increasing critical path (CP) costs through a graph. + // + // Usage: + // * Client increases the cost and/or CP at a node or small set of nodes + // (often a pair in practice, eg. edge contraction.) + // * Client calls PropagateCp::cpHasIncreased() one or more times. + // Each call indicates that the inclusive CP of some "seed" vertex + // has increased to a given value. + // * NOTE: PropagateCp will neither read nor modify the cost + // or CPs at the seed vertices, it only accesses and modifies + // vertices wayward from the seeds. + // * Client calls PropagateCp::go(). Internally, this iteratively + // propagates the new CPs wayward through the graph. + // + + // TYPES + + // We keep pending vertices in a heap during critical path propagation + struct PendingKey final { + LogicMTask* m_mtaskp; // The vertex in the heap + uint64_t m_score; // The score of this entry + void increase(uint64_t score) { + UDEBUGONLY(UASSERT(score >= m_score, "Must increase");); + m_score = score; + } + bool operator<(const PendingKey& other) const { + if (m_score != other.m_score) return m_score < other.m_score; + return *m_mtaskp < *other.m_mtaskp; + } + }; + + using PendingHeap = PairingHeap; + using PendingHeapNode = typename PendingHeap::Node; + + // MEMBERS + PendingHeap m_pendingHeap; // Heap of pending rescores + + // We allocate this many heap nodes at once + static constexpr size_t ALLOC_CHUNK_SIZE = 128; + PendingHeapNode* m_freep = nullptr; // List of free heap nodes + std::vector> m_allocated; // Allocated heap nodes + + const bool m_slowAsserts; // Enable nontrivial asserts + // Used only with slow asserts to check MTasks visited only once + std::unordered_set m_seen; + +public: + // CONSTRUCTORS + explicit PropagateCp(bool slowAsserts) + : m_slowAsserts{slowAsserts} {} + + // METHODS +private: + // Allocate a HeapNode for the given element + PendingHeapNode* allocNode() { + // If no free nodes available, then make some + if (!m_freep) { + // Allocate in chunks for efficiency + m_allocated.emplace_back(new PendingHeapNode[ALLOC_CHUNK_SIZE]); + // Set up free list pointer + m_freep = m_allocated.back().get(); + // Set up free list chain + for (size_t i = 1; i < ALLOC_CHUNK_SIZE; ++i) { + m_freep[i - 1].m_next.m_ptr = &m_freep[i]; + } + // Clear the next pointer of the last entry + m_freep[ALLOC_CHUNK_SIZE - 1].m_next.m_ptr = nullptr; + } + // Free nodes are available, pick up the first one + PendingHeapNode* const resultp = m_freep; + m_freep = resultp->m_next.m_ptr; + resultp->m_next.m_ptr = nullptr; + return resultp; + } + + // Release a heap node (make it available for future allocation) + void freeNode(PendingHeapNode* nodep) { + // Re-use the existing link pointers and simply prepend it to the free list + nodep->m_next.m_ptr = m_freep; + m_freep = nodep; + } + +public: + void cpHasIncreased(V3GraphVertex* vxp, uint64_t newInclusiveCp) { + constexpr GraphWay way{N_Way}; + constexpr GraphWay inv{way.invert()}; + + // For *vxp, whose CP-inclusive has just increased to + // newInclusiveCp, iterate to all wayward nodes, update the edges + // of each, and add each to m_pending if its overall CP has grown. + for (V3GraphEdge& graphEdge : vxp->edges()) { + MTaskEdge& edge = static_cast(graphEdge); + + LogicMTask* const relativep = edge.furtherMTaskp(); + EdgeHeap::Node& edgeHeapNode = edge.m_edgeHeapNode[inv]; + if (newInclusiveCp > edgeHeapNode.key().m_score) { + relativep->m_edgeHeap[inv].increaseKey(&edgeHeapNode, newInclusiveCp); + } + + const uint64_t critPathCost = relativep->critPathCost(way); + + if (critPathCost >= newInclusiveCp) continue; + + // relativep's critPathCost() is out of step with its longest !wayward edge. + // Schedule that to be resolved. + const uint64_t newVal = newInclusiveCp - critPathCost; + + void*& pendingNodepRef = relativep->m_propagateHeapNodep; + if (PendingHeapNode* const nodep = static_cast(pendingNodepRef)) { + // Already in heap. Increase score if needed. + if (newVal > nodep->key().m_score) m_pendingHeap.increaseKey(nodep, newVal); + continue; + } + + // Add to heap + PendingHeapNode* const nodep = allocNode(); + pendingNodepRef = nodep; + m_pendingHeap.insert(nodep, {relativep, newVal}); + } + } + + void go() { + constexpr GraphWay way{N_Way}; + constexpr GraphWay inv{way.invert()}; + + // m_pending maps each pending vertex to the amount that it wayward + // CP will grow. + // + // We can iterate over the pending set in reverse order, always + // choosing the nodes with the largest pending CP-growth. + // + // The intuition is: if the original seed node had its CP grow by + // 50, the most any wayward node can possibly grow is also 50. So + // for anything pending to grow by 50, we know we can process it + // once and we won't have to grow its CP again on the current pass. + // After we're done with all the grow-by-50s, nothing else will + // grow by 50 again on the current pass, and we can process the + // grow-by-49s and we know we'll only have to process each one + // once. And so on. + // + // This generalizes to multiple seed nodes also. + while (!m_pendingHeap.empty()) { + // Pop max element from heap + PendingHeapNode* const maxp = m_pendingHeap.max(); + m_pendingHeap.remove(maxp); + // Pick up values + LogicMTask* const mtaskp = maxp->key().m_mtaskp; + const uint64_t cpGrowBy = maxp->key().m_score; + // Free the heap node, we are done with it + freeNode(maxp); + mtaskp->m_propagateHeapNodep = nullptr; + // Update the critPathCost of mtaskp, that was out-of-date with respect to its edges + const uint64_t startCp = mtaskp->critPathCost(way); + const uint64_t newCp = startCp + cpGrowBy; + if (VL_UNLIKELY(m_slowAsserts)) { + // Check that CP matches that of the longest edge wayward of vxp. + const uint64_t edgeCp = mtaskp->m_edgeHeap[inv].max()->key().m_score; + UASSERT_OBJ(edgeCp == newCp, mtaskp, "CP doesn't match longest wayward edge"); + // Confirm that we only set each node's CP once. That's an + // important property of PropagateCp which allows it to be far + // faster than a recursive algorithm on some graphs. + const bool first = m_seen.insert(mtaskp).second; + UASSERT_OBJ(first, mtaskp, "Set CP on node twice"); + } + mtaskp->setCritPathCost(way, newCp); + cpHasIncreased(mtaskp, newCp + mtaskp->stepCost()); + } + + if (VL_UNLIKELY(m_slowAsserts)) m_seen.clear(); + } + +private: + VL_UNCOPYABLE(PropagateCp); +}; + +//###################################################################### +// Contraction + +// Perform edge or sibling contraction on the partition graph +class Contraction final { + // TYPES + // New CP information for mtaskp reflecting an upcoming merge + struct NewCp final { + uint64_t cp; + uint64_t propagateCp; + bool propagate; + }; + + // MEMBERS + OrderMTaskGraph& m_mTaskGraph; // The Mtask graph + uint64_t m_scoreLimit; // Sloppy score allowed when picking merges + // Next score rescore at + uint64_t m_scoreLimitBeforeRescore = std::numeric_limits::max(); + unsigned m_mergesSinceRescore = 0; // Merges since last rescore + const bool m_slowAsserts{v3Global.opt.debugPartition()}; // Take extra time to validate steps + MergeCandidateScoreboard m_sb; // Scoreboard + // Auxiliary per-MTask data (the SiblingMC lists) attached to each MTask via its user pointer. + // Owned here for the lifetime of this Contraction. A single array, as the number of MTasks is + // fixed for that lifetime: merging only ever deletes vertices, never creates them. + std::unique_ptr m_mtaskDatap; + + PropagateCp m_forwardPropagator{m_slowAsserts}; // Forward propagator + PropagateCp m_reversePropagator{m_slowAsserts}; // Reverse propagator + + // Singular source vertex of the OrderMTaskGraph + LogicMTask* const m_entryMTaskp = m_mTaskGraph.entryp(); + // Singular sink vertex of the dependency graph + LogicMTask* const m_exitMTaskp = m_mTaskGraph.exitp(); + + // Merge edges from a LogicMtask, keeping the merge candidate scoreboard in sync. + static void partRedirectEdgesFrom(V3Graph& graph, LogicMTask* recipientp, LogicMTask* donorp, + MergeCandidateScoreboard& sb) { + // This code removes adjacent edges. When this occurs, mark it in need + // of a rescore, in case its score has fallen and we need to move it up + // toward the front of the scoreboard. + // + // Wait, what? Shouldn't the scores only increase as we merge nodes? Well + // that's almost true. But there is one exception. + // + // Suppose we have A->B, B->C, and A->C. + // + // The A->C edge is a "transitive" edge. It's ineligible to be merged, as + // the merge would create a cycle. We score it on the scoreboard like any + // other edge. + // + // However, our "score" estimate for A->C is bogus, because the forward + // critical path to C and the reverse critical path to A both contain the + // same node (B) so we overestimate the score of A->C. At first this + // doesn't matter, since transitive edges aren't eligible to merge anyway. + // + // Later, suppose the edge contractor decides to merge the B->C edge, with + // B donating all its incoming edges into C, say. (So we reach this + // function.) + // + // With B going away, the A->C edge will no longer be transitive and it + // will become eligible to merge. But if we don't mark it for rescore, + // it'll stay in the scoreboard with its old (overestimate) score. We'll + // merge it too late due to the bogus score. When we finally merge it, we + // fail the assert in the main edge contraction loop which checks that the + // actual score did not fall below the scoreboard's score. + // + // Another way of stating this: this code ensures that scores of + // non-transitive edges only ever increase. + + // Process outgoing edges + while (MTaskEdge* const edgep = static_cast(donorp->outEdges().frontp())) { + LogicMTask* const relativep = edgep->toMTaskp(); + + relativep->removeRelativeEdge(edgep); + + if (recipientp->hasRelativeMTask(relativep)) { + // An edge already exists between recipient and relative of donor. + // Mark it in need of a rescore + // The donor edge is going away, so remove it from the scoreboard + if (edgep->userp()) sb.removeEdge(edgep); + MTaskEdge* const existMTaskEdgep = static_cast( + recipientp->findConnectingEdgep(relativep)); + UDEBUGONLY(UASSERT(existMTaskEdgep, "findConnectingEdge didn't find edge");); + // The existing edge is no longer transitive, so may need a rescore + if (EdgeMC* const existEdgeMCp = edgeMC(existMTaskEdgep)) { + sb.hintScoreChanged(existEdgeMCp); + } + VL_DO_DANGLING(edgep->unlinkDelete(), edgep); + } else { + // No existing edge between recipient and relative of donor. + // Redirect the edge from donor<->relative to recipient<->relative. + edgep->relinkFromp(recipientp); + recipientp->addRelativeMTask(relativep); + recipientp->stealRelativeEdge(edgep); + relativep->addRelativeEdge(edgep); + // The redirected edge is a merge candidate again + if (EdgeMC* const edgeMCp = edgeMC(edgep)) { + sb.hintScoreChanged(edgeMCp); + } else { + sb.addEdge(edgep); + } + } + } + + // Process incoming edges + while (MTaskEdge* const edgep = static_cast(donorp->inEdges().frontp())) { + LogicMTask* const relativep = edgep->fromMTaskp(); + + relativep->removeRelativeMTask(donorp); + relativep->removeRelativeEdge(edgep); + + if (relativep->hasRelativeMTask(recipientp)) { + // An edge already exists between recipient and relative of donor. + // Mark it in need of a rescore + // The donor edge is going away, so remove it from the scoreboard + if (edgep->userp()) sb.removeEdge(edgep); + MTaskEdge* const existMTaskEdgep = static_cast( + recipientp->findConnectingEdgep(relativep)); + UDEBUGONLY(UASSERT(existMTaskEdgep, "findConnectingEdge didn't find edge");); + // The existing edge is no longer transitive, so may need a rescore + if (EdgeMC* const existEdgeMCp = edgeMC(existMTaskEdgep)) { + sb.hintScoreChanged(existEdgeMCp); + } + VL_DO_DANGLING(edgep->unlinkDelete(), edgep); + } else { + // No existing edge between recipient and relative of donor. + // Redirect the edge from donor<->relative to recipient<->relative. + edgep->relinkTop(recipientp); + relativep->addRelativeMTask(recipientp); + relativep->addRelativeEdge(edgep); + recipientp->stealRelativeEdge(edgep); + // The redirected edge is a merge candidate again + if (EdgeMC* const edgeMCp = edgeMC(edgep)) { + sb.hintScoreChanged(edgeMCp); + } else { + sb.addEdge(edgep); + } + } + } + + // Remove donorp from the graph + VL_DO_DANGLING(donorp->unlinkDelete(&graph), donorp); + } + + template + NewCp newCp(const LogicMTask* mtaskp, const LogicMTask* otherp, const MTaskEdge* mergeEdgep) { + constexpr GraphWay way{N_Way}; + // Return new wayward-CP for mtaskp reflecting its upcoming merge + // with otherp. Set 'result.propagate' if mtaskp's wayward + // relatives will see a new wayward CP from this merge. + uint64_t newCp; + if (mergeEdgep) { + if (mtaskp == mergeEdgep->furtherp()) { + newCp = std::max(otherp->critPathCost(way), + mtaskp->critPathCostWithout(mergeEdgep)); + } else { + newCp = std::max(mtaskp->critPathCost(way), + otherp->critPathCostWithout(mergeEdgep)); + } + } else { + newCp = std::max(otherp->critPathCost(way), mtaskp->critPathCost(way)); + } + + const uint64_t origRelativesCp = mtaskp->critPathCost(way) + mtaskp->stepCost(); + const uint64_t newRelativesCp + = newCp + LogicMTask::stepCost(mtaskp->cost() + otherp->cost()); + + NewCp result; + result.cp = newCp; + result.propagate = (newRelativesCp > origRelativesCp); + result.propagateCp = newRelativesCp; + return result; + } + + void removeSiblingMCsWith(LogicMTask* mtaskp) { + // Note: 'removeSibling' unlinks the candidate from both of its MTasks' lists, so taking + // the front element repeatedly does terminate. It also erases the candidate from the + // owning (higher id) MTask's sibling set as it goes, so both the sets and the lists are + // left consistent, whichever side of the candidate 'mtaskp' happens to be on. + while (SiblingMC* const smcp = mtaskData(mtaskp).aSiblingMCs.frontp()) { + m_sb.removeSibling(smcp); + } + while (SiblingMC* const smcp = mtaskData(mtaskp).bSiblingMCs.frontp()) { + m_sb.removeSibling(smcp); + } + } + + void removeSiblingMCs(LogicMTask* recipientp, LogicMTask* donorp) { + // These two can share a SiblingMC (an edge between them does not preclude one). That is + // fine: 'removeSiblingMCsWith' unlinks each candidate from both sides, so the shared one + // is gone by the time we get to the donor. + // + // This also leaves both sibling sets empty, so they need no separate clearing: each entry + // in an MTask's sibling set is added by 'makeSiblingMC' together with a SiblingMC on that + // same MTask's 'aSiblingMCs' list, and draining that list erases the matching entry (see + // 'SiblingMC::unlinkA'). The slow assert in 'makeSiblingMC' catches it if that ever + // diverges, as a stale set entry there suppresses creating the SiblingMC it stands for. + removeSiblingMCsWith(recipientp); + removeSiblingMCsWith(donorp); + } + + void contract(MergeCandidate* mergeCanp) { + LogicMTask* top = nullptr; + LogicMTask* fromp = nullptr; + EdgeMC* const mergeEdgeMCp = mergeCanp->toEdgeMC(); + MTaskEdge* const mergeEdgep = mergeEdgeMCp ? mergeEdgeMCp->edgep() : nullptr; + SiblingMC* const mergeSibsp = mergeCanp->toSiblingMC(); + if (mergeEdgep) { + top = mergeEdgep->toMTaskp(); + fromp = mergeEdgep->fromMTaskp(); + } else { + top = mergeSibsp->ap(); + fromp = mergeSibsp->bp(); + } + + // Merge the smaller mtask into the larger mtask. If one of them + // is much larger, this will save time in partRedirectEdgesFrom(). + // Assume the more costly mtask has more edges. + // + // [TODO: now that we have edge maps, we could count the edges + // exactly without a linear search.] + LogicMTask* recipientp; + LogicMTask* donorp; + if (fromp->cost() > top->cost()) { + recipientp = fromp; + donorp = top; + } else { + donorp = fromp; + recipientp = top; + } + VL_DANGLING(fromp); + VL_DANGLING(top); // Use donorp and recipientp now instead + + // Recursively update forward and reverse CP numbers. + // + // Doing this before merging the MTasks lets us often avoid + // recursing through either incoming or outgoing edges on one or + // both MTasks. + // + // These 'NewCp' objects carry a bit indicating whether we must + // propagate CP for each of the four cases: + const NewCp recipientNewCpFwd = newCp(recipientp, donorp, mergeEdgep); + const NewCp donorNewCpFwd = newCp(donorp, recipientp, mergeEdgep); + const NewCp recipientNewCpRev = newCp(recipientp, donorp, mergeEdgep); + const NewCp donorNewCpRev = newCp(donorp, recipientp, mergeEdgep); + + if (mergeEdgep) { + // Remove and free the connecting edge. Must do this before propagating CP's below. + m_sb.removeEdge(mergeEdgep); + mergeEdgep->fromMTaskp()->removeRelativeMTask(mergeEdgep->toMTaskp()); + mergeEdgep->fromMTaskp()->removeRelativeEdge(mergeEdgep); + mergeEdgep->toMTaskp()->removeRelativeEdge(mergeEdgep); + VL_DO_DANGLING(mergeEdgep->unlinkDelete(), mergeEdgep); + } else { + // Remove the siblingMC + m_sb.removeSibling(mergeSibsp); + } + + // This also updates cost and stepCost on recipientp + recipientp->moveAllVerticesFrom(donorp); + + UINFO(9, "recipient = " << recipientp->id() << ", donor = " << donorp->id() + << ", mergeEdgep = " << mergeEdgep << "\n" + << "recipientNewCpFwd = " << recipientNewCpFwd.cp + << (recipientNewCpFwd.propagate ? " true " : " false ") + << recipientNewCpFwd.propagateCp << "\n" + << "donorNewCpFwd = " << donorNewCpFwd.cp + << (donorNewCpFwd.propagate ? " true " : " false ") + << donorNewCpFwd.propagateCp); + + recipientp->setCritPathCost(GraphWay::FORWARD, recipientNewCpFwd.cp); + if (recipientNewCpFwd.propagate) { + m_forwardPropagator.cpHasIncreased(recipientp, recipientNewCpFwd.propagateCp); + } + recipientp->setCritPathCost(GraphWay::REVERSE, recipientNewCpRev.cp); + if (recipientNewCpRev.propagate) { + m_reversePropagator.cpHasIncreased(recipientp, recipientNewCpRev.propagateCp); + } + if (donorNewCpFwd.propagate) { + m_forwardPropagator.cpHasIncreased(donorp, donorNewCpFwd.propagateCp); + } + if (donorNewCpRev.propagate) { + m_reversePropagator.cpHasIncreased(donorp, donorNewCpRev.propagateCp); + } + m_forwardPropagator.go(); + m_reversePropagator.go(); + + // Remove all other SiblingMCs that include recipientp or donorp. We remove all siblingMCs + // of recipientp so we do not get huge numbers of SiblingMCs. We'll recreate them below, up + // to a bounded number. + removeSiblingMCs(recipientp, donorp); + + // Redirect all edges, delete donorp + partRedirectEdgesFrom(m_mTaskGraph, recipientp, donorp, m_sb); + + ++m_mergesSinceRescore; + + // Do an expensive check, confirm we haven't botched the CP + // updates. + if (m_slowAsserts) partCheckCriticalPaths(m_mTaskGraph); + + // Finally, make new sibling pairs as needed: + // - prereqs and postreqs of recipientp + // - prereqs of recipientp's postreqs + // - postreqs of recipientp's prereqs + // Note that this depends on the updated critical paths (above). + siblingPairFromRelatives(recipientp); + siblingPairFromRelatives(recipientp); + unsigned edges = 0; + for (V3GraphEdge& edge : recipientp->outEdges()) { + LogicMTask* const postreqp = static_cast(edge.top()); + siblingPairFromRelatives(postreqp); + ++edges; + if (edges >= PART_SIBLING_EDGE_LIMIT) break; + } + edges = 0; + for (V3GraphEdge& edge : recipientp->inEdges()) { + LogicMTask* const prereqp = static_cast(edge.fromp()); + siblingPairFromRelatives(prereqp); + ++edges; + if (edges >= PART_SIBLING_EDGE_LIMIT) break; + } + } + + void doRescore() { + // During rescore, we know that graph isn't changing, so allow + // the critPathCost*Without() routines to cache some data in + // each LogicMTask. This is just an optimization, things should + // behave identically without the caching (just slower) + + m_sb.rescore(); + UINFO(6, "Did rescore. Merges since previous = " << m_mergesSinceRescore); + + m_mergesSinceRescore = 0; + m_scoreLimitBeforeRescore + = std::numeric_limits::max(); + } + + void makeSiblingMC(LogicMTask* ap, LogicMTask* bp) { + if (ap->id() < bp->id()) std::swap(ap, bp); + // The higher id vertex owns the association set + const auto first = mtaskData(ap).siblings.insert(bp).second; + if (first) { + m_sb.addSibling(ap, bp); + return; + } + + if (VL_UNLIKELY(m_slowAsserts)) { + // It's fine if we already have this SiblingMC, we may have + // created it earlier. Just confirm that we have associated data. + bool found = false; + for (const SiblingMC& smc : mtaskData(ap).aSiblingMCs) { + UASSERT_OBJ(smc.ap() == ap, ap, "Inconsistent SiblingMC"); + if (smc.bp() == bp) found = true; + } + UASSERT_OBJ(found, ap, "Sibling not found"); + } + } + + template + void siblingPairFromRelatives(V3GraphVertex* mtaskp) { + constexpr GraphWay way{N_Way}; + // Need at least 2 edges + auto& edges = mtaskp->edges(); + if (!edges.hasMultipleElements()) return; + + std::array neighbors; + + // This is a hot method, so we want so sort as efficiently as possible. We pre-load + // all data (critical path cost and id) required for determining ordering into an aligned + // structure. There is not enough space next to these to keep a whole pointer within 16 + // bytes, so we store an index into the neighbors buffer instead. We can then compare + // and swap these sorting records very efficiently. With this the standard library sorting + // functions are efficient enough and using more optimized methods (e.g.: sorting networks) + // has no measurable benefit. + struct alignas(16) SortingRecord final { + uint64_t m_cp; + uint32_t m_id; + uint8_t m_idx; + static_assert(PART_SIBLING_EDGE_LIMIT <= std::numeric_limits::max(), + "m_idx must fit all indices into 'neighbors'"); + bool operator<(const SortingRecord& that) const { + return m_cp < that.m_cp || (m_cp == that.m_cp && m_id < that.m_id); + } + }; + static_assert(sizeof(SortingRecord) <= 16, "How could this be padded to more than 16?"); + + std::array sortRecs; + size_t n = 0; + + // Populate the buffers + for (V3GraphEdge& edge : mtaskp->edges()) { + LogicMTask* const otherp = static_cast(edge.furtherp()); + neighbors[n] = otherp; + sortRecs[n].m_id = otherp->id(); + sortRecs[n].m_cp = otherp->critPathCost(way) + otherp->cost(); + sortRecs[n].m_idx = n; + ++n; + // Prevent nodes with huge numbers of edges from massively slowing down us down + if (n >= PART_SIBLING_EDGE_LIMIT) break; + } + + // Don't make all possible pairs of siblings when not requested (non-exhaustive). + // Just make a few pairs. + constexpr size_t MAX_NONEXHAUSTIVE_PAIRS = 3; + + if (N_Exhaustive || n <= 2 * MAX_NONEXHAUSTIVE_PAIRS) { + const size_t end = n & ~static_cast(1); // Round down to even, (we want pairs) + std::sort(sortRecs.begin(), sortRecs.begin() + n); + for (size_t i = 0; i < end; i += 2) { + makeSiblingMC(neighbors[sortRecs[i].m_idx], neighbors[sortRecs[i + 1].m_idx]); + } + } else { + constexpr size_t end = 2 * MAX_NONEXHAUSTIVE_PAIRS; + std::partial_sort(sortRecs.begin(), sortRecs.begin() + end, sortRecs.begin() + n); + for (size_t i = 0; i < end; i += 2) { + makeSiblingMC(neighbors[sortRecs[i].m_idx], neighbors[sortRecs[i + 1].m_idx]); + } + } + } + + // CONSTRUCTORS + Contraction(OrderMTaskGraph& mTaskGraph, uint64_t scoreLimit) + : m_mTaskGraph{mTaskGraph} + , m_scoreLimit{scoreLimit} { + + if (m_slowAsserts) { + // Check there are no redundant edges + for (V3GraphVertex& vtx : m_mTaskGraph.vertices()) { + std::unordered_set neighbors; + for (V3GraphEdge& edge : vtx.outEdges()) { + const bool first = neighbors.insert(edge.top()).second; + UASSERT_OBJ(first, &vtx, "Redundant edge found in input to Contraction()"); + } + } + } + + // Set up the critical path into and out of each node, then coarsen the graph. + partInitCriticalPaths(mTaskGraph); + + const uint32_t maxMTasks = []() -> uint32_t { + // If specified, use the given value + const int given = v3Global.opt.threadsMaxMTasks(); + if (given > 0) return given; + // Unspecified so estimate + return PART_DEFAULT_MAX_MTASKS_PER_THREAD * v3Global.opt.threads(); + }(); + + // OPTIMIZATION PASS: Edge contraction and sibling contraction. + // - Score pairs of LogicMTask which are a candidate to merge. + // * Each edge defines such a candidate pair + // * Two LogicMTask that are prereqs or postreqs of a common third + // vertex are "siblings", these are also a candidate pair. + // - Build a list of MergeCandidates, sorted by score. + // - Merge the best pair. + // - Incrementally recompute critical paths near the merged mtask. + + // Allocate and assign the auxiliary data for every LogicMTask. + { + const size_t nMTasks = m_mTaskGraph.vertices().size(); + m_mtaskDatap.reset(new MTaskContractionData[nMTasks]); + size_t i = 0; + for (V3GraphVertex& vtx : m_mTaskGraph.vertices()) vtx.userp(&m_mtaskDatap[i++]); + UASSERT(i == nMTasks, "Inconsistent MTask count"); + } + + // Add initial candidates + for (V3GraphVertex& vtx : m_mTaskGraph.vertices()) { + for (V3GraphEdge& edge : vtx.outEdges()) m_sb.addEdge(static_cast(&edge)); + siblingPairFromRelatives(&vtx); + siblingPairFromRelatives(&vtx); + } + + // Set initial scores in scoreboard + doRescore(); + + while (true) { + // This is the best edge to merge, with the lowest score (shortest local critical path) + MergeCandidate* const mergeCanp = m_sb.best(); + if (!mergeCanp) { + if (!m_sb.needsRescore()) break; // No more eligible candidates + // Rescore the scoreboard and try again + doRescore(); + continue; + } + + UASSERT(!m_sb.needsRescore(mergeCanp), + "Need-rescore items should not be returned by bestp"); + + const uint64_t cachedScore = mergeCanp->score(); + mergeCanp->rescore(); + const uint64_t actualScore = mergeCanp->score(); + + // If cached score is out-of-date, mark this elem as in need of a rescore and continue. + // cppcheck-suppress knownConditionTrueFalse // they are in fact different + if (actualScore > cachedScore) { + m_sb.hintScoreChanged(mergeCanp); + continue; + } + + // ... we'll also confirm that actualScore hasn't shrunk relative + // to cached score, after the mergeWouldCreateCycle() check. + + if (actualScore > m_scoreLimit) { + // Our best option isn't good enough + if (m_sb.needsRescore()) { + // Some pairs need a rescore, maybe those will be + // eligible to merge afterward. + doRescore(); + continue; + } + + // We've exhausted everything below m_scoreLimit; stop. + + // Except, if we have too many LogicMTasks, raise the score limit and keep going... + const unsigned mtaskCount = m_mTaskGraph.vertices().size(); + if (mtaskCount > maxMTasks) { + const uint64_t oldLimit = m_scoreLimit; + m_scoreLimit = (m_scoreLimit * 120) / 100; + FileLine* const flp = v3Global.rootp()->fileline(); + if (!flp->warnIsOff(V3ErrorCode::UNOPTTHREADS)) { + flp->v3warn(UNOPTTHREADS, + "Thread scheduler is unable to provide requested " + "parallelism; suggest asking for fewer threads."); + flp->modifyWarnOff(V3ErrorCode::UNOPTTHREADS, true); + } + UINFO(6, "Critical path limit was=" << oldLimit << " now=" << m_scoreLimit); + continue; + } + + // Really stop + break; + } + + // If time to rescore, that will result in a higher scoreLimitBeforeRescore, and + // possibly lower-scoring elements returned from bestp(). + if (actualScore > m_scoreLimitBeforeRescore) { + doRescore(); + continue; + } + + // Avoid merging the entry/exit nodes. This would create serialization, by forcing the + // merged MTask to run before/after everything else. Empirically this helps performance + // in a modest way by allowing other MTasks to start earlier. + if (EdgeMC* const edgeMCp = mergeCanp->toEdgeMC()) { + MTaskEdge* const edgep = edgeMCp->edgep(); + if (edgep->fromp() == m_entryMTaskp || edgep->top() == m_exitMTaskp) { + m_sb.removeEdge(edgep); + continue; + } + } + + // Avoid merging any edge that would create a cycle. + // + // For example suppose we begin with vertices A, B, C and edges + // A->B, B->C, A->C. + // + // Suppose we want to merge A->C into a single vertex. + // New edges would be AC->B and B->AC which is not a DAG. + // Do not allow this. + if (mergeCanp->mergeWouldCreateCycle()) { + // Remove this candidate from scoreboard so we don't keep + // reconsidering it on every loop. + if (SiblingMC* const smcp = mergeCanp->toSiblingMC()) { + m_sb.removeSibling(smcp); + } else { + m_sb.removeEdge(mergeCanp->toEdgeMC()->edgep()); + } + continue; + } + + partCheckCachedScoreVsActual(cachedScore, actualScore); + + // Finally there's no cycle risk, no need to rescore, we're + // within m_scoreLimit and m_scoreLimitBeforeRescore. + // This is the edge to merge. + + // Bookkeeping: if this is the first edge we'll merge since + // the last rescore, compute the new m_scoreLimitBeforeRescore + // to be somewhat higher than this edge's score. + if (!m_mergesSinceRescore) m_scoreLimitBeforeRescore = actualScore; + + // Finally merge this candidate. + contract(mergeCanp); + } + + // Free all remaining merge candidates. As an EdgeMC exists exactly while its edge is on + // the scoreboard, draining the scoreboard here frees every remaining EdgeMC; edges removed + // from the scoreboard earlier already had theirs freed. Note 'best' only ever returns + // candidates with a known score, so this only drains the scoreboard completely if nothing + // is left with an unknown score. Every 'break' out of the loop above is guarded on that, + // but assert it here, as otherwise we would leak candidates. + UASSERT(!m_sb.needsRescore(), "Should have no unknown score candidates at this point"); + while (MergeCandidate* const mergeCanp = m_sb.best()) { + if (SiblingMC* const smcp = mergeCanp->toSiblingMC()) { + m_sb.removeSibling(smcp); + } else { + m_sb.removeEdge(mergeCanp->toEdgeMC()->edgep()); + } + } + } + +public: + static void apply(OrderMTaskGraph& mTaskGraph, uint64_t scoreLimit) { + Contraction{mTaskGraph, scoreLimit}; + } +}; + +//###################################################################### +// OrderMTaskGraph entry point + +void OrderMTaskGraph::contract(OrderMTaskGraph& mtaskGraph, uint64_t scoreLimit) { + Contraction::apply(mtaskGraph, scoreLimit); +} diff --git a/src/V3OrderMTaskFixHazards.cpp b/src/V3OrderMTaskFixHazards.cpp new file mode 100644 index 000000000..ecd8ab468 --- /dev/null +++ b/src/V3OrderMTaskFixHazards.cpp @@ -0,0 +1,390 @@ +// -*- mode: C++; c-file-style: "cc-mode" -*- +//************************************************************************* +// DESCRIPTION: Verilator: Multi-threaded MTask graph data hazard fixing +// +// Code available from: https://verilator.org +// +//************************************************************************* +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of either the GNU Lesser General Public License Version 3 +// or the Perl Artistic License Version 2.0. +// SPDX-FileCopyrightText: 2003-2026 Wilson Snyder +// SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0 +// +//************************************************************************* + +#include "V3PchAstNoMT.h" // VL_MT_DISABLED_CODE_UNIT + +#include "V3Control.h" +#include "V3Global.h" +#include "V3Graph.h" +#include "V3GraphStream.h" +#include "V3OrderGraph.h" +#include "V3OrderMTaskGraph.h" + +#include +#include +#include +#include + +VL_DEFINE_DEBUG_FUNCTIONS; + +//###################################################################### +// DpiImportCallVisitor + +// Scan node, indicate whether it contains a call to a DPI imported routine. +class DpiImportCallVisitor final : public VNVisitor { + bool m_hasDpiHazard = false; // Found a DPI import call. + bool m_tracingCall = false; // Iterating into a CCall to a CFunc + // METHODS + void visit(AstCFunc* nodep) override { + if (!m_tracingCall) return; + m_tracingCall = false; + if (nodep->dpiImportWrapper()) { + if (nodep->dpiPure() ? !v3Global.opt.threadsDpiPure() + : !v3Global.opt.threadsDpiUnpure()) { + // If hierarchical DPI wrapper cost is not found or is of a 0 cost, + // we have a normal DPI which induces DPI hazard by default. + m_hasDpiHazard = V3Control::getProfileData(nodep->cname()) == 0; + UINFO(9, "DPI wrapper '" << nodep->cname() + << "' has dpi hazard = " << m_hasDpiHazard); + } + } + iterateChildren(nodep); + } + void visit(AstNodeCCall* nodep) override { + iterateChildren(nodep); + // Enter the function and trace it + m_tracingCall = true; + iterate(nodep->funcp()); + } + void visit(AstNode* nodep) override { iterateChildren(nodep); } + + // CONSTRUCTORS + explicit DpiImportCallVisitor(AstNode* nodep) { iterate(nodep); } + +public: + static bool hasDpiHazard(AstNode* nodep) { return DpiImportCallVisitor{nodep}.m_hasDpiHazard; } +}; + +//###################################################################### +// FixDataHazards + +class FixDataHazards final { + // + // Fix data hazards in the MTask graph. + // + // The fine-grained graph from V3Order may contain data hazards which are + // not a problem for serial mode, but which would be a problem in parallel + // mode. + // + // There are basically two classes: unordered pairs of writes, and + // unordered write-read pairs. We fix both here, with a combination of + // MTask-merges and new edges to ensure no such unordered pairs remain. + // + // ABOUT UNORDERED WRITE-WRITE PAIRS + // + // The V3Order dependency graph treats these as unordered events: + // + // a) sig[15:8] = stuff; + // ... + // b) sig[7:0] = other_stuff; + // + // Seems OK right? They are writes to disjoint bits of the same + // signal. They can run in either order, in serial mode, and the result + // will be the same. + // + // The resulting C code for each of this isn't a pure write, it's + // actually an R-M-W sequence: + // + // a) sig = (sig & 0xff) | (0xff00 & (stuff << 8)); + // ... + // b) sig = (sig & 0xff00) | (0xff & other_stuff); + // + // In serial mode, order doesn't matter so long as these run serially. + // In parallel mode, we must serialize these RMW's to avoid a race. + // + // We don't actually check here if each write would involve an R-M-W, we + // just assume that it would. If this routine ever causes a drastic + // increase in critical path, it could be optimized to make a better + // prediction (with all the risk that word implies!) about whether a + // given write is likely to turn into an R-M-W. + // + // ABOUT UNORDERED WRITE-READ PAIRS + // + // If we don't put unordered write-read pairs into some order at Verilation + // time, we risk a runtime race. + // + // How do such unordered writer/reader pairs happen? Here's a partial list + // of scenarios: + // + // Case 1: Circular logic + // + // If the design has circular logic, V3Order has by now generated some + // dependency cycles, and also cut some of the edges to make it + // acyclic. + // + // For serial mode, that was fine. We can break logic circles at an + // arbitrary point. At runtime, we'll repeat the _eval() until no + // changes are detected, which papers over the discarded dependency. + // + // For parallel mode, this situation can lead to unordered reads and + // writes of the same variable, causing a data race. For example if the + // original code is this: + // + // assign b = b | a << 2; + // assign out = b; + // + // ... there's originally a dependency edge which records that 'b' + // depends on the first assign. V3Order may cut this edge, making the + // statements unordered. In serial mode that's fine, they can run in + // either order. In parallel mode it's a reader/writer race. + // + // Case 2: Race Condition in Verilog Sources + // + // If the input has races, eg. blocking assignments in always blocks + // that share variables, the graph at this point will contain unordered + // writes and reads (or unordered write-write pairs) reflecting that. + + // TYPES + // Sort LogicMTask objects into deterministic order by calling id() + // which is a unique and stable serial number. + struct MTaskIdLessThan final { + bool operator()(const LogicMTask* lhsp, const LogicMTask* rhsp) const { + return *lhsp < *rhsp; + } + }; + using TasksByRank = std::map>; + + // MEMBERS + OrderMTaskGraph& m_mTaskGraph; // The Mtask graph + + // METHODS + + // Redirect all edges of 'donorp' onto 'recipientp' + static void redirectEdgesFrom(LogicMTask* recipientp, LogicMTask* donorp) { + // Process outgoing edges + while (MTaskEdge* const edgep = static_cast(donorp->outEdges().frontp())) { + LogicMTask* const top = edgep->toMTaskp(); + top->removeRelativeEdge(edgep); + + // If an edge already exists between recipient and sink of donor, drop the duplicate. + if (recipientp->hasRelativeMTask(top)) { + VL_DO_DANGLING(edgep->unlinkDelete(), edgep); + continue; + } + + // Otherwise redirect the edge from donorp->top to recipientp->top. + edgep->relinkFromp(recipientp); + recipientp->addRelativeMTask(top); + recipientp->stealRelativeEdge(edgep); + top->addRelativeEdge(edgep); + } + + // Process incoming edges + while (MTaskEdge* const edgep = static_cast(donorp->inEdges().frontp())) { + LogicMTask* const fromp = edgep->fromMTaskp(); + fromp->removeRelativeMTask(donorp); + fromp->removeRelativeEdge(edgep); + + // If an edge already exists between recipient and source of donor, drop the duplicate. + if (fromp->hasRelativeMTask(recipientp)) { + VL_DO_DANGLING(edgep->unlinkDelete(), edgep); + continue; + } + + // Otherwise redirect the edge from fromp->donorp to fromp->recipientp. + edgep->relinkTop(recipientp); + fromp->addRelativeMTask(recipientp); + fromp->addRelativeEdge(edgep); + recipientp->stealRelativeEdge(edgep); + } + } + + void findAdjacentTasks(const OrderVarStdVertex* varVtxp, TasksByRank& tasksByRank) { + // Find all writer tasks for this variable, group by rank. + for (const V3GraphEdge& edge : varVtxp->inEdges()) { + if (const auto* const logicVtxp = edge.fromp()->cast()) { + LogicMTask* const writerMtaskp = static_cast(logicVtxp->userp()); + tasksByRank[writerMtaskp->rank()].insert(writerMtaskp); + } + } + // Note: Find all reader tasks for this variable, group by rank. + // There was "broken" code here to find readers, but fixing it to + // work properly harmed performance on some tests, see issue #3360. + } + + void mergeSameRankTasks(const TasksByRank& tasksByRank) { + LogicMTask* lastRecipientp = nullptr; + for (const auto& pair : tasksByRank) { + // Find the largest node at this rank, merge into it. (If we + // happen to find a huge node, this saves time in + // redirectEdgesFrom() versus merging into an arbitrary node.) + LogicMTask* recipientp = nullptr; + for (LogicMTask* const mtaskp : pair.second) { + if (!recipientp || (recipientp->cost() < mtaskp->cost())) recipientp = mtaskp; + } + UASSERT_OBJ(!lastRecipientp || (lastRecipientp->rank() < recipientp->rank()), + recipientp, "Merging must be on lower rank"); + + for (LogicMTask* const donorp : pair.second) { + // Merge donor into recipient. + if (donorp == recipientp) continue; + // Fix up the map, so donor's OLVs map to recipientp + for (const OrderMoveVertex& vtx : donorp->vertexList()) { + vtx.logicp()->userp(recipientp); + } + // Move all vertices from donorp to recipientp + recipientp->moveAllVerticesFrom(donorp); + // Redirect edges from donorp to recipientp + redirectEdgesFrom(recipientp, donorp); + // Remove donorp from the graph + VL_DO_DANGLING(donorp->unlinkDelete(&m_mTaskGraph), donorp); + } + + if (lastRecipientp && !lastRecipientp->hasRelativeMTask(recipientp)) { + new MTaskEdge{&m_mTaskGraph, lastRecipientp, recipientp, 1}; + } + lastRecipientp = recipientp; + } + } + + bool hasDpiHazard(LogicMTask* mtaskp) { + for (const OrderMoveVertex& mVtx : mtaskp->vertexList()) { + OrderLogicVertex* const lvtxp = mVtx.logicp(); + if (!lvtxp) continue; + // NOTE: We don't handle DPI exports. If testbench code calls a DPI-exported function + // at any time during eval() we may have a data hazard. (Likewise in non-threaded mode + // if an export messes with an ordered variable we're broken.) + + // Find all calls to DPI-imported functions, we can put those into a serial order at + // least. That should solve the most likely DPI-related data hazards. + if (DpiImportCallVisitor::hasDpiHazard(lvtxp->nodep())) return true; + } + return false; + } + + // CONSTRUCTOR + FixDataHazards(OrderMTaskGraph& mTaskGraph) + : m_mTaskGraph{mTaskGraph} { + // Rank the graph. DGS is faster than V3GraphAlg's recursive rank, and also allows us to + // set up the OrderLogicVertex -> LogicMTask map at the same time. + { + GraphStreamUnordered serialize{&m_mTaskGraph}; + while (LogicMTask* const mtaskp + = const_cast(static_cast(serialize.nextp()))) { + // Compute and assign rank + uint32_t rank = 0; + for (V3GraphEdge& edge : mtaskp->inEdges()) { + rank = std::max(edge.fromp()->rank() + 1, rank); + } + mtaskp->rank(rank); + + // Set up the OrderLogicVertex -> LogicMTask map + // Entry and exit MTasks have no MTaskMoveVertices under them, so move on + if (mtaskp->vertexList().empty()) continue; + // Otherwise there should be only one OrderMoveVertex in each MTask at this stage + const OrderMoveVertex::List& vertexList = mtaskp->vertexList(); + UASSERT_OBJ(vertexList.hasSingleElement(), mtaskp, "Multiple OrderMoveVertex"); + const OrderMoveVertex* const mVtxp = vertexList.frontp(); + // Set up mapping back to the MTask from the OrderLogicVertex + if (OrderLogicVertex* const lvtxp = mVtxp->logicp()) lvtxp->userp(mtaskp); + } + } + + // Gather all variables. SystemC vars will be handled slightly specially, so keep separate. + const OrderGraph& orderGraph = m_mTaskGraph.moveGraph().orderGraph(); + std::vector regularVars; + std::vector systemCVars; + for (const V3GraphVertex& vtx : orderGraph.vertices()) { + // Only consider OrderVarStdVertex which reflects + // an actual lvalue assignment; the others do not. + if (const OrderVarStdVertex* const vvtxp = vtx.cast()) { + if (vvtxp->vscp()->varp()->isSc()) { + systemCVars.push_back(vvtxp); + } else { + regularVars.push_back(vvtxp); + } + } + } + + // For each OrderVarVertex, look at its writer and reader MTasks. + // + // If there's a set of writers and readers at the same rank, we + // know these are unordered with respect to one another, so merge + // those MTasks all together. + // + // At this point, we have at most one merged mtask per rank (for a + // given OVV.) Create edges across these remaining MTasks to ensure + // they run in serial order (going along with the existing ranks.) + // + // NOTE: we don't update the CP's stored in the LogicMTasks to + // reflect the changes we make to the graph. That's OK, as we + // haven't yet initialized CPs when we call this routine. + for (const OrderVarStdVertex* const varVtxp : regularVars) { + // Build a set of MTasks, per rank, which access this var. + // Within a rank, sort by MTaskID to avoid nondeterminism. + TasksByRank tasksByRank; + + // Find all reader and writer tasks for this variable, add to + // tasksByRank. + findAdjacentTasks(varVtxp, tasksByRank); + + // Merge all writer and reader tasks from same rank together. + // + // NOTE: Strictly speaking, we don't need to merge all the + // readers together. That may lead to extra serialization. The + // least amount of ordering we could impose here would be to + // merge all writers at a given rank together; then make edges + // from the merged writer node to each reader node at the same + // rank; and then from each reader node to the merged writer at + // the next rank. + // + // Whereas, merging all readers and writers at the same rank + // together is "the simplest thing that could possibly work" + // and it seems to. It also creates fairly few edges. We don't + // want to create tons of edges here, doing so is not nice to + // the main edge contraction pass. + mergeSameRankTasks(tasksByRank); + } + + // Handle SystemC vars just a little differently. Instead of + // treating each var as an independent entity, and serializing + // writes to that one var, we treat ALL systemC vars as a single + // entity and serialize writes (and, conservatively, reads) across + // all of them. + // + // Reasoning: writing a systemC var actually turns into a call to a + // var.write() method, which under the hood is accessing some data + // structure that's shared by many SC vars. It's not thread safe. + // + // Hopefully we only have a few SC vars -- top level ports, probably. + { + TasksByRank tasksByRank; + for (const OrderVarStdVertex* const varVtxp : systemCVars) { + findAdjacentTasks(varVtxp, tasksByRank); + } + mergeSameRankTasks(tasksByRank); + } + + // Handle nodes containing DPI calls, we want to serialize those + // by default unless user gave '--threads-dpi none'. + // Same basic strategy as above to serialize access to SC vars. + if (!v3Global.opt.threadsDpiPure() || !v3Global.opt.threadsDpiUnpure()) { + TasksByRank tasksByRank; + for (V3GraphVertex& vtx : m_mTaskGraph.vertices()) { + LogicMTask& mtask = static_cast(vtx); + if (hasDpiHazard(&mtask)) tasksByRank[mtask.rank()].insert(&mtask); + } + mergeSameRankTasks(tasksByRank); + } + } + +public: + static void apply(OrderMTaskGraph& mTaskGraph) { FixDataHazards{mTaskGraph}; } +}; + +void OrderMTaskGraph::fixDataHazards(OrderMTaskGraph& mtaskGraph) { + FixDataHazards::apply(mtaskGraph); +} diff --git a/src/V3OrderMTaskGraph.cpp b/src/V3OrderMTaskGraph.cpp new file mode 100644 index 000000000..5c32ad275 --- /dev/null +++ b/src/V3OrderMTaskGraph.cpp @@ -0,0 +1,182 @@ +// -*- mode: C++; c-file-style: "cc-mode" -*- +//************************************************************************* +// DESCRIPTION: Verilator: OrderMTask graph construction +// +// Code available from: https://verilator.org +// +//************************************************************************* +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of either the GNU Lesser General Public License Version 3 +// or the Perl Artistic License Version 2.0. +// SPDX-FileCopyrightText: 2003-2026 Wilson Snyder +// SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0 +// +//************************************************************************* + +#include "V3PchAstNoMT.h" // VL_MT_DISABLED_CODE_UNIT + +#include "V3OrderMTaskGraph.h" + +#include "V3InstrCount.h" + +VL_DEFINE_DEBUG_FUNCTIONS; + +//###################################################################### +// OrderMTaskGraph + +OrderMTaskGraph::OrderMTaskGraph(OrderMoveGraph& moveGraph) + : m_moveGraph{moveGraph} + , m_entryp{new LogicMTask{*this, nullptr}} + , m_exitp{new LogicMTask{*this, nullptr}} {} + +uint64_t OrderMTaskGraph::totalCost() const { + uint64_t cost = 0; + for (const V3GraphVertex& vtx : vertices()) cost += static_cast(vtx).cost(); + return cost; +} + +//###################################################################### +// LogicMTask + +uint32_t LogicMTask::s_nextId = 1; // Start at 1, so that 0 indicates no mtask. + +LogicMTask::LogicMTask(OrderMTaskGraph& graph, OrderMoveVertex* mVtxp) + : V3GraphVertex{&graph} { + UASSERT(s_nextId < 0xFFFFFFFFUL, "Too many LogicMTask instances"); + if (!mVtxp) return; + m_mVertices.linkBack(mVtxp); + if (const OrderLogicVertex* const olvp = mVtxp->logicp()) { + m_cost += V3InstrCount::count(olvp->nodep(), true); + } +} + +//###################################################################### +// OrderMTaskGraphBuilder + +class OrderMTaskGraphBuilder final { + // NODE STATE + // Used by V3InstrCount::count within the LogicMTask constructor only + const VNUser1InUse m_user1InUse; + + // MEMBERS + OrderMTaskGraph& m_mtaskGraph; // Output OrderMTaskGraph + + // METHODS + + // Predicate function to determine what OrderMoveVertex to bypass when constructing the MTask + // graph. The OrderMoveGraph is a bipartite graph of: + // - 1. OrderMoveVertex instances containing logic via OrderLogicVertex + // (OrderMoveVertex::logicp() != nullptr) + // - 2. OrderMoveVertex instances containing an (OrderVarVertex, domain) pair + // The goal is to order the logic vertices. The second type of variable/domain vertices only + // carry dependencies and are eventually discarded. In order to reduce the working set size, + // we 'bypass' and not create LogicMTask vertices for some variable vertices, and instead add + // the transitive dependencies directly, but only if adding the transitive edges directly does + // not require more dependency edges than keeping the intermediate vertex. That is, we bypass a + // variable vertex if fanIn * fanOut <= fanIn + fanOut. This is true if fanIn or fanOut are 1, + // or if they are both 2. This can significantly reduce the initial size of OrderMTaskGraph. + static bool bypassOk(OrderMoveVertex* mvtxp) { + // Need to keep all logic vertices + if (mvtxp->logicp()) return false; + // Count fan-in, up to 3 + unsigned fanIn = 0; + auto& inEdges = mvtxp->inEdges(); + for (auto it = inEdges.begin(); it != inEdges.end(); ++it) { + if (++fanIn == 3) break; + } + // If fanIn no more than one, bypass + if (fanIn <= 1) return true; + // Count fan-out, up to 3 + unsigned fanOut = 0; + auto& outEdges = mvtxp->outEdges(); + for (auto it = outEdges.begin(); it != outEdges.end(); ++it) { + if (++fanOut == 3) break; + } + // If fan-out no more than one, bypass + if (fanOut <= 1) return true; + // They can only be (2, 2), (2, 3), (3, 2), (3, 3) at this point, bypass if (2, 2) + return fanIn + fanOut == 4; + } + + // Add an edge to the graph, if there is not already an edge between the two vertices. + void addEdge(LogicMTask& src, LogicMTask& dst) { + UASSERT_OBJ(&src != &dst, &src, "Should not create self-edges"); + if (src.hasRelativeMTask(&dst)) return; // Don't create redundant edges. + new MTaskEdge{&m_mtaskGraph, &src, &dst, 1}; + } + + // CONSTRUCTORS + explicit OrderMTaskGraphBuilder(OrderMTaskGraph& mtaskGraph) + : m_mtaskGraph{mtaskGraph} { + + // Create the LogicMTasks for each OrderMoveVertex + for (V3GraphVertex& vtx : mtaskGraph.moveGraph().vertices()) { + OrderMoveVertex& mVtx = static_cast(vtx); + if (bypassOk(&mVtx)) { + mVtx.userp(nullptr); // Set to nullptr to mark as bypassed + } else { + mVtx.userp(new LogicMTask{mtaskGraph, &mVtx}); // Create vertex and set userp + } + } + + LogicMTask& entry = *mtaskGraph.entryp(); + LogicMTask& exit = *mtaskGraph.exitp(); + + // Create the MTask dependency edges based on the OrderMoveGraph dependencies + for (V3GraphVertex& vtx : mtaskGraph.vertices()) { + LogicMTask& mtask = static_cast(vtx); + + // Entry and exit vertices handled separately + if (VL_UNLIKELY((&mtask == &entry) || (&mtask == &exit))) continue; + + OrderMoveVertex::List& vertexList = mtask.vertexList(); + // At this point, there should only be one OrderMoveVertex per LogicMTask + UASSERT_OBJ(vertexList.hasSingleElement(), &mtask, "Multiple OrderMoveVertex"); + OrderMoveVertex* const mVtxp = vertexList.frontp(); + UASSERT_OBJ(mVtxp->userp(), &mtask, "Bypassed OrderMoveVertex should not have MTask"); + + // Iterate downstream direct dependents + for (const V3GraphEdge& dEdge : mVtxp->outEdges()) { + V3GraphVertex* const top = dEdge.top(); + + // If the opposite end of the edge is not a bypassed vertex, add direct dependency + if (LogicMTask* const otherp = static_cast(top->userp())) { + addEdge(mtask, *otherp); + continue; + } + + // The opposite end of the edge is a bypassed vertex, add transitive dependencies + for (const V3GraphEdge& tEdge : top->outEdges()) { + LogicMTask* const transp = static_cast(tEdge.top()->userp()); + // The Move graph is bipartite (logic <-> var), and logic is never + // bypassed, hence 'transp' must be non-nullptr. + UASSERT_OBJ(transp, mVtxp, "This cannot be a bypassed vertex"); + addEdge(mtask, *transp); + } + } + } + + // Create Dependencies to/from the entry/exit vertices, so all vertices are + // reachable from the entry point and flow to the exit point. + for (V3GraphVertex& vtx : mtaskGraph.vertices()) { + LogicMTask& mtask = static_cast(vtx); + if (VL_UNLIKELY((&mtask == &entry) || (&mtask == &exit))) continue; + // Add the entry/exit edges if not otherwise connected + if (mtask.inEmpty()) addEdge(entry, mtask); + if (mtask.outEmpty()) addEdge(mtask, exit); + } + } + ~OrderMTaskGraphBuilder() = default; + VL_UNCOPYABLE(OrderMTaskGraphBuilder); + VL_UNMOVABLE(OrderMTaskGraphBuilder); + +public: + static void apply(OrderMTaskGraph& mtaskGraph) { OrderMTaskGraphBuilder{mtaskGraph}; } +}; + +std::unique_ptr OrderMTaskGraph::build(OrderMoveGraph& moveGraph) { + std::unique_ptr resp{new OrderMTaskGraph{moveGraph}}; + OrderMTaskGraphBuilder::apply(*resp); + return resp; +} diff --git a/src/V3OrderMTaskGraph.h b/src/V3OrderMTaskGraph.h new file mode 100644 index 000000000..201011333 --- /dev/null +++ b/src/V3OrderMTaskGraph.h @@ -0,0 +1,421 @@ +// -*- mode: C++; c-file-style: "cc-mode" -*- +//************************************************************************* +// DESCRIPTION: Verilator: MTask graph for multi-threaded ordering +// +// Code available from: https://verilator.org +// +//************************************************************************* +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of either the GNU Lesser General Public License Version 3 +// or the Perl Artistic License Version 2.0. +// SPDX-FileCopyrightText: 2003-2026 Wilson Snyder +// SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0 +// +//************************************************************************* +// +// LogicMTask and MTaskEdge are the vertex and edge of the mtask +// graph built and coarsened by the multi-threaded partitioner (see +// V3OrderParallel.cpp). They are independent of the partitioner's merge +// candidate machinery: any auxiliary data the algorithms need is attached +// externally via the vertex/edge user pointers. +// +//************************************************************************* + +#ifndef VERILATOR_V3ORDERMTASKGRAPH_H_ +#define VERILATOR_V3ORDERMTASKGRAPH_H_ + +#include "config_build.h" +#include "verilatedos.h" + +#include "V3Graph.h" +#include "V3OrderMoveGraph.h" +#include "V3PairingHeap.h" + +#include +#include +#include +#include +#include + +class LogicMTask; +template +class PropagateCp; + +// When computing critical path costs, use a step function on the actual underlying vertex cost. +// +// If there are huge vertices, when a tiny vertex merges into a huge vertex, we can often avoid +// increasing the huge vertex's stepped cost. If the stepped cost hasn't increased, and the +// critical path into the huge vertex hasn't increased, we can avoid propagating a new critical +// path to vertices past the huge vertex. Since huge vertices tend to have huge lists of children +// and parents, this can be a substantial savings. +// +// Does not seem to reduce the quality of the partitioner's output. +// +// If you have huge vertices, leave this 'true', it is the major setting that allows the +// partitioner to handle such difficult graphs on anything like a human time scale. +// +// If you don't have huge vertices, the 'true' value doesn't help much but should cost almost +// nothing in terms of partitioner quality. +// +// If you want the most aggressive possible partition, set it "false" and be prepared to be +// disappointed when the improvement in the partition is negligible / in the noise. +// +// Q) Why retain the control, if there is really no downside? +// +// A) Cost stepping can lead to corner cases. A developer may wish to disable cost stepping to +// rule it out as the cause of unexpected behavior. +#define PART_STEPPED_COST true + +//###################################################################### +// Misc graph and assertion utilities + +inline void partCheckCachedScoreVsActual(uint64_t cached, uint64_t actual) { +#if PART_STEPPED_COST + // Cached CP might be a little bigger than actual, due to stepped CPs. + // Example: + // Let's say we have a parent with stepped_cost 40 and a grandparent + // with stepped_cost 27. Our forward-cp is 67. Then our parent and + // grandparent get merged, the merged node has stepped cost 66. We + // won't propagate that new CP to children as it hasn't grown. So, + // children may continue to think that the CP coming through this path + // is a little higher than it really is; permit that. + UASSERT((((cached * 10) <= (actual * 11)) && (cached * 11) >= (actual * 10)), + "Calculation error in scoring (approximate, may need tweak)"); +#else + UASSERT(cached == actual, "Calculation error in scoring"); +#endif +} + +//============================================================================= +// OrderMTaskGraph + +// The graph of LogicMTask vertices and MTaskEdge edges, used during multi-threaded scheduling. +class OrderMTaskGraph final : public V3Graph { + OrderMoveGraph& m_moveGraph; // The OrderMoveGraph this graph is built from + LogicMTask* const m_entryp; // The singular entry point vertex + LogicMTask* const m_exitp; // The singular exit point vertex + + // CONSTRUCTOR + explicit OrderMTaskGraph(OrderMoveGraph& moveGraph); // Used by build(), hence private + VL_UNCOPYABLE(OrderMTaskGraph); + VL_UNMOVABLE(OrderMTaskGraph); + +public: + // ACCESSORS + OrderMoveGraph& moveGraph() const { return m_moveGraph; } + LogicMTask* entryp() const { return m_entryp; } + LogicMTask* exitp() const { return m_exitp; } + + // METHODS + uint64_t totalCost() const; // O(V), called once + + // STATIC METHODS + // Build an MTask graph from 'moveGraph' + static std::unique_ptr build(OrderMoveGraph& moveGraph) VL_MT_DISABLED; + // Fix data hazards in the MTask graph + static void fixDataHazards(OrderMTaskGraph& mtaskGraph) VL_MT_DISABLED; + // Coarsen the MTask graph by merging MTasks until the given critical-path limit is reached + static void contract(OrderMTaskGraph& mtaskGraph, uint64_t scoreLimit) VL_MT_DISABLED; +}; + +//============================================================================= +// We keep MTaskEdge graph edges in a PairingHeap, sorted by score and id + +struct EdgeKey final { + uint64_t m_score; // Score part of edge key + uint64_t m_id; // Unique ID part of edge key + void increase(uint64_t score) { + UDEBUGONLY(UASSERT(score >= m_score, "Must increase");); + m_score = score; + } + // Sort first by Score then by ID + bool operator<(const EdgeKey& other) const { + if (m_score != other.m_score) return m_score < other.m_score; + return m_id < other.m_id; + } +}; + +using EdgeHeap = PairingHeap; + +//============================================================================= +// GraphEdge for the MTask graph + +class MTaskEdge final : public V3GraphEdge { + VL_RTTI_IMPL(MTaskEdge, V3GraphEdge) + + friend class LogicMTask; + template + friend class PropagateCp; + + // MEMBERS + // This edge can be in 2 EdgeHeaps, one forward and one reverse. We allocate the heap nodes + // directly within the edge as they are always required and this makes association cheap. + std::array m_edgeHeapNode; + + // Note: The edge's contraction merge candidate (if any) is held in the inherited user pointer + // (V3GraphEdge::userp), managed entirely by the partitioner; see edgeMC() and + // MergeCandidateScoreboard. Kept out of MTaskEdge so it does not depend on the MergeCandidate + // hierarchy. + +public: + // CONSTRUCTORS + inline MTaskEdge(OrderMTaskGraph* graphp, LogicMTask* fromp, LogicMTask* top, int weight); + VL_UNCOPYABLE(MTaskEdge); + VL_UNMOVABLE(MTaskEdge); + + // METHODS + template + inline LogicMTask* furtherMTaskp() const; + inline LogicMTask* fromMTaskp() const; + inline LogicMTask* toMTaskp() const; + + // Following initial assignment of critical paths, clear this MTaskEdge + // out of the edge-map for each node and reinsert at a new location + // with updated critical path. + inline void resetCriticalPaths(); + uint64_t cachedCp(GraphWay way) const { return m_edgeHeapNode[way].key().m_score; } + // Convert from the address of the m_edgeHeapNode[way] in an MTaskEdge back to the MTaskEdge + static const MTaskEdge* toMTaskEdge(GraphWay way, const EdgeHeap::Node* nodep) { + const size_t offset = VL_OFFSETOF(MTaskEdge, m_edgeHeapNode[way]); + return reinterpret_cast(reinterpret_cast(nodep) - offset); + } +}; + +//============================================================================= +// LogicMTask + +class LogicMTask final : public V3GraphVertex { + VL_RTTI_IMPL(LogicMTask, V3GraphVertex) + + template + friend class PropagateCp; + + // MEMBERS + + // List of OrderMoveVertex's assigned to this mtask. LogicMTask does not own the + // OrderMoveVertex objects, we merely keep them in a list here. + OrderMoveVertex::List m_mVertices; + + // Cost estimate for this LogicMTask, derived from V3InstrCount, in abstract time units. + uint64_t m_cost = 0; + + // Cost of critical paths going FORWARD from graph-start to the start + // of this vertex, and also going REVERSE from the end of the graph to + // the end of the vertex. Same units as m_cost. + std::array m_critPathCost = {}; + + static uint32_t s_nextId; // Next ID number to use + const uint32_t m_id = s_nextId++; // Unique LogicMTask ID number for stable comparison + + // Count "generations" which are just operations that scan through the + // graph. We'll mark each node with the last generation that scanned + // it. We can use this to avoid recursing through the same node twice + // while searching for a path. + uint64_t m_generation = 0; + + // Store a set of forward relatives so we can quickly check if we have a given child + std::unordered_set m_edgeSet; + // Store the outgoing and incoming edges in a heap sorted by the critical path length + std::array m_edgeHeap; + + // Scratch pointer used only by PropagateCp: this MTask's node in the pending heap, or nullptr + // if this MTask is not pending. Type erased, as the heap node type is private to PropagateCp, + // and differs between its two instantiations (which never run concurrently). + void* m_propagateHeapNodep = nullptr; + +public: + // CONSTRUCTORS + LogicMTask(OrderMTaskGraph& graph, OrderMoveVertex* mVtxp) VL_MT_DISABLED; + VL_UNCOPYABLE(LogicMTask); + VL_UNMOVABLE(LogicMTask); + + // ACCESSORS + OrderMoveVertex::List& vertexList() { return m_mVertices; } + const OrderMoveVertex::List& vertexList() const { return m_mVertices; } + uint32_t id() const { return m_id; } + uint64_t cost() const VL_MT_SAFE { return m_cost; } + static uint64_t stepCost(uint64_t cost) { +#if PART_STEPPED_COST + // Round cost up to the nearest 5%. Use this when computing all critical paths. The idea is + // that critical path changes don't need to propagate when they don't exceed the next step, + // saving a lot of recursion. + if (cost == 0) return 0; + + double logcost = log(cost); + // log(1.05) is about 0.05, so round logcost up to the next 0.05 boundary + logcost *= 20.0; + logcost = ceil(logcost); + logcost = logcost / 20.0; + + const uint64_t sCost = static_cast(exp(logcost)); + UDEBUGONLY(UASSERT_STATIC(sCost >= cost, "stepped cost error exceeded");); + UDEBUGONLY(UASSERT_STATIC(sCost <= ((cost * 11 / 10)), "stepped cost error exceeded");); + return sCost; +#else + return cost; +#endif + } + uint64_t stepCost() const { return stepCost(m_cost); } + uint64_t critPathCost(GraphWay way) const { return m_critPathCost[way]; } + void setCritPathCost(GraphWay way, uint64_t cost) { m_critPathCost[way] = cost; } + + // METHODS + bool operator<(const LogicMTask& rhs) const { return id() < rhs.id(); } + + void moveAllVerticesFrom(LogicMTask* otherp) { + m_mVertices.splice(m_mVertices.end(), otherp->vertexList()); + m_cost += otherp->m_cost; + } + + template + void addRelativeEdge(MTaskEdge* edgep) { + constexpr GraphWay way{N_Way}; + constexpr GraphWay inv = way.invert(); + // Add to the edge heap + LogicMTask* const relativep = edgep->furtherMTaskp(); + // Value is !way cp to this edge + const uint64_t cp = relativep->stepCost() + relativep->critPathCost(inv); + m_edgeHeap[way].insert(&edgep->m_edgeHeapNode[way], {cp, relativep->id()}); + } + template + void stealRelativeEdge(MTaskEdge* edgep) { + constexpr GraphWay way{N_Way}; + // Make heap node insertable, ruining the heap it is currently in. + edgep->m_edgeHeapNode[way].yank(); + // Add the edge as new + addRelativeEdge(edgep); + } + template + void removeRelativeEdge(MTaskEdge* edgep) { + constexpr GraphWay way{N_Way}; + // Remove from the edge heap + m_edgeHeap[way].remove(&edgep->m_edgeHeapNode[way]); + } + + void addRelativeMTask(LogicMTask* relativep) { + // Add the relative to connecting edge map + const bool exits = !m_edgeSet.emplace(relativep).second; + UDEBUGONLY(UASSERT(!exits, "Adding existing relative");); + } + void removeRelativeMTask(LogicMTask* relativep) { + const size_t removed = m_edgeSet.erase(relativep); + UDEBUGONLY(UASSERT(removed, "Relative should have been in set");); + } + bool hasRelativeMTask(LogicMTask* relativep) const { return m_edgeSet.count(relativep); } + + template + void checkRelativesCp() const { + constexpr GraphWay way{N_Way}; + for (const V3GraphEdge& edge : edges()) { + const LogicMTask* const relativep + = static_cast(edge.furtherp()); + const uint64_t cachedCp = static_cast(edge).cachedCp(way); + const uint64_t cp = relativep->critPathCost(way.invert()) + relativep->stepCost(); + partCheckCachedScoreVsActual(cachedCp, cp); + } + } + + template + uint64_t critPathCostWithout(const V3GraphEdge* withoutp) const { + const GraphWay way{N_Way}; + const GraphWay inv = way.invert(); + // Compute the critical path cost wayward to this node, without considering edge + // 'withoutp'. We need to look at two edges at most, the critical path if that is not via + // 'withoutp', or the second-worst path, if the critical path is via 'withoutp'. + UDEBUGONLY(UASSERT(withoutp->furtherp() == this, + "In critPathCostWithout(), edge 'withoutp' must further to 'this'");); + const EdgeHeap& edgeHeap = m_edgeHeap[inv]; + const EdgeHeap::Node* const maxp = edgeHeap.max(); + if (!maxp) return 0; + if (MTaskEdge::toMTaskEdge(inv, maxp) != withoutp) return maxp->key().m_score; + const EdgeHeap::Node* const secp = edgeHeap.secondMax(); + if (!secp) return 0; + return secp->key().m_score; + } + +private: + // This takes LogicMTask instead of generic V3GraphVertex. We will use the critical + // paths known to LogicMTask to prune the recursion for speed. Also store 'generation' in + // LogicMTask::m_generation so we can prune the search and avoid recursing through the same + // node more than once in a single search. + static bool pathExistsFromInternal(LogicMTask* fromp, LogicMTask* top, + const MTaskEdge* excludedEdgep, uint64_t generation) { + + // If already looked at this node in the current search, since we're back again, + // we must not have found a path on the first go. + if (fromp->m_generation == generation) return false; + + // Mark visited + fromp->m_generation = generation; + + // Base case: we found a path. + if (fromp == top) return true; + + // Base case: fromp is too late, cannot possibly be a prereq for top. + if (fromp->critPathCost(GraphWay::REVERSE) + < (top->critPathCost(GraphWay::REVERSE) + top->stepCost())) { + return false; + } + if ((fromp->critPathCost(GraphWay::FORWARD) + fromp->stepCost()) + > top->critPathCost(GraphWay::FORWARD)) { + return false; + } + + // Recursively look for a path + for (const V3GraphEdge& follow : fromp->outEdges()) { + if (&follow == excludedEdgep) continue; + LogicMTask* const nextp = static_cast(follow.top()); + if (pathExistsFromInternal(nextp, top, nullptr, generation)) return true; + } + return false; + } + +public: + // True if there's a path from 'fromp' to 'top' excluding 'excludedEdgep', false otherwise. + // 'excludedEdgep' may be nullptr in which case no edge is excluded. If 'excludedEdgep' is + // non-nullptr it must connect fromp and top. + static bool pathExistsFrom(LogicMTask* fromp, LogicMTask* top, + const MTaskEdge* excludedEdgep) { + static uint64_t s_generation = 0; + return pathExistsFromInternal(fromp, top, excludedEdgep, ++s_generation); + } + + // For Graphviz dumps only + std::string name() const override VL_MT_STABLE { + std::ostringstream out; + out << "mt" << m_id // + << " | fwdCP " << m_critPathCost[GraphWay::FORWARD] // + << " | revCP " << m_critPathCost[GraphWay::REVERSE] // + << " | cost " << cost(); + return out.str(); + } +}; + +//============================================================================= +// MTaskEdge method definitions (need the full definition of LogicMTask) + +MTaskEdge::MTaskEdge(OrderMTaskGraph* graphp, LogicMTask* fromp, LogicMTask* top, int weight) + : V3GraphEdge{graphp, fromp, top, weight} { + fromp->addRelativeMTask(top); + fromp->addRelativeEdge(this); + top->addRelativeEdge(this); +} + +template +LogicMTask* MTaskEdge::furtherMTaskp() const { + return static_cast(this->furtherp()); +} +LogicMTask* MTaskEdge::fromMTaskp() const { return static_cast(fromp()); } +LogicMTask* MTaskEdge::toMTaskp() const { return static_cast(top()); } + +void MTaskEdge::resetCriticalPaths() { + LogicMTask* const fromp = fromMTaskp(); + LogicMTask* const top = toMTaskp(); + fromp->removeRelativeEdge(this); + top->removeRelativeEdge(this); + fromp->addRelativeEdge(this); + top->addRelativeEdge(this); +} + +#endif // Guard diff --git a/src/V3OrderMoveGraph.cpp b/src/V3OrderMoveGraph.cpp index 3c19c8382..e69a790b8 100644 --- a/src/V3OrderMoveGraph.cpp +++ b/src/V3OrderMoveGraph.cpp @@ -55,7 +55,8 @@ class OrderMoveGraphBuilder final { // MEMBERS OrderGraph& m_orderGraph; // Input OrderGraph - std::unique_ptr m_moveGraphp{new OrderMoveGraph}; // Output OrderMoveGraph + // Output OrderMoveGraph + std::unique_ptr m_moveGraphp{new OrderMoveGraph{m_orderGraph}}; // Map from Trigger reference AstSenItem to the original AstSenTree const V3Order::TrigToSenMap& m_trigToSen; // Storage for domain -> OrderMoveVertex, maps held in OrderVarVertex::userp() diff --git a/src/V3OrderMoveGraph.h b/src/V3OrderMoveGraph.h index 80432c55b..b793684eb 100644 --- a/src/V3OrderMoveGraph.h +++ b/src/V3OrderMoveGraph.h @@ -82,11 +82,21 @@ public: // OrderMoveGraph is constructed from the fine-grained OrderGraph. // It is a slightly coarsened representation of dependencies used to drive serialization. class OrderMoveGraph final : public V3Graph { + OrderGraph& m_orderGraph; // The OrderGraph this move graph was built from + public: + explicit OrderMoveGraph(OrderGraph& orderGraph) + : m_orderGraph{orderGraph} {} + + OrderGraph& orderGraph() const { return m_orderGraph; } + // Build an OrderMoveGraph from an OrderGraph static std::unique_ptr build(OrderGraph&, const V3Order::TrigToSenMap&); }; +//====================================================================== +// OrderMoveDomScope + // Information stored for each unique (domain, scope) pair. Mainly a list of ready vertices under // that (domain, scope). OrderMoveDomScope instances are themselves organized into a global ready // list if they have ready vertices. diff --git a/src/V3OrderParallel.cpp b/src/V3OrderParallel.cpp index 02ab788d0..bca6c36f1 100644 --- a/src/V3OrderParallel.cpp +++ b/src/V3OrderParallel.cpp @@ -20,1736 +20,104 @@ #include "V3PchAstNoMT.h" // VL_MT_DISABLED_CODE_UNIT +#include "V3Ast.h" #include "V3Control.h" #include "V3ExecGraph.h" -#include "V3File.h" #include "V3Graph.h" #include "V3GraphStream.h" -#include "V3InstrCount.h" -#include "V3List.h" #include "V3OrderCFuncEmitter.h" #include "V3OrderInternal.h" -#include "V3Os.h" -#include "V3PairingHeap.h" -#include "V3Scoreboard.h" -#include "V3Stats.h" +#include "V3OrderMTaskGraph.h" -#include #include -#include #include -#include -#include VL_DEFINE_DEBUG_FUNCTIONS; -class LogicMTask; -class MTaskEdge; -class MergeCandidate; -class SiblingMC; - -// ###################################################################### -// Partitioner tunable settings: -// -// Before describing these settings, a bit of background: -// -// Early during the development of the partitioner, V3Split was failing to -// split large always blocks (with ~100K assignments) so we had to handle -// very large vertices with ~100K incoming and outgoing edges. -// -// The partitioner attempts to deal with such densely connected -// graphs. Some of the tuning parameters below reference "huge vertices", -// that's what they're talking about, vertices with tens of thousands of -// edges in and out. Whereas most graphs have only tens of edges in and out -// of most vertices. -// -// V3Split has since been fixed to more reliably split large always -// blocks. It's kind of an open question whether the partitioner must -// handle huge nodes gracefully. Maybe not! But it still can, given -// appropriate tuning. - -// PART_SIBLING_EDGE_LIMIT (integer) -// -// Arbitrarily limit the number of edges on a single vertex that will be -// considered when enumerating siblings, to the given value. This protects -// the partitioner runtime in the presence of huge vertices. -// -// The sibling-merge is less important than the edge merge. (You can -// totally disable the sibling merge and get halfway decent partitions; you -// can't disable edge merges, those are fundamental to the process.) So, -// skipping the enumeration of some siblings on a few vertices does not -// have a large impact on the result of the partitioner. -// -// If your vertices are small, the limit (at 26) approaches a no-op. Hence -// there's basically no cost to applying this limit even when we don't -// expect huge vertices. -// -// If you don't care about partitioner runtime and you want the most -// aggressive partition, set the limit very high. If you have huge -// vertices, leave this as is. -constexpr unsigned PART_SIBLING_EDGE_LIMIT = 26; - -// PART_STEPPED_COST (defined/undef) -// -// When computing critical path costs, use a step function on the actual -// underlying vertex cost. -// -// If there are huge vertices, when a tiny vertex merges into a huge -// vertex, we can often avoid increasing the huge vertex's stepped cost. -// If the stepped cost hasn't increased, and the critical path into the huge -// vertex hasn't increased, we can avoid propagating a new critical path to -// vertices past the huge vertex. Since huge vertices tend to have huge lists -// of children and parents, this can be a substantial savings. -// -// Does not seem to reduce the quality of the partitioner's output. -// -// If you have huge vertices, leave this 'true', it is the major setting -// that allows the partitioner to handle such difficult graphs on anything -// like a human time scale. -// -// If you don't have huge vertices, the 'true' value doesn't help much but -// should cost almost nothing in terms of partitioner quality. -// -// If you want the most aggressive possible partition, set it "false" and -// be prepared to be disappointed when the improvement in the partition is -// negligible / in the noise. -// -// Q) Why retain the control, if there is really no downside? -// -// A) Cost stepping can lead to corner cases. A developer may wish to -// disable cost stepping to rule it out as the cause of unexpected -// behavior. -#define PART_STEPPED_COST true - -// Don't produce more than a certain maximum number of MTasks. This helps -// the TSP variable sort not to blow up (a concern for some of the tests) -// and we probably don't want a huge number of mTaskGraphp in practice anyway -// (50 to 100 is typical.) -// -// If the user doesn't give one with '--threads-max-mtasks', we'll set the -// maximum # of MTasks to -// (# of threads * PART_DEFAULT_MAX_MTASKS_PER_THREAD) -constexpr unsigned PART_DEFAULT_MAX_MTASKS_PER_THREAD = 50; - -// end tunables. - //###################################################################### -// Misc graph and assertion utilities +// Partitioner implementation -static void partCheckCachedScoreVsActual(uint64_t cached, uint64_t actual) { -#if PART_STEPPED_COST - // Cached CP might be a little bigger than actual, due to stepped CPs. - // Example: - // Let's say we have a parent with stepped_cost 40 and a grandparent - // with stepped_cost 27. Our forward-cp is 67. Then our parent and - // grandparent get merged, the merged node has stepped cost 66. We - // won't propagate that new CP to children as it hasn't grown. So, - // children may continue to think that the CP coming through this path - // is a little higher than it really is; permit that. - UASSERT((((cached * 10) <= (actual * 11)) && (cached * 11) >= (actual * 10)), - "Calculation error in scoring (approximate, may need tweak)"); -#else - UASSERT(cached == actual, "Calculation error in scoring"); -#endif -} +// Partitioner takes the fine-grained OrderMoveGraph from V3Order and collapses +// it into a coarse-grained graph of LogicMTask's, each of which contains of set +// of the logic nodes from the fine-grained graph. -//============================================================================= -// We keep MTaskEdge graph edges in a PairingHeap, sorted by score and id +static std::unique_ptr partition(OrderMoveGraph& moveGraph) { + // Build the initial MTask graph. Initially, each MTask just wraps one OrderMoveVertex. We will + // merge MTasks together and eventually each MTask will wrap a large number of OrderMoveVertex + // (and the logic nodes therein). + std::unique_ptr mTaskGraphp = OrderMTaskGraph::build(moveGraph); + mTaskGraphp->hashGraphDebug("initial MTask graph"); -struct EdgeKey final { - // Node: Structure layout chosen to minimize padding in PairingHeap<*>::Node - uint64_t m_id; // Unique ID part of edge score - uint64_t m_score; // Score part of ID - void increase(uint64_t score) { - UDEBUGONLY(UASSERT(score >= m_score, "Must increase");); - m_score = score; - } - bool operator<(const EdgeKey& other) const { - // First by Score then by ID - return m_score < other.m_score || (m_score == other.m_score && m_id < other.m_id); - } -}; + // Merge nodes that could present data hazards + OrderMTaskGraph::fixDataHazards(*mTaskGraphp); + mTaskGraphp->hashGraphDebug("MTask graph after fixDataHazards()"); -using EdgeHeap = PairingHeap; - -//###################################################################### -// MTask utility classes - -struct MergeCandidateKey final { - // Note: Structure layout chosen to minimize padding in PairingHeap<*>::Node - uint64_t m_id; // Unique ID part of edge score - uint64_t m_score; // Score part of ID - bool operator<(const MergeCandidateKey& other) const { - // First by Score then by ID, but notice that we want minimums using a max-heap, so reverse - return m_score > other.m_score || (m_score == other.m_score && m_id > other.m_id); - } -}; - -using MergeCandidateScoreboard = V3Scoreboard; - -// Information associated with scoreboarding a merge candidate -class MergeCandidate VL_NOT_FINAL : public MergeCandidateScoreboard::Node { - // Only the known subclasses can create or delete one of these - friend class SiblingMC; - friend class MTaskEdge; - - // This structure is extremely hot. To save 8 bytes we pack - // one bit indicating removedFromSb with the id. To save another - // 8 bytes by not having a virtual function table, we implement the - // few polymorphic methods over the two known subclasses explicitly, - // using another bit of the id to denote the actual subtype. - - // By using the bottom bits for flags, we can still use < to compare IDs without masking. - // <63:1> Serial number for ordering, <0> subtype (SiblingMC) - static constexpr uint64_t IS_SIBLING_MASK = 1ULL << 0; - static constexpr uint64_t ID_INCREMENT = 1ULL << 1; - - bool isSiblingMC() const { return m_key.m_id & IS_SIBLING_MASK; } - - // CONSTRUCTORS - explicit MergeCandidate(bool isSiblingMC) { - static uint64_t s_serial = 0; - s_serial += ID_INCREMENT; // +ID_INCREMENT so doesn't set the special bottom bits - m_key.m_id = s_serial | (isSiblingMC * IS_SIBLING_MASK); - } - ~MergeCandidate() = default; - -public: - // METHODS - SiblingMC* toSiblingMC(); // Instead of cast<>/as<> - MTaskEdge* toMTaskEdge(); // Instead of cast<>/as<> - bool mergeWouldCreateCycle() const; // Instead of virtual method - - inline void rescore(); - uint64_t score() const { return m_key.m_score; } - - static MergeCandidate* heapNodeToElem(MergeCandidateScoreboard::Node* nodep) { - return static_cast(nodep); - } -}; - -static_assert(sizeof(MergeCandidate) == sizeof(MergeCandidateScoreboard::Node), - "Should not have a vtable"); - -// A pair of associated LogicMTask's that are merge candidates for sibling -// contraction -class SiblingMC final : public MergeCandidate { - LogicMTask* const m_ap; - LogicMTask* const m_bp; - - V3ListLinks m_aLinks; // List links to store instances of this class - V3ListLinks m_bLinks; // List links to store instances of this class - - V3ListLinks& aLinks() { return m_aLinks; } - V3ListLinks& bLinks() { return m_bLinks; } - -public: - // List type to store instances of this class - using AList = V3List; - using BList = V3List; - - // CONSTRUCTORS - SiblingMC(LogicMTask* ap, LogicMTask* bp); - ~SiblingMC() = default; - - // METHODS - void unlinkA(); - void unlinkB(); - - LogicMTask* ap() const { return m_ap; } - LogicMTask* bp() const { return m_bp; } - bool mergeWouldCreateCycle() const; -}; - -static_assert(!std::is_polymorphic::value, "Should not have a vtable"); - -// GraphEdge for the MTask graph -class MTaskEdge final : public V3GraphEdge, public MergeCandidate { - VL_RTTI_IMPL(MTaskEdge, V3GraphEdge) - friend class LogicMTask; - template - friend class PropagateCp; - - // MEMBERS - // This edge can be in 2 EdgeHeaps, one forward and one reverse. We allocate the heap nodes - // directly within the edge as they are always required and this makes association cheap. - std::array m_edgeHeapNode; - -public: - // CONSTRUCTORS - MTaskEdge(V3Graph* graphp, LogicMTask* fromp, LogicMTask* top, int weight); - // METHODS - template - inline LogicMTask* furtherMTaskp() const; - inline LogicMTask* fromMTaskp() const; - inline LogicMTask* toMTaskp() const; - bool mergeWouldCreateCycle() const; - // Following initial assignment of critical paths, clear this MTaskEdge - // out of the edge-map for each node and reinsert at a new location - // with updated critical path. - void resetCriticalPaths(); - - uint64_t cachedCp(GraphWay way) const { return m_edgeHeapNode[way].key().m_score; } - - // Convert from the address of the m_edgeHeapNode[way] in an MTaskEdge back to the MTaskEdge - static const MTaskEdge* toMTaskEdge(GraphWay way, const EdgeHeap::Node* nodep) { - const size_t offset = VL_OFFSETOF(MTaskEdge, m_edgeHeapNode[way]); - return reinterpret_cast(reinterpret_cast(nodep) - offset); - } - -private: - VL_UNCOPYABLE(MTaskEdge); -}; - -//============================================================================= -// LogicMTask - -class LogicMTask final : public V3GraphVertex { - VL_RTTI_IMPL(LogicMTask, V3GraphVertex) - template - friend class PropagateCp; - -public: - // TYPES - struct CmpLogicMTask final { - bool operator()(const LogicMTask* ap, const LogicMTask* bp) const { - return ap->id() < bp->id(); - } - }; - -private: - // MEMBERS - - // List of OrderMoveVertex's assigned to this mtask. LogicMTask does not - // own the OrderMoveVertex objects, we merely keep them in a list here. - OrderMoveVertex::List m_mVertices; - - // Cost estimate for this LogicMTask, derived from V3InstrCount. - // In abstract time units. - uint64_t m_cost = 0; - - // Cost of critical paths going FORWARD from graph-start to the start - // of this vertex, and also going REVERSE from the end of the graph to - // the end of the vertex. Same units as m_cost. - std::array m_critPathCost = {}; - - const uint32_t m_id; // Unique LogicMTask ID number - static uint32_t s_nextId; // Next ID number to use - - // Count "generations" which are just operations that scan through the - // graph. We'll mark each node with the last generation that scanned - // it. We can use this to avoid recursing through the same node twice - // while searching for a path. - uint64_t m_generation = 0; - - // Store a set of forward relatives so we can quickly check if we have a given child - std::unordered_set m_edgeSet; - // Store the outgoing and incoming edges in a heap sorted by the critical path length - std::array m_edgeHeap; - - // MTasks for which a SiblingMC exists with 'this' as the higher ID MTask (m_ap in SiblingMC) - std::set m_siblings; - // List of SiblingMCs for which this is the higher ID MTask (m_ap in SiblingMC) - SiblingMC::AList m_aSiblingMCs; - // List of SiblingMCs for which this is the lower ID MTask (m_bp in SiblingMC) - SiblingMC::BList m_bSiblingMCs; - -public: - // CONSTRUCTORS - // cppcheck-suppress constParameterCallback - LogicMTask(V3Graph* graphp, OrderMoveVertex* mVtxp) - : V3GraphVertex{graphp} - , m_id{s_nextId++} { - UASSERT(s_nextId < 0xFFFFFFFFUL, "Too many mTaskGraphp"); - if (mVtxp) { - m_mVertices.linkBack(mVtxp); - if (const OrderLogicVertex* const olvp = mVtxp->logicp()) { - m_cost += V3InstrCount::count(olvp->nodep(), true); - } - } - } - - // METHODS - std::set& siblings() { return m_siblings; }; - SiblingMC::AList& aSiblingMCs() { return m_aSiblingMCs; }; - SiblingMC::BList& bSiblingMCs() { return m_bSiblingMCs; }; - - OrderMoveVertex::List& vertexList() { return m_mVertices; } - const OrderMoveVertex::List& vertexList() const { return m_mVertices; } - void moveAllVerticesFrom(LogicMTask* otherp) { - m_mVertices.splice(m_mVertices.end(), otherp->vertexList()); - m_cost += otherp->m_cost; - } - static uint64_t incGeneration() { - static uint64_t s_generation = 0; - ++s_generation; - return s_generation; - } - - // Use this instead of pointer-compares to compare LogicMTasks. Avoids - // nondeterministic output. Also name mTaskGraphp based on this number in - // the final C++ output. - uint32_t id() const { return m_id; } - // Abstract cost of every logic mtask - uint64_t cost() const VL_MT_SAFE { return m_cost; } - void setCost(uint64_t cost) { m_cost = cost; } // For tests only - uint64_t stepCost() const { return stepCost(m_cost); } - static uint64_t stepCost(uint64_t cost) { -#if PART_STEPPED_COST - // Round cost up to the nearest 5%. Use this when computing all - // critical paths. The idea is that critical path changes don't - // need to propagate when they don't exceed the next step, saving a - // lot of recursion. - if (cost == 0) return 0; - - double logcost = log(cost); - // log(1.05) is about 0.05 - // So, round logcost up to the next 0.05 boundary - logcost *= 20.0; - logcost = ceil(logcost); - logcost = logcost / 20.0; - - const uint64_t stepCost = static_cast(exp(logcost)); - UDEBUGONLY(UASSERT_STATIC(stepCost >= cost, "stepped cost error exceeded");); - UDEBUGONLY(UASSERT_STATIC(stepCost <= ((cost * 11 / 10)), "stepped cost error exceeded");); - return stepCost; -#else - return cost; -#endif - } - - template - void addRelativeEdge(MTaskEdge* edgep) { - constexpr GraphWay way{N_Way}; - constexpr GraphWay inv = way.invert(); - // Add to the edge heap - LogicMTask* const relativep = edgep->furtherMTaskp(); - // Value is !way cp to this edge - const uint64_t cp = relativep->stepCost() + relativep->critPathCost(inv); - // - m_edgeHeap[way].insert(&edgep->m_edgeHeapNode[way], {relativep->id(), cp}); - } - template - void stealRelativeEdge(MTaskEdge* edgep) { - constexpr GraphWay way{N_Way}; - // Make heap node insertable, ruining the heap it is currently in. - edgep->m_edgeHeapNode[way].yank(); - // Add the edge as new - addRelativeEdge(edgep); - } - template - void removeRelativeEdge(MTaskEdge* edgep) { - constexpr GraphWay way{N_Way}; - // Remove from the edge heap - m_edgeHeap[way].remove(&edgep->m_edgeHeapNode[way]); - } - - void addRelativeMTask(LogicMTask* relativep) { - // Add the relative to connecting edge map - const bool exits = !m_edgeSet.emplace(relativep).second; - UDEBUGONLY(UASSERT(!exits, "Adding existing relative");); - } - void removeRelativeMTask(LogicMTask* relativep) { - const size_t removed = m_edgeSet.erase(relativep); - UDEBUGONLY(UASSERT(removed, "Relative should have been in set");); - } - bool hasRelativeMTask(LogicMTask* relativep) const { return m_edgeSet.count(relativep); } - - template - void checkRelativesCp() const { - constexpr GraphWay way{N_Way}; - for (const V3GraphEdge& edge : edges()) { - const LogicMTask* const relativep - = static_cast(edge.furtherp()); - const uint64_t cachedCp = static_cast(edge).cachedCp(way); - const uint64_t cp = relativep->critPathCost(way.invert()) + relativep->stepCost(); - partCheckCachedScoreVsActual(cachedCp, cp); - } - } - - string name() const override VL_MT_STABLE { - // Display forward and reverse critical path costs. This gives a quick - // read on whether graph partitioning looks reasonable or bad. - std::ostringstream out; - out << "mt" << m_id << "." << this << " [b" << m_critPathCost[GraphWay::FORWARD] << " a" - << m_critPathCost[GraphWay::REVERSE] << " c" << cost(); - return out.str(); - } - - void setCritPathCost(GraphWay way, uint64_t cost) { m_critPathCost[way] = cost; } - uint64_t critPathCost(GraphWay way) const { return m_critPathCost[way]; } - template - uint64_t critPathCostWithout(const V3GraphEdge* withoutp) const { - const GraphWay way{N_Way}; - const GraphWay inv = way.invert(); - // Compute the critical path cost wayward to this node, without considering edge - // 'withoutp'. We need to look at two edges at most, the critical path if that is not via - // 'withoutp', or the second-worst path, if the critical path is via 'withoutp'. - UDEBUGONLY(UASSERT(withoutp->furtherp() == this, - "In critPathCostWithout(), edge 'withoutp' must further to 'this'");); - const EdgeHeap& edgeHeap = m_edgeHeap[inv]; - const EdgeHeap::Node* const maxp = edgeHeap.max(); - if (!maxp) return 0; - if (MTaskEdge::toMTaskEdge(inv, maxp) != withoutp) return maxp->key().m_score; - const EdgeHeap::Node* const secp = edgeHeap.secondMax(); - if (!secp) return 0; - return secp->key().m_score; - } - -private: - static bool pathExistsFromInternal(LogicMTask* fromp, LogicMTask* top, - const V3GraphEdge* excludedEdgep, uint64_t generation) { - // Q) Why does this take LogicMTask instead of generic V3GraphVertex? - // A) We'll use the critical paths known to LogicMTask to prune the - // recursion for speed. Also store 'generation' in - // LogicMTask::m_generation so we can prune the search and avoid - // recursing through the same node more than once in a single - // search. - - if (fromp->m_generation == generation) { - // Already looked at this node in the current search. - // Since we're back again, we must not have found a path on the - // first go. - return false; - } - fromp->m_generation = generation; - - // Base case: we found a path. - if (fromp == top) return true; - - // Base case: fromp is too late, cannot possibly be a prereq for top. - if (fromp->critPathCost(GraphWay::REVERSE) - < (top->critPathCost(GraphWay::REVERSE) + top->stepCost())) { - return false; - } - if ((fromp->critPathCost(GraphWay::FORWARD) + fromp->stepCost()) - > top->critPathCost(GraphWay::FORWARD)) { - return false; - } - - // Recursively look for a path - for (const V3GraphEdge& follow : fromp->outEdges()) { - if (&follow == excludedEdgep) continue; - LogicMTask* const nextp = static_cast(follow.top()); - if (pathExistsFromInternal(nextp, top, nullptr, generation)) return true; - } - return false; - } - - // True if there's a path from 'fromp' to 'top' excluding - // 'excludedEdgep', false otherwise. + // Order the graph. We know it's already ranked from fixDataHazards() so we don't need to rank + // it again. // - // 'excludedEdgep' may be nullptr in which case no edge is excluded. If - // 'excludedEdgep' is non-nullptr it must connect fromp and top. + // On at least some models, ordering the graph here seems to help performance. (Why? Is it just + // triggering noise in a lucky direction? Is it just as likely to harm results?) // - // TODO: consider changing this API to the 'isTransitiveEdge' API - // used by GraphPathChecker -public: - static bool pathExistsFrom(LogicMTask* fromp, LogicMTask* top, - const V3GraphEdge* excludedEdgep) { - return pathExistsFromInternal(fromp, top, excludedEdgep, incGeneration()); + // More diversity of models that can build with --threads will eventually tell us. For now keep + // the order() so we don't forget about it, in case it actually helps. TODO: get more data and + // maybe remove this later if it doesn't really help. + mTaskGraphp->orderPreRanked(); + mTaskGraphp->hashGraphDebug("MTask graph after orderPreRanked()"); + + // Merge MTask nodes together, repeatedly, until the critical path budget is reached. Coarsens + // the graph, usually by several orders of magnitude. Some tests disable this for stability, + // it should always be enabled in production. + if (v3Global.opt.threadsCoarsen()) { + const int nThreads = v3Global.opt.threads(); + UASSERT(nThreads >= 2, "Should not reach Partitioner when --threads <= 1"); + + // Set critical path limit to roughly totalGraphCost / nThreads. Actually set it slighly + // lower, by a hardcoded fudge factor. This results in a smaller graph, which helps reduce + // fragmentation when scheduling them. TODO: What does this sentence mean? + const uint64_t fudgeNum = 3; + const uint64_t fudgeDen = 5; + const uint64_t limit = (mTaskGraphp->totalCost() * fudgeNum) / (nThreads * fudgeDen); + UINFO(4, "Partitioner set critical path limit = " << limit); + + OrderMTaskGraph::contract(*mTaskGraphp, limit); + mTaskGraphp->hashGraphDebug("MTask graph after contract()"); } - static void dumpCpFilePrefixed(const V3Graph& graph, const string& nameComment) { - const string filename = v3Global.debugFilename(nameComment) + ".txt"; - UINFO(1, "Writing " << filename); - const std::unique_ptr ofp{V3File::new_ofstream(filename)}; - std::ostream* const osp = &(*ofp); // &* needed to deref unique_ptr - if (osp->fail()) v3fatalStatic("Can't write file: " << filename); - - // Find start vertex with longest CP - const LogicMTask* startp = nullptr; - for (const V3GraphVertex& vtx : graph.vertices()) { - const LogicMTask& mtask = static_cast(vtx); - if (!startp) { - startp = &mtask; - continue; - } - if (mtask.cost() + mtask.critPathCost(GraphWay::REVERSE) - > startp->cost() + startp->critPathCost(GraphWay::REVERSE)) { - startp = &mtask; - } - } - - // Follow the entire critical path - std::vector path; - uint64_t totalCost = 0; - for (const LogicMTask* nextp = startp; nextp;) { - path.push_back(nextp); - totalCost += nextp->cost(); - - if (EdgeHeap::Node* const maxp = nextp->m_edgeHeap[GraphWay::FORWARD].max()) { - nextp = MTaskEdge::toMTaskEdge(GraphWay::FORWARD, maxp)->toMTaskp(); - } else { - nextp = nullptr; - } - } - - *osp << "totalCost = " << totalCost - << " (should match the computed critical path cost (CP) for the graph)\n"; - - // Dump - for (const LogicMTask* mtaskp : path) { - *osp << "begin mtask with cost " << mtaskp->cost() << '\n'; - for (const OrderMoveVertex& mVtx : mtaskp->vertexList()) { - const OrderLogicVertex* const logicp = mVtx.logicp(); - if (!logicp) continue; - // Show nodes with hierarchical costs - V3InstrCount::count(logicp->nodep(), false, osp); - } - } - } - -private: - VL_UNCOPYABLE(LogicMTask); -}; - -// Start at 1, so that 0 indicates no mtask. -uint32_t LogicMTask::s_nextId = 1; - -// Instead of dynamic cast -SiblingMC* MergeCandidate::toSiblingMC() { - return isSiblingMC() ? static_cast(this) : nullptr; -} - -MTaskEdge* MergeCandidate::toMTaskEdge() { - return isSiblingMC() ? nullptr : static_cast(this); -} - -// Normally this would be a virtual function, but we save space by not having a vtable, -// and we know we only have 2 possible subclasses. -bool MergeCandidate::mergeWouldCreateCycle() const { - return isSiblingMC() ? static_cast(this)->mergeWouldCreateCycle() - : static_cast(this)->mergeWouldCreateCycle(); -} - -static uint64_t siblingScore(const SiblingMC* sibsp) { - const LogicMTask* const ap = sibsp->ap(); - const LogicMTask* const bp = sibsp->bp(); - const uint64_t mergedCpCostFwd - = std::max(ap->critPathCost(GraphWay::FORWARD), bp->critPathCost(GraphWay::FORWARD)); - const uint64_t mergedCpCostRev - = std::max(ap->critPathCost(GraphWay::REVERSE), bp->critPathCost(GraphWay::REVERSE)); - return mergedCpCostRev + mergedCpCostFwd + LogicMTask::stepCost(ap->cost() + bp->cost()); -} - -static uint64_t edgeScore(const MTaskEdge* edgep) { - // Score this edge. Lower is better. The score is the new local CP - // length if we merge these mTaskGraphp. ("Local" means the longest - // critical path running through the merged node.) - const LogicMTask* const top = edgep->toMTaskp(); - const LogicMTask* const fromp = edgep->fromMTaskp(); - const uint64_t mergedCpCostFwd = std::max(fromp->critPathCost(GraphWay::FORWARD), - top->critPathCostWithout(edgep)); - const uint64_t mergedCpCostRev = std::max(fromp->critPathCostWithout(edgep), - top->critPathCost(GraphWay::REVERSE)); - return mergedCpCostRev + mergedCpCostFwd + LogicMTask::stepCost(fromp->cost() + top->cost()); -} - -void MergeCandidate::rescore() { - if (const SiblingMC* const sibp = toSiblingMC()) { - m_key.m_score = siblingScore(sibp); - } else { - // The '1 +' favors merging a SiblingMC over an otherwise- - // equal-scoring MTaskEdge. The comment on selfTest() talks - // about why. - m_key.m_score = 1 + edgeScore(static_cast(this)); - } -} - -SiblingMC::SiblingMC(LogicMTask* ap, LogicMTask* bp) - : MergeCandidate{/* isSiblingMC: */ true} - , m_ap{ap} - , m_bp{bp} { - // Storage management depends on this - UASSERT(ap->id() > bp->id(), "Should be ordered"); - UDEBUGONLY(UASSERT(ap->siblings().count(bp), "Should be in sibling map");); - m_ap->aSiblingMCs().linkBack(this); - m_bp->bSiblingMCs().linkBack(this); -} - -void SiblingMC::unlinkA() { - VL_ATTR_UNUSED const size_t removed = m_ap->siblings().erase(m_bp); - UDEBUGONLY(UASSERT(removed == 1, "Should have been in sibling set");); - m_ap->aSiblingMCs().unlink(this); -} - -void SiblingMC::unlinkB() { m_bp->bSiblingMCs().unlink(this); } - -// cppcheck-suppress duplInheritedMember -bool SiblingMC::mergeWouldCreateCycle() const { - return (LogicMTask::pathExistsFrom(m_ap, m_bp, nullptr) - || LogicMTask::pathExistsFrom(m_bp, m_ap, nullptr)); -} - -MTaskEdge::MTaskEdge(V3Graph* graphp, LogicMTask* fromp, LogicMTask* top, int weight) - : V3GraphEdge{graphp, fromp, top, weight} - , MergeCandidate{/* isSiblingMC: */ false} { - fromp->addRelativeMTask(top); - fromp->addRelativeEdge(this); - top->addRelativeEdge(this); -} - -template -LogicMTask* MTaskEdge::furtherMTaskp() const { - return static_cast(this->furtherp()); -} -LogicMTask* MTaskEdge::fromMTaskp() const { return static_cast(fromp()); } -LogicMTask* MTaskEdge::toMTaskp() const { return static_cast(top()); } - -// cppcheck-suppress duplInheritedMember -bool MTaskEdge::mergeWouldCreateCycle() const { - return LogicMTask::pathExistsFrom(fromMTaskp(), toMTaskp(), this); -} -// Following initial assignment of critical paths, clear this MTaskEdge -// out of the edge-map for each node and reinsert at a new location -// with updated critical path. -void MTaskEdge::resetCriticalPaths() { - LogicMTask* const fromp = fromMTaskp(); - LogicMTask* const top = toMTaskp(); - fromp->removeRelativeEdge(this); - top->removeRelativeEdge(this); - fromp->addRelativeEdge(this); - top->addRelativeEdge(this); -} - -//###################################################################### - -// Look at vertex costs (in one way) to form critical paths for each -// vertex. -template -static void partInitHalfCriticalPaths(V3Graph& mTaskGraph, bool checkOnly) { - constexpr GraphWay way{N_Way}; - constexpr GraphWay rev = way.invert(); - GraphStreamUnordered order{&mTaskGraph, way}; - for (const V3GraphVertex* vertexp; (vertexp = order.nextp());) { - const LogicMTask* const mtaskcp = static_cast(vertexp); - LogicMTask* const mtaskp = const_cast(mtaskcp); - uint64_t cpCost = 0; -#if VL_DEBUG - std::unordered_set relatives; -#endif - for (const V3GraphEdge& edge : vertexp->edges()) { -#if VL_DEBUG - // Run a few asserts on the initial mtask graph, - // while we're iterating through... - UASSERT_OBJ(edge.weight() != 0, mtaskp, "Should be no cut edges in mTaskGraphp graph"); - UASSERT_OBJ(relatives.find(edge.furtherp()) == relatives.end(), mtaskp, - "Should be no redundant edges in mTaskGraphp graph"); - relatives.insert(edge.furtherp()); -#endif - const LogicMTask* const relativep = static_cast(edge.furtherp()); - cpCost = std::max(cpCost, (relativep->critPathCost(way) - + static_cast(relativep->stepCost()))); - } - if (checkOnly) { - partCheckCachedScoreVsActual(mtaskp->critPathCost(way), cpCost); - } else { - mtaskp->setCritPathCost(way, cpCost); - } - } -} - -// Look at vertex costs to form critical paths for each vertex. -static void partInitCriticalPaths(V3Graph& mTaskGraph) { - partInitHalfCriticalPaths(mTaskGraph, false); - partInitHalfCriticalPaths(mTaskGraph, false); - - // Reset all MTaskEdges so that 'm_edges' will show correct CP numbers. - // They would have been all zeroes on initial creation of the MTaskEdges. - for (V3GraphVertex& vtx : mTaskGraph.vertices()) { - for (V3GraphEdge& edge : vtx.outEdges()) edge.as()->resetCriticalPaths(); - } -} - -// Do an EXPENSIVE check to make sure that all incremental CP updates have -// gone correctly. -static void partCheckCriticalPaths(V3Graph& mTaskGraph) { - partInitHalfCriticalPaths(mTaskGraph, true); - partInitHalfCriticalPaths(mTaskGraph, true); - for (const V3GraphVertex& vtx : mTaskGraph.vertices()) { - const LogicMTask& mtask = static_cast(vtx); - mtask.checkRelativesCp(); - mtask.checkRelativesCp(); - } -} - -// ###################################################################### -// PropagateCp - -template -class PropagateCp final { - // Propagate increasing critical path (CP) costs through a graph. - // - // Usage: - // * Client increases the cost and/or CP at a node or small set of nodes - // (often a pair in practice, eg. edge contraction.) - // * Client calls PropagateCp::cpHasIncreased() one or more times. - // Each call indicates that the inclusive CP of some "seed" vertex - // has increased to a given value. - // * NOTE: PropagateCp will neither read nor modify the cost - // or CPs at the seed vertices, it only accesses and modifies - // vertices wayward from the seeds. - // * Client calls PropagateCp::go(). Internally, this iteratively - // propagates the new CPs wayward through the graph. - // - - // TYPES - - // We keep pending vertices in a heap during critical path propagation - struct PendingKey final { - LogicMTask* m_mtaskp; // The vertex in the heap - uint64_t m_score; // The score of this entry - void increase(uint64_t score) { - UDEBUGONLY(UASSERT(score >= m_score, "Must increase");); - m_score = score; - } - bool operator<(const PendingKey& other) const { - if (m_score != other.m_score) return m_score < other.m_score; - return LogicMTask::CmpLogicMTask{}(m_mtaskp, other.m_mtaskp); - } - }; - - using PendingHeap = PairingHeap; - using PendingHeapNode = typename PendingHeap::Node; - - // MEMBERS - PendingHeap m_pendingHeap; // Heap of pending rescores - - // We allocate this many heap nodes at once - static constexpr size_t ALLOC_CHUNK_SIZE = 128; - PendingHeapNode* m_freep = nullptr; // List of free heap nodes - std::vector> m_allocated; // Allocated heap nodes - - const bool m_slowAsserts; // Enable nontrivial asserts - // Used only with slow asserts to check mTaskGraphp visited only once - std::set m_seen; - -public: - // CONSTRUCTORS - explicit PropagateCp(bool slowAsserts) - : m_slowAsserts{slowAsserts} {} - - // METHODS -private: - // Allocate a HeapNode for the given element - PendingHeapNode* allocNode() { - // If no free nodes available, then make some - if (!m_freep) { - // Allocate in chunks for efficiency - m_allocated.emplace_back(new PendingHeapNode[ALLOC_CHUNK_SIZE]); - // Set up free list pointer - m_freep = m_allocated.back().get(); - // Set up free list chain - for (size_t i = 1; i < ALLOC_CHUNK_SIZE; ++i) { - m_freep[i - 1].m_next.m_ptr = &m_freep[i]; - } - // Clear the next pointer of the last entry - m_freep[ALLOC_CHUNK_SIZE - 1].m_next.m_ptr = nullptr; - } - // Free nodes are available, pick up the first one - PendingHeapNode* const resultp = m_freep; - m_freep = resultp->m_next.m_ptr; - resultp->m_next.m_ptr = nullptr; - return resultp; - } - - // Release a heap node (make it available for future allocation) - void freeNode(PendingHeapNode* nodep) { - // Re-use the existing link pointers and simply prepend it to the free list - nodep->m_next.m_ptr = m_freep; - m_freep = nodep; - } - -public: - void cpHasIncreased(V3GraphVertex* vxp, uint64_t newInclusiveCp) { - constexpr GraphWay way{N_Way}; - constexpr GraphWay inv{way.invert()}; - - // For *vxp, whose CP-inclusive has just increased to - // newInclusiveCp, iterate to all wayward nodes, update the edges - // of each, and add each to m_pending if its overall CP has grown. - for (V3GraphEdge& graphEdge : vxp->edges()) { - MTaskEdge& edge = static_cast(graphEdge); - - LogicMTask* const relativep = edge.furtherMTaskp(); - EdgeHeap::Node& edgeHeapNode = edge.m_edgeHeapNode[inv]; - if (newInclusiveCp > edgeHeapNode.key().m_score) { - relativep->m_edgeHeap[inv].increaseKey(&edgeHeapNode, newInclusiveCp); - } - - const uint64_t critPathCost = relativep->critPathCost(way); - - if (critPathCost >= newInclusiveCp) continue; - - // relativep's critPathCost() is out of step with its longest !wayward edge. - // Schedule that to be resolved. - const uint64_t newVal = newInclusiveCp - critPathCost; - - if (PendingHeapNode* const nodep = static_cast(relativep->userp())) { - // Already in heap. Increase score if needed. - if (newVal > nodep->key().m_score) m_pendingHeap.increaseKey(nodep, newVal); - continue; - } - - // Add to heap - PendingHeapNode* const nodep = allocNode(); - relativep->userp(nodep); - m_pendingHeap.insert(nodep, {relativep, newVal}); - } - } - - void go() { - constexpr GraphWay way{N_Way}; - constexpr GraphWay inv{way.invert()}; - - // m_pending maps each pending vertex to the amount that it wayward - // CP will grow. - // - // We can iterate over the pending set in reverse order, always - // choosing the nodes with the largest pending CP-growth. - // - // The intuition is: if the original seed node had its CP grow by - // 50, the most any wayward node can possibly grow is also 50. So - // for anything pending to grow by 50, we know we can process it - // once and we won't have to grow its CP again on the current pass. - // After we're done with all the grow-by-50s, nothing else will - // grow by 50 again on the current pass, and we can process the - // grow-by-49s and we know we'll only have to process each one - // once. And so on. - // - // This generalizes to multiple seed nodes also. - while (!m_pendingHeap.empty()) { - // Pop max element from heap - PendingHeapNode* const maxp = m_pendingHeap.max(); - m_pendingHeap.remove(maxp); - // Pick up values - LogicMTask* const mtaskp = maxp->key().m_mtaskp; - const uint64_t cpGrowBy = maxp->key().m_score; - // Free the heap node, we are done with it - freeNode(maxp); - mtaskp->userp(nullptr); - // Update the critPathCost of mtaskp, that was out-of-date with respect to its edges - const uint64_t startCp = mtaskp->critPathCost(way); - const uint64_t newCp = startCp + cpGrowBy; - if (VL_UNLIKELY(m_slowAsserts)) { - // Check that CP matches that of the longest edge wayward of vxp. - const uint64_t edgeCp = mtaskp->m_edgeHeap[inv].max()->key().m_score; - UASSERT_OBJ(edgeCp == newCp, mtaskp, "CP doesn't match longest wayward edge"); - // Confirm that we only set each node's CP once. That's an - // important property of PropagateCp which allows it to be far - // faster than a recursive algorithm on some graphs. - const bool first = m_seen.insert(mtaskp).second; - UASSERT_OBJ(first, mtaskp, "Set CP on node twice"); - } - mtaskp->setCritPathCost(way, newCp); - cpHasIncreased(mtaskp, newCp + mtaskp->stepCost()); - } - - if (VL_UNLIKELY(m_slowAsserts)) m_seen.clear(); - } - -private: - VL_UNCOPYABLE(PropagateCp); - -public: - static void selfTest() { - V3Graph graph; // A graph - std::array vx; // All vertices within the graph - - // Generate a pseudo-random graph - std::array rngState - = {{0x12345678ULL, 0x9abcdef0ULL}}; // GCC 3.8.0 wants {{}} - // Create 50 vertices - for (auto& i : vx) { - i = new LogicMTask{&graph, nullptr}; - i->setCost(1); - } - // Create 250 edges at random. Edges must go from - // lower-to-higher index vertices, so we get a DAG. - for (unsigned i = 0; i < 250; ++i) { - const unsigned idx1 = V3Os::rand64(rngState) % 50; - const unsigned idx2 = V3Os::rand64(rngState) % 50; - if (idx1 > idx2) { - if (!vx[idx2]->hasRelativeMTask(vx[idx1])) { - new MTaskEdge{&graph, vx[idx2], vx[idx1], 1}; - } - } else if (idx2 > idx1) { - if (!vx[idx1]->hasRelativeMTask(vx[idx2])) { - new MTaskEdge{&graph, vx[idx1], vx[idx2], 1}; - } - } - } - - partInitCriticalPaths(graph); - - PropagateCp prop{true}; - - // Seed the propagator with every input node; - // This should result in the complete graph getting all CP's assigned. - for (const auto& i : vx) { - if (i->inEmpty()) prop.cpHasIncreased(i, 1 /* inclusive CP starts at 1 */); - } - - // Run the propagator. - prop.go(); - - // Finally, confirm that the entire graph appears to have correct CPs. - partCheckCriticalPaths(graph); - } -}; - -// Merge edges from a LogicMtask. -static void partRedirectEdgesFrom(V3Graph& graph, LogicMTask* recipientp, LogicMTask* donorp, - MergeCandidateScoreboard* sbp) { - // This code removes adjacent edges. When this occurs, mark it in need - // of a rescore, in case its score has fallen and we need to move it up - // toward the front of the scoreboard. - // - // Wait, what? Shouldn't the scores only increase as we merge nodes? Well - // that's almost true. But there is one exception. - // - // Suppose we have A->B, B->C, and A->C. - // - // The A->C edge is a "transitive" edge. It's ineligible to be merged, as - // the merge would create a cycle. We score it on the scoreboard like any - // other edge. - // - // However, our "score" estimate for A->C is bogus, because the forward - // critical path to C and the reverse critical path to A both contain the - // same node (B) so we overestimate the score of A->C. At first this - // doesn't matter, since transitive edges aren't eligible to merge anyway. - // - // Later, suppose the edge contractor decides to merge the B->C edge, with - // B donating all its incoming edges into C, say. (So we reach this - // function.) - // - // With B going away, the A->C edge will no longer be transitive and it - // will become eligible to merge. But if we don't mark it for rescore, - // it'll stay in the scoreboard with its old (overestimate) score. We'll - // merge it too late due to the bogus score. When we finally merge it, we - // fail the assert in the main edge contraction loop which checks that the - // actual score did not fall below the scoreboard's score. - // - // Another way of stating this: this code ensures that scores of - // non-transitive edges only ever increase. - - // Process outgoing edges - while (MTaskEdge* const edgep = static_cast(donorp->outEdges().frontp())) { - LogicMTask* const relativep = edgep->toMTaskp(); - - relativep->removeRelativeEdge(edgep); - - if (recipientp->hasRelativeMTask(relativep)) { - // An edge already exists between recipient and relative of donor. - // Mark it in need of a rescore - if (sbp) { - if (sbp->contains(edgep)) sbp->remove(edgep); - MTaskEdge* const existMTaskEdgep = static_cast( - recipientp->findConnectingEdgep(relativep)); - UDEBUGONLY(UASSERT(existMTaskEdgep, "findConnectingEdge didn't find edge");); - if (sbp->contains(existMTaskEdgep)) sbp->hintScoreChanged(existMTaskEdgep); - } - VL_DO_DANGLING(edgep->unlinkDelete(), edgep); - } else { - // No existing edge between recipient and relative of donor. - // Redirect the edge from donor<->relative to recipient<->relative. - edgep->relinkFromp(recipientp); - recipientp->addRelativeMTask(relativep); - recipientp->stealRelativeEdge(edgep); - relativep->addRelativeEdge(edgep); - if (sbp) { - if (!sbp->contains(edgep)) { - sbp->add(edgep); - } else { - sbp->hintScoreChanged(edgep); - } - } - } - } - - // Process incoming edges - while (MTaskEdge* const edgep = static_cast(donorp->inEdges().frontp())) { - LogicMTask* const relativep = edgep->fromMTaskp(); - - relativep->removeRelativeMTask(donorp); - relativep->removeRelativeEdge(edgep); - - if (relativep->hasRelativeMTask(recipientp)) { - // An edge already exists between recipient and relative of donor. - // Mark it in need of a rescore - if (sbp) { - if (sbp->contains(edgep)) sbp->remove(edgep); - MTaskEdge* const existMTaskEdgep = static_cast( - recipientp->findConnectingEdgep(relativep)); - UDEBUGONLY(UASSERT(existMTaskEdgep, "findConnectingEdge didn't find edge");); - if (sbp->contains(existMTaskEdgep)) sbp->hintScoreChanged(existMTaskEdgep); - } - VL_DO_DANGLING(edgep->unlinkDelete(), edgep); - } else { - // No existing edge between recipient and relative of donor. - // Redirect the edge from donor<->relative to recipient<->relative. - edgep->relinkTop(recipientp); - relativep->addRelativeMTask(recipientp); - relativep->addRelativeEdge(edgep); - recipientp->stealRelativeEdge(edgep); - if (sbp) { - if (!sbp->contains(edgep)) { - sbp->add(edgep); - } else { - sbp->hintScoreChanged(edgep); - } - } - } - } - - // Remove donorp from the graph - VL_DO_DANGLING(donorp->unlinkDelete(&graph), donorp); -} - -//###################################################################### -// Contraction - -// Perform edge or sibling contraction on the partition graph -class Contraction final { - // TYPES - // New CP information for mtaskp reflecting an upcoming merge - struct NewCp final { - uint64_t cp; - uint64_t propagateCp; - bool propagate; - }; - - // MEMBERS - V3Graph& m_mTaskGraph; // The Mtask graph - uint64_t m_scoreLimit; // Sloppy score allowed when picking merges - uint64_t m_scoreLimitBeforeRescore - = std::numeric_limits::max(); // Next score rescore - // at - unsigned m_mergesSinceRescore = 0; // Merges since last rescore - const bool m_slowAsserts; // Take extra time to validate algorithm - MergeCandidateScoreboard m_sb; // Scoreboard - - PropagateCp m_forwardPropagator{m_slowAsserts}; // Forward propagator - PropagateCp m_reversePropagator{m_slowAsserts}; // Reverse propagator - - LogicMTask* const m_entryMTaskp; // Singular source vertex of the dependency graph - LogicMTask* const m_exitMTaskp; // Singular sink vertex of the dependency graph - -public: - // CONSTRUCTORS - Contraction(V3Graph& mTaskGraph, uint64_t scoreLimit, LogicMTask* entryMTaskp, - LogicMTask* exitMTaskp, bool slowAsserts) - : m_mTaskGraph{mTaskGraph} - , m_scoreLimit{scoreLimit} - , m_slowAsserts{slowAsserts} - , m_entryMTaskp{entryMTaskp} - , m_exitMTaskp{exitMTaskp} { - if (m_slowAsserts) { - // Check there are no redundant edges - for (V3GraphVertex& vtx : m_mTaskGraph.vertices()) { - std::unordered_set neighbors; - for (V3GraphEdge& edge : vtx.outEdges()) { - const bool first = neighbors.insert(edge.top()).second; - UASSERT_OBJ(first, &vtx, "Redundant edge found in input to Contraction()"); - } - } - } - - unsigned maxMTasks = v3Global.opt.threadsMaxMTasks(); - if (maxMTasks == 0) { // Unspecified so estimate - if (v3Global.opt.threads() > 1) { - maxMTasks = (PART_DEFAULT_MAX_MTASKS_PER_THREAD * v3Global.opt.threads()); - } else { - // Running Contraction with --threads <= 1 means self-test - maxMTasks = 500; - } - } - - // OPTIMIZATION PASS: Edge contraction and sibling contraction. - // - Score each pair of mTaskGraphp which is a candidate to merge. - // * Each edge defines such a candidate pair - // * Two mTaskGraphp that are prereqs or postreqs of a common third - // vertex are "siblings", these are also a candidate pair. - // - Build a list of MergeCandidates, sorted by score. - // - Merge the best pair. - // - Incrementally recompute critical paths near the merged mtask. - - for (V3GraphVertex& vtx : m_mTaskGraph.vertices()) { - vtx.userp(nullptr); // Reset user value while we are here. Used by PropagateCp. - for (V3GraphEdge& edge : vtx.outEdges()) m_sb.add(static_cast(&edge)); - siblingPairFromRelatives(&vtx); - siblingPairFromRelatives(&vtx); - } - - doRescore(); // Set initial scores in scoreboard - - while (true) { - // This is the best edge to merge, with the lowest - // score (shortest local critical path) - MergeCandidate* const mergeCanp = m_sb.best(); - if (!mergeCanp) { - // Scoreboard found no eligible merges. Maybe a rescore - // will produce some merge-able pairs? - if (m_sb.needsRescore()) { - doRescore(); - continue; - } + mTaskGraphp->removeTransitiveEdges(); + mTaskGraphp->hashGraphDebug("MTask graph after removeTransitiveEdges()"); + + // Remove MTasks that have no logic in it, rerouting the edges. Set user to indicate the + // mtask on every underlying OrderMoveVertex. Clear vertex lists (used later). + moveGraph.userClearVertices(); + for (V3GraphVertex* const vtxp : mTaskGraphp->vertices().unlinkable()) { + LogicMTask* const mtaskp = vtxp->as(); + OrderMoveVertex::List& vertexList = mtaskp->vertexList(); + // Check if MTask is empty + bool empty = true; + for (const OrderMoveVertex& mVtx : vertexList) { + if (mVtx.logicp()) { + empty = false; break; } - - if (m_slowAsserts) { - UASSERT(!m_sb.needsRescore(mergeCanp), - "Need-rescore items should not be returned by bestp"); - } - const uint64_t cachedScore = mergeCanp->score(); - mergeCanp->rescore(); - const uint64_t actualScore = mergeCanp->score(); - - // cppcheck-suppress knownConditionTrueFalse // they are in fact different - if (actualScore > cachedScore) { - // Cached score is out-of-date. - // Mark this elem as in need of a rescore and continue. - m_sb.hintScoreChanged(mergeCanp); - continue; - } - // ... we'll also confirm that actualScore hasn't shrunk relative - // to cached score, after the mergeWouldCreateCycle() check. - - if (actualScore > m_scoreLimit) { - // Our best option isn't good enough - if (m_sb.needsRescore()) { - // Some pairs need a rescore, maybe those will be - // eligible to merge afterward. - doRescore(); - continue; - } else { - // We've exhausted everything below m_scoreLimit; stop. - - // Except, if we have too many mTaskGraphp, raise the score - // limit and keep going... - const unsigned mtaskCount = m_mTaskGraph.vertices().size(); - if (mtaskCount > maxMTasks) { - const uint64_t oldLimit = m_scoreLimit; - m_scoreLimit = (m_scoreLimit * 120) / 100; - FileLine* const flp = v3Global.rootp()->fileline(); - if (!flp->warnIsOff(V3ErrorCode::UNOPTTHREADS)) { - flp->v3warn(UNOPTTHREADS, - "Thread scheduler is unable to provide requested " - "parallelism; suggest asking for fewer threads."); - flp->modifyWarnOff(V3ErrorCode::UNOPTTHREADS, true); - } - UINFO(6, - "Critical path limit was=" << oldLimit << " now=" << m_scoreLimit); - continue; - } - // Really stop - break; - } - } - if (actualScore > m_scoreLimitBeforeRescore) { - // Time to rescore, that will result in a higher - // scoreLimitBeforeRescore, and possibly lower-scoring - // elements returned from bestp(). - doRescore(); - continue; - } - - // Avoid merging the entry/exit nodes. This would create serialization, by forcing the - // merged MTask to run before/after everything else. Empirically this helps - // performance in a modest way by allowing other MTasks to start earlier. - if (MTaskEdge* const edgep = mergeCanp->toMTaskEdge()) { - if (edgep->fromp() == m_entryMTaskp || edgep->top() == m_exitMTaskp) { - m_sb.remove(mergeCanp); - continue; - } - } - - // Avoid merging any edge that would create a cycle. - // - // For example suppose we begin with vertices A, B, C and edges - // A->B, B->C, A->C. - // - // Suppose we want to merge A->C into a single vertex. - // New edges would be AC->B and B->AC which is not a DAG. - // Do not allow this. - if (mergeCanp->mergeWouldCreateCycle()) { - // Remove this candidate from scoreboard so we don't keep - // reconsidering it on every loop. - m_sb.remove(mergeCanp); - if (SiblingMC* const smcp = mergeCanp->toSiblingMC()) { - smcp->unlinkA(); - smcp->unlinkB(); - VL_DO_DANGLING(delete smcp, smcp); - } - continue; - } - - partCheckCachedScoreVsActual(cachedScore, actualScore); - - // Finally there's no cycle risk, no need to rescore, we're - // within m_scoreLimit and m_scoreLimitBeforeRescore. - // This is the edge to merge. - // - // Bookkeeping: if this is the first edge we'll merge since - // the last rescore, compute the new m_scoreLimitBeforeRescore - // to be somewhat higher than this edge's score. - if (m_mergesSinceRescore == 0) { -#if PART_STEPPED_RESCORELIMIT - m_scoreLimitBeforeRescore = (actualScore * 105) / 100; -#else - m_scoreLimitBeforeRescore = actualScore; -#endif - - // This print can serve as a progress indicator, as it - // increases from low numbers up toward cpLimit. It may be - // helpful to see progress during slow partitions. Maybe - // display something by default even? - UINFO(6, "New scoreLimitBeforeRescore: " << m_scoreLimitBeforeRescore); - } - - // Finally merge this candidate. - contract(mergeCanp); } - - // Free remaining SiblingMCs - while (MergeCandidate* const mergeCanp = m_sb.best()) { - m_sb.remove(mergeCanp); - if (SiblingMC* const smcp = mergeCanp->toSiblingMC()) { - smcp->unlinkA(); - smcp->unlinkB(); - VL_DO_DANGLING(delete smcp, smcp); - } + // If empty remove it now + if (empty) { + mtaskp->rerouteEdges(mTaskGraphp.get()); + VL_DO_DANGLING(mtaskp->unlinkDelete(mTaskGraphp.get()), mtaskp); + continue; } + // Annotate the underlying OrderMoveVertex vertices and unlink them + while (OrderMoveVertex* const mVtxp = vertexList.unlinkFront()) mVtxp->userp(mtaskp); } + mTaskGraphp->removeRedundantEdgesSum(&V3GraphEdge::followAlwaysTrue); -private: - template - NewCp newCp(const LogicMTask* mtaskp, const LogicMTask* otherp, const MTaskEdge* mergeEdgep) { - constexpr GraphWay way{N_Way}; - // Return new wayward-CP for mtaskp reflecting its upcoming merge - // with otherp. Set 'result.propagate' if mtaskp's wayward - // relatives will see a new wayward CP from this merge. - uint64_t newCp; - if (mergeEdgep) { - if (mtaskp == mergeEdgep->furtherp()) { - newCp = std::max(otherp->critPathCost(way), - mtaskp->critPathCostWithout(mergeEdgep)); - } else { - newCp = std::max(mtaskp->critPathCost(way), - otherp->critPathCostWithout(mergeEdgep)); - } - } else { - newCp = std::max(otherp->critPathCost(way), mtaskp->critPathCost(way)); - } - - const uint64_t origRelativesCp = mtaskp->critPathCost(way) + mtaskp->stepCost(); - const uint64_t newRelativesCp - = newCp + LogicMTask::stepCost(mtaskp->cost() + otherp->cost()); - - NewCp result; - result.cp = newCp; - result.propagate = (newRelativesCp > origRelativesCp); - result.propagateCp = newRelativesCp; - return result; - } - - void removeSiblingMCsWith(LogicMTask* mtaskp) { - while (SiblingMC* const smcp = mtaskp->aSiblingMCs().unlinkFront()) { - m_sb.remove(smcp); - smcp->unlinkB(); - VL_DO_DANGLING(delete smcp, smcp); - } - while (SiblingMC* const smcp = mtaskp->bSiblingMCs().unlinkFront()) { - m_sb.remove(smcp); - smcp->unlinkA(); - VL_DO_DANGLING(delete smcp, smcp); - } - } - - void removeSiblingMCs(LogicMTask* recipientp, LogicMTask* donorp) { - // The lists here should be disjoint (there should be only one SiblingMC involving these - // two MTasks, and we removed that elsewhere), so no need for unlinking from the lists we - // are clearing. - removeSiblingMCsWith(recipientp); - removeSiblingMCsWith(donorp); - - // Clear the sibling map of the recipient. The donor will be deleted anyway, so we can - // leave that in a corrupt for efficiency. - recipientp->siblings().clear(); - } - - void contract(MergeCandidate* mergeCanp) { - LogicMTask* top = nullptr; - LogicMTask* fromp = nullptr; - MTaskEdge* const mergeEdgep = mergeCanp->toMTaskEdge(); - SiblingMC* const mergeSibsp = mergeCanp->toSiblingMC(); - if (mergeEdgep) { - top = mergeEdgep->toMTaskp(); - fromp = mergeEdgep->fromMTaskp(); - } else { - top = mergeSibsp->ap(); - fromp = mergeSibsp->bp(); - } - - // Merge the smaller mtask into the larger mtask. If one of them - // is much larger, this will save time in partRedirectEdgesFrom(). - // Assume the more costly mtask has more edges. - // - // [TODO: now that we have edge maps, we could count the edges - // exactly without a linear search.] - LogicMTask* recipientp; - LogicMTask* donorp; - if (fromp->cost() > top->cost()) { - recipientp = fromp; - donorp = top; - } else { - donorp = fromp; - recipientp = top; - } - VL_DANGLING(fromp); - VL_DANGLING(top); // Use donorp and recipientp now instead - - // Recursively update forward and reverse CP numbers. - // - // Doing this before merging the mTaskGraphp lets us often avoid - // recursing through either incoming or outgoing edges on one or - // both mTaskGraphp. - // - // These 'NewCp' objects carry a bit indicating whether we must - // propagate CP for each of the four cases: - const NewCp recipientNewCpFwd = newCp(recipientp, donorp, mergeEdgep); - const NewCp donorNewCpFwd = newCp(donorp, recipientp, mergeEdgep); - const NewCp recipientNewCpRev = newCp(recipientp, donorp, mergeEdgep); - const NewCp donorNewCpRev = newCp(donorp, recipientp, mergeEdgep); - - m_sb.remove(mergeCanp); - - if (mergeEdgep) { - // Remove and free the connecting edge. Must do this before propagating CP's below. - mergeEdgep->fromMTaskp()->removeRelativeMTask(mergeEdgep->toMTaskp()); - mergeEdgep->fromMTaskp()->removeRelativeEdge(mergeEdgep); - mergeEdgep->toMTaskp()->removeRelativeEdge(mergeEdgep); - VL_DO_DANGLING(mergeEdgep->unlinkDelete(), mergeEdgep); - } else { - // Remove the siblingMC - mergeSibsp->unlinkA(); - mergeSibsp->unlinkB(); - VL_DO_DANGLING(delete mergeSibsp, mergeSibsp); - } - - // This also updates cost and stepCost on recipientp - recipientp->moveAllVerticesFrom(donorp); - - UINFO(9, "recipient = " << recipientp->id() << ", donor = " << donorp->id() - << ", mergeEdgep = " << mergeEdgep << "\n" - << "recipientNewCpFwd = " << recipientNewCpFwd.cp - << (recipientNewCpFwd.propagate ? " true " : " false ") - << recipientNewCpFwd.propagateCp << "\n" - << "donorNewCpFwd = " << donorNewCpFwd.cp - << (donorNewCpFwd.propagate ? " true " : " false ") - << donorNewCpFwd.propagateCp); - - recipientp->setCritPathCost(GraphWay::FORWARD, recipientNewCpFwd.cp); - if (recipientNewCpFwd.propagate) { - m_forwardPropagator.cpHasIncreased(recipientp, recipientNewCpFwd.propagateCp); - } - recipientp->setCritPathCost(GraphWay::REVERSE, recipientNewCpRev.cp); - if (recipientNewCpRev.propagate) { - m_reversePropagator.cpHasIncreased(recipientp, recipientNewCpRev.propagateCp); - } - if (donorNewCpFwd.propagate) { - m_forwardPropagator.cpHasIncreased(donorp, donorNewCpFwd.propagateCp); - } - if (donorNewCpRev.propagate) { - m_reversePropagator.cpHasIncreased(donorp, donorNewCpRev.propagateCp); - } - m_forwardPropagator.go(); - m_reversePropagator.go(); - - // Remove all other SiblingMCs that include recipientp or donorp. We remove all siblingMCs - // of recipientp so we do not get huge numbers of SiblingMCs. We'll recreate them below, up - // to a bounded number. - removeSiblingMCs(recipientp, donorp); - - // Redirect all edges, delete donorp - partRedirectEdgesFrom(m_mTaskGraph, recipientp, donorp, &m_sb); - - ++m_mergesSinceRescore; - - // Do an expensive check, confirm we haven't botched the CP - // updates. - if (m_slowAsserts) partCheckCriticalPaths(m_mTaskGraph); - - // Finally, make new sibling pairs as needed: - // - prereqs and postreqs of recipientp - // - prereqs of recipientp's postreqs - // - postreqs of recipientp's prereqs - // Note that this depends on the updated critical paths (above). - siblingPairFromRelatives(recipientp); - siblingPairFromRelatives(recipientp); - unsigned edges = 0; - for (V3GraphEdge& edge : recipientp->outEdges()) { - LogicMTask* const postreqp = static_cast(edge.top()); - siblingPairFromRelatives(postreqp); - ++edges; - if (edges >= PART_SIBLING_EDGE_LIMIT) break; - } - edges = 0; - for (V3GraphEdge& edge : recipientp->inEdges()) { - LogicMTask* const prereqp = static_cast(edge.fromp()); - siblingPairFromRelatives(prereqp); - ++edges; - if (edges >= PART_SIBLING_EDGE_LIMIT) break; - } - } - - void doRescore() { - // During rescore, we know that graph isn't changing, so allow - // the critPathCost*Without() routines to cache some data in - // each LogicMTask. This is just an optimization, things should - // behave identically without the caching (just slower) - - m_sb.rescore(); - UINFO(6, "Did rescore. Merges since previous = " << m_mergesSinceRescore); - - m_mergesSinceRescore = 0; - m_scoreLimitBeforeRescore - = std::numeric_limits::max(); - } - - void makeSiblingMC(LogicMTask* ap, LogicMTask* bp) { - if (ap->id() < bp->id()) std::swap(ap, bp); - // The higher id vertex owns the association set - const auto first = ap->siblings().insert(bp).second; - if (first) { - m_sb.add(new SiblingMC{ap, bp}); - } else if (VL_UNLIKELY(m_slowAsserts)) { - // It's fine if we already have this SiblingMC, we may have - // created it earlier. Just confirm that we have associated data. - bool found = false; - for (const SiblingMC& smc : ap->aSiblingMCs()) { - UASSERT_OBJ(smc.ap() == ap, ap, "Inconsistent SiblingMC"); - UASSERT_OBJ(m_sb.contains(&smc), ap, "Must be on the scoreboard"); - if (smc.bp() == bp) found = true; - } - UASSERT_OBJ(found, ap, "Sibling not found"); - } - } - - template - void siblingPairFromRelatives(V3GraphVertex* mtaskp) { - constexpr GraphWay way{N_Way}; - // Need at least 2 edges - auto& edges = mtaskp->edges(); - if (!edges.hasMultipleElements()) return; - - std::array neighbors; - - // This is a hot method, so we want so sort as efficiently as possible. We pre-load - // all data (critical path cost and id) required for determining ordering into an aligned - // structure. There is not enough space next to these to keep a whole pointer within 16 - // bytes, so we store an index into the neighbors buffer instead. We can then compare - // and swap these sorting records very efficiently. With this the standard library sorting - // functions are efficient enough and using more optimized methods (e.g.: sorting networks) - // has no measurable benefit. - struct alignas(16) SortingRecord final { - uint64_t m_cp; - uint32_t m_id; - uint8_t m_idx; - static_assert(PART_SIBLING_EDGE_LIMIT <= std::numeric_limits::max(), - "m_idx must fit all indices into 'neighbors'"); - bool operator<(const SortingRecord& that) const { - return m_cp < that.m_cp || (m_cp == that.m_cp && m_id < that.m_id); - } - }; - static_assert(sizeof(SortingRecord) <= 16, "How could this be padded to more than 16?"); - - std::array sortRecs; - size_t n = 0; - - // Populate the buffers - for (V3GraphEdge& edge : mtaskp->edges()) { - LogicMTask* const otherp = static_cast(edge.furtherp()); - neighbors[n] = otherp; - sortRecs[n].m_id = otherp->id(); - sortRecs[n].m_cp = otherp->critPathCost(way) + otherp->cost(); - sortRecs[n].m_idx = n; - ++n; - // Prevent nodes with huge numbers of edges from massively slowing down us down - if (n >= PART_SIBLING_EDGE_LIMIT) break; - } - - // Don't make all possible pairs of siblings when not requested (non-exhaustive). - // Just make a few pairs. - constexpr size_t MAX_NONEXHAUSTIVE_PAIRS = 3; - - if (N_Exhaustive || n <= 2 * MAX_NONEXHAUSTIVE_PAIRS) { - const size_t end = n & ~static_cast(1); // Round down to even, (we want pairs) - std::sort(sortRecs.begin(), sortRecs.begin() + n); - for (size_t i = 0; i < end; i += 2) { - makeSiblingMC(neighbors[sortRecs[i].m_idx], neighbors[sortRecs[i + 1].m_idx]); - } - } else { - constexpr size_t end = 2 * MAX_NONEXHAUSTIVE_PAIRS; - std::partial_sort(sortRecs.begin(), sortRecs.begin() + end, sortRecs.begin() + n); - for (size_t i = 0; i < end; i += 2) { - makeSiblingMC(neighbors[sortRecs[i].m_idx], neighbors[sortRecs[i + 1].m_idx]); - } - } - } - - // SELF TESTS - - // This is a performance test, its intent is to demonstrate that the - // partitioner doesn't run on this chain in N^2 time or worse. Overall - // runtime should be N*log(N) for a chain-shaped graph. - // - static void selfTestChain() { - const uint64_t usecsSmall = partitionChainUsecs(5); - const uint64_t usecsLarge = partitionChainUsecs(500); - // Large input is 50x bigger than small input. - // Its runtime should be about 10x longer -- not about 2500x longer - // or worse which would suggest N^2 scaling or worse. - UASSERT(usecsLarge < (usecsSmall * 1500), - "selfTestChain() took longer than expected. Small input runtime = " - << usecsSmall << ", large input runtime = " << usecsLarge); - } - - static uint64_t partitionChainUsecs(unsigned chain_len) { - // NOTE: To get a dot file run with --debugi-Partitioner 4 or more. - const uint64_t startUsecs = V3Os::timeUsecs(); - V3Graph mTaskGraph; - LogicMTask* lastp = nullptr; - for (unsigned i = 0; i < chain_len; ++i) { - LogicMTask* const mtp = new LogicMTask{&mTaskGraph, nullptr}; - mtp->setCost(1); - if (lastp) new MTaskEdge{&mTaskGraph, lastp, mtp, 1}; - lastp = mtp; - } - partInitCriticalPaths(mTaskGraph); - - // Since slowAsserts mode is *expected* to cause N^2 runtime, and the - // intent of this test is to demonstrate better-than-N^2 runtime, disable - // slowAsserts. - Contraction::apply(mTaskGraph, - // Any CP limit >chain_len should work: - chain_len * 2, nullptr, nullptr, /* slowAsserts: */ false); - - // All vertices should merge into one - UASSERT_SELFTEST(const bool, mTaskGraph.vertices().hasSingleElement(), true); - - const uint64_t endUsecs = V3Os::timeUsecs(); - const uint64_t elapsedUsecs = endUsecs - startUsecs; - - return elapsedUsecs; - } - - // This test defends against a particular failure mode that the - // partitioner exhibited during development: - // - // At one time, the partitioner consistently favored edge-merges over - // equal-scoring sibling merges. Every edge and sibling merge in this - // test starts out with an equal score. If you only do edge-merges, all - // possible merges will continue to have equal score as the center node - // grows and grows. Soon the critical path budget is exhausted by a - // large center node, and we still have many small leaf nodes -- it's - // literally the worst partition possible. - // - // Now, instead, the partitioner gives slight favoritism to sibling - // merges in the event that scores are tied. This is better for the - // test and also real designs. - static void selfTestX() { - // NOTE: To get a dot file run with --debugi-Partitioner 4 or more. - V3Graph mTaskGraph; - LogicMTask* const centerp = new LogicMTask{&mTaskGraph, nullptr}; - centerp->setCost(1); - unsigned i; - for (i = 0; i < 50; ++i) { - LogicMTask* const mtp = new LogicMTask{&mTaskGraph, nullptr}; - mtp->setCost(1); - // Edge from every input -> centerp - new MTaskEdge{&mTaskGraph, mtp, centerp, 1}; - } - for (i = 0; i < 50; ++i) { - LogicMTask* const mtp = new LogicMTask{&mTaskGraph, nullptr}; - mtp->setCost(1); - // Edge from centerp -> every output - new MTaskEdge{&mTaskGraph, centerp, mtp, 1}; - } - - partInitCriticalPaths(mTaskGraph); - Contraction::apply(mTaskGraph, 20, nullptr, nullptr, true); - - const auto report = mTaskGraph.parallelismReport( - [](const V3GraphVertex* vtxp) { return vtxp->as()->cost(); }); - - // Checking exact values here is maybe overly precise. What we're - // mostly looking for is a healthy reduction in the number of mTaskGraphp. - UASSERT_SELFTEST(const uint64_t, report.criticalPathCost(), 19); - UASSERT_SELFTEST(const uint64_t, report.totalGraphCost(), 101); - UASSERT_SELFTEST(const uint64_t, report.vertexCount(), 14); - UASSERT_SELFTEST(const uint64_t, report.edgeCount(), 13); - } - -public: - static void selfTest() { - selfTestX(); - selfTestChain(); - } - - static void apply(V3Graph& mTaskGraph, uint64_t scoreLimit, LogicMTask* entryMTaskp, - LogicMTask* exitMTaskp, bool slowAsserts) { - Contraction{mTaskGraph, scoreLimit, entryMTaskp, exitMTaskp, slowAsserts}; - } -}; + // Return the resulting MTask graph + return mTaskGraphp; +} //###################################################################### -// DpiImportCallVisitor +// DpiThreadsVisitor - Finds number of threads used by an ExecMTask -// Scan node, indicate whether it contains a call to a DPI imported -// routine. -class DpiImportCallVisitor final : public VNVisitor { - bool m_hasDpiHazard = false; // Found a DPI import call. - bool m_tracingCall = false; // Iterating into a CCall to a CFunc - // METHODS - void visit(AstCFunc* nodep) override { - if (!m_tracingCall) return; - m_tracingCall = false; - if (nodep->dpiImportWrapper()) { - if (nodep->dpiPure() ? !v3Global.opt.threadsDpiPure() - : !v3Global.opt.threadsDpiUnpure()) { - // If hierarchical DPI wrapper cost is not found or is of a 0 cost, - // we have a normal DPI which induces DPI hazard by default. - m_hasDpiHazard = V3Control::getProfileData(nodep->cname()) == 0; - UINFO(9, "DPI wrapper '" << nodep->cname() - << "' has dpi hazard = " << m_hasDpiHazard); - } - } - iterateChildren(nodep); - } - void visit(AstNodeCCall* nodep) override { - iterateChildren(nodep); - // Enter the function and trace it - m_tracingCall = true; - iterate(nodep->funcp()); - } - void visit(AstNode* nodep) override { iterateChildren(nodep); } - -public: - // CONSTRUCTORS - explicit DpiImportCallVisitor(AstNode* nodep) { iterate(nodep); } - bool hasDpiHazard() const { return m_hasDpiHazard; } - ~DpiImportCallVisitor() override = default; - -private: - VL_UNCOPYABLE(DpiImportCallVisitor); -}; - -//###################################################################### -// DpiThreadsVisitor - -// Get number of threads occupied by this mtask class DpiThreadsVisitor final : public VNVisitorConst { int m_threads = 1; // Max number of threads used by this mtask @@ -1761,649 +129,32 @@ class DpiThreadsVisitor final : public VNVisitorConst { void visit(AstNodeCCall* nodep) override { iterateConst(nodep->funcp()); } void visit(AstNode* nodep) override { iterateChildrenConst(nodep); } -public: // CONSTRUCTORS explicit DpiThreadsVisitor(AstCFunc* nodep) { iterateConst(nodep); } - int threads() const { return m_threads; } ~DpiThreadsVisitor() override = default; - -private: VL_UNCOPYABLE(DpiThreadsVisitor); -}; - -//###################################################################### -// FixDataHazards - -class FixDataHazards final { - // - // Fix data hazards in the MTask graph. - // - // The fine-grained graph from V3Order may contain data hazards which are - // not a problem for serial mode, but which would be a problem in parallel - // mode. - // - // There are basically two classes: unordered pairs of writes, and - // unordered write-read pairs. We fix both here, with a combination of - // MTask-merges and new edges to ensure no such unordered pairs remain. - // - // ABOUT UNORDERED WRITE-WRITE PAIRS - // - // The V3Order dependency graph treats these as unordered events: - // - // a) sig[15:8] = stuff; - // ... - // b) sig[7:0] = other_stuff; - // - // Seems OK right? They are writes to disjoint bits of the same - // signal. They can run in either order, in serial mode, and the result - // will be the same. - // - // The resulting C code for each of this isn't a pure write, it's - // actually an R-M-W sequence: - // - // a) sig = (sig & 0xff) | (0xff00 & (stuff << 8)); - // ... - // b) sig = (sig & 0xff00) | (0xff & other_stuff); - // - // In serial mode, order doesn't matter so long as these run serially. - // In parallel mode, we must serialize these RMW's to avoid a race. - // - // We don't actually check here if each write would involve an R-M-W, we - // just assume that it would. If this routine ever causes a drastic - // increase in critical path, it could be optimized to make a better - // prediction (with all the risk that word implies!) about whether a - // given write is likely to turn into an R-M-W. - // - // ABOUT UNORDERED WRITE-READ PAIRS - // - // If we don't put unordered write-read pairs into some order at Verilation - // time, we risk a runtime race. - // - // How do such unordered writer/reader pairs happen? Here's a partial list - // of scenarios: - // - // Case 1: Circular logic - // - // If the design has circular logic, V3Order has by now generated some - // dependency cycles, and also cut some of the edges to make it - // acyclic. - // - // For serial mode, that was fine. We can break logic circles at an - // arbitrary point. At runtime, we'll repeat the _eval() until no - // changes are detected, which papers over the discarded dependency. - // - // For parallel mode, this situation can lead to unordered reads and - // writes of the same variable, causing a data race. For example if the - // original code is this: - // - // assign b = b | a << 2; - // assign out = b; - // - // ... there's originally a dependency edge which records that 'b' - // depends on the first assign. V3Order may cut this edge, making the - // statements unordered. In serial mode that's fine, they can run in - // either order. In parallel mode it's a reader/writer race. - // - // Case 2: Race Condition in Verilog Sources - // - // If the input has races, eg. blocking assignments in always blocks - // that share variables, the graph at this point will contain unordered - // writes and reads (or unordered write-write pairs) reflecting that. - // - // Case 3: Interesting V3Order Behavior - // - // There's code in V3Order that explicitly avoids making a dependency - // edge from a clock-gater signal to the logic node that produces the - // clock signal. This leads to unordered reader/writer pairs in - // parallel mode. - // - - // TYPES - // Sort LogicMTask objects into deterministic order by calling id() - // which is a unique and stable serial number. - struct MTaskIdLessThan final { - bool operator()(const LogicMTask* lhsp, const LogicMTask* rhsp) const { - return lhsp->id() < rhsp->id(); - } - }; - using TasksByRank = std::map>; - - // MEMBERS - V3Graph& m_mTaskGraph; // The Mtask graph - - // CONSTRUCTORs - FixDataHazards(const OrderGraph& orderGraph, V3Graph& mTaskGraph) - : m_mTaskGraph{mTaskGraph} { - // Rank the graph. DGS is faster than V3GraphAlg's recursive rank, and also allows us to - // set up the OrderLogicVertex -> LogicMTask map at the same time. - { - GraphStreamUnordered serialize{&m_mTaskGraph}; - while (LogicMTask* const mtaskp - = const_cast(static_cast(serialize.nextp()))) { - // Compute and assign rank - uint32_t rank = 0; - for (V3GraphEdge& edge : mtaskp->inEdges()) { - rank = std::max(edge.fromp()->rank() + 1, rank); - } - mtaskp->rank(rank); - - // Set up the OrderLogicVertex -> LogicMTask map - // Entry and exit MTasks have no MTaskMoveVertices under them, so move on - if (mtaskp->vertexList().empty()) continue; - // Otherwise there should be only one OrderMoveVertex in each MTask at this stage - const OrderMoveVertex::List& vertexList = mtaskp->vertexList(); - UASSERT_OBJ(vertexList.hasSingleElement(), mtaskp, "Multiple OrderMoveVertex"); - const OrderMoveVertex* const mVtxp = vertexList.frontp(); - // Set up mapping back to the MTask from the OrderLogicVertex - if (OrderLogicVertex* const lvtxp = mVtxp->logicp()) lvtxp->userp(mtaskp); - } - } - - // Gather all variables. SystemC vars will be handled slightly specially, so keep separate. - std::vector regularVars; - std::vector systemCVars; - for (const V3GraphVertex& vtx : orderGraph.vertices()) { - // Only consider OrderVarStdVertex which reflects - // an actual lvalue assignment; the others do not. - if (const OrderVarStdVertex* const vvtxp = vtx.cast()) { - if (vvtxp->vscp()->varp()->isSc()) { - systemCVars.push_back(vvtxp); - } else { - regularVars.push_back(vvtxp); - } - } - } - - // For each OrderVarVertex, look at its writer and reader mTaskGraphp. - // - // If there's a set of writers and readers at the same rank, we - // know these are unordered with respect to one another, so merge - // those mTaskGraphp all together. - // - // At this point, we have at most one merged mtask per rank (for a - // given OVV.) Create edges across these remaining mTaskGraphp to ensure - // they run in serial order (going along with the existing ranks.) - // - // NOTE: we don't update the CP's stored in the LogicMTasks to - // reflect the changes we make to the graph. That's OK, as we - // haven't yet initialized CPs when we call this routine. - for (const OrderVarStdVertex* const varVtxp : regularVars) { - // Build a set of mTaskGraphp, per rank, which access this var. - // Within a rank, sort by MTaskID to avoid nondeterminism. - TasksByRank tasksByRank; - - // Find all reader and writer tasks for this variable, add to - // tasksByRank. - findAdjacentTasks(varVtxp, tasksByRank); - - // Merge all writer and reader tasks from same rank together. - // - // NOTE: Strictly speaking, we don't need to merge all the - // readers together. That may lead to extra serialization. The - // least amount of ordering we could impose here would be to - // merge all writers at a given rank together; then make edges - // from the merged writer node to each reader node at the same - // rank; and then from each reader node to the merged writer at - // the next rank. - // - // Whereas, merging all readers and writers at the same rank - // together is "the simplest thing that could possibly work" - // and it seems to. It also creates fairly few edges. We don't - // want to create tons of edges here, doing so is not nice to - // the main edge contraction pass. - mergeSameRankTasks(tasksByRank); - } - - // Handle SystemC vars just a little differently. Instead of - // treating each var as an independent entity, and serializing - // writes to that one var, we treat ALL systemC vars as a single - // entity and serialize writes (and, conservatively, reads) across - // all of them. - // - // Reasoning: writing a systemC var actually turns into a call to a - // var.write() method, which under the hood is accessing some data - // structure that's shared by many SC vars. It's not thread safe. - // - // Hopefully we only have a few SC vars -- top level ports, probably. - { - TasksByRank tasksByRank; - for (const OrderVarStdVertex* const varVtxp : systemCVars) { - findAdjacentTasks(varVtxp, tasksByRank); - } - mergeSameRankTasks(tasksByRank); - } - - // Handle nodes containing DPI calls, we want to serialize those - // by default unless user gave '--threads-dpi none'. - // Same basic strategy as above to serialize access to SC vars. - if (!v3Global.opt.threadsDpiPure() || !v3Global.opt.threadsDpiUnpure()) { - TasksByRank tasksByRank; - for (V3GraphVertex& vtx : m_mTaskGraph.vertices()) { - LogicMTask& mtask = static_cast(vtx); - if (hasDpiHazard(&mtask)) tasksByRank[mtask.rank()].insert(&mtask); - } - mergeSameRankTasks(tasksByRank); - } - } - - // METHODS - void findAdjacentTasks(const OrderVarStdVertex* varVtxp, TasksByRank& tasksByRank) { - // Find all writer tasks for this variable, group by rank. - for (const V3GraphEdge& edge : varVtxp->inEdges()) { - if (const auto* const logicVtxp = edge.fromp()->cast()) { - LogicMTask* const writerMtaskp = static_cast(logicVtxp->userp()); - tasksByRank[writerMtaskp->rank()].insert(writerMtaskp); - } - } - // Note: Find all reader tasks for this variable, group by rank. - // There was "broken" code here to find readers, but fixing it to - // work properly harmed performance on some tests, see issue #3360. - } - void mergeSameRankTasks(const TasksByRank& tasksByRank) { - LogicMTask* lastRecipientp = nullptr; - for (const auto& pair : tasksByRank) { - // Find the largest node at this rank, merge into it. (If we - // happen to find a huge node, this saves time in - // partRedirectEdgesFrom() versus merging into an arbitrary node.) - LogicMTask* recipientp = nullptr; - for (LogicMTask* const mtaskp : pair.second) { - if (!recipientp || (recipientp->cost() < mtaskp->cost())) recipientp = mtaskp; - } - UASSERT_OBJ(!lastRecipientp || (lastRecipientp->rank() < recipientp->rank()), - recipientp, "Merging must be on lower rank"); - - for (LogicMTask* const donorp : pair.second) { - // Merge donor into recipient. - if (donorp == recipientp) continue; - // Fix up the map, so donor's OLVs map to recipientp - for (const OrderMoveVertex& vtx : donorp->vertexList()) { - vtx.logicp()->userp(recipientp); - } - // Move all vertices from donorp to recipientp - recipientp->moveAllVerticesFrom(donorp); - // Redirect edges from donorp to recipientp, delete donorp - partRedirectEdgesFrom(m_mTaskGraph, recipientp, donorp, nullptr); - } - - if (lastRecipientp && !lastRecipientp->hasRelativeMTask(recipientp)) { - new MTaskEdge{&m_mTaskGraph, lastRecipientp, recipientp, 1}; - } - lastRecipientp = recipientp; - } - } - bool hasDpiHazard(LogicMTask* mtaskp) { - for (const OrderMoveVertex& mVtx : mtaskp->vertexList()) { - if (OrderLogicVertex* const lvtxp = mVtx.logicp()) { - // NOTE: We don't handle DPI exports. If testbench code calls a - // DPI-exported function at any time during eval() we may have - // a data hazard. (Likewise in non-threaded mode if an export - // messes with an ordered variable we're broken.) - - // Find all calls to DPI-imported functions, we can put those - // into a serial order at least. That should solve the most - // likely DPI-related data hazards. - if (DpiImportCallVisitor{lvtxp->nodep()}.hasDpiHazard()) return true; - } - } - return false; - } - - VL_UNCOPYABLE(FixDataHazards); public: - static void apply(const OrderGraph& orderGraph, V3Graph& mTaskGraph) { - FixDataHazards(orderGraph, mTaskGraph); + // Number of threads occupied by the given MTask + static int apply(const ExecMTask* mTaskp) { + return DpiThreadsVisitor{mTaskp->funcp()}.m_threads; } }; //###################################################################### -// Partitioner implementation +// Entry point -// Print debug stats about graphp whose nodes must be LogicMTask's. -static void debugMTaskGraphStats(V3Graph& graph, const string& stage) { - if (!debug() && !dumpLevel() && !dumpGraphLevel()) return; - - UINFO(4, "\n"); - UINFO(4, " Stats for " << stage); - uint64_t mtaskCount = 0; - uint64_t totalCost = 0; - constexpr int scoreBits = std::numeric_limits::digits; - std::array mtaskCostHist{}; - for (const V3GraphVertex& mtask : graph.vertices()) { - ++mtaskCount; - uint64_t mtaskCost = mtask.as()->cost(); - totalCost += mtaskCost; - - unsigned log2Cost = 0; - while (mtaskCost >>= 1) ++log2Cost; - UASSERT(log2Cost < scoreBits, "log2Cost overflow in debugMTaskGraphStats"); - ++mtaskCostHist[log2Cost]; - } - UINFO(4, " Total mtask cost = " << totalCost); - UINFO(4, " Mtask count = " << mtaskCount); - UINFO(4, " Avg cost / mtask = " << ((mtaskCount > 0) ? cvtToStr(totalCost / mtaskCount) - : "INF!")); - UINFO(4, " Histogram of mtask costs:"); - for (unsigned i = 0; i < scoreBits; ++i) { - if (mtaskCostHist[i]) { - UINFO(4, " 2^" << i << ": " << mtaskCostHist[i]); - V3Stats::addStat("MTask graph, " + stage + ", mtask cost 2^" + (i < 10 ? " " : "") - + cvtToStr(i), - mtaskCostHist[i]); - } - } - - if (mtaskCount < 1000) { - string filePrefix("ordermv_"); - filePrefix += stage; - if (dumpGraphLevel() >= 4) graph.dumpDotFilePrefixedAlways(filePrefix); - } - - // Look only at the cost of each mtask, neglect communication cost. - // This will show us how much parallelism we expect, assuming cache-miss - // costs are minor and the cost of running logic is the dominant cost. - const auto report = graph.parallelismReport( - [](const V3GraphVertex* vtxp) { return vtxp->as()->cost(); }); - V3Stats::addStat("MTask graph, " + stage + ", critical path cost", report.criticalPathCost()); - V3Stats::addStat("MTask graph, " + stage + ", total graph cost", report.totalGraphCost()); - V3Stats::addStat("MTask graph, " + stage + ", mtask count", report.vertexCount()); - V3Stats::addStat("MTask graph, " + stage + ", edge count", report.edgeCount()); - V3Stats::addStat("MTask graph, " + stage + ", parallelism factor", report.parallelismFactor()); - if (debug() >= 4) { - UINFO(0, "\n"); - UINFO(0, " MTask Parallelism estimate based costs at stage" << stage << ":"); - UINFO(0, " Critical path cost = " << report.criticalPathCost()); - UINFO(0, " Total graph cost = " << report.totalGraphCost()); - UINFO(0, " MTask vertex count = " << report.vertexCount()); - UINFO(0, " Edge count = " << report.edgeCount()); - UINFO(0, " Parallelism factor = " << report.parallelismFactor()); - } -} - -// Print a hash of the shape of graphp. If you are battling -// nondeterminism, this can help to pinpoint where in the pipeline it's -// creeping in. -static void hashGraphDebug(const V3Graph& graph, const char* debugName) { - // Disabled when there are no nondeterminism issues in flight. - if (!v3Global.opt.debugNondeterminism()) return; - - std::unordered_map vx2Id; - unsigned id = 0; - for (const V3GraphVertex& vtx : graph.vertices()) vx2Id[&vtx] = id++; - unsigned hash = 0; - for (const V3GraphVertex& vtx : graph.vertices()) { - for (const V3GraphEdge& edge : vtx.outEdges()) { - hash = vx2Id[edge.top()] + 31U * hash; // The K&R hash function - } - } - UINFO(0, "Hash of shape (not contents) of " << debugName << " = " << cvtToStr(hash)); -} - -//************************************************************************* -// Partitioner takes the fine-grained logic graph from V3Order and -// collapses it into a coarse-grained graph of LogicMTask's, each -// of which contains of set of the logic nodes from the fine-grained -// graph. - -class Partitioner final { - // MEMBERS - OrderMoveGraph& m_moveGraph; // Fine-grained dependency graph - std::unique_ptr m_mTaskGraphp{new V3Graph{}}; // The resulting MTask graph - - LogicMTask* m_entryMTaskp = nullptr; // Singular source vertex of the dependency graph - LogicMTask* m_exitMTaskp = nullptr; // Singular sink vertex of the dependency graph - - // METHODS - - // Predicate function to determine what OrderMoveVertex to bypass when constructing the MTask - // graph. The fine-grained dependency graph of OrderMoveVertex vertices is a bipartite graph - // of: - // - 1. OrderMoveVertex instances containing logic via OrderLogicVertex - // (OrderMoveVertex::logicp() != nullptr) - // - 2. OrderMoveVertex instances containing an (OrderVarVertex, domain) pair - // Our goal is to order the logic vertices. The second type of variable/domain vertices only - // carry dependencies and are eventually discarded. In order to reduce the working set size of - // Contraction, we 'bypass' and not create LogicMTask vertices for the variable vertices, - // and instead add the transitive dependencies directly, but only if adding the transitive - // edges directly does not require more dependency edges than keeping the intermediate vertex. - // That is, we bypass a variable vertex if fanIn * fanOut <= fanIn + fanOut. This can only be - // true if fanIn or fanOut are 1, or if they are both 2. This can cause significant reduction - // in working set size. - static bool bypassOk(OrderMoveVertex* mvtxp) { - // Need to keep all logic vertices - if (mvtxp->logicp()) return false; - // Count fan-in, up to 3 - unsigned fanIn = 0; - auto& inEdges = mvtxp->inEdges(); - for (auto it = inEdges.begin(); it != inEdges.end(); ++it) { - if (++fanIn == 3) break; - } - UDEBUGONLY(UASSERT_OBJ(fanIn <= 3, mvtxp, "Should have stopped counting fanIn");); - // If fanInn no more than one, bypass - if (fanIn <= 1) return true; - // Count fan-out, up to 3 - unsigned fanOut = 0; - auto& outEdges = mvtxp->outEdges(); - for (auto it = outEdges.begin(); it != outEdges.end(); ++it) { - if (++fanOut == 3) break; - } - UDEBUGONLY(UASSERT_OBJ(fanOut <= 3, mvtxp, "Should have stopped counting fanOut");); - // If fan-out no more than one, bypass - if (fanOut <= 1) return true; - // They can only be (2, 2), (2, 3), (3, 2), (3, 3) at this point, bypass if (2, 2) - return fanIn + fanOut == 4; - } - - uint64_t setupMTaskDeps() VL_MT_DISABLED { - uint64_t totalGraphCost = 0; - - // Artificial single entry point vertex in the MTask graph to allow sibling merges. - // This is required as otherwise disjoint sub-graphs could not be merged, but the - // coarsening algorithm assumes that the graph is connected. - m_entryMTaskp = new LogicMTask{m_mTaskGraphp.get(), nullptr}; - - // The V3InstrCount within LogicMTask will set user1 on each AST - // node, to assert that we never count any node twice. - const VNUser1InUse user1inUse; - - // Create the LogicMTasks for each OrderMoveVertex - for (V3GraphVertex& vtx : m_moveGraph.vertices()) { - OrderMoveVertex& mVtx = static_cast(vtx); - if (bypassOk(&mVtx)) { - mVtx.userp(nullptr); // Set to nullptr to mark as bypassed - } else { - LogicMTask* const mtaskp = new LogicMTask{m_mTaskGraphp.get(), &mVtx}; - mVtx.userp(mtaskp); - totalGraphCost += mtaskp->cost(); - } - } - - // Artificial single exit point vertex in the MTask graph to allow sibling merges. - // this enables merging MTasks with no downstream dependents if that is the ideal merge. - m_exitMTaskp = new LogicMTask{m_mTaskGraphp.get(), nullptr}; - - // Create the mtask->mtask dependency edges based on the dependencies between - // OrderMoveVertex vertices. - for (V3GraphVertex& vtx : m_mTaskGraphp->vertices()) { - LogicMTask& mtask = static_cast(vtx); - - // Entry and exit vertices handled separately - if (VL_UNLIKELY((&mtask == m_entryMTaskp) || (&mtask == m_exitMTaskp))) continue; - - OrderMoveVertex::List& vertexList = mtask.vertexList(); - // At this point, there should only be one OrderMoveVertex per LogicMTask - UASSERT_OBJ(vertexList.hasSingleElement(), &mtask, "Multiple OrderMoveVertex"); - OrderMoveVertex* const mVtxp = vertexList.frontp(); - UASSERT_OBJ(mVtxp->userp(), &mtask, "Bypassed OrderMoveVertex should not have MTask"); - - // Function to add a edge to a dependent from 'mtaskp' - const auto addEdge = [this, &mtask](LogicMTask* otherp) { - UASSERT_OBJ(otherp != &mtask, &mtask, "Would create a cycle edge"); - if (mtask.hasRelativeMTask(otherp)) return; // Don't create redundant edges. - new MTaskEdge{m_mTaskGraphp.get(), &mtask, otherp, 1}; - }; - - // Iterate downstream direct dependents - for (const V3GraphEdge& dEdge : mVtxp->outEdges()) { - V3GraphVertex* const top = dEdge.top(); - if (LogicMTask* const otherp = static_cast(top->userp())) { - // The opposite end of the edge is not a bypassed vertex, add as direct - // dependent - addEdge(otherp); - } else { - // The opposite end of the edge is a bypassed vertex, add transitive dependents - for (const V3GraphEdge& tEdge : top->outEdges()) { - LogicMTask* const transp = static_cast(tEdge.top()->userp()); - // The Move graph is bipartite (logic <-> var), and logic is never - // bypassed, hence 'transp' must be non-nullptr. - UASSERT_OBJ(transp, mVtxp, "This cannot be a bypassed vertex"); - addEdge(transp); - } - } - } - } - - // Create Dependencies to/from the entry/exit vertices. - for (V3GraphVertex& vtx : m_mTaskGraphp->vertices()) { - LogicMTask& mtask = static_cast(vtx); - - if (VL_UNLIKELY((&mtask == m_entryMTaskp) || (&mtask == m_exitMTaskp))) continue; - - // Add the entry/exit edges - if (mtask.inEmpty()) new MTaskEdge{m_mTaskGraphp.get(), m_entryMTaskp, &mtask, 1}; - if (mtask.outEmpty()) new MTaskEdge{m_mTaskGraphp.get(), &mtask, m_exitMTaskp, 1}; - } - - return totalGraphCost; - } - - // CONSTRUCTORS - Partitioner(const OrderGraph& orderGraph, OrderMoveGraph& moveGraph) - : m_moveGraph{moveGraph} { - // Fill in the m_mTaskGraphp with LogicMTask's and their interdependencies. - - // Called by V3Order - hashGraphDebug(m_moveGraph, "v3partition initial fine-grained deps"); - - // Create the first MTasks. Initially, each MTask just wraps one - // OrderMoveVertex. Over time, we'll merge MTasks together and - // eventually each MTask will wrap a large number of MTaskMoveVertices - // (and the logic nodes therein.) - const uint64_t totalGraphCost = setupMTaskDeps(); - - debugMTaskGraphStats(*m_mTaskGraphp, "initial"); - - // For debug: print out the longest critical path. This allows us to - // verify that the costs look reasonable, that we aren't combining - // nodes that should probably be split, etc. - if (dumpLevel() >= 3) LogicMTask::dumpCpFilePrefixed(*m_mTaskGraphp, "cp"); - - // Merge nodes that could present data hazards; see comment within. - FixDataHazards::apply(orderGraph, *m_mTaskGraphp); - debugMTaskGraphStats(*m_mTaskGraphp, "hazards"); - hashGraphDebug(*m_mTaskGraphp, "mTaskGraphpp after fixDataHazards()"); - - // Setup the critical path into and out of each node. - partInitCriticalPaths(*m_mTaskGraphp); - hashGraphDebug(*m_mTaskGraphp, "after partInitCriticalPaths()"); - - // Order the graph. We know it's already ranked from fixDataHazards() - // so we don't need to rank it again. - // - // On at least some models, ordering the graph here seems to help - // performance. (Why? Is it just triggering noise in a lucky direction? - // Is it just as likely to harm results?) - // - // More diversity of models that can build with --threads will - // eventually tell us. For now keep the order() so we don't forget - // about it, in case it actually helps. TODO: get more data and maybe - // remove this later if it doesn't really help. - m_mTaskGraphp->orderPreRanked(); - - // Merge MTask nodes together, repeatedly, until the CP budget is - // reached. Coarsens the graph, usually by several orders of - // magnitude. - // - // Some tests disable this, hence the test on threadsCoarsen(). - // Coarsening is always enabled in production. - if (v3Global.opt.threadsCoarsen()) { - const int targetParFactor = v3Global.opt.threads(); - UASSERT(targetParFactor >= 2, "Should not reach Partitioner when --threads <= 1"); - - // Set cpLimit to roughly totalGraphCost / nThreads - // - // Actually set it a bit lower, by a hardcoded fudge factor. This - // results in more smaller mTaskGraphp, which helps reduce fragmentation - // when scheduling them. - const unsigned fudgeNumerator = 3; - const unsigned fudgeDenominator = 5; - const uint64_t cpLimit - = ((totalGraphCost * fudgeNumerator) / (targetParFactor * fudgeDenominator)); - UINFO(4, "Partitioner set cpLimit = " << cpLimit); - - Contraction::apply(*m_mTaskGraphp, cpLimit, m_entryMTaskp, m_exitMTaskp, - // --debugPartition is used by tests - // to enable slow assertions. - v3Global.opt.debugPartition()); - debugMTaskGraphStats(*m_mTaskGraphp, "contraction"); - } - - m_mTaskGraphp->removeTransitiveEdges(); - debugMTaskGraphStats(*m_mTaskGraphp, "transitive1"); - - // Remove MTasks that have no logic in it rerouting the edges. Set user to indicate the - // mtask on every underlying OrderMoveVertex. Clear vertex lists (used later). - m_moveGraph.userClearVertices(); - for (V3GraphVertex* const vtxp : m_mTaskGraphp->vertices().unlinkable()) { - LogicMTask* const mtaskp = vtxp->as(); - OrderMoveVertex::List& vertexList = mtaskp->vertexList(); - // Check if MTask is empty - bool empty = true; - for (const OrderMoveVertex& mVtx : vertexList) { - if (mVtx.logicp()) { - empty = false; - break; - } - } - // If empty remove it now - if (empty) { - mtaskp->rerouteEdges(m_mTaskGraphp.get()); - VL_DO_DANGLING(mtaskp->unlinkDelete(m_mTaskGraphp.get()), mtaskp); - continue; - } - // Annotate the underlying OrderMoveVertex vertices and unlink them - while (OrderMoveVertex* const mVtxp = vertexList.unlinkFront()) mVtxp->userp(mtaskp); - } - m_mTaskGraphp->removeRedundantEdgesSum(&V3GraphEdge::followAlwaysTrue); - } - ~Partitioner() = default; - VL_UNCOPYABLE(Partitioner); - VL_UNMOVABLE(Partitioner); - -public: - static std::unique_ptr apply(const OrderGraph& orderGraph, - OrderMoveGraph& moveGraph) { - return std::move(Partitioner{orderGraph, moveGraph}.m_mTaskGraphp); - } -}; - -// Sort LogicMTask vertices by their serial IDs. -struct MTaskVxIdLessThan final { - bool operator()(const V3GraphVertex* lhsp, const V3GraphVertex* rhsp) const { - return lhsp->as()->id() < rhsp->as()->id(); - } -}; - -AstNodeStmt* V3Order::createParallel(const OrderGraph& orderGraph, OrderMoveGraph& moveGraph, - const std::string& tag, bool slow) { +AstNodeStmt* V3Order::createParallel(OrderMoveGraph& moveGraph, const std::string& tag, + bool slow) { UINFO(2, " Constructing parallel code for '" + tag + "'"); - // For nondeterminism debug: - hashGraphDebug(orderGraph, "V3OrderParallel's input OrderGraph"); + // For nondeterminism debugging + moveGraph.hashGraphDebug("V3Order::createParallel input OrderMoveGraph"); + moveGraph.orderGraph().hashGraphDebug("V3Order::createParallel input OrderGraph"); // Partition moveGraph into LogicMTask's. The partitioner will set userp() on each logic // vertex in the moveGraph to the MTask it belongs to. - const std::unique_ptr mTaskGraphp = Partitioner::apply(orderGraph, moveGraph); + const std::unique_ptr mTaskGraphp = partition(moveGraph); if (dumpGraphLevel() >= 9) moveGraph.dumpDotFilePrefixed(tag + "_ordermv_mtasks"); // Some variable OrderMoveVertices are not assigned to an MTask. Reroute and delete these. @@ -2441,6 +192,12 @@ AstNodeStmt* V3Order::createParallel(const OrderGraph& orderGraph, OrderMoveGrap std::unordered_map logicMTaskToExecMTask; OrderMoveGraphSerializer serializer{moveGraph}; V3OrderCFuncEmitter emitter{tag, slow}; + // Sort LogicMTask vertices by their serial IDs. + struct MTaskVxIdLessThan final { + bool operator()(const V3GraphVertex* lhsp, const V3GraphVertex* rhsp) const { + return lhsp->as()->id() < rhsp->as()->id(); + } + }; GraphStream mtaskStream{mTaskGraphp.get()}; while (const V3GraphVertex* const vtxp = mtaskStream.nextp()) { const LogicMTask* const cMTaskp = vtxp->as(); @@ -2472,7 +229,7 @@ AstNodeStmt* V3Order::createParallel(const OrderGraph& orderGraph, OrderMoveGrap // Create the ExecMTask ExecMTask* const execMTaskp = new ExecMTask{execGraphp, scopep, emitter.getStmts()}; if (!v3Global.opt.hierBlocks().empty()) { - execMTaskp->threads(DpiThreadsVisitor{execMTaskp->funcp()}.threads()); + execMTaskp->threads(DpiThreadsVisitor::apply(execMTaskp)); } const bool newEntry = logicMTaskToExecMTask.emplace(mTaskp, execMTaskp).second; UASSERT_OBJ(newEntry, mTaskp, "LogicMTasks should be processed in dependencyorder"); @@ -2503,10 +260,3 @@ AstNodeStmt* V3Order::createParallel(const OrderGraph& orderGraph, OrderMoveGrap return execGraphp; } - -void V3Order::selfTestParallel() { - UINFO(2, __FUNCTION__ << ":"); - PropagateCp::selfTest(); - PropagateCp::selfTest(); - Contraction::selfTest(); -} diff --git a/src/V3Scoreboard.cpp b/src/V3Scoreboard.cpp deleted file mode 100644 index a4ee5ae6c..000000000 --- a/src/V3Scoreboard.cpp +++ /dev/null @@ -1,96 +0,0 @@ -// -*- mode: C++; c-file-style: "cc-mode" -*- -//************************************************************************* -// DESCRIPTION: Verilator: Threading's element scoreboarding -// -// Code available from: https://verilator.org -// -//************************************************************************* -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of either the GNU Lesser General Public License Version 3 -// or the Perl Artistic License Version 2.0. -// SPDX-FileCopyrightText: 2003-2026 Wilson Snyder -// SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0 -// -//************************************************************************* - -#include "V3PchAstNoMT.h" // VL_MT_DISABLED_CODE_UNIT - -#include "V3Scoreboard.h" - -class ScoreboardTestElem; - -struct Key final { - // Node: Structure layout chosen to minimize padding in PairingHeao<*>::Node - uint64_t m_id; // Unique ID part of edge score - uint32_t m_score; // Score part of ID - bool operator<(const Key& other) const { - // First by Score then by ID, but notice that we want minimums using a max-heap, so reverse - return m_score > other.m_score || (m_score == other.m_score && m_id > other.m_id); - } -}; - -using Scoreboard = V3Scoreboard; - -class ScoreboardTestElem final : public Scoreboard::Node { -public: - uint32_t m_newScore; - // CONSTRUCTORS - explicit ScoreboardTestElem(uint32_t score) - : m_newScore{score} { - m_key.m_score = m_newScore; - static uint32_t s_serial = 0; - m_key.m_id = ++s_serial; - } - ScoreboardTestElem() = delete; - - uint64_t id() const { return m_key.m_id; } - void rescore() { m_key.m_score = m_newScore; } - uint32_t score() const { return m_key.m_score; } - static ScoreboardTestElem* heapNodeToElem(Scoreboard::Node* nodep) { - return static_cast(nodep); - } -}; - -void V3ScoreboardBase::selfTest() { - Scoreboard sb; - - UASSERT(!sb.needsRescore(), "SelfTest: Empty sb should not need rescore."); - - ScoreboardTestElem e1{10}; - ScoreboardTestElem e2{20}; - ScoreboardTestElem e3{30}; - - sb.add(&e1); - sb.add(&e2); - sb.add(&e3); - - UASSERT(sb.needsRescore(), "SelfTest: Newly filled sb should need a rescore."); - UASSERT(sb.needsRescore(&e1), "SelfTest: Individual newly-added element should need rescore"); - UASSERT(nullptr == sb.best(), - "SelfTest: Newly filled sb should have nothing eligible for Bestp()"); - - sb.rescore(); - - UASSERT(!sb.needsRescore(), "SelfTest: Newly rescored sb should not need rescore"); - UASSERT(!sb.needsRescore(&e1), - "SelfTest: Newly rescored sb should not need an element rescored"); - UASSERT(&e1 == sb.best(), "SelfTest: Should return element with lowest (best) score"); - - // Change one element's score - sb.hintScoreChanged(&e2); - e2.m_newScore = 21; - UASSERT(sb.needsRescore(&e2), "SelfTest: Should need rescore on elem after hintScoreChanged"); - - // Remove an element - UASSERT(sb.contains(&e1), "SelfTest: e1 should be there"); - sb.remove(&e1); - UASSERT(!sb.contains(&e1), "SelfTest: e1 should be gone"); - UASSERT(sb.contains(&e2), "SelfTest: e2 should be there, despite needing rescore"); - - // Now e3 should be our best-scoring element, even though - // e2 has a better score, since e2 is pending rescore. - UASSERT(&e3 == sb.best(), "SelfTest: Expect e3 as best element with known score."); - sb.rescore(); - UASSERT(&e2 == sb.best(), "SelfTest: Expect e2 as best element again after Rescore"); -} diff --git a/src/V3Scoreboard.h b/src/V3Scoreboard.h deleted file mode 100644 index c9accfe54..000000000 --- a/src/V3Scoreboard.h +++ /dev/null @@ -1,145 +0,0 @@ -// -*- mode: C++; c-file-style: "cc-mode" -*- -//************************************************************************* -// DESCRIPTION: Verilator: Scoreboard for mtask coarsening -// -// Code available from: https://verilator.org -// -//************************************************************************* -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of either the GNU Lesser General Public License Version 3 -// or the Perl Artistic License Version 2.0. -// SPDX-FileCopyrightText: 2003-2026 Wilson Snyder -// SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0 -// -//************************************************************************* - -#ifndef VERILATOR_V3SCOREBOARD_H_ -#define VERILATOR_V3SCOREBOARD_H_ - -#include "config_build.h" -#include "verilatedos.h" - -#include "V3Error.h" -#include "V3PairingHeap.h" - -//=============================================================================================== -// V3Scoreboard is essentially a heap that can be hinted that some elements have changed keys, at -// which points those elements will be deferred as 'unknown' until the next 'rescore' call. We -// largely reuse the implementation of the slightly more generic PairingHeap, but we do rely on the -// internal structure of the PairingHeap so changing that class requires changing this. -// -// For efficiency, the elements themselves must be the heap nodes, by deriving them from -// V3Scoreboard::Node. This also means a single element can only be associated with -// a single scoreboard. - -template -class V3Scoreboard final { - // TYPES - using Heap = PairingHeap; - -public: - using Node = typename Heap::Node; - -private: - using Link = typename Heap::Link; - - // Note: T_Elem is incomplete here, so we cannot assert 'std::is_base_of::value' - - // MEMBERS - Heap m_known; // The heap of entries with known scores - Link m_unknown; // List of entries with unknown scores - -public: - // CONSTRUCTORS - explicit V3Scoreboard() = default; - ~V3Scoreboard() = default; - -private: - VL_UNCOPYABLE(V3Scoreboard); - - // METHODSs - void addUnknown(T_Elem* nodep) { - // Just prepend it to the list of unknown entries - nodep->m_next.link(m_unknown.unlink()); - m_unknown.linkNonNull(nodep); - // We mark nodes on the unknown list by making their child pointer point to themselves - nodep->m_kids.m_ptr = nodep; - } - -public: - // Returns true if the element is present in the scoreboard, false otherwise. Every other - // method that takes a T_Elem* (except for 'add') has undefined behavior if the element is not - // in this scoreboard. Furthermore, this method is only valid if the element can only possibly - // be in this scoreboard. That is: if the element might be in another scoreboard, the behaviour - // of this method is undefined. - static bool contains(const T_Elem* nodep) { return nodep->m_ownerpp; } - - // Add an element to the scoreboard. This will not be returned before the next 'rescore' call. - void add(T_Elem* nodep) { -#if VL_DEBUG - UASSERT(!contains(nodep), "Adding element to scoreboard that was already in a scoreboard"); -#endif - addUnknown(nodep); - } - - // Remove element from scoreboard. - void remove(T_Elem* nodep) { - if (nodep->m_kids.m_ptr == nodep) { - // Node is on the unknown list, replace with next - nodep->replaceWith(nodep->m_next.unlink()); - return; - } - // Node is in the known heap, remove it - m_known.remove(nodep); - } - - // Get the known element with the highest score (as we are using a max-heap), or nullptr if - // there are no elements with known entries. This does not automatically 'rescore'. The client - // must call 'rescore' appropriately to ensure all elements in the scoreboard are reflected in - // the result of this method. - T_Elem* best() const { return T_Elem::heapNodeToElem(m_known.max()); } - - // Tell the scoreboard that this element's score may have changed. At the time of this call, - // the element's score becomes 'unknown' to the scoreboard. Unknown elements will not be - // returned by 'best until the next call to 'rescore'. - void hintScoreChanged(T_Elem* nodep) { - // If it's already in the unknown list, then nothing to do - if (nodep->m_kids.m_ptr == nodep) return; - // Otherwise it was in the heap, remove it - m_known.remove(nodep); - // Prepend it to the unknown list - addUnknown(nodep); - } - - // True if we have elements with unknown score - bool needsRescore() const { return m_unknown; } - - // True if the element's score is unknown, false otherwise. - static bool needsRescore(const T_Elem* nodep) { return nodep->m_kids.m_ptr == nodep; } - - // For each element whose score is unknown, recompute the score and add to the known heap - void rescore() { - // Rescore and insert all unknown elements - for (Node *nodep = m_unknown.unlink(), *nextp; nodep; nodep = nextp) { - // Pick up next - nextp = nodep->m_next.ptr(); - // Reset pointers - nodep->m_next.m_ptr = nullptr; - nodep->m_kids.m_ptr = nullptr; - nodep->m_ownerpp = nullptr; - // Re-compute the score of the element - T_Elem::heapNodeToElem(nodep)->rescore(); - // re-insert into the heap - m_known.insert(nodep); - } - } -}; - -// ###################################################################### - -namespace V3ScoreboardBase { -void selfTest() VL_MT_DISABLED; -} // namespace V3ScoreboardBase - -#endif // Guard diff --git a/src/Verilator.cpp b/src/Verilator.cpp index b694691dd..58af01afa 100644 --- a/src/Verilator.cpp +++ b/src/Verilator.cpp @@ -96,7 +96,6 @@ #include "V3Sampled.h" #include "V3Sched.h" #include "V3Scope.h" -#include "V3Scoreboard.h" #include "V3Slice.h" #include "V3Split.h" #include "V3SplitVar.h" @@ -744,8 +743,6 @@ static bool verilate(const string& argString) { VHashSha256::selfTest(); VSpellCheck::selfTest(); V3Graph::selfTest(); - V3ScoreboardBase::selfTest(); - V3Order::selfTestParallel(); V3ExecGraph::selfTest(); V3PreShell::selfTest(); V3Broken::selfTest(); diff --git a/test_regress/t/t_dotfiles.py b/test_regress/t/t_dotfiles.py index 3145748f6..ee7d60dfc 100755 --- a/test_regress/t/t_dotfiles.py +++ b/test_regress/t/t_dotfiles.py @@ -18,8 +18,7 @@ test.compile(v_flags2=["--dumpi-graph 6"], threads=2) for dotname in [ "linkcells", "task_call", "gate_graph", "gate_final", "acyc_simp", "orderg_pre", - "orderg_acyc", "orderg_order", "orderg_domain", "ordermv_initial", "ordermv_hazards", - "ordermv_contraction", "ordermv_transitive1", "orderg_done", "pack", "schedule" + "orderg_acyc", "orderg_order", "orderg_domain", "orderg_done", "pack", "schedule" ]: # Some files with identical prefix are generated multiple times during # Verilation. Ensure that at least one of each dotname-prefixed file is generated.