Internals: Refactor MT scheduling (#8012)
Prep for fixing test added in #7913. This is a large scale no functional change refactor, however, MT output is perturbed as tied scores will be broken differently due to ordering changes (still deterministic). Split multi-threaded scheduling out of the monolithic V3OrderParallel.cpp, into relatively independent parts, simplify the data structures, and drop redundant or unused code. New translation units: - V3OrderMTaskGraph.h/.cpp: OrderMTaskGraph, the graph of LogicMTask vertices and MTaskEdge edges. LogicMTask and MTaskEdge no longer depend on the coarsening algorithm's merge candidate types; per-algorithm auxiliary data is attached externally via the vertex and edge user pointers. - V3OrderMTaskFixHazards.cpp: data hazard fixup, was FixDataHazards. - V3OrderMTaskContraction.cpp: graph coarsening, was Partitioner together with PropagateCp and the merge candidate types. - V3OrderParallel.cpp: now just the partitioning driver and ExecMTask graph construction. Data structure changes: - Delete V3Scoreboard.h/.cpp. The generic template had a single user, now a file-local MergeCandidateScoreboard in V3OrderMTaskContraction.cpp. - Merge candidates are now MergeCandidate/SiblingMC/EdgeMC, distinguished by a bit in the candidate id rather than by a vtable, and allocated by the scoreboard, which owns their lifetime. This removes the multiple inheritance previously used by MTaskEdge. Move `hashGraphDebug` which prints the hash of a graph's shape for debugging to generic `V3Graph::hashGraphDebug`. Removed (can be added back later): - Unnecesasry self tests that force special data stucture requirements. - Per stage --stats output under --debug. (Final figures still reported.) - Various debug dumps
This commit is contained in:
Geza Lore
GitHub
parent
40323cc02c
commit
781f6d90bf
|
|
@ -149,6 +149,7 @@ set(HEADERS
|
||||||
V3OrderGraph.h
|
V3OrderGraph.h
|
||||||
V3OrderInternal.h
|
V3OrderInternal.h
|
||||||
V3OrderMoveGraph.h
|
V3OrderMoveGraph.h
|
||||||
|
V3OrderMTaskGraph.h
|
||||||
V3Os.h
|
V3Os.h
|
||||||
V3PairingHeap.h
|
V3PairingHeap.h
|
||||||
V3Param.h
|
V3Param.h
|
||||||
|
|
@ -170,7 +171,6 @@ set(HEADERS
|
||||||
V3Sampled.h
|
V3Sampled.h
|
||||||
V3Sched.h
|
V3Sched.h
|
||||||
V3Scope.h
|
V3Scope.h
|
||||||
V3Scoreboard.h
|
|
||||||
V3SenExprBuilder.h
|
V3SenExprBuilder.h
|
||||||
V3SenTree.h
|
V3SenTree.h
|
||||||
V3Simulate.h
|
V3Simulate.h
|
||||||
|
|
@ -319,6 +319,9 @@ set(COMMON_SOURCES
|
||||||
V3Order.cpp
|
V3Order.cpp
|
||||||
V3OrderGraphBuilder.cpp
|
V3OrderGraphBuilder.cpp
|
||||||
V3OrderMoveGraph.cpp
|
V3OrderMoveGraph.cpp
|
||||||
|
V3OrderMTaskContraction.cpp
|
||||||
|
V3OrderMTaskFixHazards.cpp
|
||||||
|
V3OrderMTaskGraph.cpp
|
||||||
V3OrderParallel.cpp
|
V3OrderParallel.cpp
|
||||||
V3OrderProcessDomains.cpp
|
V3OrderProcessDomains.cpp
|
||||||
V3OrderSerial.cpp
|
V3OrderSerial.cpp
|
||||||
|
|
@ -346,7 +349,6 @@ set(COMMON_SOURCES
|
||||||
V3SchedUtil.cpp
|
V3SchedUtil.cpp
|
||||||
V3SchedVirtIface.cpp
|
V3SchedVirtIface.cpp
|
||||||
V3Scope.cpp
|
V3Scope.cpp
|
||||||
V3Scoreboard.cpp
|
|
||||||
V3Slice.cpp
|
V3Slice.cpp
|
||||||
V3Split.cpp
|
V3Split.cpp
|
||||||
V3SplitVar.cpp
|
V3SplitVar.cpp
|
||||||
|
|
|
||||||
|
|
@ -309,6 +309,9 @@ RAW_OBJS_PCH_ASTNOMT = \
|
||||||
V3Order.o \
|
V3Order.o \
|
||||||
V3OrderGraphBuilder.o \
|
V3OrderGraphBuilder.o \
|
||||||
V3OrderMoveGraph.o \
|
V3OrderMoveGraph.o \
|
||||||
|
V3OrderMTaskContraction.o \
|
||||||
|
V3OrderMTaskFixHazards.o \
|
||||||
|
V3OrderMTaskGraph.o \
|
||||||
V3OrderParallel.o \
|
V3OrderParallel.o \
|
||||||
V3OrderProcessDomains.o \
|
V3OrderProcessDomains.o \
|
||||||
V3OrderSerial.o \
|
V3OrderSerial.o \
|
||||||
|
|
@ -330,7 +333,6 @@ RAW_OBJS_PCH_ASTNOMT = \
|
||||||
V3SchedUtil.o \
|
V3SchedUtil.o \
|
||||||
V3SchedVirtIface.o \
|
V3SchedVirtIface.o \
|
||||||
V3Scope.o \
|
V3Scope.o \
|
||||||
V3Scoreboard.o \
|
|
||||||
V3Slice.o \
|
V3Slice.o \
|
||||||
V3Split.o \
|
V3Split.o \
|
||||||
V3SplitVar.o \
|
V3SplitVar.o \
|
||||||
|
|
|
||||||
|
|
@ -879,8 +879,9 @@ void finalizeCosts(V3Graph* execMTaskGraphp) {
|
||||||
execMTaskGraphp->removeTransitiveEdges();
|
execMTaskGraphp->removeTransitiveEdges();
|
||||||
|
|
||||||
// Record summary stats for final m_tasks graph.
|
// Record summary stats for final m_tasks graph.
|
||||||
const auto report = execMTaskGraphp->parallelismReport(
|
const auto report = execMTaskGraphp->parallelismReport([](const V3GraphVertex* vtxp) { //
|
||||||
[](const V3GraphVertex* vtxp) { return vtxp->as<const ExecMTask>()->cost(); });
|
return vtxp->as<const ExecMTask>()->cost();
|
||||||
|
});
|
||||||
V3Stats::addStat("MTask graph, final, critical path cost", report.criticalPathCost());
|
V3Stats::addStat("MTask graph, final, critical path cost", report.criticalPathCost());
|
||||||
V3Stats::addStat("MTask graph, final, total graph cost", report.totalGraphCost());
|
V3Stats::addStat("MTask graph, final, total graph cost", report.totalGraphCost());
|
||||||
V3Stats::addStat("MTask graph, final, mtask count", report.vertexCount());
|
V3Stats::addStat("MTask graph, final, mtask count", report.vertexCount());
|
||||||
|
|
|
||||||
|
|
@ -388,3 +388,18 @@ void V3Graph::dumpDotFile(const string& filename, bool colorAsSubgraph) const {
|
||||||
|
|
||||||
cout << "dot -Tpdf -o ~/a.pdf " << filename << "\n";
|
cout << "dot -Tpdf -o ~/a.pdf " << filename << "\n";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void V3Graph::hashGraphDebug(const char* debugName) const {
|
||||||
|
// Disabled when there are no nondeterminism issues in flight.
|
||||||
|
if (!v3Global.opt.debugNondeterminism()) return;
|
||||||
|
|
||||||
|
// Assign a unique ID to each vertex for pointer stability, then hash
|
||||||
|
uint32_t id = 1;
|
||||||
|
std::unordered_map<const V3GraphVertex*, uint32_t> vx2Id;
|
||||||
|
for (const V3GraphVertex& vtx : vertices()) vx2Id[&vtx] = ++id;
|
||||||
|
V3Hash hash;
|
||||||
|
for (const V3GraphVertex& vtx : vertices()) {
|
||||||
|
for (const V3GraphEdge& edge : vtx.outEdges()) hash += vx2Id[edge.top()];
|
||||||
|
}
|
||||||
|
UINFO(0, "Hash of shape (not contents) of " << debugName << " = " << cvtToHex(hash.value()));
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -450,6 +450,10 @@ public:
|
||||||
void dumpDotFilePrefixedAlways(const string& nameComment,
|
void dumpDotFilePrefixedAlways(const string& nameComment,
|
||||||
bool colorAsSubgraph = false) const VL_MT_DISABLED;
|
bool colorAsSubgraph = false) const VL_MT_DISABLED;
|
||||||
void dumpEdges(std::ostream& os, const V3GraphVertex& vertex) const VL_MT_DISABLED;
|
void dumpEdges(std::ostream& os, const V3GraphVertex& vertex) const VL_MT_DISABLED;
|
||||||
|
// Print a hash of the shape of graphp. When debugging nondeterminism, this can help
|
||||||
|
// pinpoint where it's coming from.
|
||||||
|
void hashGraphDebug(const char* debugName) const VL_MT_DISABLED;
|
||||||
|
|
||||||
static void selfTest() VL_MT_DISABLED;
|
static void selfTest() VL_MT_DISABLED;
|
||||||
|
|
||||||
class ParallelismReport final {
|
class ParallelismReport final {
|
||||||
|
|
|
||||||
|
|
@ -121,7 +121,7 @@ AstCFunc* V3Order::order(AstNetlist* netlistp, //
|
||||||
AstNodeStmt* stmtsp = nullptr;
|
AstNodeStmt* stmtsp = nullptr;
|
||||||
if (!moveGraphp->empty()) {
|
if (!moveGraphp->empty()) {
|
||||||
if (parallel) {
|
if (parallel) {
|
||||||
stmtsp = createParallel(*graph, *moveGraphp, tag, slow);
|
stmtsp = createParallel(*moveGraphp, tag, slow);
|
||||||
} else {
|
} else {
|
||||||
stmtsp = createSerial(*moveGraphp, tag, slow);
|
stmtsp = createSerial(*moveGraphp, tag, slow);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -51,8 +51,6 @@ AstCFunc* order(AstNetlist* netlistp, //
|
||||||
bool slow, //
|
bool slow, //
|
||||||
const ExternalDomainsProvider& externalDomains) VL_MT_DISABLED;
|
const ExternalDomainsProvider& externalDomains) VL_MT_DISABLED;
|
||||||
|
|
||||||
void selfTestParallel();
|
|
||||||
|
|
||||||
}; // namespace V3Order
|
}; // namespace V3Order
|
||||||
|
|
||||||
#endif // Guard
|
#endif // Guard
|
||||||
|
|
|
||||||
|
|
@ -51,14 +51,9 @@ void processDomains(AstNetlist* netlistp, //
|
||||||
const std::string& tag, //
|
const std::string& tag, //
|
||||||
const ExternalDomainsProvider& externalDomains);
|
const ExternalDomainsProvider& externalDomains);
|
||||||
|
|
||||||
AstNodeStmt* createSerial(OrderMoveGraph& moveGraph, //
|
AstNodeStmt* createSerial(OrderMoveGraph& moveGraph, const std::string& tag, bool slow);
|
||||||
const std::string& tag, //
|
|
||||||
bool slow);
|
|
||||||
|
|
||||||
AstNodeStmt* createParallel(const OrderGraph& orderGraph, //
|
AstNodeStmt* createParallel(OrderMoveGraph& moveGraph, const std::string& tag, bool slow);
|
||||||
OrderMoveGraph& moveGraph, //
|
|
||||||
const std::string& tag, //
|
|
||||||
bool slow);
|
|
||||||
|
|
||||||
}; // namespace V3Order
|
}; // namespace V3Order
|
||||||
|
|
||||||
|
|
|
||||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,390 @@
|
||||||
|
// -*- mode: C++; c-file-style: "cc-mode" -*-
|
||||||
|
//*************************************************************************
|
||||||
|
// DESCRIPTION: Verilator: Multi-threaded MTask graph data hazard fixing
|
||||||
|
//
|
||||||
|
// Code available from: https://verilator.org
|
||||||
|
//
|
||||||
|
//*************************************************************************
|
||||||
|
//
|
||||||
|
// This program is free software; you can redistribute it and/or modify it
|
||||||
|
// under the terms of either the GNU Lesser General Public License Version 3
|
||||||
|
// or the Perl Artistic License Version 2.0.
|
||||||
|
// SPDX-FileCopyrightText: 2003-2026 Wilson Snyder
|
||||||
|
// SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0
|
||||||
|
//
|
||||||
|
//*************************************************************************
|
||||||
|
|
||||||
|
#include "V3PchAstNoMT.h" // VL_MT_DISABLED_CODE_UNIT
|
||||||
|
|
||||||
|
#include "V3Control.h"
|
||||||
|
#include "V3Global.h"
|
||||||
|
#include "V3Graph.h"
|
||||||
|
#include "V3GraphStream.h"
|
||||||
|
#include "V3OrderGraph.h"
|
||||||
|
#include "V3OrderMTaskGraph.h"
|
||||||
|
|
||||||
|
#include <algorithm>
|
||||||
|
#include <map>
|
||||||
|
#include <set>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
VL_DEFINE_DEBUG_FUNCTIONS;
|
||||||
|
|
||||||
|
//######################################################################
|
||||||
|
// DpiImportCallVisitor
|
||||||
|
|
||||||
|
// Scan node, indicate whether it contains a call to a DPI imported routine.
|
||||||
|
class DpiImportCallVisitor final : public VNVisitor {
|
||||||
|
bool m_hasDpiHazard = false; // Found a DPI import call.
|
||||||
|
bool m_tracingCall = false; // Iterating into a CCall to a CFunc
|
||||||
|
// METHODS
|
||||||
|
void visit(AstCFunc* nodep) override {
|
||||||
|
if (!m_tracingCall) return;
|
||||||
|
m_tracingCall = false;
|
||||||
|
if (nodep->dpiImportWrapper()) {
|
||||||
|
if (nodep->dpiPure() ? !v3Global.opt.threadsDpiPure()
|
||||||
|
: !v3Global.opt.threadsDpiUnpure()) {
|
||||||
|
// If hierarchical DPI wrapper cost is not found or is of a 0 cost,
|
||||||
|
// we have a normal DPI which induces DPI hazard by default.
|
||||||
|
m_hasDpiHazard = V3Control::getProfileData(nodep->cname()) == 0;
|
||||||
|
UINFO(9, "DPI wrapper '" << nodep->cname()
|
||||||
|
<< "' has dpi hazard = " << m_hasDpiHazard);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
iterateChildren(nodep);
|
||||||
|
}
|
||||||
|
void visit(AstNodeCCall* nodep) override {
|
||||||
|
iterateChildren(nodep);
|
||||||
|
// Enter the function and trace it
|
||||||
|
m_tracingCall = true;
|
||||||
|
iterate(nodep->funcp());
|
||||||
|
}
|
||||||
|
void visit(AstNode* nodep) override { iterateChildren(nodep); }
|
||||||
|
|
||||||
|
// CONSTRUCTORS
|
||||||
|
explicit DpiImportCallVisitor(AstNode* nodep) { iterate(nodep); }
|
||||||
|
|
||||||
|
public:
|
||||||
|
static bool hasDpiHazard(AstNode* nodep) { return DpiImportCallVisitor{nodep}.m_hasDpiHazard; }
|
||||||
|
};
|
||||||
|
|
||||||
|
//######################################################################
|
||||||
|
// FixDataHazards
|
||||||
|
|
||||||
|
class FixDataHazards final {
|
||||||
|
//
|
||||||
|
// Fix data hazards in the MTask graph.
|
||||||
|
//
|
||||||
|
// The fine-grained graph from V3Order may contain data hazards which are
|
||||||
|
// not a problem for serial mode, but which would be a problem in parallel
|
||||||
|
// mode.
|
||||||
|
//
|
||||||
|
// There are basically two classes: unordered pairs of writes, and
|
||||||
|
// unordered write-read pairs. We fix both here, with a combination of
|
||||||
|
// MTask-merges and new edges to ensure no such unordered pairs remain.
|
||||||
|
//
|
||||||
|
// ABOUT UNORDERED WRITE-WRITE PAIRS
|
||||||
|
//
|
||||||
|
// The V3Order dependency graph treats these as unordered events:
|
||||||
|
//
|
||||||
|
// a) sig[15:8] = stuff;
|
||||||
|
// ...
|
||||||
|
// b) sig[7:0] = other_stuff;
|
||||||
|
//
|
||||||
|
// Seems OK right? They are writes to disjoint bits of the same
|
||||||
|
// signal. They can run in either order, in serial mode, and the result
|
||||||
|
// will be the same.
|
||||||
|
//
|
||||||
|
// The resulting C code for each of this isn't a pure write, it's
|
||||||
|
// actually an R-M-W sequence:
|
||||||
|
//
|
||||||
|
// a) sig = (sig & 0xff) | (0xff00 & (stuff << 8));
|
||||||
|
// ...
|
||||||
|
// b) sig = (sig & 0xff00) | (0xff & other_stuff);
|
||||||
|
//
|
||||||
|
// In serial mode, order doesn't matter so long as these run serially.
|
||||||
|
// In parallel mode, we must serialize these RMW's to avoid a race.
|
||||||
|
//
|
||||||
|
// We don't actually check here if each write would involve an R-M-W, we
|
||||||
|
// just assume that it would. If this routine ever causes a drastic
|
||||||
|
// increase in critical path, it could be optimized to make a better
|
||||||
|
// prediction (with all the risk that word implies!) about whether a
|
||||||
|
// given write is likely to turn into an R-M-W.
|
||||||
|
//
|
||||||
|
// ABOUT UNORDERED WRITE-READ PAIRS
|
||||||
|
//
|
||||||
|
// If we don't put unordered write-read pairs into some order at Verilation
|
||||||
|
// time, we risk a runtime race.
|
||||||
|
//
|
||||||
|
// How do such unordered writer/reader pairs happen? Here's a partial list
|
||||||
|
// of scenarios:
|
||||||
|
//
|
||||||
|
// Case 1: Circular logic
|
||||||
|
//
|
||||||
|
// If the design has circular logic, V3Order has by now generated some
|
||||||
|
// dependency cycles, and also cut some of the edges to make it
|
||||||
|
// acyclic.
|
||||||
|
//
|
||||||
|
// For serial mode, that was fine. We can break logic circles at an
|
||||||
|
// arbitrary point. At runtime, we'll repeat the _eval() until no
|
||||||
|
// changes are detected, which papers over the discarded dependency.
|
||||||
|
//
|
||||||
|
// For parallel mode, this situation can lead to unordered reads and
|
||||||
|
// writes of the same variable, causing a data race. For example if the
|
||||||
|
// original code is this:
|
||||||
|
//
|
||||||
|
// assign b = b | a << 2;
|
||||||
|
// assign out = b;
|
||||||
|
//
|
||||||
|
// ... there's originally a dependency edge which records that 'b'
|
||||||
|
// depends on the first assign. V3Order may cut this edge, making the
|
||||||
|
// statements unordered. In serial mode that's fine, they can run in
|
||||||
|
// either order. In parallel mode it's a reader/writer race.
|
||||||
|
//
|
||||||
|
// Case 2: Race Condition in Verilog Sources
|
||||||
|
//
|
||||||
|
// If the input has races, eg. blocking assignments in always blocks
|
||||||
|
// that share variables, the graph at this point will contain unordered
|
||||||
|
// writes and reads (or unordered write-write pairs) reflecting that.
|
||||||
|
|
||||||
|
// TYPES
|
||||||
|
// Sort LogicMTask objects into deterministic order by calling id()
|
||||||
|
// which is a unique and stable serial number.
|
||||||
|
struct MTaskIdLessThan final {
|
||||||
|
bool operator()(const LogicMTask* lhsp, const LogicMTask* rhsp) const {
|
||||||
|
return *lhsp < *rhsp;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
using TasksByRank = std::map<uint32_t /*rank*/, std::set<LogicMTask*, MTaskIdLessThan>>;
|
||||||
|
|
||||||
|
// MEMBERS
|
||||||
|
OrderMTaskGraph& m_mTaskGraph; // The Mtask graph
|
||||||
|
|
||||||
|
// METHODS
|
||||||
|
|
||||||
|
// Redirect all edges of 'donorp' onto 'recipientp'
|
||||||
|
static void redirectEdgesFrom(LogicMTask* recipientp, LogicMTask* donorp) {
|
||||||
|
// Process outgoing edges
|
||||||
|
while (MTaskEdge* const edgep = static_cast<MTaskEdge*>(donorp->outEdges().frontp())) {
|
||||||
|
LogicMTask* const top = edgep->toMTaskp();
|
||||||
|
top->removeRelativeEdge<GraphWay::REVERSE>(edgep);
|
||||||
|
|
||||||
|
// If an edge already exists between recipient and sink of donor, drop the duplicate.
|
||||||
|
if (recipientp->hasRelativeMTask(top)) {
|
||||||
|
VL_DO_DANGLING(edgep->unlinkDelete(), edgep);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Otherwise redirect the edge from donorp->top to recipientp->top.
|
||||||
|
edgep->relinkFromp(recipientp);
|
||||||
|
recipientp->addRelativeMTask(top);
|
||||||
|
recipientp->stealRelativeEdge<GraphWay::FORWARD>(edgep);
|
||||||
|
top->addRelativeEdge<GraphWay::REVERSE>(edgep);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Process incoming edges
|
||||||
|
while (MTaskEdge* const edgep = static_cast<MTaskEdge*>(donorp->inEdges().frontp())) {
|
||||||
|
LogicMTask* const fromp = edgep->fromMTaskp();
|
||||||
|
fromp->removeRelativeMTask(donorp);
|
||||||
|
fromp->removeRelativeEdge<GraphWay::FORWARD>(edgep);
|
||||||
|
|
||||||
|
// If an edge already exists between recipient and source of donor, drop the duplicate.
|
||||||
|
if (fromp->hasRelativeMTask(recipientp)) {
|
||||||
|
VL_DO_DANGLING(edgep->unlinkDelete(), edgep);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Otherwise redirect the edge from fromp->donorp to fromp->recipientp.
|
||||||
|
edgep->relinkTop(recipientp);
|
||||||
|
fromp->addRelativeMTask(recipientp);
|
||||||
|
fromp->addRelativeEdge<GraphWay::FORWARD>(edgep);
|
||||||
|
recipientp->stealRelativeEdge<GraphWay::REVERSE>(edgep);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void findAdjacentTasks(const OrderVarStdVertex* varVtxp, TasksByRank& tasksByRank) {
|
||||||
|
// Find all writer tasks for this variable, group by rank.
|
||||||
|
for (const V3GraphEdge& edge : varVtxp->inEdges()) {
|
||||||
|
if (const auto* const logicVtxp = edge.fromp()->cast<OrderLogicVertex>()) {
|
||||||
|
LogicMTask* const writerMtaskp = static_cast<LogicMTask*>(logicVtxp->userp());
|
||||||
|
tasksByRank[writerMtaskp->rank()].insert(writerMtaskp);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Note: Find all reader tasks for this variable, group by rank.
|
||||||
|
// There was "broken" code here to find readers, but fixing it to
|
||||||
|
// work properly harmed performance on some tests, see issue #3360.
|
||||||
|
}
|
||||||
|
|
||||||
|
void mergeSameRankTasks(const TasksByRank& tasksByRank) {
|
||||||
|
LogicMTask* lastRecipientp = nullptr;
|
||||||
|
for (const auto& pair : tasksByRank) {
|
||||||
|
// Find the largest node at this rank, merge into it. (If we
|
||||||
|
// happen to find a huge node, this saves time in
|
||||||
|
// redirectEdgesFrom() versus merging into an arbitrary node.)
|
||||||
|
LogicMTask* recipientp = nullptr;
|
||||||
|
for (LogicMTask* const mtaskp : pair.second) {
|
||||||
|
if (!recipientp || (recipientp->cost() < mtaskp->cost())) recipientp = mtaskp;
|
||||||
|
}
|
||||||
|
UASSERT_OBJ(!lastRecipientp || (lastRecipientp->rank() < recipientp->rank()),
|
||||||
|
recipientp, "Merging must be on lower rank");
|
||||||
|
|
||||||
|
for (LogicMTask* const donorp : pair.second) {
|
||||||
|
// Merge donor into recipient.
|
||||||
|
if (donorp == recipientp) continue;
|
||||||
|
// Fix up the map, so donor's OLVs map to recipientp
|
||||||
|
for (const OrderMoveVertex& vtx : donorp->vertexList()) {
|
||||||
|
vtx.logicp()->userp(recipientp);
|
||||||
|
}
|
||||||
|
// Move all vertices from donorp to recipientp
|
||||||
|
recipientp->moveAllVerticesFrom(donorp);
|
||||||
|
// Redirect edges from donorp to recipientp
|
||||||
|
redirectEdgesFrom(recipientp, donorp);
|
||||||
|
// Remove donorp from the graph
|
||||||
|
VL_DO_DANGLING(donorp->unlinkDelete(&m_mTaskGraph), donorp);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (lastRecipientp && !lastRecipientp->hasRelativeMTask(recipientp)) {
|
||||||
|
new MTaskEdge{&m_mTaskGraph, lastRecipientp, recipientp, 1};
|
||||||
|
}
|
||||||
|
lastRecipientp = recipientp;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bool hasDpiHazard(LogicMTask* mtaskp) {
|
||||||
|
for (const OrderMoveVertex& mVtx : mtaskp->vertexList()) {
|
||||||
|
OrderLogicVertex* const lvtxp = mVtx.logicp();
|
||||||
|
if (!lvtxp) continue;
|
||||||
|
// NOTE: We don't handle DPI exports. If testbench code calls a DPI-exported function
|
||||||
|
// at any time during eval() we may have a data hazard. (Likewise in non-threaded mode
|
||||||
|
// if an export messes with an ordered variable we're broken.)
|
||||||
|
|
||||||
|
// Find all calls to DPI-imported functions, we can put those into a serial order at
|
||||||
|
// least. That should solve the most likely DPI-related data hazards.
|
||||||
|
if (DpiImportCallVisitor::hasDpiHazard(lvtxp->nodep())) return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// CONSTRUCTOR
|
||||||
|
FixDataHazards(OrderMTaskGraph& mTaskGraph)
|
||||||
|
: m_mTaskGraph{mTaskGraph} {
|
||||||
|
// Rank the graph. DGS is faster than V3GraphAlg's recursive rank, and also allows us to
|
||||||
|
// set up the OrderLogicVertex -> LogicMTask map at the same time.
|
||||||
|
{
|
||||||
|
GraphStreamUnordered serialize{&m_mTaskGraph};
|
||||||
|
while (LogicMTask* const mtaskp
|
||||||
|
= const_cast<LogicMTask*>(static_cast<const LogicMTask*>(serialize.nextp()))) {
|
||||||
|
// Compute and assign rank
|
||||||
|
uint32_t rank = 0;
|
||||||
|
for (V3GraphEdge& edge : mtaskp->inEdges()) {
|
||||||
|
rank = std::max(edge.fromp()->rank() + 1, rank);
|
||||||
|
}
|
||||||
|
mtaskp->rank(rank);
|
||||||
|
|
||||||
|
// Set up the OrderLogicVertex -> LogicMTask map
|
||||||
|
// Entry and exit MTasks have no MTaskMoveVertices under them, so move on
|
||||||
|
if (mtaskp->vertexList().empty()) continue;
|
||||||
|
// Otherwise there should be only one OrderMoveVertex in each MTask at this stage
|
||||||
|
const OrderMoveVertex::List& vertexList = mtaskp->vertexList();
|
||||||
|
UASSERT_OBJ(vertexList.hasSingleElement(), mtaskp, "Multiple OrderMoveVertex");
|
||||||
|
const OrderMoveVertex* const mVtxp = vertexList.frontp();
|
||||||
|
// Set up mapping back to the MTask from the OrderLogicVertex
|
||||||
|
if (OrderLogicVertex* const lvtxp = mVtxp->logicp()) lvtxp->userp(mtaskp);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Gather all variables. SystemC vars will be handled slightly specially, so keep separate.
|
||||||
|
const OrderGraph& orderGraph = m_mTaskGraph.moveGraph().orderGraph();
|
||||||
|
std::vector<const OrderVarStdVertex*> regularVars;
|
||||||
|
std::vector<const OrderVarStdVertex*> systemCVars;
|
||||||
|
for (const V3GraphVertex& vtx : orderGraph.vertices()) {
|
||||||
|
// Only consider OrderVarStdVertex which reflects
|
||||||
|
// an actual lvalue assignment; the others do not.
|
||||||
|
if (const OrderVarStdVertex* const vvtxp = vtx.cast<const OrderVarStdVertex>()) {
|
||||||
|
if (vvtxp->vscp()->varp()->isSc()) {
|
||||||
|
systemCVars.push_back(vvtxp);
|
||||||
|
} else {
|
||||||
|
regularVars.push_back(vvtxp);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// For each OrderVarVertex, look at its writer and reader MTasks.
|
||||||
|
//
|
||||||
|
// If there's a set of writers and readers at the same rank, we
|
||||||
|
// know these are unordered with respect to one another, so merge
|
||||||
|
// those MTasks all together.
|
||||||
|
//
|
||||||
|
// At this point, we have at most one merged mtask per rank (for a
|
||||||
|
// given OVV.) Create edges across these remaining MTasks to ensure
|
||||||
|
// they run in serial order (going along with the existing ranks.)
|
||||||
|
//
|
||||||
|
// NOTE: we don't update the CP's stored in the LogicMTasks to
|
||||||
|
// reflect the changes we make to the graph. That's OK, as we
|
||||||
|
// haven't yet initialized CPs when we call this routine.
|
||||||
|
for (const OrderVarStdVertex* const varVtxp : regularVars) {
|
||||||
|
// Build a set of MTasks, per rank, which access this var.
|
||||||
|
// Within a rank, sort by MTaskID to avoid nondeterminism.
|
||||||
|
TasksByRank tasksByRank;
|
||||||
|
|
||||||
|
// Find all reader and writer tasks for this variable, add to
|
||||||
|
// tasksByRank.
|
||||||
|
findAdjacentTasks(varVtxp, tasksByRank);
|
||||||
|
|
||||||
|
// Merge all writer and reader tasks from same rank together.
|
||||||
|
//
|
||||||
|
// NOTE: Strictly speaking, we don't need to merge all the
|
||||||
|
// readers together. That may lead to extra serialization. The
|
||||||
|
// least amount of ordering we could impose here would be to
|
||||||
|
// merge all writers at a given rank together; then make edges
|
||||||
|
// from the merged writer node to each reader node at the same
|
||||||
|
// rank; and then from each reader node to the merged writer at
|
||||||
|
// the next rank.
|
||||||
|
//
|
||||||
|
// Whereas, merging all readers and writers at the same rank
|
||||||
|
// together is "the simplest thing that could possibly work"
|
||||||
|
// and it seems to. It also creates fairly few edges. We don't
|
||||||
|
// want to create tons of edges here, doing so is not nice to
|
||||||
|
// the main edge contraction pass.
|
||||||
|
mergeSameRankTasks(tasksByRank);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handle SystemC vars just a little differently. Instead of
|
||||||
|
// treating each var as an independent entity, and serializing
|
||||||
|
// writes to that one var, we treat ALL systemC vars as a single
|
||||||
|
// entity and serialize writes (and, conservatively, reads) across
|
||||||
|
// all of them.
|
||||||
|
//
|
||||||
|
// Reasoning: writing a systemC var actually turns into a call to a
|
||||||
|
// var.write() method, which under the hood is accessing some data
|
||||||
|
// structure that's shared by many SC vars. It's not thread safe.
|
||||||
|
//
|
||||||
|
// Hopefully we only have a few SC vars -- top level ports, probably.
|
||||||
|
{
|
||||||
|
TasksByRank tasksByRank;
|
||||||
|
for (const OrderVarStdVertex* const varVtxp : systemCVars) {
|
||||||
|
findAdjacentTasks(varVtxp, tasksByRank);
|
||||||
|
}
|
||||||
|
mergeSameRankTasks(tasksByRank);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handle nodes containing DPI calls, we want to serialize those
|
||||||
|
// by default unless user gave '--threads-dpi none'.
|
||||||
|
// Same basic strategy as above to serialize access to SC vars.
|
||||||
|
if (!v3Global.opt.threadsDpiPure() || !v3Global.opt.threadsDpiUnpure()) {
|
||||||
|
TasksByRank tasksByRank;
|
||||||
|
for (V3GraphVertex& vtx : m_mTaskGraph.vertices()) {
|
||||||
|
LogicMTask& mtask = static_cast<LogicMTask&>(vtx);
|
||||||
|
if (hasDpiHazard(&mtask)) tasksByRank[mtask.rank()].insert(&mtask);
|
||||||
|
}
|
||||||
|
mergeSameRankTasks(tasksByRank);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public:
|
||||||
|
static void apply(OrderMTaskGraph& mTaskGraph) { FixDataHazards{mTaskGraph}; }
|
||||||
|
};
|
||||||
|
|
||||||
|
void OrderMTaskGraph::fixDataHazards(OrderMTaskGraph& mtaskGraph) {
|
||||||
|
FixDataHazards::apply(mtaskGraph);
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,182 @@
|
||||||
|
// -*- mode: C++; c-file-style: "cc-mode" -*-
|
||||||
|
//*************************************************************************
|
||||||
|
// DESCRIPTION: Verilator: OrderMTask graph construction
|
||||||
|
//
|
||||||
|
// Code available from: https://verilator.org
|
||||||
|
//
|
||||||
|
//*************************************************************************
|
||||||
|
//
|
||||||
|
// This program is free software; you can redistribute it and/or modify it
|
||||||
|
// under the terms of either the GNU Lesser General Public License Version 3
|
||||||
|
// or the Perl Artistic License Version 2.0.
|
||||||
|
// SPDX-FileCopyrightText: 2003-2026 Wilson Snyder
|
||||||
|
// SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0
|
||||||
|
//
|
||||||
|
//*************************************************************************
|
||||||
|
|
||||||
|
#include "V3PchAstNoMT.h" // VL_MT_DISABLED_CODE_UNIT
|
||||||
|
|
||||||
|
#include "V3OrderMTaskGraph.h"
|
||||||
|
|
||||||
|
#include "V3InstrCount.h"
|
||||||
|
|
||||||
|
VL_DEFINE_DEBUG_FUNCTIONS;
|
||||||
|
|
||||||
|
//######################################################################
|
||||||
|
// OrderMTaskGraph
|
||||||
|
|
||||||
|
OrderMTaskGraph::OrderMTaskGraph(OrderMoveGraph& moveGraph)
|
||||||
|
: m_moveGraph{moveGraph}
|
||||||
|
, m_entryp{new LogicMTask{*this, nullptr}}
|
||||||
|
, m_exitp{new LogicMTask{*this, nullptr}} {}
|
||||||
|
|
||||||
|
uint64_t OrderMTaskGraph::totalCost() const {
|
||||||
|
uint64_t cost = 0;
|
||||||
|
for (const V3GraphVertex& vtx : vertices()) cost += static_cast<const LogicMTask&>(vtx).cost();
|
||||||
|
return cost;
|
||||||
|
}
|
||||||
|
|
||||||
|
//######################################################################
|
||||||
|
// LogicMTask
|
||||||
|
|
||||||
|
uint32_t LogicMTask::s_nextId = 1; // Start at 1, so that 0 indicates no mtask.
|
||||||
|
|
||||||
|
LogicMTask::LogicMTask(OrderMTaskGraph& graph, OrderMoveVertex* mVtxp)
|
||||||
|
: V3GraphVertex{&graph} {
|
||||||
|
UASSERT(s_nextId < 0xFFFFFFFFUL, "Too many LogicMTask instances");
|
||||||
|
if (!mVtxp) return;
|
||||||
|
m_mVertices.linkBack(mVtxp);
|
||||||
|
if (const OrderLogicVertex* const olvp = mVtxp->logicp()) {
|
||||||
|
m_cost += V3InstrCount::count(olvp->nodep(), true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
//######################################################################
|
||||||
|
// OrderMTaskGraphBuilder
|
||||||
|
|
||||||
|
class OrderMTaskGraphBuilder final {
|
||||||
|
// NODE STATE
|
||||||
|
// Used by V3InstrCount::count within the LogicMTask constructor only
|
||||||
|
const VNUser1InUse m_user1InUse;
|
||||||
|
|
||||||
|
// MEMBERS
|
||||||
|
OrderMTaskGraph& m_mtaskGraph; // Output OrderMTaskGraph
|
||||||
|
|
||||||
|
// METHODS
|
||||||
|
|
||||||
|
// Predicate function to determine what OrderMoveVertex to bypass when constructing the MTask
|
||||||
|
// graph. The OrderMoveGraph is a bipartite graph of:
|
||||||
|
// - 1. OrderMoveVertex instances containing logic via OrderLogicVertex
|
||||||
|
// (OrderMoveVertex::logicp() != nullptr)
|
||||||
|
// - 2. OrderMoveVertex instances containing an (OrderVarVertex, domain) pair
|
||||||
|
// The goal is to order the logic vertices. The second type of variable/domain vertices only
|
||||||
|
// carry dependencies and are eventually discarded. In order to reduce the working set size,
|
||||||
|
// we 'bypass' and not create LogicMTask vertices for some variable vertices, and instead add
|
||||||
|
// the transitive dependencies directly, but only if adding the transitive edges directly does
|
||||||
|
// not require more dependency edges than keeping the intermediate vertex. That is, we bypass a
|
||||||
|
// variable vertex if fanIn * fanOut <= fanIn + fanOut. This is true if fanIn or fanOut are 1,
|
||||||
|
// or if they are both 2. This can significantly reduce the initial size of OrderMTaskGraph.
|
||||||
|
static bool bypassOk(OrderMoveVertex* mvtxp) {
|
||||||
|
// Need to keep all logic vertices
|
||||||
|
if (mvtxp->logicp()) return false;
|
||||||
|
// Count fan-in, up to 3
|
||||||
|
unsigned fanIn = 0;
|
||||||
|
auto& inEdges = mvtxp->inEdges();
|
||||||
|
for (auto it = inEdges.begin(); it != inEdges.end(); ++it) {
|
||||||
|
if (++fanIn == 3) break;
|
||||||
|
}
|
||||||
|
// If fanIn no more than one, bypass
|
||||||
|
if (fanIn <= 1) return true;
|
||||||
|
// Count fan-out, up to 3
|
||||||
|
unsigned fanOut = 0;
|
||||||
|
auto& outEdges = mvtxp->outEdges();
|
||||||
|
for (auto it = outEdges.begin(); it != outEdges.end(); ++it) {
|
||||||
|
if (++fanOut == 3) break;
|
||||||
|
}
|
||||||
|
// If fan-out no more than one, bypass
|
||||||
|
if (fanOut <= 1) return true;
|
||||||
|
// They can only be (2, 2), (2, 3), (3, 2), (3, 3) at this point, bypass if (2, 2)
|
||||||
|
return fanIn + fanOut == 4;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add an edge to the graph, if there is not already an edge between the two vertices.
|
||||||
|
void addEdge(LogicMTask& src, LogicMTask& dst) {
|
||||||
|
UASSERT_OBJ(&src != &dst, &src, "Should not create self-edges");
|
||||||
|
if (src.hasRelativeMTask(&dst)) return; // Don't create redundant edges.
|
||||||
|
new MTaskEdge{&m_mtaskGraph, &src, &dst, 1};
|
||||||
|
}
|
||||||
|
|
||||||
|
// CONSTRUCTORS
|
||||||
|
explicit OrderMTaskGraphBuilder(OrderMTaskGraph& mtaskGraph)
|
||||||
|
: m_mtaskGraph{mtaskGraph} {
|
||||||
|
|
||||||
|
// Create the LogicMTasks for each OrderMoveVertex
|
||||||
|
for (V3GraphVertex& vtx : mtaskGraph.moveGraph().vertices()) {
|
||||||
|
OrderMoveVertex& mVtx = static_cast<OrderMoveVertex&>(vtx);
|
||||||
|
if (bypassOk(&mVtx)) {
|
||||||
|
mVtx.userp(nullptr); // Set to nullptr to mark as bypassed
|
||||||
|
} else {
|
||||||
|
mVtx.userp(new LogicMTask{mtaskGraph, &mVtx}); // Create vertex and set userp
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
LogicMTask& entry = *mtaskGraph.entryp();
|
||||||
|
LogicMTask& exit = *mtaskGraph.exitp();
|
||||||
|
|
||||||
|
// Create the MTask dependency edges based on the OrderMoveGraph dependencies
|
||||||
|
for (V3GraphVertex& vtx : mtaskGraph.vertices()) {
|
||||||
|
LogicMTask& mtask = static_cast<LogicMTask&>(vtx);
|
||||||
|
|
||||||
|
// Entry and exit vertices handled separately
|
||||||
|
if (VL_UNLIKELY((&mtask == &entry) || (&mtask == &exit))) continue;
|
||||||
|
|
||||||
|
OrderMoveVertex::List& vertexList = mtask.vertexList();
|
||||||
|
// At this point, there should only be one OrderMoveVertex per LogicMTask
|
||||||
|
UASSERT_OBJ(vertexList.hasSingleElement(), &mtask, "Multiple OrderMoveVertex");
|
||||||
|
OrderMoveVertex* const mVtxp = vertexList.frontp();
|
||||||
|
UASSERT_OBJ(mVtxp->userp(), &mtask, "Bypassed OrderMoveVertex should not have MTask");
|
||||||
|
|
||||||
|
// Iterate downstream direct dependents
|
||||||
|
for (const V3GraphEdge& dEdge : mVtxp->outEdges()) {
|
||||||
|
V3GraphVertex* const top = dEdge.top();
|
||||||
|
|
||||||
|
// If the opposite end of the edge is not a bypassed vertex, add direct dependency
|
||||||
|
if (LogicMTask* const otherp = static_cast<LogicMTask*>(top->userp())) {
|
||||||
|
addEdge(mtask, *otherp);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// The opposite end of the edge is a bypassed vertex, add transitive dependencies
|
||||||
|
for (const V3GraphEdge& tEdge : top->outEdges()) {
|
||||||
|
LogicMTask* const transp = static_cast<LogicMTask*>(tEdge.top()->userp());
|
||||||
|
// The Move graph is bipartite (logic <-> var), and logic is never
|
||||||
|
// bypassed, hence 'transp' must be non-nullptr.
|
||||||
|
UASSERT_OBJ(transp, mVtxp, "This cannot be a bypassed vertex");
|
||||||
|
addEdge(mtask, *transp);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create Dependencies to/from the entry/exit vertices, so all vertices are
|
||||||
|
// reachable from the entry point and flow to the exit point.
|
||||||
|
for (V3GraphVertex& vtx : mtaskGraph.vertices()) {
|
||||||
|
LogicMTask& mtask = static_cast<LogicMTask&>(vtx);
|
||||||
|
if (VL_UNLIKELY((&mtask == &entry) || (&mtask == &exit))) continue;
|
||||||
|
// Add the entry/exit edges if not otherwise connected
|
||||||
|
if (mtask.inEmpty()) addEdge(entry, mtask);
|
||||||
|
if (mtask.outEmpty()) addEdge(mtask, exit);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
~OrderMTaskGraphBuilder() = default;
|
||||||
|
VL_UNCOPYABLE(OrderMTaskGraphBuilder);
|
||||||
|
VL_UNMOVABLE(OrderMTaskGraphBuilder);
|
||||||
|
|
||||||
|
public:
|
||||||
|
static void apply(OrderMTaskGraph& mtaskGraph) { OrderMTaskGraphBuilder{mtaskGraph}; }
|
||||||
|
};
|
||||||
|
|
||||||
|
std::unique_ptr<OrderMTaskGraph> OrderMTaskGraph::build(OrderMoveGraph& moveGraph) {
|
||||||
|
std::unique_ptr<OrderMTaskGraph> resp{new OrderMTaskGraph{moveGraph}};
|
||||||
|
OrderMTaskGraphBuilder::apply(*resp);
|
||||||
|
return resp;
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,421 @@
|
||||||
|
// -*- mode: C++; c-file-style: "cc-mode" -*-
|
||||||
|
//*************************************************************************
|
||||||
|
// DESCRIPTION: Verilator: MTask graph for multi-threaded ordering
|
||||||
|
//
|
||||||
|
// Code available from: https://verilator.org
|
||||||
|
//
|
||||||
|
//*************************************************************************
|
||||||
|
//
|
||||||
|
// This program is free software; you can redistribute it and/or modify it
|
||||||
|
// under the terms of either the GNU Lesser General Public License Version 3
|
||||||
|
// or the Perl Artistic License Version 2.0.
|
||||||
|
// SPDX-FileCopyrightText: 2003-2026 Wilson Snyder
|
||||||
|
// SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0
|
||||||
|
//
|
||||||
|
//*************************************************************************
|
||||||
|
//
|
||||||
|
// LogicMTask and MTaskEdge are the vertex and edge of the mtask
|
||||||
|
// graph built and coarsened by the multi-threaded partitioner (see
|
||||||
|
// V3OrderParallel.cpp). They are independent of the partitioner's merge
|
||||||
|
// candidate machinery: any auxiliary data the algorithms need is attached
|
||||||
|
// externally via the vertex/edge user pointers.
|
||||||
|
//
|
||||||
|
//*************************************************************************
|
||||||
|
|
||||||
|
#ifndef VERILATOR_V3ORDERMTASKGRAPH_H_
|
||||||
|
#define VERILATOR_V3ORDERMTASKGRAPH_H_
|
||||||
|
|
||||||
|
#include "config_build.h"
|
||||||
|
#include "verilatedos.h"
|
||||||
|
|
||||||
|
#include "V3Graph.h"
|
||||||
|
#include "V3OrderMoveGraph.h"
|
||||||
|
#include "V3PairingHeap.h"
|
||||||
|
|
||||||
|
#include <array>
|
||||||
|
#include <cmath>
|
||||||
|
#include <memory>
|
||||||
|
#include <sstream>
|
||||||
|
#include <unordered_set>
|
||||||
|
|
||||||
|
class LogicMTask;
|
||||||
|
template <GraphWay::en N_Way>
|
||||||
|
class PropagateCp;
|
||||||
|
|
||||||
|
// When computing critical path costs, use a step function on the actual underlying vertex cost.
|
||||||
|
//
|
||||||
|
// If there are huge vertices, when a tiny vertex merges into a huge vertex, we can often avoid
|
||||||
|
// increasing the huge vertex's stepped cost. If the stepped cost hasn't increased, and the
|
||||||
|
// critical path into the huge vertex hasn't increased, we can avoid propagating a new critical
|
||||||
|
// path to vertices past the huge vertex. Since huge vertices tend to have huge lists of children
|
||||||
|
// and parents, this can be a substantial savings.
|
||||||
|
//
|
||||||
|
// Does not seem to reduce the quality of the partitioner's output.
|
||||||
|
//
|
||||||
|
// If you have huge vertices, leave this 'true', it is the major setting that allows the
|
||||||
|
// partitioner to handle such difficult graphs on anything like a human time scale.
|
||||||
|
//
|
||||||
|
// If you don't have huge vertices, the 'true' value doesn't help much but should cost almost
|
||||||
|
// nothing in terms of partitioner quality.
|
||||||
|
//
|
||||||
|
// If you want the most aggressive possible partition, set it "false" and be prepared to be
|
||||||
|
// disappointed when the improvement in the partition is negligible / in the noise.
|
||||||
|
//
|
||||||
|
// Q) Why retain the control, if there is really no downside?
|
||||||
|
//
|
||||||
|
// A) Cost stepping can lead to corner cases. A developer may wish to disable cost stepping to
|
||||||
|
// rule it out as the cause of unexpected behavior.
|
||||||
|
#define PART_STEPPED_COST true
|
||||||
|
|
||||||
|
//######################################################################
|
||||||
|
// Misc graph and assertion utilities
|
||||||
|
|
||||||
|
inline void partCheckCachedScoreVsActual(uint64_t cached, uint64_t actual) {
|
||||||
|
#if PART_STEPPED_COST
|
||||||
|
// Cached CP might be a little bigger than actual, due to stepped CPs.
|
||||||
|
// Example:
|
||||||
|
// Let's say we have a parent with stepped_cost 40 and a grandparent
|
||||||
|
// with stepped_cost 27. Our forward-cp is 67. Then our parent and
|
||||||
|
// grandparent get merged, the merged node has stepped cost 66. We
|
||||||
|
// won't propagate that new CP to children as it hasn't grown. So,
|
||||||
|
// children may continue to think that the CP coming through this path
|
||||||
|
// is a little higher than it really is; permit that.
|
||||||
|
UASSERT((((cached * 10) <= (actual * 11)) && (cached * 11) >= (actual * 10)),
|
||||||
|
"Calculation error in scoring (approximate, may need tweak)");
|
||||||
|
#else
|
||||||
|
UASSERT(cached == actual, "Calculation error in scoring");
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
//=============================================================================
|
||||||
|
// OrderMTaskGraph
|
||||||
|
|
||||||
|
// The graph of LogicMTask vertices and MTaskEdge edges, used during multi-threaded scheduling.
|
||||||
|
class OrderMTaskGraph final : public V3Graph {
|
||||||
|
OrderMoveGraph& m_moveGraph; // The OrderMoveGraph this graph is built from
|
||||||
|
LogicMTask* const m_entryp; // The singular entry point vertex
|
||||||
|
LogicMTask* const m_exitp; // The singular exit point vertex
|
||||||
|
|
||||||
|
// CONSTRUCTOR
|
||||||
|
explicit OrderMTaskGraph(OrderMoveGraph& moveGraph); // Used by build(), hence private
|
||||||
|
VL_UNCOPYABLE(OrderMTaskGraph);
|
||||||
|
VL_UNMOVABLE(OrderMTaskGraph);
|
||||||
|
|
||||||
|
public:
|
||||||
|
// ACCESSORS
|
||||||
|
OrderMoveGraph& moveGraph() const { return m_moveGraph; }
|
||||||
|
LogicMTask* entryp() const { return m_entryp; }
|
||||||
|
LogicMTask* exitp() const { return m_exitp; }
|
||||||
|
|
||||||
|
// METHODS
|
||||||
|
uint64_t totalCost() const; // O(V), called once
|
||||||
|
|
||||||
|
// STATIC METHODS
|
||||||
|
// Build an MTask graph from 'moveGraph'
|
||||||
|
static std::unique_ptr<OrderMTaskGraph> build(OrderMoveGraph& moveGraph) VL_MT_DISABLED;
|
||||||
|
// Fix data hazards in the MTask graph
|
||||||
|
static void fixDataHazards(OrderMTaskGraph& mtaskGraph) VL_MT_DISABLED;
|
||||||
|
// Coarsen the MTask graph by merging MTasks until the given critical-path limit is reached
|
||||||
|
static void contract(OrderMTaskGraph& mtaskGraph, uint64_t scoreLimit) VL_MT_DISABLED;
|
||||||
|
};
|
||||||
|
|
||||||
|
//=============================================================================
|
||||||
|
// We keep MTaskEdge graph edges in a PairingHeap, sorted by score and id
|
||||||
|
|
||||||
|
struct EdgeKey final {
|
||||||
|
uint64_t m_score; // Score part of edge key
|
||||||
|
uint64_t m_id; // Unique ID part of edge key
|
||||||
|
void increase(uint64_t score) {
|
||||||
|
UDEBUGONLY(UASSERT(score >= m_score, "Must increase"););
|
||||||
|
m_score = score;
|
||||||
|
}
|
||||||
|
// Sort first by Score then by ID
|
||||||
|
bool operator<(const EdgeKey& other) const {
|
||||||
|
if (m_score != other.m_score) return m_score < other.m_score;
|
||||||
|
return m_id < other.m_id;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
using EdgeHeap = PairingHeap<EdgeKey>;
|
||||||
|
|
||||||
|
//=============================================================================
|
||||||
|
// GraphEdge for the MTask graph
|
||||||
|
|
||||||
|
class MTaskEdge final : public V3GraphEdge {
|
||||||
|
VL_RTTI_IMPL(MTaskEdge, V3GraphEdge)
|
||||||
|
|
||||||
|
friend class LogicMTask;
|
||||||
|
template <GraphWay::en N_Way>
|
||||||
|
friend class PropagateCp;
|
||||||
|
|
||||||
|
// MEMBERS
|
||||||
|
// This edge can be in 2 EdgeHeaps, one forward and one reverse. We allocate the heap nodes
|
||||||
|
// directly within the edge as they are always required and this makes association cheap.
|
||||||
|
std::array<EdgeHeap::Node, GraphWay::NUM_WAYS> m_edgeHeapNode;
|
||||||
|
|
||||||
|
// Note: The edge's contraction merge candidate (if any) is held in the inherited user pointer
|
||||||
|
// (V3GraphEdge::userp), managed entirely by the partitioner; see edgeMC() and
|
||||||
|
// MergeCandidateScoreboard. Kept out of MTaskEdge so it does not depend on the MergeCandidate
|
||||||
|
// hierarchy.
|
||||||
|
|
||||||
|
public:
|
||||||
|
// CONSTRUCTORS
|
||||||
|
inline MTaskEdge(OrderMTaskGraph* graphp, LogicMTask* fromp, LogicMTask* top, int weight);
|
||||||
|
VL_UNCOPYABLE(MTaskEdge);
|
||||||
|
VL_UNMOVABLE(MTaskEdge);
|
||||||
|
|
||||||
|
// METHODS
|
||||||
|
template <GraphWay::en N_Way>
|
||||||
|
inline LogicMTask* furtherMTaskp() const;
|
||||||
|
inline LogicMTask* fromMTaskp() const;
|
||||||
|
inline LogicMTask* toMTaskp() const;
|
||||||
|
|
||||||
|
// Following initial assignment of critical paths, clear this MTaskEdge
|
||||||
|
// out of the edge-map for each node and reinsert at a new location
|
||||||
|
// with updated critical path.
|
||||||
|
inline void resetCriticalPaths();
|
||||||
|
uint64_t cachedCp(GraphWay way) const { return m_edgeHeapNode[way].key().m_score; }
|
||||||
|
// Convert from the address of the m_edgeHeapNode[way] in an MTaskEdge back to the MTaskEdge
|
||||||
|
static const MTaskEdge* toMTaskEdge(GraphWay way, const EdgeHeap::Node* nodep) {
|
||||||
|
const size_t offset = VL_OFFSETOF(MTaskEdge, m_edgeHeapNode[way]);
|
||||||
|
return reinterpret_cast<const MTaskEdge*>(reinterpret_cast<uintptr_t>(nodep) - offset);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
//=============================================================================
|
||||||
|
// LogicMTask
|
||||||
|
|
||||||
|
class LogicMTask final : public V3GraphVertex {
|
||||||
|
VL_RTTI_IMPL(LogicMTask, V3GraphVertex)
|
||||||
|
|
||||||
|
template <GraphWay::en N_Way>
|
||||||
|
friend class PropagateCp;
|
||||||
|
|
||||||
|
// MEMBERS
|
||||||
|
|
||||||
|
// List of OrderMoveVertex's assigned to this mtask. LogicMTask does not own the
|
||||||
|
// OrderMoveVertex objects, we merely keep them in a list here.
|
||||||
|
OrderMoveVertex::List m_mVertices;
|
||||||
|
|
||||||
|
// Cost estimate for this LogicMTask, derived from V3InstrCount, in abstract time units.
|
||||||
|
uint64_t m_cost = 0;
|
||||||
|
|
||||||
|
// Cost of critical paths going FORWARD from graph-start to the start
|
||||||
|
// of this vertex, and also going REVERSE from the end of the graph to
|
||||||
|
// the end of the vertex. Same units as m_cost.
|
||||||
|
std::array<uint64_t, GraphWay::NUM_WAYS> m_critPathCost = {};
|
||||||
|
|
||||||
|
static uint32_t s_nextId; // Next ID number to use
|
||||||
|
const uint32_t m_id = s_nextId++; // Unique LogicMTask ID number for stable comparison
|
||||||
|
|
||||||
|
// Count "generations" which are just operations that scan through the
|
||||||
|
// graph. We'll mark each node with the last generation that scanned
|
||||||
|
// it. We can use this to avoid recursing through the same node twice
|
||||||
|
// while searching for a path.
|
||||||
|
uint64_t m_generation = 0;
|
||||||
|
|
||||||
|
// Store a set of forward relatives so we can quickly check if we have a given child
|
||||||
|
std::unordered_set<LogicMTask*> m_edgeSet;
|
||||||
|
// Store the outgoing and incoming edges in a heap sorted by the critical path length
|
||||||
|
std::array<EdgeHeap, GraphWay::NUM_WAYS> m_edgeHeap;
|
||||||
|
|
||||||
|
// Scratch pointer used only by PropagateCp: this MTask's node in the pending heap, or nullptr
|
||||||
|
// if this MTask is not pending. Type erased, as the heap node type is private to PropagateCp,
|
||||||
|
// and differs between its two instantiations (which never run concurrently).
|
||||||
|
void* m_propagateHeapNodep = nullptr;
|
||||||
|
|
||||||
|
public:
|
||||||
|
// CONSTRUCTORS
|
||||||
|
LogicMTask(OrderMTaskGraph& graph, OrderMoveVertex* mVtxp) VL_MT_DISABLED;
|
||||||
|
VL_UNCOPYABLE(LogicMTask);
|
||||||
|
VL_UNMOVABLE(LogicMTask);
|
||||||
|
|
||||||
|
// ACCESSORS
|
||||||
|
OrderMoveVertex::List& vertexList() { return m_mVertices; }
|
||||||
|
const OrderMoveVertex::List& vertexList() const { return m_mVertices; }
|
||||||
|
uint32_t id() const { return m_id; }
|
||||||
|
uint64_t cost() const VL_MT_SAFE { return m_cost; }
|
||||||
|
static uint64_t stepCost(uint64_t cost) {
|
||||||
|
#if PART_STEPPED_COST
|
||||||
|
// Round cost up to the nearest 5%. Use this when computing all critical paths. The idea is
|
||||||
|
// that critical path changes don't need to propagate when they don't exceed the next step,
|
||||||
|
// saving a lot of recursion.
|
||||||
|
if (cost == 0) return 0;
|
||||||
|
|
||||||
|
double logcost = log(cost);
|
||||||
|
// log(1.05) is about 0.05, so round logcost up to the next 0.05 boundary
|
||||||
|
logcost *= 20.0;
|
||||||
|
logcost = ceil(logcost);
|
||||||
|
logcost = logcost / 20.0;
|
||||||
|
|
||||||
|
const uint64_t sCost = static_cast<uint64_t>(exp(logcost));
|
||||||
|
UDEBUGONLY(UASSERT_STATIC(sCost >= cost, "stepped cost error exceeded"););
|
||||||
|
UDEBUGONLY(UASSERT_STATIC(sCost <= ((cost * 11 / 10)), "stepped cost error exceeded"););
|
||||||
|
return sCost;
|
||||||
|
#else
|
||||||
|
return cost;
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
uint64_t stepCost() const { return stepCost(m_cost); }
|
||||||
|
uint64_t critPathCost(GraphWay way) const { return m_critPathCost[way]; }
|
||||||
|
void setCritPathCost(GraphWay way, uint64_t cost) { m_critPathCost[way] = cost; }
|
||||||
|
|
||||||
|
// METHODS
|
||||||
|
bool operator<(const LogicMTask& rhs) const { return id() < rhs.id(); }
|
||||||
|
|
||||||
|
void moveAllVerticesFrom(LogicMTask* otherp) {
|
||||||
|
m_mVertices.splice(m_mVertices.end(), otherp->vertexList());
|
||||||
|
m_cost += otherp->m_cost;
|
||||||
|
}
|
||||||
|
|
||||||
|
template <GraphWay::en N_Way>
|
||||||
|
void addRelativeEdge(MTaskEdge* edgep) {
|
||||||
|
constexpr GraphWay way{N_Way};
|
||||||
|
constexpr GraphWay inv = way.invert();
|
||||||
|
// Add to the edge heap
|
||||||
|
LogicMTask* const relativep = edgep->furtherMTaskp<N_Way>();
|
||||||
|
// Value is !way cp to this edge
|
||||||
|
const uint64_t cp = relativep->stepCost() + relativep->critPathCost(inv);
|
||||||
|
m_edgeHeap[way].insert(&edgep->m_edgeHeapNode[way], {cp, relativep->id()});
|
||||||
|
}
|
||||||
|
template <GraphWay::en N_Way>
|
||||||
|
void stealRelativeEdge(MTaskEdge* edgep) {
|
||||||
|
constexpr GraphWay way{N_Way};
|
||||||
|
// Make heap node insertable, ruining the heap it is currently in.
|
||||||
|
edgep->m_edgeHeapNode[way].yank();
|
||||||
|
// Add the edge as new
|
||||||
|
addRelativeEdge<N_Way>(edgep);
|
||||||
|
}
|
||||||
|
template <GraphWay::en N_Way>
|
||||||
|
void removeRelativeEdge(MTaskEdge* edgep) {
|
||||||
|
constexpr GraphWay way{N_Way};
|
||||||
|
// Remove from the edge heap
|
||||||
|
m_edgeHeap[way].remove(&edgep->m_edgeHeapNode[way]);
|
||||||
|
}
|
||||||
|
|
||||||
|
void addRelativeMTask(LogicMTask* relativep) {
|
||||||
|
// Add the relative to connecting edge map
|
||||||
|
const bool exits = !m_edgeSet.emplace(relativep).second;
|
||||||
|
UDEBUGONLY(UASSERT(!exits, "Adding existing relative"););
|
||||||
|
}
|
||||||
|
void removeRelativeMTask(LogicMTask* relativep) {
|
||||||
|
const size_t removed = m_edgeSet.erase(relativep);
|
||||||
|
UDEBUGONLY(UASSERT(removed, "Relative should have been in set"););
|
||||||
|
}
|
||||||
|
bool hasRelativeMTask(LogicMTask* relativep) const { return m_edgeSet.count(relativep); }
|
||||||
|
|
||||||
|
template <GraphWay::en N_Way>
|
||||||
|
void checkRelativesCp() const {
|
||||||
|
constexpr GraphWay way{N_Way};
|
||||||
|
for (const V3GraphEdge& edge : edges<N_Way>()) {
|
||||||
|
const LogicMTask* const relativep
|
||||||
|
= static_cast<const LogicMTask*>(edge.furtherp<N_Way>());
|
||||||
|
const uint64_t cachedCp = static_cast<const MTaskEdge&>(edge).cachedCp(way);
|
||||||
|
const uint64_t cp = relativep->critPathCost(way.invert()) + relativep->stepCost();
|
||||||
|
partCheckCachedScoreVsActual(cachedCp, cp);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
template <GraphWay::en N_Way>
|
||||||
|
uint64_t critPathCostWithout(const V3GraphEdge* withoutp) const {
|
||||||
|
const GraphWay way{N_Way};
|
||||||
|
const GraphWay inv = way.invert();
|
||||||
|
// Compute the critical path cost wayward to this node, without considering edge
|
||||||
|
// 'withoutp'. We need to look at two edges at most, the critical path if that is not via
|
||||||
|
// 'withoutp', or the second-worst path, if the critical path is via 'withoutp'.
|
||||||
|
UDEBUGONLY(UASSERT(withoutp->furtherp<N_Way>() == this,
|
||||||
|
"In critPathCostWithout(), edge 'withoutp' must further to 'this'"););
|
||||||
|
const EdgeHeap& edgeHeap = m_edgeHeap[inv];
|
||||||
|
const EdgeHeap::Node* const maxp = edgeHeap.max();
|
||||||
|
if (!maxp) return 0;
|
||||||
|
if (MTaskEdge::toMTaskEdge(inv, maxp) != withoutp) return maxp->key().m_score;
|
||||||
|
const EdgeHeap::Node* const secp = edgeHeap.secondMax();
|
||||||
|
if (!secp) return 0;
|
||||||
|
return secp->key().m_score;
|
||||||
|
}
|
||||||
|
|
||||||
|
private:
|
||||||
|
// This takes LogicMTask instead of generic V3GraphVertex. We will use the critical
|
||||||
|
// paths known to LogicMTask to prune the recursion for speed. Also store 'generation' in
|
||||||
|
// LogicMTask::m_generation so we can prune the search and avoid recursing through the same
|
||||||
|
// node more than once in a single search.
|
||||||
|
static bool pathExistsFromInternal(LogicMTask* fromp, LogicMTask* top,
|
||||||
|
const MTaskEdge* excludedEdgep, uint64_t generation) {
|
||||||
|
|
||||||
|
// If already looked at this node in the current search, since we're back again,
|
||||||
|
// we must not have found a path on the first go.
|
||||||
|
if (fromp->m_generation == generation) return false;
|
||||||
|
|
||||||
|
// Mark visited
|
||||||
|
fromp->m_generation = generation;
|
||||||
|
|
||||||
|
// Base case: we found a path.
|
||||||
|
if (fromp == top) return true;
|
||||||
|
|
||||||
|
// Base case: fromp is too late, cannot possibly be a prereq for top.
|
||||||
|
if (fromp->critPathCost(GraphWay::REVERSE)
|
||||||
|
< (top->critPathCost(GraphWay::REVERSE) + top->stepCost())) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if ((fromp->critPathCost(GraphWay::FORWARD) + fromp->stepCost())
|
||||||
|
> top->critPathCost(GraphWay::FORWARD)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Recursively look for a path
|
||||||
|
for (const V3GraphEdge& follow : fromp->outEdges()) {
|
||||||
|
if (&follow == excludedEdgep) continue;
|
||||||
|
LogicMTask* const nextp = static_cast<LogicMTask*>(follow.top());
|
||||||
|
if (pathExistsFromInternal(nextp, top, nullptr, generation)) return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
public:
|
||||||
|
// True if there's a path from 'fromp' to 'top' excluding 'excludedEdgep', false otherwise.
|
||||||
|
// 'excludedEdgep' may be nullptr in which case no edge is excluded. If 'excludedEdgep' is
|
||||||
|
// non-nullptr it must connect fromp and top.
|
||||||
|
static bool pathExistsFrom(LogicMTask* fromp, LogicMTask* top,
|
||||||
|
const MTaskEdge* excludedEdgep) {
|
||||||
|
static uint64_t s_generation = 0;
|
||||||
|
return pathExistsFromInternal(fromp, top, excludedEdgep, ++s_generation);
|
||||||
|
}
|
||||||
|
|
||||||
|
// For Graphviz dumps only
|
||||||
|
std::string name() const override VL_MT_STABLE {
|
||||||
|
std::ostringstream out;
|
||||||
|
out << "mt" << m_id //
|
||||||
|
<< " | fwdCP " << m_critPathCost[GraphWay::FORWARD] //
|
||||||
|
<< " | revCP " << m_critPathCost[GraphWay::REVERSE] //
|
||||||
|
<< " | cost " << cost();
|
||||||
|
return out.str();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
//=============================================================================
|
||||||
|
// MTaskEdge method definitions (need the full definition of LogicMTask)
|
||||||
|
|
||||||
|
MTaskEdge::MTaskEdge(OrderMTaskGraph* graphp, LogicMTask* fromp, LogicMTask* top, int weight)
|
||||||
|
: V3GraphEdge{graphp, fromp, top, weight} {
|
||||||
|
fromp->addRelativeMTask(top);
|
||||||
|
fromp->addRelativeEdge<GraphWay::FORWARD>(this);
|
||||||
|
top->addRelativeEdge<GraphWay::REVERSE>(this);
|
||||||
|
}
|
||||||
|
|
||||||
|
template <GraphWay::en N_Way>
|
||||||
|
LogicMTask* MTaskEdge::furtherMTaskp() const {
|
||||||
|
return static_cast<LogicMTask*>(this->furtherp<N_Way>());
|
||||||
|
}
|
||||||
|
LogicMTask* MTaskEdge::fromMTaskp() const { return static_cast<LogicMTask*>(fromp()); }
|
||||||
|
LogicMTask* MTaskEdge::toMTaskp() const { return static_cast<LogicMTask*>(top()); }
|
||||||
|
|
||||||
|
void MTaskEdge::resetCriticalPaths() {
|
||||||
|
LogicMTask* const fromp = fromMTaskp();
|
||||||
|
LogicMTask* const top = toMTaskp();
|
||||||
|
fromp->removeRelativeEdge<GraphWay::FORWARD>(this);
|
||||||
|
top->removeRelativeEdge<GraphWay::REVERSE>(this);
|
||||||
|
fromp->addRelativeEdge<GraphWay::FORWARD>(this);
|
||||||
|
top->addRelativeEdge<GraphWay::REVERSE>(this);
|
||||||
|
}
|
||||||
|
|
||||||
|
#endif // Guard
|
||||||
|
|
@ -55,7 +55,8 @@ class OrderMoveGraphBuilder final {
|
||||||
|
|
||||||
// MEMBERS
|
// MEMBERS
|
||||||
OrderGraph& m_orderGraph; // Input OrderGraph
|
OrderGraph& m_orderGraph; // Input OrderGraph
|
||||||
std::unique_ptr<OrderMoveGraph> m_moveGraphp{new OrderMoveGraph}; // Output OrderMoveGraph
|
// Output OrderMoveGraph
|
||||||
|
std::unique_ptr<OrderMoveGraph> m_moveGraphp{new OrderMoveGraph{m_orderGraph}};
|
||||||
// Map from Trigger reference AstSenItem to the original AstSenTree
|
// Map from Trigger reference AstSenItem to the original AstSenTree
|
||||||
const V3Order::TrigToSenMap& m_trigToSen;
|
const V3Order::TrigToSenMap& m_trigToSen;
|
||||||
// Storage for domain -> OrderMoveVertex, maps held in OrderVarVertex::userp()
|
// Storage for domain -> OrderMoveVertex, maps held in OrderVarVertex::userp()
|
||||||
|
|
|
||||||
|
|
@ -82,11 +82,21 @@ public:
|
||||||
// OrderMoveGraph is constructed from the fine-grained OrderGraph.
|
// OrderMoveGraph is constructed from the fine-grained OrderGraph.
|
||||||
// It is a slightly coarsened representation of dependencies used to drive serialization.
|
// It is a slightly coarsened representation of dependencies used to drive serialization.
|
||||||
class OrderMoveGraph final : public V3Graph {
|
class OrderMoveGraph final : public V3Graph {
|
||||||
|
OrderGraph& m_orderGraph; // The OrderGraph this move graph was built from
|
||||||
|
|
||||||
public:
|
public:
|
||||||
|
explicit OrderMoveGraph(OrderGraph& orderGraph)
|
||||||
|
: m_orderGraph{orderGraph} {}
|
||||||
|
|
||||||
|
OrderGraph& orderGraph() const { return m_orderGraph; }
|
||||||
|
|
||||||
// Build an OrderMoveGraph from an OrderGraph
|
// Build an OrderMoveGraph from an OrderGraph
|
||||||
static std::unique_ptr<OrderMoveGraph> build(OrderGraph&, const V3Order::TrigToSenMap&);
|
static std::unique_ptr<OrderMoveGraph> build(OrderGraph&, const V3Order::TrigToSenMap&);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
//======================================================================
|
||||||
|
// OrderMoveDomScope
|
||||||
|
|
||||||
// Information stored for each unique (domain, scope) pair. Mainly a list of ready vertices under
|
// Information stored for each unique (domain, scope) pair. Mainly a list of ready vertices under
|
||||||
// that (domain, scope). OrderMoveDomScope instances are themselves organized into a global ready
|
// that (domain, scope). OrderMoveDomScope instances are themselves organized into a global ready
|
||||||
// list if they have ready vertices.
|
// list if they have ready vertices.
|
||||||
|
|
|
||||||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,96 +0,0 @@
|
||||||
// -*- mode: C++; c-file-style: "cc-mode" -*-
|
|
||||||
//*************************************************************************
|
|
||||||
// DESCRIPTION: Verilator: Threading's element scoreboarding
|
|
||||||
//
|
|
||||||
// Code available from: https://verilator.org
|
|
||||||
//
|
|
||||||
//*************************************************************************
|
|
||||||
//
|
|
||||||
// This program is free software; you can redistribute it and/or modify it
|
|
||||||
// under the terms of either the GNU Lesser General Public License Version 3
|
|
||||||
// or the Perl Artistic License Version 2.0.
|
|
||||||
// SPDX-FileCopyrightText: 2003-2026 Wilson Snyder
|
|
||||||
// SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0
|
|
||||||
//
|
|
||||||
//*************************************************************************
|
|
||||||
|
|
||||||
#include "V3PchAstNoMT.h" // VL_MT_DISABLED_CODE_UNIT
|
|
||||||
|
|
||||||
#include "V3Scoreboard.h"
|
|
||||||
|
|
||||||
class ScoreboardTestElem;
|
|
||||||
|
|
||||||
struct Key final {
|
|
||||||
// Node: Structure layout chosen to minimize padding in PairingHeao<*>::Node
|
|
||||||
uint64_t m_id; // Unique ID part of edge score
|
|
||||||
uint32_t m_score; // Score part of ID
|
|
||||||
bool operator<(const Key& other) const {
|
|
||||||
// First by Score then by ID, but notice that we want minimums using a max-heap, so reverse
|
|
||||||
return m_score > other.m_score || (m_score == other.m_score && m_id > other.m_id);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
using Scoreboard = V3Scoreboard<ScoreboardTestElem, Key>;
|
|
||||||
|
|
||||||
class ScoreboardTestElem final : public Scoreboard::Node {
|
|
||||||
public:
|
|
||||||
uint32_t m_newScore;
|
|
||||||
// CONSTRUCTORS
|
|
||||||
explicit ScoreboardTestElem(uint32_t score)
|
|
||||||
: m_newScore{score} {
|
|
||||||
m_key.m_score = m_newScore;
|
|
||||||
static uint32_t s_serial = 0;
|
|
||||||
m_key.m_id = ++s_serial;
|
|
||||||
}
|
|
||||||
ScoreboardTestElem() = delete;
|
|
||||||
|
|
||||||
uint64_t id() const { return m_key.m_id; }
|
|
||||||
void rescore() { m_key.m_score = m_newScore; }
|
|
||||||
uint32_t score() const { return m_key.m_score; }
|
|
||||||
static ScoreboardTestElem* heapNodeToElem(Scoreboard::Node* nodep) {
|
|
||||||
return static_cast<ScoreboardTestElem*>(nodep);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
void V3ScoreboardBase::selfTest() {
|
|
||||||
Scoreboard sb;
|
|
||||||
|
|
||||||
UASSERT(!sb.needsRescore(), "SelfTest: Empty sb should not need rescore.");
|
|
||||||
|
|
||||||
ScoreboardTestElem e1{10};
|
|
||||||
ScoreboardTestElem e2{20};
|
|
||||||
ScoreboardTestElem e3{30};
|
|
||||||
|
|
||||||
sb.add(&e1);
|
|
||||||
sb.add(&e2);
|
|
||||||
sb.add(&e3);
|
|
||||||
|
|
||||||
UASSERT(sb.needsRescore(), "SelfTest: Newly filled sb should need a rescore.");
|
|
||||||
UASSERT(sb.needsRescore(&e1), "SelfTest: Individual newly-added element should need rescore");
|
|
||||||
UASSERT(nullptr == sb.best(),
|
|
||||||
"SelfTest: Newly filled sb should have nothing eligible for Bestp()");
|
|
||||||
|
|
||||||
sb.rescore();
|
|
||||||
|
|
||||||
UASSERT(!sb.needsRescore(), "SelfTest: Newly rescored sb should not need rescore");
|
|
||||||
UASSERT(!sb.needsRescore(&e1),
|
|
||||||
"SelfTest: Newly rescored sb should not need an element rescored");
|
|
||||||
UASSERT(&e1 == sb.best(), "SelfTest: Should return element with lowest (best) score");
|
|
||||||
|
|
||||||
// Change one element's score
|
|
||||||
sb.hintScoreChanged(&e2);
|
|
||||||
e2.m_newScore = 21;
|
|
||||||
UASSERT(sb.needsRescore(&e2), "SelfTest: Should need rescore on elem after hintScoreChanged");
|
|
||||||
|
|
||||||
// Remove an element
|
|
||||||
UASSERT(sb.contains(&e1), "SelfTest: e1 should be there");
|
|
||||||
sb.remove(&e1);
|
|
||||||
UASSERT(!sb.contains(&e1), "SelfTest: e1 should be gone");
|
|
||||||
UASSERT(sb.contains(&e2), "SelfTest: e2 should be there, despite needing rescore");
|
|
||||||
|
|
||||||
// Now e3 should be our best-scoring element, even though
|
|
||||||
// e2 has a better score, since e2 is pending rescore.
|
|
||||||
UASSERT(&e3 == sb.best(), "SelfTest: Expect e3 as best element with known score.");
|
|
||||||
sb.rescore();
|
|
||||||
UASSERT(&e2 == sb.best(), "SelfTest: Expect e2 as best element again after Rescore");
|
|
||||||
}
|
|
||||||
|
|
@ -1,145 +0,0 @@
|
||||||
// -*- mode: C++; c-file-style: "cc-mode" -*-
|
|
||||||
//*************************************************************************
|
|
||||||
// DESCRIPTION: Verilator: Scoreboard for mtask coarsening
|
|
||||||
//
|
|
||||||
// Code available from: https://verilator.org
|
|
||||||
//
|
|
||||||
//*************************************************************************
|
|
||||||
//
|
|
||||||
// This program is free software; you can redistribute it and/or modify it
|
|
||||||
// under the terms of either the GNU Lesser General Public License Version 3
|
|
||||||
// or the Perl Artistic License Version 2.0.
|
|
||||||
// SPDX-FileCopyrightText: 2003-2026 Wilson Snyder
|
|
||||||
// SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0
|
|
||||||
//
|
|
||||||
//*************************************************************************
|
|
||||||
|
|
||||||
#ifndef VERILATOR_V3SCOREBOARD_H_
|
|
||||||
#define VERILATOR_V3SCOREBOARD_H_
|
|
||||||
|
|
||||||
#include "config_build.h"
|
|
||||||
#include "verilatedos.h"
|
|
||||||
|
|
||||||
#include "V3Error.h"
|
|
||||||
#include "V3PairingHeap.h"
|
|
||||||
|
|
||||||
//===============================================================================================
|
|
||||||
// V3Scoreboard is essentially a heap that can be hinted that some elements have changed keys, at
|
|
||||||
// which points those elements will be deferred as 'unknown' until the next 'rescore' call. We
|
|
||||||
// largely reuse the implementation of the slightly more generic PairingHeap, but we do rely on the
|
|
||||||
// internal structure of the PairingHeap so changing that class requires changing this.
|
|
||||||
//
|
|
||||||
// For efficiency, the elements themselves must be the heap nodes, by deriving them from
|
|
||||||
// V3Scoreboard<T_Elem, T_Key>::Node. This also means a single element can only be associated with
|
|
||||||
// a single scoreboard.
|
|
||||||
|
|
||||||
template <typename T_Elem, typename T_Key>
|
|
||||||
class V3Scoreboard final {
|
|
||||||
// TYPES
|
|
||||||
using Heap = PairingHeap<T_Key>;
|
|
||||||
|
|
||||||
public:
|
|
||||||
using Node = typename Heap::Node;
|
|
||||||
|
|
||||||
private:
|
|
||||||
using Link = typename Heap::Link;
|
|
||||||
|
|
||||||
// Note: T_Elem is incomplete here, so we cannot assert 'std::is_base_of<Node, T_Elem>::value'
|
|
||||||
|
|
||||||
// MEMBERS
|
|
||||||
Heap m_known; // The heap of entries with known scores
|
|
||||||
Link m_unknown; // List of entries with unknown scores
|
|
||||||
|
|
||||||
public:
|
|
||||||
// CONSTRUCTORS
|
|
||||||
explicit V3Scoreboard() = default;
|
|
||||||
~V3Scoreboard() = default;
|
|
||||||
|
|
||||||
private:
|
|
||||||
VL_UNCOPYABLE(V3Scoreboard);
|
|
||||||
|
|
||||||
// METHODSs
|
|
||||||
void addUnknown(T_Elem* nodep) {
|
|
||||||
// Just prepend it to the list of unknown entries
|
|
||||||
nodep->m_next.link(m_unknown.unlink());
|
|
||||||
m_unknown.linkNonNull(nodep);
|
|
||||||
// We mark nodes on the unknown list by making their child pointer point to themselves
|
|
||||||
nodep->m_kids.m_ptr = nodep;
|
|
||||||
}
|
|
||||||
|
|
||||||
public:
|
|
||||||
// Returns true if the element is present in the scoreboard, false otherwise. Every other
|
|
||||||
// method that takes a T_Elem* (except for 'add') has undefined behavior if the element is not
|
|
||||||
// in this scoreboard. Furthermore, this method is only valid if the element can only possibly
|
|
||||||
// be in this scoreboard. That is: if the element might be in another scoreboard, the behaviour
|
|
||||||
// of this method is undefined.
|
|
||||||
static bool contains(const T_Elem* nodep) { return nodep->m_ownerpp; }
|
|
||||||
|
|
||||||
// Add an element to the scoreboard. This will not be returned before the next 'rescore' call.
|
|
||||||
void add(T_Elem* nodep) {
|
|
||||||
#if VL_DEBUG
|
|
||||||
UASSERT(!contains(nodep), "Adding element to scoreboard that was already in a scoreboard");
|
|
||||||
#endif
|
|
||||||
addUnknown(nodep);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Remove element from scoreboard.
|
|
||||||
void remove(T_Elem* nodep) {
|
|
||||||
if (nodep->m_kids.m_ptr == nodep) {
|
|
||||||
// Node is on the unknown list, replace with next
|
|
||||||
nodep->replaceWith(nodep->m_next.unlink());
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
// Node is in the known heap, remove it
|
|
||||||
m_known.remove(nodep);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get the known element with the highest score (as we are using a max-heap), or nullptr if
|
|
||||||
// there are no elements with known entries. This does not automatically 'rescore'. The client
|
|
||||||
// must call 'rescore' appropriately to ensure all elements in the scoreboard are reflected in
|
|
||||||
// the result of this method.
|
|
||||||
T_Elem* best() const { return T_Elem::heapNodeToElem(m_known.max()); }
|
|
||||||
|
|
||||||
// Tell the scoreboard that this element's score may have changed. At the time of this call,
|
|
||||||
// the element's score becomes 'unknown' to the scoreboard. Unknown elements will not be
|
|
||||||
// returned by 'best until the next call to 'rescore'.
|
|
||||||
void hintScoreChanged(T_Elem* nodep) {
|
|
||||||
// If it's already in the unknown list, then nothing to do
|
|
||||||
if (nodep->m_kids.m_ptr == nodep) return;
|
|
||||||
// Otherwise it was in the heap, remove it
|
|
||||||
m_known.remove(nodep);
|
|
||||||
// Prepend it to the unknown list
|
|
||||||
addUnknown(nodep);
|
|
||||||
}
|
|
||||||
|
|
||||||
// True if we have elements with unknown score
|
|
||||||
bool needsRescore() const { return m_unknown; }
|
|
||||||
|
|
||||||
// True if the element's score is unknown, false otherwise.
|
|
||||||
static bool needsRescore(const T_Elem* nodep) { return nodep->m_kids.m_ptr == nodep; }
|
|
||||||
|
|
||||||
// For each element whose score is unknown, recompute the score and add to the known heap
|
|
||||||
void rescore() {
|
|
||||||
// Rescore and insert all unknown elements
|
|
||||||
for (Node *nodep = m_unknown.unlink(), *nextp; nodep; nodep = nextp) {
|
|
||||||
// Pick up next
|
|
||||||
nextp = nodep->m_next.ptr();
|
|
||||||
// Reset pointers
|
|
||||||
nodep->m_next.m_ptr = nullptr;
|
|
||||||
nodep->m_kids.m_ptr = nullptr;
|
|
||||||
nodep->m_ownerpp = nullptr;
|
|
||||||
// Re-compute the score of the element
|
|
||||||
T_Elem::heapNodeToElem(nodep)->rescore();
|
|
||||||
// re-insert into the heap
|
|
||||||
m_known.insert(nodep);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
// ######################################################################
|
|
||||||
|
|
||||||
namespace V3ScoreboardBase {
|
|
||||||
void selfTest() VL_MT_DISABLED;
|
|
||||||
} // namespace V3ScoreboardBase
|
|
||||||
|
|
||||||
#endif // Guard
|
|
||||||
|
|
@ -96,7 +96,6 @@
|
||||||
#include "V3Sampled.h"
|
#include "V3Sampled.h"
|
||||||
#include "V3Sched.h"
|
#include "V3Sched.h"
|
||||||
#include "V3Scope.h"
|
#include "V3Scope.h"
|
||||||
#include "V3Scoreboard.h"
|
|
||||||
#include "V3Slice.h"
|
#include "V3Slice.h"
|
||||||
#include "V3Split.h"
|
#include "V3Split.h"
|
||||||
#include "V3SplitVar.h"
|
#include "V3SplitVar.h"
|
||||||
|
|
@ -744,8 +743,6 @@ static bool verilate(const string& argString) {
|
||||||
VHashSha256::selfTest();
|
VHashSha256::selfTest();
|
||||||
VSpellCheck::selfTest();
|
VSpellCheck::selfTest();
|
||||||
V3Graph::selfTest();
|
V3Graph::selfTest();
|
||||||
V3ScoreboardBase::selfTest();
|
|
||||||
V3Order::selfTestParallel();
|
|
||||||
V3ExecGraph::selfTest();
|
V3ExecGraph::selfTest();
|
||||||
V3PreShell::selfTest();
|
V3PreShell::selfTest();
|
||||||
V3Broken::selfTest();
|
V3Broken::selfTest();
|
||||||
|
|
|
||||||
|
|
@ -18,8 +18,7 @@ test.compile(v_flags2=["--dumpi-graph 6"], threads=2)
|
||||||
|
|
||||||
for dotname in [
|
for dotname in [
|
||||||
"linkcells", "task_call", "gate_graph", "gate_final", "acyc_simp", "orderg_pre",
|
"linkcells", "task_call", "gate_graph", "gate_final", "acyc_simp", "orderg_pre",
|
||||||
"orderg_acyc", "orderg_order", "orderg_domain", "ordermv_initial", "ordermv_hazards",
|
"orderg_acyc", "orderg_order", "orderg_domain", "orderg_done", "pack", "schedule"
|
||||||
"ordermv_contraction", "ordermv_transitive1", "orderg_done", "pack", "schedule"
|
|
||||||
]:
|
]:
|
||||||
# Some files with identical prefix are generated multiple times during
|
# Some files with identical prefix are generated multiple times during
|
||||||
# Verilation. Ensure that at least one of each dotname-prefixed file is generated.
|
# Verilation. Ensure that at least one of each dotname-prefixed file is generated.
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue