mirror of
https://github.com/verilator/verilator.git
synced 2026-09-02 10:48:27 +02:00
Internals: Refactor MT scheduling (#8012)
Prep for fixing test added in #7913. This is a large scale no functional change refactor, however, MT output is perturbed as tied scores will be broken differently due to ordering changes (still deterministic). Split multi-threaded scheduling out of the monolithic V3OrderParallel.cpp, into relatively independent parts, simplify the data structures, and drop redundant or unused code. New translation units: - V3OrderMTaskGraph.h/.cpp: OrderMTaskGraph, the graph of LogicMTask vertices and MTaskEdge edges. LogicMTask and MTaskEdge no longer depend on the coarsening algorithm's merge candidate types; per-algorithm auxiliary data is attached externally via the vertex and edge user pointers. - V3OrderMTaskFixHazards.cpp: data hazard fixup, was FixDataHazards. - V3OrderMTaskContraction.cpp: graph coarsening, was Partitioner together with PropagateCp and the merge candidate types. - V3OrderParallel.cpp: now just the partitioning driver and ExecMTask graph construction. Data structure changes: - Delete V3Scoreboard.h/.cpp. The generic template had a single user, now a file-local MergeCandidateScoreboard in V3OrderMTaskContraction.cpp. - Merge candidates are now MergeCandidate/SiblingMC/EdgeMC, distinguished by a bit in the candidate id rather than by a vtable, and allocated by the scoreboard, which owns their lifetime. This removes the multiple inheritance previously used by MTaskEdge. Move `hashGraphDebug` which prints the hash of a graph's shape for debugging to generic `V3Graph::hashGraphDebug`. Removed (can be added back later): - Unnecesasry self tests that force special data stucture requirements. - Per stage --stats output under --debug. (Final figures still reported.) - Various debug dumps
This commit is contained in:
Geza Lore
@@ -0,0 +1,390 @@
|
||||
// -*- mode: C++; c-file-style: "cc-mode" -*-
|
||||
//*************************************************************************
|
||||
// DESCRIPTION: Verilator: Multi-threaded MTask graph data hazard fixing
|
||||
//
|
||||
// Code available from: https://verilator.org
|
||||
//
|
||||
//*************************************************************************
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify it
|
||||
// under the terms of either the GNU Lesser General Public License Version 3
|
||||
// or the Perl Artistic License Version 2.0.
|
||||
// SPDX-FileCopyrightText: 2003-2026 Wilson Snyder
|
||||
// SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0
|
||||
//
|
||||
//*************************************************************************
|
||||
|
||||
#include "V3PchAstNoMT.h" // VL_MT_DISABLED_CODE_UNIT
|
||||
|
||||
#include "V3Control.h"
|
||||
#include "V3Global.h"
|
||||
#include "V3Graph.h"
|
||||
#include "V3GraphStream.h"
|
||||
#include "V3OrderGraph.h"
|
||||
#include "V3OrderMTaskGraph.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <map>
|
||||
#include <set>
|
||||
#include <vector>
|
||||
|
||||
VL_DEFINE_DEBUG_FUNCTIONS;
|
||||
|
||||
//######################################################################
|
||||
// DpiImportCallVisitor
|
||||
|
||||
// Scan node, indicate whether it contains a call to a DPI imported routine.
|
||||
class DpiImportCallVisitor final : public VNVisitor {
|
||||
bool m_hasDpiHazard = false; // Found a DPI import call.
|
||||
bool m_tracingCall = false; // Iterating into a CCall to a CFunc
|
||||
// METHODS
|
||||
void visit(AstCFunc* nodep) override {
|
||||
if (!m_tracingCall) return;
|
||||
m_tracingCall = false;
|
||||
if (nodep->dpiImportWrapper()) {
|
||||
if (nodep->dpiPure() ? !v3Global.opt.threadsDpiPure()
|
||||
: !v3Global.opt.threadsDpiUnpure()) {
|
||||
// If hierarchical DPI wrapper cost is not found or is of a 0 cost,
|
||||
// we have a normal DPI which induces DPI hazard by default.
|
||||
m_hasDpiHazard = V3Control::getProfileData(nodep->cname()) == 0;
|
||||
UINFO(9, "DPI wrapper '" << nodep->cname()
|
||||
<< "' has dpi hazard = " << m_hasDpiHazard);
|
||||
}
|
||||
}
|
||||
iterateChildren(nodep);
|
||||
}
|
||||
void visit(AstNodeCCall* nodep) override {
|
||||
iterateChildren(nodep);
|
||||
// Enter the function and trace it
|
||||
m_tracingCall = true;
|
||||
iterate(nodep->funcp());
|
||||
}
|
||||
void visit(AstNode* nodep) override { iterateChildren(nodep); }
|
||||
|
||||
// CONSTRUCTORS
|
||||
explicit DpiImportCallVisitor(AstNode* nodep) { iterate(nodep); }
|
||||
|
||||
public:
|
||||
static bool hasDpiHazard(AstNode* nodep) { return DpiImportCallVisitor{nodep}.m_hasDpiHazard; }
|
||||
};
|
||||
|
||||
//######################################################################
|
||||
// FixDataHazards
|
||||
|
||||
class FixDataHazards final {
|
||||
//
|
||||
// Fix data hazards in the MTask graph.
|
||||
//
|
||||
// The fine-grained graph from V3Order may contain data hazards which are
|
||||
// not a problem for serial mode, but which would be a problem in parallel
|
||||
// mode.
|
||||
//
|
||||
// There are basically two classes: unordered pairs of writes, and
|
||||
// unordered write-read pairs. We fix both here, with a combination of
|
||||
// MTask-merges and new edges to ensure no such unordered pairs remain.
|
||||
//
|
||||
// ABOUT UNORDERED WRITE-WRITE PAIRS
|
||||
//
|
||||
// The V3Order dependency graph treats these as unordered events:
|
||||
//
|
||||
// a) sig[15:8] = stuff;
|
||||
// ...
|
||||
// b) sig[7:0] = other_stuff;
|
||||
//
|
||||
// Seems OK right? They are writes to disjoint bits of the same
|
||||
// signal. They can run in either order, in serial mode, and the result
|
||||
// will be the same.
|
||||
//
|
||||
// The resulting C code for each of this isn't a pure write, it's
|
||||
// actually an R-M-W sequence:
|
||||
//
|
||||
// a) sig = (sig & 0xff) | (0xff00 & (stuff << 8));
|
||||
// ...
|
||||
// b) sig = (sig & 0xff00) | (0xff & other_stuff);
|
||||
//
|
||||
// In serial mode, order doesn't matter so long as these run serially.
|
||||
// In parallel mode, we must serialize these RMW's to avoid a race.
|
||||
//
|
||||
// We don't actually check here if each write would involve an R-M-W, we
|
||||
// just assume that it would. If this routine ever causes a drastic
|
||||
// increase in critical path, it could be optimized to make a better
|
||||
// prediction (with all the risk that word implies!) about whether a
|
||||
// given write is likely to turn into an R-M-W.
|
||||
//
|
||||
// ABOUT UNORDERED WRITE-READ PAIRS
|
||||
//
|
||||
// If we don't put unordered write-read pairs into some order at Verilation
|
||||
// time, we risk a runtime race.
|
||||
//
|
||||
// How do such unordered writer/reader pairs happen? Here's a partial list
|
||||
// of scenarios:
|
||||
//
|
||||
// Case 1: Circular logic
|
||||
//
|
||||
// If the design has circular logic, V3Order has by now generated some
|
||||
// dependency cycles, and also cut some of the edges to make it
|
||||
// acyclic.
|
||||
//
|
||||
// For serial mode, that was fine. We can break logic circles at an
|
||||
// arbitrary point. At runtime, we'll repeat the _eval() until no
|
||||
// changes are detected, which papers over the discarded dependency.
|
||||
//
|
||||
// For parallel mode, this situation can lead to unordered reads and
|
||||
// writes of the same variable, causing a data race. For example if the
|
||||
// original code is this:
|
||||
//
|
||||
// assign b = b | a << 2;
|
||||
// assign out = b;
|
||||
//
|
||||
// ... there's originally a dependency edge which records that 'b'
|
||||
// depends on the first assign. V3Order may cut this edge, making the
|
||||
// statements unordered. In serial mode that's fine, they can run in
|
||||
// either order. In parallel mode it's a reader/writer race.
|
||||
//
|
||||
// Case 2: Race Condition in Verilog Sources
|
||||
//
|
||||
// If the input has races, eg. blocking assignments in always blocks
|
||||
// that share variables, the graph at this point will contain unordered
|
||||
// writes and reads (or unordered write-write pairs) reflecting that.
|
||||
|
||||
// TYPES
|
||||
// Sort LogicMTask objects into deterministic order by calling id()
|
||||
// which is a unique and stable serial number.
|
||||
struct MTaskIdLessThan final {
|
||||
bool operator()(const LogicMTask* lhsp, const LogicMTask* rhsp) const {
|
||||
return *lhsp < *rhsp;
|
||||
}
|
||||
};
|
||||
using TasksByRank = std::map<uint32_t /*rank*/, std::set<LogicMTask*, MTaskIdLessThan>>;
|
||||
|
||||
// MEMBERS
|
||||
OrderMTaskGraph& m_mTaskGraph; // The Mtask graph
|
||||
|
||||
// METHODS
|
||||
|
||||
// Redirect all edges of 'donorp' onto 'recipientp'
|
||||
static void redirectEdgesFrom(LogicMTask* recipientp, LogicMTask* donorp) {
|
||||
// Process outgoing edges
|
||||
while (MTaskEdge* const edgep = static_cast<MTaskEdge*>(donorp->outEdges().frontp())) {
|
||||
LogicMTask* const top = edgep->toMTaskp();
|
||||
top->removeRelativeEdge<GraphWay::REVERSE>(edgep);
|
||||
|
||||
// If an edge already exists between recipient and sink of donor, drop the duplicate.
|
||||
if (recipientp->hasRelativeMTask(top)) {
|
||||
VL_DO_DANGLING(edgep->unlinkDelete(), edgep);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Otherwise redirect the edge from donorp->top to recipientp->top.
|
||||
edgep->relinkFromp(recipientp);
|
||||
recipientp->addRelativeMTask(top);
|
||||
recipientp->stealRelativeEdge<GraphWay::FORWARD>(edgep);
|
||||
top->addRelativeEdge<GraphWay::REVERSE>(edgep);
|
||||
}
|
||||
|
||||
// Process incoming edges
|
||||
while (MTaskEdge* const edgep = static_cast<MTaskEdge*>(donorp->inEdges().frontp())) {
|
||||
LogicMTask* const fromp = edgep->fromMTaskp();
|
||||
fromp->removeRelativeMTask(donorp);
|
||||
fromp->removeRelativeEdge<GraphWay::FORWARD>(edgep);
|
||||
|
||||
// If an edge already exists between recipient and source of donor, drop the duplicate.
|
||||
if (fromp->hasRelativeMTask(recipientp)) {
|
||||
VL_DO_DANGLING(edgep->unlinkDelete(), edgep);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Otherwise redirect the edge from fromp->donorp to fromp->recipientp.
|
||||
edgep->relinkTop(recipientp);
|
||||
fromp->addRelativeMTask(recipientp);
|
||||
fromp->addRelativeEdge<GraphWay::FORWARD>(edgep);
|
||||
recipientp->stealRelativeEdge<GraphWay::REVERSE>(edgep);
|
||||
}
|
||||
}
|
||||
|
||||
void findAdjacentTasks(const OrderVarStdVertex* varVtxp, TasksByRank& tasksByRank) {
|
||||
// Find all writer tasks for this variable, group by rank.
|
||||
for (const V3GraphEdge& edge : varVtxp->inEdges()) {
|
||||
if (const auto* const logicVtxp = edge.fromp()->cast<OrderLogicVertex>()) {
|
||||
LogicMTask* const writerMtaskp = static_cast<LogicMTask*>(logicVtxp->userp());
|
||||
tasksByRank[writerMtaskp->rank()].insert(writerMtaskp);
|
||||
}
|
||||
}
|
||||
// Note: Find all reader tasks for this variable, group by rank.
|
||||
// There was "broken" code here to find readers, but fixing it to
|
||||
// work properly harmed performance on some tests, see issue #3360.
|
||||
}
|
||||
|
||||
void mergeSameRankTasks(const TasksByRank& tasksByRank) {
|
||||
LogicMTask* lastRecipientp = nullptr;
|
||||
for (const auto& pair : tasksByRank) {
|
||||
// Find the largest node at this rank, merge into it. (If we
|
||||
// happen to find a huge node, this saves time in
|
||||
// redirectEdgesFrom() versus merging into an arbitrary node.)
|
||||
LogicMTask* recipientp = nullptr;
|
||||
for (LogicMTask* const mtaskp : pair.second) {
|
||||
if (!recipientp || (recipientp->cost() < mtaskp->cost())) recipientp = mtaskp;
|
||||
}
|
||||
UASSERT_OBJ(!lastRecipientp || (lastRecipientp->rank() < recipientp->rank()),
|
||||
recipientp, "Merging must be on lower rank");
|
||||
|
||||
for (LogicMTask* const donorp : pair.second) {
|
||||
// Merge donor into recipient.
|
||||
if (donorp == recipientp) continue;
|
||||
// Fix up the map, so donor's OLVs map to recipientp
|
||||
for (const OrderMoveVertex& vtx : donorp->vertexList()) {
|
||||
vtx.logicp()->userp(recipientp);
|
||||
}
|
||||
// Move all vertices from donorp to recipientp
|
||||
recipientp->moveAllVerticesFrom(donorp);
|
||||
// Redirect edges from donorp to recipientp
|
||||
redirectEdgesFrom(recipientp, donorp);
|
||||
// Remove donorp from the graph
|
||||
VL_DO_DANGLING(donorp->unlinkDelete(&m_mTaskGraph), donorp);
|
||||
}
|
||||
|
||||
if (lastRecipientp && !lastRecipientp->hasRelativeMTask(recipientp)) {
|
||||
new MTaskEdge{&m_mTaskGraph, lastRecipientp, recipientp, 1};
|
||||
}
|
||||
lastRecipientp = recipientp;
|
||||
}
|
||||
}
|
||||
|
||||
bool hasDpiHazard(LogicMTask* mtaskp) {
|
||||
for (const OrderMoveVertex& mVtx : mtaskp->vertexList()) {
|
||||
OrderLogicVertex* const lvtxp = mVtx.logicp();
|
||||
if (!lvtxp) continue;
|
||||
// NOTE: We don't handle DPI exports. If testbench code calls a DPI-exported function
|
||||
// at any time during eval() we may have a data hazard. (Likewise in non-threaded mode
|
||||
// if an export messes with an ordered variable we're broken.)
|
||||
|
||||
// Find all calls to DPI-imported functions, we can put those into a serial order at
|
||||
// least. That should solve the most likely DPI-related data hazards.
|
||||
if (DpiImportCallVisitor::hasDpiHazard(lvtxp->nodep())) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// CONSTRUCTOR
|
||||
FixDataHazards(OrderMTaskGraph& mTaskGraph)
|
||||
: m_mTaskGraph{mTaskGraph} {
|
||||
// Rank the graph. DGS is faster than V3GraphAlg's recursive rank, and also allows us to
|
||||
// set up the OrderLogicVertex -> LogicMTask map at the same time.
|
||||
{
|
||||
GraphStreamUnordered serialize{&m_mTaskGraph};
|
||||
while (LogicMTask* const mtaskp
|
||||
= const_cast<LogicMTask*>(static_cast<const LogicMTask*>(serialize.nextp()))) {
|
||||
// Compute and assign rank
|
||||
uint32_t rank = 0;
|
||||
for (V3GraphEdge& edge : mtaskp->inEdges()) {
|
||||
rank = std::max(edge.fromp()->rank() + 1, rank);
|
||||
}
|
||||
mtaskp->rank(rank);
|
||||
|
||||
// Set up the OrderLogicVertex -> LogicMTask map
|
||||
// Entry and exit MTasks have no MTaskMoveVertices under them, so move on
|
||||
if (mtaskp->vertexList().empty()) continue;
|
||||
// Otherwise there should be only one OrderMoveVertex in each MTask at this stage
|
||||
const OrderMoveVertex::List& vertexList = mtaskp->vertexList();
|
||||
UASSERT_OBJ(vertexList.hasSingleElement(), mtaskp, "Multiple OrderMoveVertex");
|
||||
const OrderMoveVertex* const mVtxp = vertexList.frontp();
|
||||
// Set up mapping back to the MTask from the OrderLogicVertex
|
||||
if (OrderLogicVertex* const lvtxp = mVtxp->logicp()) lvtxp->userp(mtaskp);
|
||||
}
|
||||
}
|
||||
|
||||
// Gather all variables. SystemC vars will be handled slightly specially, so keep separate.
|
||||
const OrderGraph& orderGraph = m_mTaskGraph.moveGraph().orderGraph();
|
||||
std::vector<const OrderVarStdVertex*> regularVars;
|
||||
std::vector<const OrderVarStdVertex*> systemCVars;
|
||||
for (const V3GraphVertex& vtx : orderGraph.vertices()) {
|
||||
// Only consider OrderVarStdVertex which reflects
|
||||
// an actual lvalue assignment; the others do not.
|
||||
if (const OrderVarStdVertex* const vvtxp = vtx.cast<const OrderVarStdVertex>()) {
|
||||
if (vvtxp->vscp()->varp()->isSc()) {
|
||||
systemCVars.push_back(vvtxp);
|
||||
} else {
|
||||
regularVars.push_back(vvtxp);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// For each OrderVarVertex, look at its writer and reader MTasks.
|
||||
//
|
||||
// If there's a set of writers and readers at the same rank, we
|
||||
// know these are unordered with respect to one another, so merge
|
||||
// those MTasks all together.
|
||||
//
|
||||
// At this point, we have at most one merged mtask per rank (for a
|
||||
// given OVV.) Create edges across these remaining MTasks to ensure
|
||||
// they run in serial order (going along with the existing ranks.)
|
||||
//
|
||||
// NOTE: we don't update the CP's stored in the LogicMTasks to
|
||||
// reflect the changes we make to the graph. That's OK, as we
|
||||
// haven't yet initialized CPs when we call this routine.
|
||||
for (const OrderVarStdVertex* const varVtxp : regularVars) {
|
||||
// Build a set of MTasks, per rank, which access this var.
|
||||
// Within a rank, sort by MTaskID to avoid nondeterminism.
|
||||
TasksByRank tasksByRank;
|
||||
|
||||
// Find all reader and writer tasks for this variable, add to
|
||||
// tasksByRank.
|
||||
findAdjacentTasks(varVtxp, tasksByRank);
|
||||
|
||||
// Merge all writer and reader tasks from same rank together.
|
||||
//
|
||||
// NOTE: Strictly speaking, we don't need to merge all the
|
||||
// readers together. That may lead to extra serialization. The
|
||||
// least amount of ordering we could impose here would be to
|
||||
// merge all writers at a given rank together; then make edges
|
||||
// from the merged writer node to each reader node at the same
|
||||
// rank; and then from each reader node to the merged writer at
|
||||
// the next rank.
|
||||
//
|
||||
// Whereas, merging all readers and writers at the same rank
|
||||
// together is "the simplest thing that could possibly work"
|
||||
// and it seems to. It also creates fairly few edges. We don't
|
||||
// want to create tons of edges here, doing so is not nice to
|
||||
// the main edge contraction pass.
|
||||
mergeSameRankTasks(tasksByRank);
|
||||
}
|
||||
|
||||
// Handle SystemC vars just a little differently. Instead of
|
||||
// treating each var as an independent entity, and serializing
|
||||
// writes to that one var, we treat ALL systemC vars as a single
|
||||
// entity and serialize writes (and, conservatively, reads) across
|
||||
// all of them.
|
||||
//
|
||||
// Reasoning: writing a systemC var actually turns into a call to a
|
||||
// var.write() method, which under the hood is accessing some data
|
||||
// structure that's shared by many SC vars. It's not thread safe.
|
||||
//
|
||||
// Hopefully we only have a few SC vars -- top level ports, probably.
|
||||
{
|
||||
TasksByRank tasksByRank;
|
||||
for (const OrderVarStdVertex* const varVtxp : systemCVars) {
|
||||
findAdjacentTasks(varVtxp, tasksByRank);
|
||||
}
|
||||
mergeSameRankTasks(tasksByRank);
|
||||
}
|
||||
|
||||
// Handle nodes containing DPI calls, we want to serialize those
|
||||
// by default unless user gave '--threads-dpi none'.
|
||||
// Same basic strategy as above to serialize access to SC vars.
|
||||
if (!v3Global.opt.threadsDpiPure() || !v3Global.opt.threadsDpiUnpure()) {
|
||||
TasksByRank tasksByRank;
|
||||
for (V3GraphVertex& vtx : m_mTaskGraph.vertices()) {
|
||||
LogicMTask& mtask = static_cast<LogicMTask&>(vtx);
|
||||
if (hasDpiHazard(&mtask)) tasksByRank[mtask.rank()].insert(&mtask);
|
||||
}
|
||||
mergeSameRankTasks(tasksByRank);
|
||||
}
|
||||
}
|
||||
|
||||
public:
|
||||
static void apply(OrderMTaskGraph& mTaskGraph) { FixDataHazards{mTaskGraph}; }
|
||||
};
|
||||
|
||||
void OrderMTaskGraph::fixDataHazards(OrderMTaskGraph& mtaskGraph) {
|
||||
FixDataHazards::apply(mtaskGraph);
|
||||
}
|
||||
Reference in New Issue
Block a user