391 lines
17 KiB
C++
391 lines
17 KiB
C++
// -*- mode: C++; c-file-style: "cc-mode" -*-
|
|
//*************************************************************************
|
|
// DESCRIPTION: Verilator: Multi-threaded MTask graph data hazard fixing
|
|
//
|
|
// Code available from: https://verilator.org
|
|
//
|
|
//*************************************************************************
|
|
//
|
|
// This program is free software; you can redistribute it and/or modify it
|
|
// under the terms of either the GNU Lesser General Public License Version 3
|
|
// or the Perl Artistic License Version 2.0.
|
|
// SPDX-FileCopyrightText: 2003-2026 Wilson Snyder
|
|
// SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0
|
|
//
|
|
//*************************************************************************
|
|
|
|
#include "V3PchAstNoMT.h" // VL_MT_DISABLED_CODE_UNIT
|
|
|
|
#include "V3Control.h"
|
|
#include "V3Global.h"
|
|
#include "V3Graph.h"
|
|
#include "V3GraphStream.h"
|
|
#include "V3OrderGraph.h"
|
|
#include "V3OrderMTaskGraph.h"
|
|
|
|
#include <algorithm>
|
|
#include <map>
|
|
#include <set>
|
|
#include <vector>
|
|
|
|
VL_DEFINE_DEBUG_FUNCTIONS;
|
|
|
|
//######################################################################
|
|
// DpiImportCallVisitor
|
|
|
|
// Scan node, indicate whether it contains a call to a DPI imported routine.
|
|
class DpiImportCallVisitor final : public VNVisitor {
|
|
bool m_hasDpiHazard = false; // Found a DPI import call.
|
|
bool m_tracingCall = false; // Iterating into a CCall to a CFunc
|
|
// METHODS
|
|
void visit(AstCFunc* nodep) override {
|
|
if (!m_tracingCall) return;
|
|
m_tracingCall = false;
|
|
if (nodep->dpiImportWrapper()) {
|
|
if (nodep->dpiPure() ? !v3Global.opt.threadsDpiPure()
|
|
: !v3Global.opt.threadsDpiUnpure()) {
|
|
// If hierarchical DPI wrapper cost is not found or is of a 0 cost,
|
|
// we have a normal DPI which induces DPI hazard by default.
|
|
m_hasDpiHazard = V3Control::getProfileData(nodep->cname()) == 0;
|
|
UINFO(9, "DPI wrapper '" << nodep->cname()
|
|
<< "' has dpi hazard = " << m_hasDpiHazard);
|
|
}
|
|
}
|
|
iterateChildren(nodep);
|
|
}
|
|
void visit(AstNodeCCall* nodep) override {
|
|
iterateChildren(nodep);
|
|
// Enter the function and trace it
|
|
m_tracingCall = true;
|
|
iterate(nodep->funcp());
|
|
}
|
|
void visit(AstNode* nodep) override { iterateChildren(nodep); }
|
|
|
|
// CONSTRUCTORS
|
|
explicit DpiImportCallVisitor(AstNode* nodep) { iterate(nodep); }
|
|
|
|
public:
|
|
static bool hasDpiHazard(AstNode* nodep) { return DpiImportCallVisitor{nodep}.m_hasDpiHazard; }
|
|
};
|
|
|
|
//######################################################################
|
|
// FixDataHazards
|
|
|
|
class FixDataHazards final {
|
|
//
|
|
// Fix data hazards in the MTask graph.
|
|
//
|
|
// The fine-grained graph from V3Order may contain data hazards which are
|
|
// not a problem for serial mode, but which would be a problem in parallel
|
|
// mode.
|
|
//
|
|
// There are basically two classes: unordered pairs of writes, and
|
|
// unordered write-read pairs. We fix both here, with a combination of
|
|
// MTask-merges and new edges to ensure no such unordered pairs remain.
|
|
//
|
|
// ABOUT UNORDERED WRITE-WRITE PAIRS
|
|
//
|
|
// The V3Order dependency graph treats these as unordered events:
|
|
//
|
|
// a) sig[15:8] = stuff;
|
|
// ...
|
|
// b) sig[7:0] = other_stuff;
|
|
//
|
|
// Seems OK right? They are writes to disjoint bits of the same
|
|
// signal. They can run in either order, in serial mode, and the result
|
|
// will be the same.
|
|
//
|
|
// The resulting C code for each of this isn't a pure write, it's
|
|
// actually an R-M-W sequence:
|
|
//
|
|
// a) sig = (sig & 0xff) | (0xff00 & (stuff << 8));
|
|
// ...
|
|
// b) sig = (sig & 0xff00) | (0xff & other_stuff);
|
|
//
|
|
// In serial mode, order doesn't matter so long as these run serially.
|
|
// In parallel mode, we must serialize these RMW's to avoid a race.
|
|
//
|
|
// We don't actually check here if each write would involve an R-M-W, we
|
|
// just assume that it would. If this routine ever causes a drastic
|
|
// increase in critical path, it could be optimized to make a better
|
|
// prediction (with all the risk that word implies!) about whether a
|
|
// given write is likely to turn into an R-M-W.
|
|
//
|
|
// ABOUT UNORDERED WRITE-READ PAIRS
|
|
//
|
|
// If we don't put unordered write-read pairs into some order at Verilation
|
|
// time, we risk a runtime race.
|
|
//
|
|
// How do such unordered writer/reader pairs happen? Here's a partial list
|
|
// of scenarios:
|
|
//
|
|
// Case 1: Circular logic
|
|
//
|
|
// If the design has circular logic, V3Order has by now generated some
|
|
// dependency cycles, and also cut some of the edges to make it
|
|
// acyclic.
|
|
//
|
|
// For serial mode, that was fine. We can break logic circles at an
|
|
// arbitrary point. At runtime, we'll repeat the _eval() until no
|
|
// changes are detected, which papers over the discarded dependency.
|
|
//
|
|
// For parallel mode, this situation can lead to unordered reads and
|
|
// writes of the same variable, causing a data race. For example if the
|
|
// original code is this:
|
|
//
|
|
// assign b = b | a << 2;
|
|
// assign out = b;
|
|
//
|
|
// ... there's originally a dependency edge which records that 'b'
|
|
// depends on the first assign. V3Order may cut this edge, making the
|
|
// statements unordered. In serial mode that's fine, they can run in
|
|
// either order. In parallel mode it's a reader/writer race.
|
|
//
|
|
// Case 2: Race Condition in Verilog Sources
|
|
//
|
|
// If the input has races, eg. blocking assignments in always blocks
|
|
// that share variables, the graph at this point will contain unordered
|
|
// writes and reads (or unordered write-write pairs) reflecting that.
|
|
|
|
// TYPES
|
|
// Sort LogicMTask objects into deterministic order by calling id()
|
|
// which is a unique and stable serial number.
|
|
struct MTaskIdLessThan final {
|
|
bool operator()(const LogicMTask* lhsp, const LogicMTask* rhsp) const {
|
|
return *lhsp < *rhsp;
|
|
}
|
|
};
|
|
using TasksByRank = std::map<uint32_t /*rank*/, std::set<LogicMTask*, MTaskIdLessThan>>;
|
|
|
|
// MEMBERS
|
|
OrderMTaskGraph& m_mTaskGraph; // The Mtask graph
|
|
|
|
// METHODS
|
|
|
|
// Redirect all edges of 'donorp' onto 'recipientp'
|
|
static void redirectEdgesFrom(LogicMTask* recipientp, LogicMTask* donorp) {
|
|
// Process outgoing edges
|
|
while (MTaskEdge* const edgep = static_cast<MTaskEdge*>(donorp->outEdges().frontp())) {
|
|
LogicMTask* const top = edgep->toMTaskp();
|
|
top->removeRelativeEdge<GraphWay::REVERSE>(edgep);
|
|
|
|
// If an edge already exists between recipient and sink of donor, drop the duplicate.
|
|
if (recipientp->hasRelativeMTask(top)) {
|
|
VL_DO_DANGLING(edgep->unlinkDelete(), edgep);
|
|
continue;
|
|
}
|
|
|
|
// Otherwise redirect the edge from donorp->top to recipientp->top.
|
|
edgep->relinkFromp(recipientp);
|
|
recipientp->addRelativeMTask(top);
|
|
recipientp->stealRelativeEdge<GraphWay::FORWARD>(edgep);
|
|
top->addRelativeEdge<GraphWay::REVERSE>(edgep);
|
|
}
|
|
|
|
// Process incoming edges
|
|
while (MTaskEdge* const edgep = static_cast<MTaskEdge*>(donorp->inEdges().frontp())) {
|
|
LogicMTask* const fromp = edgep->fromMTaskp();
|
|
fromp->removeRelativeMTask(donorp);
|
|
fromp->removeRelativeEdge<GraphWay::FORWARD>(edgep);
|
|
|
|
// If an edge already exists between recipient and source of donor, drop the duplicate.
|
|
if (fromp->hasRelativeMTask(recipientp)) {
|
|
VL_DO_DANGLING(edgep->unlinkDelete(), edgep);
|
|
continue;
|
|
}
|
|
|
|
// Otherwise redirect the edge from fromp->donorp to fromp->recipientp.
|
|
edgep->relinkTop(recipientp);
|
|
fromp->addRelativeMTask(recipientp);
|
|
fromp->addRelativeEdge<GraphWay::FORWARD>(edgep);
|
|
recipientp->stealRelativeEdge<GraphWay::REVERSE>(edgep);
|
|
}
|
|
}
|
|
|
|
void findAdjacentTasks(const OrderVarStdVertex* varVtxp, TasksByRank& tasksByRank) {
|
|
// Find all writer tasks for this variable, group by rank.
|
|
for (const V3GraphEdge& edge : varVtxp->inEdges()) {
|
|
if (const auto* const logicVtxp = edge.fromp()->cast<OrderLogicVertex>()) {
|
|
LogicMTask* const writerMtaskp = static_cast<LogicMTask*>(logicVtxp->userp());
|
|
tasksByRank[writerMtaskp->rank()].insert(writerMtaskp);
|
|
}
|
|
}
|
|
// Note: Find all reader tasks for this variable, group by rank.
|
|
// There was "broken" code here to find readers, but fixing it to
|
|
// work properly harmed performance on some tests, see issue #3360.
|
|
}
|
|
|
|
void mergeSameRankTasks(const TasksByRank& tasksByRank) {
|
|
LogicMTask* lastRecipientp = nullptr;
|
|
for (const auto& pair : tasksByRank) {
|
|
// Find the largest node at this rank, merge into it. (If we
|
|
// happen to find a huge node, this saves time in
|
|
// redirectEdgesFrom() versus merging into an arbitrary node.)
|
|
LogicMTask* recipientp = nullptr;
|
|
for (LogicMTask* const mtaskp : pair.second) {
|
|
if (!recipientp || (recipientp->cost() < mtaskp->cost())) recipientp = mtaskp;
|
|
}
|
|
UASSERT_OBJ(!lastRecipientp || (lastRecipientp->rank() < recipientp->rank()),
|
|
recipientp, "Merging must be on lower rank");
|
|
|
|
for (LogicMTask* const donorp : pair.second) {
|
|
// Merge donor into recipient.
|
|
if (donorp == recipientp) continue;
|
|
// Fix up the map, so donor's OLVs map to recipientp
|
|
for (const OrderMoveVertex& vtx : donorp->vertexList()) {
|
|
vtx.logicp()->userp(recipientp);
|
|
}
|
|
// Move all vertices from donorp to recipientp
|
|
recipientp->moveAllVerticesFrom(donorp);
|
|
// Redirect edges from donorp to recipientp
|
|
redirectEdgesFrom(recipientp, donorp);
|
|
// Remove donorp from the graph
|
|
VL_DO_DANGLING(donorp->unlinkDelete(&m_mTaskGraph), donorp);
|
|
}
|
|
|
|
if (lastRecipientp && !lastRecipientp->hasRelativeMTask(recipientp)) {
|
|
new MTaskEdge{&m_mTaskGraph, lastRecipientp, recipientp, 1};
|
|
}
|
|
lastRecipientp = recipientp;
|
|
}
|
|
}
|
|
|
|
bool hasDpiHazard(LogicMTask* mtaskp) {
|
|
for (const OrderMoveVertex& mVtx : mtaskp->vertexList()) {
|
|
OrderLogicVertex* const lvtxp = mVtx.logicp();
|
|
if (!lvtxp) continue;
|
|
// NOTE: We don't handle DPI exports. If testbench code calls a DPI-exported function
|
|
// at any time during eval() we may have a data hazard. (Likewise in non-threaded mode
|
|
// if an export messes with an ordered variable we're broken.)
|
|
|
|
// Find all calls to DPI-imported functions, we can put those into a serial order at
|
|
// least. That should solve the most likely DPI-related data hazards.
|
|
if (DpiImportCallVisitor::hasDpiHazard(lvtxp->nodep())) return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
// CONSTRUCTOR
|
|
FixDataHazards(OrderMTaskGraph& mTaskGraph)
|
|
: m_mTaskGraph{mTaskGraph} {
|
|
// Rank the graph. DGS is faster than V3GraphAlg's recursive rank, and also allows us to
|
|
// set up the OrderLogicVertex -> LogicMTask map at the same time.
|
|
{
|
|
GraphStreamUnordered serialize{&m_mTaskGraph};
|
|
while (LogicMTask* const mtaskp
|
|
= const_cast<LogicMTask*>(static_cast<const LogicMTask*>(serialize.nextp()))) {
|
|
// Compute and assign rank
|
|
uint32_t rank = 0;
|
|
for (V3GraphEdge& edge : mtaskp->inEdges()) {
|
|
rank = std::max(edge.fromp()->rank() + 1, rank);
|
|
}
|
|
mtaskp->rank(rank);
|
|
|
|
// Set up the OrderLogicVertex -> LogicMTask map
|
|
// Entry and exit MTasks have no MTaskMoveVertices under them, so move on
|
|
if (mtaskp->vertexList().empty()) continue;
|
|
// Otherwise there should be only one OrderMoveVertex in each MTask at this stage
|
|
const OrderMoveVertex::List& vertexList = mtaskp->vertexList();
|
|
UASSERT_OBJ(vertexList.hasSingleElement(), mtaskp, "Multiple OrderMoveVertex");
|
|
const OrderMoveVertex* const mVtxp = vertexList.frontp();
|
|
// Set up mapping back to the MTask from the OrderLogicVertex
|
|
if (OrderLogicVertex* const lvtxp = mVtxp->logicp()) lvtxp->userp(mtaskp);
|
|
}
|
|
}
|
|
|
|
// Gather all variables. SystemC vars will be handled slightly specially, so keep separate.
|
|
const OrderGraph& orderGraph = m_mTaskGraph.moveGraph().orderGraph();
|
|
std::vector<const OrderVarStdVertex*> regularVars;
|
|
std::vector<const OrderVarStdVertex*> systemCVars;
|
|
for (const V3GraphVertex& vtx : orderGraph.vertices()) {
|
|
// Only consider OrderVarStdVertex which reflects
|
|
// an actual lvalue assignment; the others do not.
|
|
if (const OrderVarStdVertex* const vvtxp = vtx.cast<const OrderVarStdVertex>()) {
|
|
if (vvtxp->vscp()->varp()->isSc()) {
|
|
systemCVars.push_back(vvtxp);
|
|
} else {
|
|
regularVars.push_back(vvtxp);
|
|
}
|
|
}
|
|
}
|
|
|
|
// For each OrderVarVertex, look at its writer and reader MTasks.
|
|
//
|
|
// If there's a set of writers and readers at the same rank, we
|
|
// know these are unordered with respect to one another, so merge
|
|
// those MTasks all together.
|
|
//
|
|
// At this point, we have at most one merged mtask per rank (for a
|
|
// given OVV.) Create edges across these remaining MTasks to ensure
|
|
// they run in serial order (going along with the existing ranks.)
|
|
//
|
|
// NOTE: we don't update the CP's stored in the LogicMTasks to
|
|
// reflect the changes we make to the graph. That's OK, as we
|
|
// haven't yet initialized CPs when we call this routine.
|
|
for (const OrderVarStdVertex* const varVtxp : regularVars) {
|
|
// Build a set of MTasks, per rank, which access this var.
|
|
// Within a rank, sort by MTaskID to avoid nondeterminism.
|
|
TasksByRank tasksByRank;
|
|
|
|
// Find all reader and writer tasks for this variable, add to
|
|
// tasksByRank.
|
|
findAdjacentTasks(varVtxp, tasksByRank);
|
|
|
|
// Merge all writer and reader tasks from same rank together.
|
|
//
|
|
// NOTE: Strictly speaking, we don't need to merge all the
|
|
// readers together. That may lead to extra serialization. The
|
|
// least amount of ordering we could impose here would be to
|
|
// merge all writers at a given rank together; then make edges
|
|
// from the merged writer node to each reader node at the same
|
|
// rank; and then from each reader node to the merged writer at
|
|
// the next rank.
|
|
//
|
|
// Whereas, merging all readers and writers at the same rank
|
|
// together is "the simplest thing that could possibly work"
|
|
// and it seems to. It also creates fairly few edges. We don't
|
|
// want to create tons of edges here, doing so is not nice to
|
|
// the main edge contraction pass.
|
|
mergeSameRankTasks(tasksByRank);
|
|
}
|
|
|
|
// Handle SystemC vars just a little differently. Instead of
|
|
// treating each var as an independent entity, and serializing
|
|
// writes to that one var, we treat ALL systemC vars as a single
|
|
// entity and serialize writes (and, conservatively, reads) across
|
|
// all of them.
|
|
//
|
|
// Reasoning: writing a systemC var actually turns into a call to a
|
|
// var.write() method, which under the hood is accessing some data
|
|
// structure that's shared by many SC vars. It's not thread safe.
|
|
//
|
|
// Hopefully we only have a few SC vars -- top level ports, probably.
|
|
{
|
|
TasksByRank tasksByRank;
|
|
for (const OrderVarStdVertex* const varVtxp : systemCVars) {
|
|
findAdjacentTasks(varVtxp, tasksByRank);
|
|
}
|
|
mergeSameRankTasks(tasksByRank);
|
|
}
|
|
|
|
// Handle nodes containing DPI calls, we want to serialize those
|
|
// by default unless user gave '--threads-dpi none'.
|
|
// Same basic strategy as above to serialize access to SC vars.
|
|
if (!v3Global.opt.threadsDpiPure() || !v3Global.opt.threadsDpiUnpure()) {
|
|
TasksByRank tasksByRank;
|
|
for (V3GraphVertex& vtx : m_mTaskGraph.vertices()) {
|
|
LogicMTask& mtask = static_cast<LogicMTask&>(vtx);
|
|
if (hasDpiHazard(&mtask)) tasksByRank[mtask.rank()].insert(&mtask);
|
|
}
|
|
mergeSameRankTasks(tasksByRank);
|
|
}
|
|
}
|
|
|
|
public:
|
|
static void apply(OrderMTaskGraph& mTaskGraph) { FixDataHazards{mTaskGraph}; }
|
|
};
|
|
|
|
void OrderMTaskGraph::fixDataHazards(OrderMTaskGraph& mtaskGraph) {
|
|
FixDataHazards::apply(mtaskGraph);
|
|
}
|