diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0848e263b..4e1c9b8ae 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -219,6 +219,7 @@ set(COMMON_SOURCES V3DfgOptimizer.cpp V3DfgPasses.cpp V3DfgPeephole.cpp + V3DfgRegularize.cpp V3DupFinder.cpp V3Timing.cpp V3EmitCBase.cpp diff --git a/src/Makefile_obj.in b/src/Makefile_obj.in index 99b284757..50bf1f9da 100644 --- a/src/Makefile_obj.in +++ b/src/Makefile_obj.in @@ -233,6 +233,7 @@ RAW_OBJS_PCH_ASTNOMT = \ V3DfgOptimizer.o \ V3DfgPasses.o \ V3DfgPeephole.o \ + V3DfgRegularize.o \ V3DupFinder.o \ V3EmitCMain.o \ V3EmitCMake.o \ diff --git a/src/V3Dfg.cpp b/src/V3Dfg.cpp index cc1603724..f12c4582d 100644 --- a/src/V3Dfg.cpp +++ b/src/V3Dfg.cpp @@ -74,6 +74,8 @@ static void dumpDotVertex(std::ostream& os, const DfgVertex& vtx) { } else if (varVtxp->hasExtRefs()) { os << ", shape=box, style=filled, fillcolor=firebrick2"; // Red } else if (varVtxp->hasModRefs()) { + os << ", shape=box, style=filled, fillcolor=darkorange1"; // Orange + } else if (varVtxp->hasDfgRefs()) { os << ", shape=box, style=filled, fillcolor=gold2"; // Yellow } else if (varVtxp->keep()) { os << ", shape=box, style=filled, fillcolor=grey"; @@ -98,6 +100,8 @@ static void dumpDotVertex(std::ostream& os, const DfgVertex& vtx) { } else if (arrVtxp->hasExtRefs()) { os << ", shape=box3d, style=filled, fillcolor=firebrick2"; // Red } else if (arrVtxp->hasModRefs()) { + os << ", shape=box3d, style=filled, fillcolor=darkorange1"; // Orange + } else if (arrVtxp->hasDfgRefs()) { os << ", shape=box3d, style=filled, fillcolor=gold2"; // Yellow } else if (arrVtxp->keep()) { os << ", shape=box3d, style=filled, fillcolor=grey"; @@ -229,7 +233,7 @@ static void dumpDotUpstreamConeFromVertex(std::ostream& os, const DfgVertex& vtx // Emit all DfgVarPacked vertices that have external references driven by this vertex vtx.forEachSink([&](const DfgVertex& dst) { if (const DfgVarPacked* const varVtxp = dst.cast()) { - if (varVtxp->hasRefs()) dumpDotVertexAndSourceEdges(os, dst); + if (varVtxp->hasNonLocalRefs()) dumpDotVertexAndSourceEdges(os, dst); } }); } @@ -263,7 +267,7 @@ void DfgGraph::dumpDotAllVarConesPrefixed(const string& label) const { // Check if this vertex drives a variable referenced outside the DFG. const DfgVarPacked* const sinkp = vtx.findSink([](const DfgVarPacked& sink) { // - return sink.hasRefs(); + return sink.hasNonLocalRefs(); }); // We only dump cones driving an externally referenced variable diff --git a/src/V3Dfg.h b/src/V3Dfg.h index 79fdb3908..be9bfb617 100644 --- a/src/V3Dfg.h +++ b/src/V3Dfg.h @@ -504,9 +504,6 @@ public: // Is this a DfgConst that is all ones inline bool isOnes() const VL_MT_DISABLED; - // Should this vertex be inlined when rendering to Ast, or be stored to a temporary - inline bool inlined() const VL_MT_DISABLED; - // Methods that allow DfgVertex to participate in error reporting/messaging void v3errorEnd(std::ostringstream& str) const VL_RELEASE(V3Error::s().m_mutex) { m_filelinep->v3errorEnd(str); @@ -930,13 +927,4 @@ bool DfgVertex::isOnes() const { return false; } -bool DfgVertex::inlined() const { - // Inline vertices that drive only a single node, or are special - if (!hasMultipleSinks()) return true; - if (is()) return true; - if (is()) return true; - if (const DfgArraySel* const selp = cast()) return selp->bitp()->is(); - return false; -} - #endif diff --git a/src/V3DfgDecomposition.cpp b/src/V3DfgDecomposition.cpp index d3e7a420f..446b0b7d0 100644 --- a/src/V3DfgDecomposition.cpp +++ b/src/V3DfgDecomposition.cpp @@ -329,11 +329,13 @@ class ExtractCyclicComponents final { clonep = new DfgVarArray{m_dfg, aVtxp->varp()}; } UASSERT_OBJ(clonep, &vtx, "Unhandled 'DfgVertexVar' sub-type"); + if (vtx.hasModRefs()) clonep->setHasModRefs(); + if (vtx.hasExtRefs()) clonep->setHasExtRefs(); VertexState& cloneStatep = allocState(*clonep); cloneStatep.component = component; - // We need to mark both the original and the clone as having additional references - vtx.setHasModRefs(); - clonep->setHasModRefs(); + // We need to mark both the original and the clone as having references in other DFGs + vtx.setHasDfgRefs(); + clonep->setHasDfgRefs(); } return *clonep; } diff --git a/src/V3DfgDfgToAst.cpp b/src/V3DfgDfgToAst.cpp index 4836ca0c6..a854bac66 100644 --- a/src/V3DfgDfgToAst.cpp +++ b/src/V3DfgDfgToAst.cpp @@ -126,11 +126,6 @@ AstSliceSel* makeNodeisDrivenFullyByDfg()) return vtxp->varp(); - - // Look up map - const auto it = m_canonVars.find(vtxp->varp()); - if (it != m_canonVars.end()) return it->second; - - // Not known yet, compute it (for all vars driven fully from the same driver) - std::vector varps; - vtxp->source(0)->forEachSink([&](const DfgVertex& vtx) { - if (const DfgVarPacked* const varVtxp = vtx.cast()) { - if (varVtxp->isDrivenFullyByDfg()) varps.push_back(varVtxp); - } - }); - UASSERT_OBJ(!varps.empty(), vtxp, "The input vtxp is always available"); - std::stable_sort(varps.begin(), varps.end(), - [](const DfgVarPacked* ap, const DfgVarPacked* bp) { - if (ap->hasExtRefs() != bp->hasExtRefs()) return ap->hasExtRefs(); - const FileLine& aFl = *(ap->fileline()); - const FileLine& bFl = *(bp->fileline()); - if (const int cmp = aFl.operatorCompare(bFl)) return cmp < 0; - return ap->varp()->name() < bp->varp()->name(); - }); - AstVar* const canonVarp = varps.front()->varp(); - - // Add results to map - for (const DfgVarPacked* const varp : varps) m_canonVars.emplace(varp->varp(), canonVarp); - - // Return it - return canonVarp; - } - - // Given a DfgVertex, return an AstVar that will hold the value of the given DfgVertex once we - // are done with converting this Dfg into Ast form. - AstVar* getResultVar(DfgVertex* vtxp) { - const auto pair = m_resultVars.emplace(vtxp, nullptr); - AstVar*& varp = pair.first->second; - if (pair.second) { - // If this vertex is a DfgVarPacked, then we know the variable. If this node is not a - // DfgVarPacked, then first we try to find a DfgVarPacked driven by this node, and use - // that, otherwise we create a temporary - if (const DfgVarPacked* const thisDfgVarPackedp = vtxp->cast()) { - // This is a DfgVarPacked - varp = getCanonicalVar(thisDfgVarPackedp); - } else if (const DfgVarArray* const thisDfgVarArrayp = vtxp->cast()) { - // This is a DfgVarArray - varp = thisDfgVarArrayp->varp(); - } else if (const DfgVarPacked* const sinkDfgVarPackedp = vtxp->findSink( - [](const DfgVarPacked& var) { return var.isDrivenFullyByDfg(); })) { - // We found a DfgVarPacked driven fully by this node - varp = getCanonicalVar(sinkDfgVarPackedp); - } else { - // No DfgVarPacked driven fully by this node. Create a temporary. - // TODO: should we reuse parts when the AstVar is used as an rvalue? - const string name = m_tmpNames.get(vtxp->hash().toString()); - // Note: It is ok for these temporary variables to be always unsigned. They are - // read only by other expressions within the graph and all expressions interpret - // their operands based on the expression type, not the operand type. - AstNodeDType* const dtypep = v3Global.rootp()->findBitDType( - vtxp->width(), vtxp->width(), VSigning::UNSIGNED); - varp = new AstVar{vtxp->fileline(), VVarType::MODULETEMP, name, dtypep}; - varp->user1(true); // Mark as temporary - // Add temporary AstVar to containing module - m_modp->addStmtsp(varp); - } - // Add to map - } - return varp; - } - AstNodeExpr* convertDfgVertexToAstNodeExpr(DfgVertex* vtxp) { UASSERT_OBJ(!m_resultp, vtxp, "Result already computed"); + UASSERT_OBJ(!vtxp->hasMultipleSinks() || vtxp->is() + || vtxp->is() || vtxp->is(), + vtxp, "Intermediate DFG value with multiple uses"); iterate(vtxp); UASSERT_OBJ(m_resultp, vtxp, "Missing result"); AstNodeExpr* const resultp = m_resultp; @@ -226,25 +151,18 @@ class DfgToAstVisitor final : DfgVisitor { return resultp; } - AstNodeExpr* convertSource(DfgVertex* vtxp) { - if (vtxp->inlined()) { - // Inlined vertices are simply recursively converted - UASSERT_OBJ(vtxp->hasSinks(), vtxp, "Must have one sink: " << vtxp->typeName()); - return convertDfgVertexToAstNodeExpr(vtxp); - } else { - // Vertices that are not inlined need a variable, just return a reference - return new AstVarRef{vtxp->fileline(), getResultVar(vtxp), VAccess::READ}; - } + void addResultEquation(FileLine* flp, AstNodeExpr* lhsp, AstNodeExpr* rhsp) { + m_modp->addStmtsp(new AstAssignW{flp, lhsp, rhsp}); + ++m_ctx.m_resultEquations; } - void convertCanonicalVarDriver(const DfgVarPacked* dfgVarp) { - const auto wRef = [dfgVarp]() { - return new AstVarRef{dfgVarp->fileline(), dfgVarp->varp(), VAccess::WRITE}; - }; + void convertVarDriver(const DfgVarPacked* dfgVarp) { if (dfgVarp->isDrivenFullyByDfg()) { // Whole variable is driven. Render driver and assign directly to whole variable. + FileLine* const flp = dfgVarp->driverFileLine(0); + AstVarRef* const lhsp = new AstVarRef{flp, dfgVarp->varp(), VAccess::WRITE}; AstNodeExpr* const rhsp = convertDfgVertexToAstNodeExpr(dfgVarp->source(0)); - addResultEquation(dfgVarp->driverFileLine(0), wRef(), rhsp); + addResultEquation(flp, lhsp, rhsp); } else { // Variable is driven partially. Render each driver as a separate assignment. dfgVarp->forEachSourceEdge([&](const DfgEdge& edge, size_t idx) { @@ -253,35 +171,10 @@ class DfgToAstVisitor final : DfgVisitor { AstNodeExpr* const rhsp = convertDfgVertexToAstNodeExpr(edge.sourcep()); // Create select LValue FileLine* const flp = dfgVarp->driverFileLine(idx); + AstVarRef* const refp = new AstVarRef{flp, dfgVarp->varp(), VAccess::WRITE}; AstConst* const lsbp = new AstConst{flp, dfgVarp->driverLsb(idx)}; AstConst* const widthp = new AstConst{flp, edge.sourcep()->width()}; - AstSel* const lhsp = new AstSel{flp, wRef(), lsbp, widthp}; - // Add assignment of the value to the selected bits - addResultEquation(flp, lhsp, rhsp); - }); - } - } - - void convertDuplicateVarDriver(const DfgVarPacked* dfgVarp, AstVar* canonVarp) { - const auto rRef = [canonVarp]() { - return new AstVarRef{canonVarp->fileline(), canonVarp, VAccess::READ}; - }; - const auto wRef = [dfgVarp]() { - return new AstVarRef{dfgVarp->fileline(), dfgVarp->varp(), VAccess::WRITE}; - }; - if (dfgVarp->isDrivenFullyByDfg()) { - // Whole variable is driven. Just assign from the canonical variable. - addResultEquation(dfgVarp->driverFileLine(0), wRef(), rRef()); - } else { - // Variable is driven partially. Assign from parts of the canonical var. - dfgVarp->forEachSourceEdge([&](const DfgEdge& edge, size_t idx) { - UASSERT_OBJ(edge.sourcep(), dfgVarp, "Should have removed undriven sources"); - // Create select LValue - FileLine* const flp = dfgVarp->driverFileLine(idx); - AstConst* const lsbp = new AstConst{flp, dfgVarp->driverLsb(idx)}; - AstConst* const widthp = new AstConst{flp, edge.sourcep()->width()}; - AstSel* const rhsp = new AstSel{flp, rRef(), lsbp, widthp->cloneTreePure(false)}; - AstSel* const lhsp = new AstSel{flp, wRef(), lsbp->cloneTreePure(false), widthp}; + AstSel* const lhsp = new AstSel{flp, refp, lsbp, widthp}; // Add assignment of the value to the selected bits addResultEquation(flp, lhsp, rhsp); }); @@ -304,18 +197,13 @@ class DfgToAstVisitor final : DfgVisitor { }); } - void addResultEquation(FileLine* flp, AstNodeExpr* lhsp, AstNodeExpr* rhsp) { - m_modp->addStmtsp(new AstAssignW{flp, lhsp, rhsp}); - ++m_ctx.m_resultEquations; - } - // VISITORS void visit(DfgVertex* vtxp) override { // LCOV_EXCL_START vtxp->v3fatal("Unhandled DfgVertex: " << vtxp->typeName()); } // LCOV_EXCL_STOP void visit(DfgVarPacked* vtxp) override { - m_resultp = new AstVarRef{vtxp->fileline(), getCanonicalVar(vtxp), VAccess::READ}; + m_resultp = new AstVarRef{vtxp->fileline(), vtxp->varp(), VAccess::READ}; } void visit(DfgVarArray* vtxp) override { @@ -328,7 +216,7 @@ class DfgToAstVisitor final : DfgVisitor { void visit(DfgSel* vtxp) override { FileLine* const flp = vtxp->fileline(); - AstNodeExpr* const fromp = convertSource(vtxp->fromp()); + AstNodeExpr* const fromp = convertDfgVertexToAstNodeExpr(vtxp->fromp()); AstConst* const lsbp = new AstConst{flp, vtxp->lsb()}; AstConst* const widthp = new AstConst{flp, vtxp->width()}; m_resultp = new AstSel{flp, fromp, lsbp, widthp}; @@ -336,8 +224,8 @@ class DfgToAstVisitor final : DfgVisitor { void visit(DfgMux* vtxp) override { FileLine* const flp = vtxp->fileline(); - AstNodeExpr* const fromp = convertSource(vtxp->fromp()); - AstNodeExpr* const lsbp = convertSource(vtxp->lsbp()); + AstNodeExpr* const fromp = convertDfgVertexToAstNodeExpr(vtxp->fromp()); + AstNodeExpr* const lsbp = convertDfgVertexToAstNodeExpr(vtxp->lsbp()); AstConst* const widthp = new AstConst{flp, vtxp->width()}; m_resultp = new AstSel{flp, fromp, lsbp, widthp}; } @@ -351,13 +239,7 @@ class DfgToAstVisitor final : DfgVisitor { , m_ctx{ctx} { // Convert the graph back to combinational assignments - // Used by DfgVertex::hash - const auto userDataInUse = dfg.userDataInUse(); - - // We can eliminate some variables completely - std::vector redundantVarps; - - // First render variable assignments + // The graph must have been regularized, so we only need to render assignments for (DfgVertexVar *vtxp = dfg.varVerticesBeginp(), *nextp; vtxp; vtxp = nextp) { nextp = vtxp->verticesNext(); @@ -366,74 +248,13 @@ class DfgToAstVisitor final : DfgVisitor { // Render packed variable assignments if (const DfgVarPacked* const dfgVarp = vtxp->cast()) { - // The driver of this DfgVarPacked might drive multiple variables. Only emit one - // assignment from the driver to an arbitrarily chosen canonical variable, and - // assign the other variables from that canonical variable - AstVar* const canonVarp = getCanonicalVar(dfgVarp); - if (canonVarp == dfgVarp->varp()) { - // This is the canonical variable, so render the driver - convertCanonicalVarDriver(dfgVarp); - } else if (dfgVarp->keep()) { - // Not the canonical variable but it must be kept - convertDuplicateVarDriver(dfgVarp, canonVarp); - } else { - // Not a canonical var, and it can be removed. We will replace all references - // to it with the canonical variable, and hence this can be removed. - redundantVarps.push_back(dfgVarp->varp()); - ++m_ctx.m_replacedVars; - } - // Done + convertVarDriver(dfgVarp); continue; } // Render array variable assignments - if (const DfgVarArray* dfgVarp = vtxp->cast()) { - // We don't canonicalize arrays, so just render the drivers - convertArrayDiver(dfgVarp); - // Done - continue; - } + convertArrayDiver(vtxp->as()); } - - // Constants are always inlined, so we only need to iterate proper operations - for (DfgVertex *vtxp = dfg.opVerticesBeginp(), *nextp; vtxp; vtxp = nextp) { - nextp = vtxp->verticesNext(); - - // If the vertex is known to be inlined, then there is nothing to do - if (vtxp->inlined()) continue; - - // Check if this uses a temporary, vs one of the vars rendered above - AstVar* const resultVarp = getResultVar(vtxp); - if (resultVarp->user1()) { - // We introduced a temporary for this DfgVertex - ++m_ctx.m_intermediateVars; - FileLine* const flp = vtxp->fileline(); - // Just render the logic - AstNodeExpr* const rhsp = convertDfgVertexToAstNodeExpr(vtxp); - // The lhs is the temporary - AstNodeExpr* const lhsp = new AstVarRef{flp, resultVarp, VAccess::WRITE}; - // Add assignment of the value to the variable - addResultEquation(flp, lhsp, rhsp); - } - } - - // Remap all references to point to the canonical variables, if one exists - VNDeleter deleter; - m_modp->foreach([&](AstVarRef* refp) { - // Any variable that is written partially outside the DFG will have itself as the - // canonical var, so need not be replaced, furthermore, if a variable is traced, we - // don't want to update the write-refs we just created above, so we only replace - // read-only references to those variables to those variables we know are not written - // in non-DFG logic. - if (!refp->access().isReadOnly() || refp->varp()->user3()) return; - const auto it = m_canonVars.find(refp->varp()); - if (it == m_canonVars.end() || it->second == refp->varp()) return; - refp->replaceWith(new AstVarRef{refp->fileline(), it->second, refp->access()}); - deleter.pushDeletep(refp); - }); - - // Remove redundant variables - for (AstVar* const varp : redundantVarps) varp->unlinkFrBack()->deleteTree(); } public: diff --git a/src/V3DfgOptimizer.cpp b/src/V3DfgOptimizer.cpp index 12bbe3b62..0a407adfd 100644 --- a/src/V3DfgOptimizer.cpp +++ b/src/V3DfgOptimizer.cpp @@ -25,8 +25,6 @@ #include "V3AstUserAllocator.h" #include "V3Dfg.h" #include "V3DfgPasses.h" -#include "V3DfgPatternStats.h" -#include "V3File.h" #include "V3Graph.h" #include "V3UniqueNames.h" @@ -242,7 +240,7 @@ void V3DfgOptimizer::optimize(AstNetlist* netlistp, const string& label) { UINFO(2, __FUNCTION__ << ": " << endl); // NODE STATE - // AstVar::user1 -> Used by V3DfgPasses::astToDfg and DfgPassed::dfgToAst + // AstVar::user1 -> Used by V3DfgPasses::astToDfg // AstVar::user2 -> bool: Flag indicating referenced by AstVarXRef (set just below) // AstVar::user3 -> bool: Flag indicating written by logic not representable as DFG // (set by V3DfgPasses::astToDfg) @@ -254,8 +252,6 @@ void V3DfgOptimizer::optimize(AstNetlist* netlistp, const string& label) { V3DfgOptimizationContext ctx{label}; - V3DfgPatternStats patternStats; - // Run the optimization phase for (AstNode* nodep = netlistp->modulesp(); nodep; nodep = nodep->nextp()) { // Only optimize proper modules @@ -282,14 +278,6 @@ void V3DfgOptimizer::optimize(AstNetlist* netlistp, const string& label) { // Quick sanity check UASSERT_OBJ(dfg->size() == 0, nodep, "DfgGraph should have become empty"); - // For each cyclic component - for (auto& component : cyclicComponents) { - if (dumpDfgLevel() >= 7) component->dumpDotFilePrefixed(ctx.prefix() + "source"); - // TODO: Apply optimizations safe for cyclic graphs - // Add back under the main DFG (we will convert everything back in one go) - dfg->addGraph(*component); - } - // For each acyclic component for (auto& component : acyclicComponents) { if (dumpDfgLevel() >= 7) component->dumpDotFilePrefixed(ctx.prefix() + "source"); @@ -299,8 +287,18 @@ void V3DfgOptimizer::optimize(AstNetlist* netlistp, const string& label) { dfg->addGraph(*component); } - // Accumulate patterns from the optimized graph for reporting - if (v3Global.opt.stats()) patternStats.accumulate(*dfg); + // Eliminate redundant variables. Run this on the whole acyclic DFG. It needs to traverse + // the module to perform variable substitutions. Doing this by component would do + // redundant traversals and can be extremely slow in large modules with many components. + V3DfgPasses::eliminateVars(*dfg, ctx.m_eliminateVarsContext); + + // For each cyclic component + for (auto& component : cyclicComponents) { + if (dumpDfgLevel() >= 7) component->dumpDotFilePrefixed(ctx.prefix() + "source"); + // TODO: Apply optimizations safe for cyclic graphs + // Add back under the main DFG (we will convert everything back in one go) + dfg->addGraph(*component); + } // Convert back to Ast if (dumpDfgLevel() >= 8) dfg->dumpDotFilePrefixed(ctx.prefix() + "whole-optimized"); @@ -308,25 +306,5 @@ void V3DfgOptimizer::optimize(AstNetlist* netlistp, const string& label) { UASSERT_OBJ(resultModp == modp, modp, "Should be the same module"); } - // Print the collected patterns - if (v3Global.opt.stats()) { - // Label to lowercase, without spaces - std::string ident = label; - std::transform(ident.begin(), ident.end(), ident.begin(), [](unsigned char c) { // - return c == ' ' ? '_' : std::tolower(c); - }); - - // File to dump to - const std::string filename = v3Global.opt.hierTopDataDir() + "/" + v3Global.opt.prefix() - + "__stats_dfg_patterns__" + ident + ".txt"; - - // Open, write, close - std::ofstream* const ofp = V3File::new_ofstream(filename); - if (ofp->fail()) v3fatal("Can't write " << filename); - patternStats.dump(label, *ofp); - ofp->close(); - VL_DO_DANGLING(delete ofp, ofp); - } - V3Global::dumpCheckGlobalTree("dfg-optimize", 0, dumpTreeEitherLevel() >= 3); } diff --git a/src/V3DfgPasses.cpp b/src/V3DfgPasses.cpp index 7ee4433d1..819c00f59 100644 --- a/src/V3DfgPasses.cpp +++ b/src/V3DfgPasses.cpp @@ -19,6 +19,7 @@ #include "V3DfgPasses.h" #include "V3Dfg.h" +#include "V3File.h" #include "V3Global.h" #include "V3String.h" @@ -29,9 +30,16 @@ V3DfgCseContext::~V3DfgCseContext() { m_eliminated); } -DfgRemoveVarsContext::~DfgRemoveVarsContext() { - V3Stats::addStat("Optimizations, DFG " + m_label + " Remove vars, variables removed", - m_removed); +V3DfgRegularizeContext::~V3DfgRegularizeContext() { + V3Stats::addStat("Optimizations, DFG " + m_label + " Regularize, temporaries introduced", + m_temporariesIntroduced); +} + +V3DfgEliminateVarsContext::~V3DfgEliminateVarsContext() { + V3Stats::addStat("Optimizations, DFG " + m_label + " EliminateVars, variables replaced", + m_varsReplaced); + V3Stats::addStat("Optimizations, DFG " + m_label + " EliminateVars, variables removed", + m_varsRemoved); } static std::string getPrefix(const std::string& label) { @@ -62,10 +70,25 @@ V3DfgOptimizationContext::~V3DfgOptimizationContext() { V3Stats::addStat(prefix + "Ast2Dfg, non-representable (unknown)", m_nonRepUnknown); V3Stats::addStat(prefix + "Ast2Dfg, non-representable (var ref)", m_nonRepVarRef); V3Stats::addStat(prefix + "Ast2Dfg, non-representable (width)", m_nonRepWidth); - V3Stats::addStat(prefix + "Dfg2Ast, intermediate variables", m_intermediateVars); - V3Stats::addStat(prefix + "Dfg2Ast, replaced variables", m_replacedVars); V3Stats::addStat(prefix + "Dfg2Ast, result equations", m_resultEquations); + // Print the collected patterns + if (v3Global.opt.stats()) { + // Label to lowercase, without spaces + std::string ident = m_label; + std::transform(ident.begin(), ident.end(), ident.begin(), [](unsigned char c) { // + return c == ' ' ? '_' : std::tolower(c); + }); + + // File to dump to + const std::string filename = v3Global.opt.hierTopDataDir() + "/" + v3Global.opt.prefix() + + "__stats_dfg_patterns__" + ident + ".txt"; + // Open, write, close + const std::unique_ptr ofp{V3File::new_ofstream(filename)}; + if (ofp->fail()) v3fatal("Can't write " << filename); + m_patternStats.dump(m_label, *ofp); + } + // Check the stats are consistent UASSERT(m_inputEquations == m_representable + m_nonRepDType + m_nonRepImpure + m_nonRepTiming + m_nonRepLhs @@ -161,63 +184,6 @@ void V3DfgPasses::inlineVars(const DfgGraph& dfg) { } } -void V3DfgPasses::removeVars(DfgGraph& dfg, DfgRemoveVarsContext& ctx) { - for (DfgVertexVar *vtxp = dfg.varVerticesBeginp(), *nextp; vtxp; vtxp = nextp) { - nextp = vtxp->verticesNext(); - - // We can only eliminate DfgVarPacked vertices at the moment - DfgVarPacked* const varp = vtxp->cast(); - if (!varp) continue; - - // Can't remove if it has consumers - if (varp->hasSinks()) continue; - - // Otherwise if it has drivers - if (varp->isDrivenByDfg()) { - // Can't remove if read in the module and driven here (i.e.: it's an output of the DFG) - if (varp->hasModRefs()) continue; - - // Can't remove if referenced externally, or other special reasons - if (varp->keep()) continue; - - // If the driver of this variable is not an inlined vertex, then we would need a - // temporary when rendering the graph. Instead of introducing a temporary, keep the - // first variable that is driven by that driver. Note that we still remove if the only - // sinks we have are variables, as we might be able to remove all of them (we can be - // sure the not inlined if we have at least 2 non-variable sinks). - if (varp->isDrivenFullyByDfg()) { - DfgVertex* const driverp = varp->source(0); - if (!driverp->inlined()) { - unsigned nonVarSinks = 0; - const DfgVarPacked* firstp = nullptr; - const bool found = driverp->findSink([&](const DfgVertex& sink) { - if (const DfgVarPacked* const sinkVarp = sink.cast()) { - if (!firstp) firstp = sinkVarp; - } else { - ++nonVarSinks; - } - // We can stop as soon as we found the first var, and 2 non-var sinks - return firstp && nonVarSinks >= 2; - }); - // Keep this DfgVarPacked if needed - if (found && firstp == varp) continue; - } - } - } - - // OK, we can delete this DfgVarPacked from the graph. - - // If not referenced outside the DFG, then also delete the referenced AstVar (now unused). - if (!varp->hasRefs()) { - ++ctx.m_removed; - varp->varp()->unlinkFrBack()->deleteTree(); - } - - // Unlink and delete vertex - varp->unlinkDelete(dfg); - } -} - void V3DfgPasses::removeUnused(DfgGraph& dfg) { // DfgVertex::user is the next pointer of the work list elements const auto userDataInUse = dfg.userDataInUse(); @@ -275,6 +241,120 @@ void V3DfgPasses::removeUnused(DfgGraph& dfg) { } } +void V3DfgPasses::eliminateVars(DfgGraph& dfg, V3DfgEliminateVarsContext& ctx) { + const auto userDataInUse = dfg.userDataInUse(); + + // Head of work list. Note that we want all next pointers in the list to be non-zero + // (including that of the last element). This allows us to do two important things: detect + // if an element is in the list by checking for a non-zero next pointer, and easy + // prefetching without conditionals. The address of the graph is a good sentinel as it is a + // valid memory address, and we can easily check for the end of the list. + DfgVertex* const sentinelp = reinterpret_cast(&dfg); + DfgVertex* workListp = sentinelp; + + // Add all variables to the initial work list + for (DfgVertexVar *vtxp = dfg.varVerticesBeginp(), *nextp; vtxp; vtxp = nextp) { + nextp = vtxp->verticesNext(); + if (VL_LIKELY(nextp)) VL_PREFETCH_RW(nextp); + vtxp->setUser(workListp); + workListp = vtxp; + } + + const auto addToWorkList = [&](DfgVertex& vtx) { + // If already in work list then nothing to do + DfgVertex*& nextInWorklistp = vtx.user(); + if (nextInWorklistp) return; + // Actually add to work list. + nextInWorklistp = workListp; + workListp = &vtx; + }; + + // Variable replacements to apply in the module + std::unordered_map replacements; + + // Process the work list + while (workListp != sentinelp) { + // Pick up the head of the work list + DfgVertex* const vtxp = workListp; + // Detach the head + workListp = vtxp->getUser(); + // Prefetch next item + VL_PREFETCH_RW(workListp); + + // Remove unused non-variable vertices + if (!vtxp->is() && !vtxp->hasSinks()) { + // Add sources of removed vertex to work list + vtxp->forEachSource(addToWorkList); + // Remove the unused vertex + vtxp->unlinkDelete(dfg); + } + + // We can only eliminate DfgVarPacked vertices at the moment + DfgVarPacked* const varp = vtxp->cast(); + if (!varp) continue; + + // Can't remove if it has external drivers + if (!varp->isDrivenFullyByDfg()) continue; + + // Can't remove if must be kept (including external, non module references) + if (varp->keep()) continue; + + // Can't remove if referenced in other DFGs of the same module (otherwise might rm twice) + if (varp->hasDfgRefs()) continue; + + // If it has multiple sinks, it can't be eliminated + if (varp->hasMultipleSinks()) continue; + + if (!varp->hasModRefs()) { + // If it is only referenced in this DFG, it can be removed + ++ctx.m_varsRemoved; + varp->replaceWith(varp->source(0)); + varp->varp()->unlinkFrBack()->deleteTree(); + } else if (DfgVarPacked* const canonp = varp->source(0)->cast()) { + // If it's driven from another canonical variable, it can be replaced by that. + // However, we don't want to propagate SystemC variables into the design + if (canonp->varp()->isSc()) continue; + // Note that if this is a duplicate variable, then the canonical variable must + // be either kept or have module references. We ensured this earlier when picking + // the canonical variable in the regularize pass. Additionally, it's possible + // neither of those holds, if an otherwise unreferenced variable drives another one. + // In that case it's true that it must not have a source, so it cannot itself be + // substituted. This condition can be relaxed if needed by supporting recursive + // substitution below. + UASSERT_OBJ(canonp->keep() || canonp->hasDfgRefs() || canonp->hasModRefs() + || !canonp->isDrivenByDfg(), + varp, "Canonical variable should be kept or have module refs"); + ++ctx.m_varsReplaced; + UASSERT_OBJ(!varp->hasSinks(), varp, "Variable inlining should make this impossible"); + const bool newEntry = replacements.emplace(varp->varp(), canonp->varp()).second; + UASSERT_OBJ(newEntry, varp->varp(), "Replacement already exists"); + } else { + // Otherwise this *is* the canonical var + continue; + } + + // Add sources of redundant variable to the work list + vtxp->forEachSource(addToWorkList); + // Remove the redundant variable + vtxp->unlinkDelete(dfg); + } + + // Job done if no replacements possible + if (replacements.empty()) return; + + // Apply variable replacements in the module + VNDeleter deleter; + dfg.modulep()->foreach([&](AstVarRef* refp) { + const auto it = replacements.find(refp->varp()); + if (it == replacements.end()) return; + refp->replaceWith(new AstVarRef{refp->fileline(), it->second, refp->access()}); + deleter.pushDeletep(refp); + }); + + // Remove the replaced variables + for (const auto& pair : replacements) pair.first->unlinkFrBack()->deleteTree(); +} + void V3DfgPasses::optimize(DfgGraph& dfg, V3DfgOptimizationContext& ctx) { // There is absolutely nothing useful we can do with a graph of size 2 or less if (dfg.size() <= 2) return; @@ -301,6 +381,8 @@ void V3DfgPasses::optimize(DfgGraph& dfg, V3DfgOptimizationContext& ctx) { // We just did CSE above, so without peephole there is no need to run it again these apply(4, "cse1 ", [&]() { cse(dfg, ctx.m_cseContext1); }); } - apply(4, "removeVars ", [&]() { removeVars(dfg, ctx.m_removeVarsContext); }); + // Accumulate patterns for reporting + if (v3Global.opt.stats()) ctx.m_patternStats.accumulate(dfg); + apply(4, "regularize", [&]() { regularize(dfg, ctx.m_regularizeContext); }); if (dumpDfgLevel() >= 8) dfg.dumpDotAllVarConesPrefixed(ctx.prefix() + "optimized"); } diff --git a/src/V3DfgPasses.h b/src/V3DfgPasses.h index a67b1cca0..b5a14fabf 100644 --- a/src/V3DfgPasses.h +++ b/src/V3DfgPasses.h @@ -19,6 +19,7 @@ #include "config_build.h" +#include "V3DfgPatternStats.h" #include "V3DfgPeephole.h" #include "V3ThreadSafety.h" @@ -39,14 +40,37 @@ public: ~V3DfgCseContext() VL_MT_DISABLED; }; -class DfgRemoveVarsContext final { +class V3DfgRegularizeContext final { + const std::string m_label; // Label to apply to stats + + const std::string m_ident = [&]() { + std::string ident = m_label; + std::transform(ident.begin(), ident.end(), ident.begin(), [](unsigned char c) { // + return c == ' ' ? '_' : std::tolower(c); + }); + return ident; + }(); + +public: + VDouble0 m_temporariesIntroduced; // Number of temporaries introduced + + const std::string& ident() const { return m_ident; } + + explicit V3DfgRegularizeContext(const std::string& label) + : m_label{label} {} + ~V3DfgRegularizeContext() VL_MT_DISABLED; +}; + +class V3DfgEliminateVarsContext final { const std::string m_label; // Label to apply to stats public: - VDouble0 m_removed; // Number of redundant variables removed - explicit DfgRemoveVarsContext(const std::string& label) + VDouble0 m_varsReplaced; // Number of variables replaced + VDouble0 m_varsRemoved; // Number of variables removed + + explicit V3DfgEliminateVarsContext(const std::string& label) : m_label{label} {} - ~DfgRemoveVarsContext() VL_MT_DISABLED; + ~V3DfgEliminateVarsContext() VL_MT_DISABLED; }; class V3DfgOptimizationContext final { @@ -66,14 +90,16 @@ public: VDouble0 m_nonRepUnknown; // Equations non-representable due to unknown node VDouble0 m_nonRepVarRef; // Equations non-representable due to variable reference VDouble0 m_nonRepWidth; // Equations non-representable due to width mismatch - VDouble0 m_intermediateVars; // Number of intermediate variables introduced - VDouble0 m_replacedVars; // Number of variables replaced VDouble0 m_resultEquations; // Number of result combinational equations V3DfgCseContext m_cseContext0{m_label + " 1st"}; V3DfgCseContext m_cseContext1{m_label + " 2nd"}; V3DfgPeepholeContext m_peepholeContext{m_label}; - DfgRemoveVarsContext m_removeVarsContext{m_label}; + V3DfgRegularizeContext m_regularizeContext{m_label}; + V3DfgEliminateVarsContext m_eliminateVarsContext{m_label}; + + V3DfgPatternStats m_patternStats; + explicit V3DfgOptimizationContext(const std::string& label) VL_MT_DISABLED; ~V3DfgOptimizationContext() VL_MT_DISABLED; @@ -107,10 +133,13 @@ void cse(DfgGraph&, V3DfgCseContext&) VL_MT_DISABLED; void inlineVars(const DfgGraph&) VL_MT_DISABLED; // Peephole optimizations void peephole(DfgGraph&, V3DfgPeepholeContext&) VL_MT_DISABLED; -// Remove redundant variables -void removeVars(DfgGraph&, DfgRemoveVarsContext&) VL_MT_DISABLED; +// Regularize graph. This must be run before converting back to Ast. +void regularize(DfgGraph&, V3DfgRegularizeContext&) VL_MT_DISABLED; // Remove unused nodes void removeUnused(DfgGraph&) VL_MT_DISABLED; +// Eliminate (remove or replace) redundant variables. Also removes resulting unused logic. +void eliminateVars(DfgGraph&, V3DfgEliminateVarsContext&) VL_MT_DISABLED; + } // namespace V3DfgPasses #endif diff --git a/src/V3DfgRegularize.cpp b/src/V3DfgRegularize.cpp new file mode 100644 index 000000000..95637bfd6 --- /dev/null +++ b/src/V3DfgRegularize.cpp @@ -0,0 +1,123 @@ +// -*- mode: C++; c-file-style: "cc-mode" -*- +//************************************************************************* +// DESCRIPTION: Verilator: Convert DfgGraph to AstModule +// +// Code available from: https://verilator.org +// +//************************************************************************* +// +// Copyright 2003-2024 by Wilson Snyder. This program is free software; you +// can redistribute it and/or modify it under the terms of either the GNU +// Lesser General Public License Version 3 or the Perl Artistic License +// Version 2.0. +// SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0 +// +//************************************************************************* +// +// - Ensures intermediate values (other than simple memory references or +// constants) with multiple uses are assigned to variables +// +//************************************************************************* + +#include "V3PchAstNoMT.h" // VL_MT_DISABLED_CODE_UNIT + +#include "V3Dfg.h" +#include "V3DfgPasses.h" +#include "V3UniqueNames.h" + +VL_DEFINE_DEBUG_FUNCTIONS; + +class DfgRegularize final { + DfgGraph& m_dfg; // The graph being processed + V3DfgRegularizeContext& m_ctx; // The optimization context for stats + + // For generating temporary names + V3UniqueNames m_tmpNames{"__VdfgRegularize_" + m_ctx.ident() + "_" + m_dfg.modulep()->name() + + "_tmp"}; + + // Return canonical variable that can be used to hold the value of this vertex + DfgVarPacked* getCanonicalVariable(DfgVertex* vtxp) { + // First gather all existing variables fully written by this vertex + std::vector varVtxps; + vtxp->forEachSink([&](DfgVertex& vtx) { + if (DfgVarPacked* const varVtxp = vtx.cast()) { + if (varVtxp->isDrivenFullyByDfg()) varVtxps.push_back(varVtxp); + } + }); + + if (!varVtxps.empty()) { + // There is at least one usable, existing variable. Pick the first one in source + // order for deterministic results. + std::stable_sort(varVtxps.begin(), varVtxps.end(), + [](const DfgVarPacked* ap, const DfgVarPacked* bp) { + // Prefer those variables that must be kept anyway + const bool keepA = ap->keep() || ap->hasDfgRefs(); + const bool keepB = bp->keep() || bp->hasDfgRefs(); + if (keepA != keepB) return keepA; + // Prefer those that already have module references, so we don't + // have to support recursive substitutions. + if (ap->hasModRefs() != bp->hasModRefs()) return ap->hasModRefs(); + // Otherwise source order + const FileLine& aFl = *(ap->fileline()); + const FileLine& bFl = *(bp->fileline()); + if (const int cmp = aFl.operatorCompare(bFl)) return cmp < 0; + // Fall back on names if all else fails + return ap->varp()->name() < bp->varp()->name(); + }); + return varVtxps.front(); + } + + // We need to introduce a temporary + ++m_ctx.m_temporariesIntroduced; + + // Add temporary AstVar to containing module + FileLine* const flp = vtxp->fileline(); + const std::string name = m_tmpNames.get(vtxp->hash().toString()); + AstVar* const varp = new AstVar{flp, VVarType::MODULETEMP, name, vtxp->dtypep()}; + m_dfg.modulep()->addStmtsp(varp); + + // Create and return a variable vertex for the temporary + return new DfgVarPacked{m_dfg, varp}; + } + + // Insert intermediate variables for vertices with multiple sinks (or use an existing one) + DfgRegularize(DfgGraph& dfg, V3DfgRegularizeContext& ctx) + : m_dfg{dfg} + , m_ctx{ctx} { + + // Used by DfgVertex::hash + const auto userDataInUse = m_dfg.userDataInUse(); + + // Ensure intermediate values used multiple times are written to variables + for (DfgVertex *vtxp = m_dfg.opVerticesBeginp(), *nextp; vtxp; vtxp = nextp) { + nextp = vtxp->verticesNext(); + + // Operations without multiple sinks need no variables + if (!vtxp->hasMultipleSinks()) continue; + // Array selects need no variables, they are just memory references + if (vtxp->is()) continue; + + // This is an op which has multiple sinks. Ensure it is assigned to a variable. + DfgVarPacked* const varp = getCanonicalVariable(vtxp); + if (varp->arity()) { + // Existing variable + FileLine* const flp = varp->driverFileLine(0); + varp->sourceEdge(0)->unlinkSource(); + varp->resetSources(); + vtxp->replaceWith(varp); + varp->addDriver(flp, 0, vtxp); + } else { + // Temporary variable + vtxp->replaceWith(varp); + varp->addDriver(vtxp->fileline(), 0, vtxp); + } + } + } + +public: + static void apply(DfgGraph& dfg, V3DfgRegularizeContext& ctx) { DfgRegularize{dfg, ctx}; } +}; + +void V3DfgPasses::regularize(DfgGraph& dfg, V3DfgRegularizeContext& ctx) { + DfgRegularize::apply(dfg, ctx); +} diff --git a/src/V3DfgVertices.h b/src/V3DfgVertices.h index cdcbb34a4..9f892d46a 100644 --- a/src/V3DfgVertices.h +++ b/src/V3DfgVertices.h @@ -40,6 +40,7 @@ class DfgVertexVar VL_NOT_FINAL : public DfgVertexVariadic { AstVar* const m_varp; // The AstVar associated with this vertex (not owned by this vertex) + bool m_hasDfgRefs = false; // This AstVar is referenced in a different DFG of the module bool m_hasModRefs = false; // This AstVar is referenced outside the DFG, but in the module bool m_hasExtRefs = false; // This AstVar is referenced from outside the module @@ -62,11 +63,13 @@ public: bool isDrivenByDfg() const { return arity() > 0; } AstVar* varp() const { return m_varp; } + bool hasDfgRefs() const { return m_hasDfgRefs; } + void setHasDfgRefs() { m_hasDfgRefs = true; } bool hasModRefs() const { return m_hasModRefs; } void setHasModRefs() { m_hasModRefs = true; } bool hasExtRefs() const { return m_hasExtRefs; } void setHasExtRefs() { m_hasExtRefs = true; } - bool hasRefs() const { return m_hasModRefs || m_hasExtRefs; } + bool hasNonLocalRefs() const { return hasDfgRefs() || hasModRefs() || hasExtRefs(); } // Variable cannot be removed, even if redundant in the DfgGraph (might be used externally) bool keep() const { diff --git a/src/astgen b/src/astgen index 275614e38..1fa0ca3ca 100755 --- a/src/astgen +++ b/src/astgen @@ -1294,7 +1294,7 @@ def write_dfg_dfg_to_ast(filename): "void visit(Dfg{t}* vtxp) override {{\n".format(t=node.name)) for i in range(node.arity): fh.write( - " AstNodeExpr* const op{j}p = convertSource(vtxp->source<{i}>());\n" + " AstNodeExpr* const op{j}p = convertDfgVertexToAstNodeExpr(vtxp->source<{i}>());\n" .format(i=i, j=i + 1)) fh.write( " m_resultp = makeNode(vtxp".format(t=node.name)) diff --git a/test_regress/t/t_json_only_first.out b/test_regress/t/t_json_only_first.out index c40fe0178..d28a05347 100644 --- a/test_regress/t/t_json_only_first.out +++ b/test_regress/t/t_json_only_first.out @@ -47,7 +47,7 @@ {"type":"VARREF","name":"d","addr":"(IB)","loc":"d,49:16,49:17","dtypep":"(G)","access":"RD","varp":"(Z)","varScopep":"UNLINKED","classOrPackagep":"UNLINKED"} ], "lhsp": [ - {"type":"VARREF","name":"q","addr":"(JB)","loc":"d,50:22,50:23","dtypep":"(G)","access":"WR","varp":"(CB)","varScopep":"UNLINKED","classOrPackagep":"UNLINKED"} + {"type":"VARREF","name":"q","addr":"(JB)","loc":"d,53:13,53:14","dtypep":"(G)","access":"WR","varp":"(CB)","varScopep":"UNLINKED","classOrPackagep":"UNLINKED"} ],"timingControlp": [],"strengthSpecp": []} ],"activesp": []}, {"type":"MODULE","name":"mod1__W4","addr":"(M)","loc":"d,31:8,31:12","origName":"mod1","level":3,"modPublic":false,"inLibrary":false,"dead":false,"recursiveClone":false,"recursive":false,"timeunit":"1ps","inlinesp": [], diff --git a/test_regress/t/t_json_only_flat.out b/test_regress/t/t_json_only_flat.out index e735e5a7e..a4ca9fc3c 100644 --- a/test_regress/t/t_json_only_flat.out +++ b/test_regress/t/t_json_only_flat.out @@ -131,7 +131,7 @@ {"type":"VARREF","name":"t.between","addr":"(ZC)","loc":"d,17:22,17:29","dtypep":"(H)","access":"RD","varp":"(O)","varScopep":"(HB)","classOrPackagep":"UNLINKED"} ], "lhsp": [ - {"type":"VARREF","name":"q","addr":"(AD)","loc":"d,15:22,15:23","dtypep":"(H)","access":"WR","varp":"(G)","varScopep":"(BB)","classOrPackagep":"UNLINKED"} + {"type":"VARREF","name":"q","addr":"(AD)","loc":"d,53:13,53:14","dtypep":"(H)","access":"WR","varp":"(G)","varScopep":"(BB)","classOrPackagep":"UNLINKED"} ],"timingControlp": [],"strengthSpecp": []} ]} ]} diff --git a/test_regress/t/t_xml_first.out b/test_regress/t/t_xml_first.out index 1b4130a9b..511de3c4b 100644 --- a/test_regress/t/t_xml_first.out +++ b/test_regress/t/t_xml_first.out @@ -51,7 +51,7 @@ - + diff --git a/test_regress/t/t_xml_flat.out b/test_regress/t/t_xml_flat.out index 42fd9a767..df60ede82 100644 --- a/test_regress/t/t_xml_flat.out +++ b/test_regress/t/t_xml_flat.out @@ -100,7 +100,7 @@ - +