From ccb4ab9fe3cb63ff2ccb8b4337c62e283017272b Mon Sep 17 00:00:00 2001 From: Geza Lore Date: Fri, 18 Sep 2026 19:19:53 +0200 Subject: [PATCH] Optimize Dfg vertex input edge storage (#8313) Store input edges of fixed arity vertices inline in the vertex class. This reduces heap allocations, memory fragmentation, and pointer chasing and speeds up Dfg passes. The extra branch introduced in inputEdgep() is well predictable and profiling shows branchless alternatives are a loss. --- src/V3Dfg.cpp | 7 ++-- src/V3Dfg.h | 61 +++++++++++++++++++++++++--------- src/V3DfgVertices.h | 80 +++++++++++++++++++++++++++++---------------- 3 files changed, 102 insertions(+), 46 deletions(-) diff --git a/src/V3Dfg.cpp b/src/V3Dfg.cpp index 66bbc205e..124bffda8 100644 --- a/src/V3Dfg.cpp +++ b/src/V3Dfg.cpp @@ -473,8 +473,11 @@ DfgGraph::neighborhood(const std::vector& vtxps, size_t n) con //------------------------------------------------------------------------------ // DfgVertex -DfgVertex::DfgVertex(DfgGraph& dfg, VDfgType type, FileLine* flp, const DfgDataType& dt) - : m_filelinep{flp} +DfgVertex::DfgVertex(DfgGraph& dfg, VDfgType type, FileLine* flp, const DfgDataType& dt, + DfgEdge* inlineInputsp, uint32_t nInputs) + : m_inlineInputsp{inlineInputsp} + , m_nInputs{nInputs} + , m_filelinep{flp} , m_dtype{dt} , m_type{type} { dfg.addVertex(*this); diff --git a/src/V3Dfg.h b/src/V3Dfg.h index 6bbb7bf2f..7ae3e0334 100644 --- a/src/V3Dfg.h +++ b/src/V3Dfg.h @@ -120,6 +120,28 @@ public: inline void relinkSrcp(DfgVertex* srcp); }; +//------------------------------------------------------------------------------ +// Input edge storage of a fixed arity vertices, embedded in the vertex itself. +template +class DfgInlineEdgeStorage final { + static_assert(N_Edges > 0, "'DfgInlineEdgeStorage' must hold at least one edge"); + VL_UNCOPYABLE(DfgInlineEdgeStorage); + VL_UNMOVABLE(DfgInlineEdgeStorage); + +public: + union { // Union, for manual memory management, so DfgEdge need no implicit constructor + DfgEdge m_edges[N_Edges]; // The input edges of the owning vertex + }; + + explicit DfgInlineEdgeStorage(DfgVertex* vtxp) { + for (uint32_t i = 0; i < N_Edges; ++i) new (&m_edges[i]) DfgEdge{vtxp}; + } + ~DfgInlineEdgeStorage() { + for (uint32_t i = 0; i < N_Edges; ++i) m_edges[i].~DfgEdge(); + } + DfgInlineEdgeStorage() = delete; +}; + //------------------------------------------------------------------------------ // Dataflow graph vertex class DfgVertex VL_NOT_FINAL { @@ -128,10 +150,12 @@ class DfgVertex VL_NOT_FINAL { friend class DfgVisitor; template friend class DfgUserMap; + friend class DfgVertexVariadic; // STATE V3ListLinks m_links; // V3List links in the DfgGraph - std::vector> m_inputps; // Input edges, as vector, for fast indexing + DfgEdge* const m_inlineInputsp; // Input edges stored inline in the vertex (iff fixed arity) + uint32_t m_nInputs; // Number of input edges DfgEdge::List m_sinks; // List of sink edges of this vertex FileLine* const m_filelinep; // Source location @@ -161,25 +185,22 @@ public: protected: // CONSTRUCTOR - DfgVertex(DfgGraph& dfg, VDfgType type, FileLine* flp, const DfgDataType& dt) VL_MT_DISABLED; + DfgVertex(DfgGraph& dfg, VDfgType type, FileLine* flp, const DfgDataType& dt, + DfgEdge* inlineInputsp, uint32_t nInputs) VL_MT_DISABLED; // Use unlinkDelete instead virtual ~DfgVertex() VL_MT_DISABLED = default; - // Create a new input edge and return it - DfgEdge* newInput() { - m_inputps.emplace_back(new DfgEdge{this}); - return m_inputps.back().get(); - } +private: + // Get input edge 'i' + inline DfgEdge* inputEdgep(size_t i) const; public: // Get input 'i' - DfgVertex* inputp(size_t i) const { return m_inputps[i]->srcp(); } + DfgVertex* inputp(size_t i) const { return inputEdgep(i)->srcp(); } // Relink input 'i' - void inputp(size_t i, DfgVertex* vtxp) { m_inputps[i]->relinkSrcp(vtxp); } + void inputp(size_t i, DfgVertex* vtxp) { inputEdgep(i)->relinkSrcp(vtxp); } // The number of inputs this vertex has. Some might be unconnected. - size_t nInputs() const { return m_inputps.size(); } - // Unlink all inputs and reset to no inputs - use very carefully - void resetInputs() { m_inputps.clear(); } + size_t nInputs() const { return m_nInputs; } // The type of this vertex VDfgType type() const { return m_type; } @@ -252,8 +273,8 @@ public: bool foreachSource(T_Callable&& f) { static_assert(vlstd::is_invocable_r::value, "T_Callable 'f' must have a signature compatible with 'bool(DfgVertex&)'"); - for (const std::unique_ptr& edgep : m_inputps) { - if (DfgVertex* const srcp = edgep->srcp()) { + for (size_t i = 0; i < m_nInputs; ++i) { + if (DfgVertex* const srcp = inputEdgep(i)->srcp()) { if (f(*srcp)) return true; } } @@ -268,8 +289,8 @@ public: static_assert( vlstd::is_invocable_r::value, "T_Callable 'f' must have a signature compatible with 'bool(const DfgVertex&)'"); - for (const std::unique_ptr& edgep : m_inputps) { - if (const DfgVertex* const srcp = edgep->srcp()) { + for (size_t i = 0; i < m_nInputs; ++i) { + if (const DfgVertex* const srcp = inputEdgep(i)->srcp()) { if (f(*srcp)) return true; } } @@ -826,6 +847,14 @@ void DfgEdge::relinkSrcp(DfgVertex* srcp) { // DfgVertex {{{ +DfgEdge* DfgVertex::inputEdgep(size_t i) const { + UDEBUGONLY(UASSERT_OBJ(i < m_nInputs, this, "Input index out of range");); + if (VL_LIKELY(m_inlineInputsp)) return m_inlineInputsp + i; + // 'm_inlineInputsp' is null exactly for a DfgVertexVariadic + UDEBUGONLY(UASSERT_OBJ(is(), this, "Vertex without input edge storage");); + return static_cast(this)->m_edgeps[i].get(); +} + bool DfgVertex::isCheaperThanLoad() const { // Constants if (is()) return true; diff --git a/src/V3DfgVertices.h b/src/V3DfgVertices.h index 13dbe3f92..f260afd5e 100644 --- a/src/V3DfgVertices.h +++ b/src/V3DfgVertices.h @@ -48,6 +48,7 @@ class DfgVertexVar VL_NOT_FINAL : public DfgVertex { // Represents a variable. It has 2 optional inputs, 'srcp' and 'defaultp'. + DfgInlineEdgeStorage<2> m_inputs{this}; // Input edges AstVarScope* const m_vscp; // The AstVarScope associated with this vertex (not owned) // Location of driver of this variable. Only used for converting back to Ast. Might be nullptr. FileLine* m_driverFileLine = nullptr; @@ -58,15 +59,16 @@ class DfgVertexVar VL_NOT_FINAL : public DfgVertex { protected: DfgVertexVar(DfgGraph& dfg, VDfgType type, AstVarScope* vscp) - : DfgVertex{dfg, type, vscp->varp()->fileline(), - *DfgDataType::fromAst(vscp->varp()->dtypep())} + : DfgVertex{dfg, + type, + vscp->varp()->fileline(), + *DfgDataType::fromAst(vscp->varp()->dtypep()), + m_inputs.m_edges, + 2} , m_vscp{vscp} { // Increment reference count m_vscp->user1(m_vscp->user1() + 0x40); UASSERT_OBJ((m_vscp->user1() >> 6) > 0, m_vscp, "Reference count overflow"); - // Allocate sources - newInput(); - newInput(); } public: @@ -183,8 +185,12 @@ class DfgPrev final : public DfgVertex { public: DfgPrev(DfgGraph& dfg, AstVarScope* vscp) - : DfgVertex{dfg, dfgType(), vscp->varp()->fileline(), - *DfgDataType::fromAst(vscp->varp()->dtypep())} + : DfgVertex{dfg, + dfgType(), + vscp->varp()->fileline(), + *DfgDataType::fromAst(vscp->varp()->dtypep()), + nullptr, + 0} , m_vscp{vscp} { UASSERT_OBJ(!DfgVertexVar::hasPrev(vscp), vscp, "Variable already has a DfgPrev"); m_vscp->user1(m_vscp->user1() | 0x20); // Mark having a DfgPrev @@ -209,8 +215,14 @@ class DfgVertexAst VL_NOT_FINAL : public DfgVertex { AstNodeExpr* m_exprp; // The AstNodeExpr representing this reference public: - DfgVertexAst(DfgGraph& dfg, VDfgType type, AstNodeExpr* exprp) - : DfgVertex{dfg, type, exprp->fileline(), *DfgDataType::fromAst(exprp->dtypep())} + DfgVertexAst(DfgGraph& dfg, VDfgType type, AstNodeExpr* exprp, DfgEdge* inlineInputsp, + uint32_t nInputs) + : DfgVertex{dfg, + type, + exprp->fileline(), + *DfgDataType::fromAst(exprp->dtypep()), + inlineInputsp, + nInputs} , m_exprp{exprp} {} ASTGEN_MEMBERS_DfgVertexAst; @@ -222,17 +234,15 @@ class DfgAstRd final : public DfgVertexAst { friend class DfgVertex; friend class DfgVisitor; + DfgInlineEdgeStorage<1> m_inputs{this}; // Input edges const bool m_inSenItem; // Reference is in a sensitivity list const bool m_inLoop; // Reference is in a loop public: DfgAstRd(DfgGraph& dfg, AstNodeExpr* exprp, bool inSenItem, bool inLoop) - : DfgVertexAst{dfg, dfgType(), exprp} + : DfgVertexAst{dfg, dfgType(), exprp, m_inputs.m_edges, 1} , m_inSenItem{inSenItem} - , m_inLoop{inLoop} { - // Allocate sources - newInput(); - } + , m_inLoop{inLoop} {} ASTGEN_MEMBERS_DfgAstRd; DfgVertex* srcp() const { return inputp(0); } @@ -248,7 +258,7 @@ public: class DfgVertexNullary VL_NOT_FINAL : public DfgVertex { protected: DfgVertexNullary(DfgGraph& dfg, VDfgType type, FileLine* flp, const DfgDataType& dtype) - : DfgVertex{dfg, type, flp, dtype} {} + : DfgVertex{dfg, type, flp, dtype, nullptr, 0} {} public: ASTGEN_MEMBERS_DfgVertexNullary; @@ -298,11 +308,11 @@ public: // Unary vertices - 1 inputs class DfgVertexUnary VL_NOT_FINAL : public DfgVertex { + DfgInlineEdgeStorage<1> m_inputs{this}; // Input edges + protected: DfgVertexUnary(DfgGraph& dfg, VDfgType type, FileLine* flp, const DfgDataType& dtype) - : DfgVertex{dfg, type, flp, dtype} { - newInput(); - } + : DfgVertex{dfg, type, flp, dtype, m_inputs.m_edges, 1} {} public: ASTGEN_MEMBERS_DfgVertexUnary; @@ -356,12 +366,11 @@ public: // Binary vertices - 2 inputs class DfgVertexBinary VL_NOT_FINAL : public DfgVertex { + DfgInlineEdgeStorage<2> m_inputs{this}; // Input edges + protected: DfgVertexBinary(DfgGraph& dfg, VDfgType type, FileLine* flp, const DfgDataType& dtype) - : DfgVertex{dfg, type, flp, dtype} { - newInput(); - newInput(); - } + : DfgVertex{dfg, type, flp, dtype, m_inputs.m_edges, 2} {} public: ASTGEN_MEMBERS_DfgVertexBinary; @@ -403,13 +412,11 @@ public: // Ternary vertices - 3 inputs class DfgVertexTernary VL_NOT_FINAL : public DfgVertex { + DfgInlineEdgeStorage<3> m_inputs{this}; // Input edges + protected: DfgVertexTernary(DfgGraph& dfg, VDfgType type, FileLine* flp, const DfgDataType& dtype) - : DfgVertex{dfg, type, flp, dtype} { - newInput(); - newInput(); - newInput(); - } + : DfgVertex{dfg, type, flp, dtype, m_inputs.m_edges, 3} {} public: ASTGEN_MEMBERS_DfgVertexTernary; @@ -419,9 +426,26 @@ public: // Variadic vertices - variable number of inputs class DfgVertexVariadic VL_NOT_FINAL : public DfgVertex { + friend class DfgVertex; // For 'DfgVertex::inputEdgep' to access 'm_edgeps' + + std::vector> m_edgeps; // Input edges + protected: DfgVertexVariadic(DfgGraph& dfg, VDfgType type, FileLine* flp, const DfgDataType& dtype) - : DfgVertex{dfg, type, flp, dtype} {} + : DfgVertex{dfg, type, flp, dtype, nullptr, 0} {} + + // Create a new input edge and return it + DfgEdge* newInput() { + m_edgeps.emplace_back(new DfgEdge{this}); + m_nInputs = m_edgeps.size(); + return m_edgeps.back().get(); + } + + // Unlink all inputs and reset to no inputs - use very carefully + void resetInputs() { + m_edgeps.clear(); + m_nInputs = 0; + } public: ASTGEN_MEMBERS_DfgVertexVariadic;