diff --git a/src/V3Dfg.cpp b/src/V3Dfg.cpp index 66bbc205e..124bffda8 100644 --- a/src/V3Dfg.cpp +++ b/src/V3Dfg.cpp @@ -473,8 +473,11 @@ DfgGraph::neighborhood(const std::vector& vtxps, size_t n) con //------------------------------------------------------------------------------ // DfgVertex -DfgVertex::DfgVertex(DfgGraph& dfg, VDfgType type, FileLine* flp, const DfgDataType& dt) - : m_filelinep{flp} +DfgVertex::DfgVertex(DfgGraph& dfg, VDfgType type, FileLine* flp, const DfgDataType& dt, + DfgEdge* inlineInputsp, uint32_t nInputs) + : m_inlineInputsp{inlineInputsp} + , m_nInputs{nInputs} + , m_filelinep{flp} , m_dtype{dt} , m_type{type} { dfg.addVertex(*this); diff --git a/src/V3Dfg.h b/src/V3Dfg.h index 6bbb7bf2f..7ae3e0334 100644 --- a/src/V3Dfg.h +++ b/src/V3Dfg.h @@ -120,6 +120,28 @@ public: inline void relinkSrcp(DfgVertex* srcp); }; +//------------------------------------------------------------------------------ +// Input edge storage of a fixed arity vertices, embedded in the vertex itself. +template +class DfgInlineEdgeStorage final { + static_assert(N_Edges > 0, "'DfgInlineEdgeStorage' must hold at least one edge"); + VL_UNCOPYABLE(DfgInlineEdgeStorage); + VL_UNMOVABLE(DfgInlineEdgeStorage); + +public: + union { // Union, for manual memory management, so DfgEdge need no implicit constructor + DfgEdge m_edges[N_Edges]; // The input edges of the owning vertex + }; + + explicit DfgInlineEdgeStorage(DfgVertex* vtxp) { + for (uint32_t i = 0; i < N_Edges; ++i) new (&m_edges[i]) DfgEdge{vtxp}; + } + ~DfgInlineEdgeStorage() { + for (uint32_t i = 0; i < N_Edges; ++i) m_edges[i].~DfgEdge(); + } + DfgInlineEdgeStorage() = delete; +}; + //------------------------------------------------------------------------------ // Dataflow graph vertex class DfgVertex VL_NOT_FINAL { @@ -128,10 +150,12 @@ class DfgVertex VL_NOT_FINAL { friend class DfgVisitor; template friend class DfgUserMap; + friend class DfgVertexVariadic; // STATE V3ListLinks m_links; // V3List links in the DfgGraph - std::vector> m_inputps; // Input edges, as vector, for fast indexing + DfgEdge* const m_inlineInputsp; // Input edges stored inline in the vertex (iff fixed arity) + uint32_t m_nInputs; // Number of input edges DfgEdge::List m_sinks; // List of sink edges of this vertex FileLine* const m_filelinep; // Source location @@ -161,25 +185,22 @@ public: protected: // CONSTRUCTOR - DfgVertex(DfgGraph& dfg, VDfgType type, FileLine* flp, const DfgDataType& dt) VL_MT_DISABLED; + DfgVertex(DfgGraph& dfg, VDfgType type, FileLine* flp, const DfgDataType& dt, + DfgEdge* inlineInputsp, uint32_t nInputs) VL_MT_DISABLED; // Use unlinkDelete instead virtual ~DfgVertex() VL_MT_DISABLED = default; - // Create a new input edge and return it - DfgEdge* newInput() { - m_inputps.emplace_back(new DfgEdge{this}); - return m_inputps.back().get(); - } +private: + // Get input edge 'i' + inline DfgEdge* inputEdgep(size_t i) const; public: // Get input 'i' - DfgVertex* inputp(size_t i) const { return m_inputps[i]->srcp(); } + DfgVertex* inputp(size_t i) const { return inputEdgep(i)->srcp(); } // Relink input 'i' - void inputp(size_t i, DfgVertex* vtxp) { m_inputps[i]->relinkSrcp(vtxp); } + void inputp(size_t i, DfgVertex* vtxp) { inputEdgep(i)->relinkSrcp(vtxp); } // The number of inputs this vertex has. Some might be unconnected. - size_t nInputs() const { return m_inputps.size(); } - // Unlink all inputs and reset to no inputs - use very carefully - void resetInputs() { m_inputps.clear(); } + size_t nInputs() const { return m_nInputs; } // The type of this vertex VDfgType type() const { return m_type; } @@ -252,8 +273,8 @@ public: bool foreachSource(T_Callable&& f) { static_assert(vlstd::is_invocable_r::value, "T_Callable 'f' must have a signature compatible with 'bool(DfgVertex&)'"); - for (const std::unique_ptr& edgep : m_inputps) { - if (DfgVertex* const srcp = edgep->srcp()) { + for (size_t i = 0; i < m_nInputs; ++i) { + if (DfgVertex* const srcp = inputEdgep(i)->srcp()) { if (f(*srcp)) return true; } } @@ -268,8 +289,8 @@ public: static_assert( vlstd::is_invocable_r::value, "T_Callable 'f' must have a signature compatible with 'bool(const DfgVertex&)'"); - for (const std::unique_ptr& edgep : m_inputps) { - if (const DfgVertex* const srcp = edgep->srcp()) { + for (size_t i = 0; i < m_nInputs; ++i) { + if (const DfgVertex* const srcp = inputEdgep(i)->srcp()) { if (f(*srcp)) return true; } } @@ -826,6 +847,14 @@ void DfgEdge::relinkSrcp(DfgVertex* srcp) { // DfgVertex {{{ +DfgEdge* DfgVertex::inputEdgep(size_t i) const { + UDEBUGONLY(UASSERT_OBJ(i < m_nInputs, this, "Input index out of range");); + if (VL_LIKELY(m_inlineInputsp)) return m_inlineInputsp + i; + // 'm_inlineInputsp' is null exactly for a DfgVertexVariadic + UDEBUGONLY(UASSERT_OBJ(is(), this, "Vertex without input edge storage");); + return static_cast(this)->m_edgeps[i].get(); +} + bool DfgVertex::isCheaperThanLoad() const { // Constants if (is()) return true; diff --git a/src/V3DfgVertices.h b/src/V3DfgVertices.h index 13dbe3f92..f260afd5e 100644 --- a/src/V3DfgVertices.h +++ b/src/V3DfgVertices.h @@ -48,6 +48,7 @@ class DfgVertexVar VL_NOT_FINAL : public DfgVertex { // Represents a variable. It has 2 optional inputs, 'srcp' and 'defaultp'. + DfgInlineEdgeStorage<2> m_inputs{this}; // Input edges AstVarScope* const m_vscp; // The AstVarScope associated with this vertex (not owned) // Location of driver of this variable. Only used for converting back to Ast. Might be nullptr. FileLine* m_driverFileLine = nullptr; @@ -58,15 +59,16 @@ class DfgVertexVar VL_NOT_FINAL : public DfgVertex { protected: DfgVertexVar(DfgGraph& dfg, VDfgType type, AstVarScope* vscp) - : DfgVertex{dfg, type, vscp->varp()->fileline(), - *DfgDataType::fromAst(vscp->varp()->dtypep())} + : DfgVertex{dfg, + type, + vscp->varp()->fileline(), + *DfgDataType::fromAst(vscp->varp()->dtypep()), + m_inputs.m_edges, + 2} , m_vscp{vscp} { // Increment reference count m_vscp->user1(m_vscp->user1() + 0x40); UASSERT_OBJ((m_vscp->user1() >> 6) > 0, m_vscp, "Reference count overflow"); - // Allocate sources - newInput(); - newInput(); } public: @@ -183,8 +185,12 @@ class DfgPrev final : public DfgVertex { public: DfgPrev(DfgGraph& dfg, AstVarScope* vscp) - : DfgVertex{dfg, dfgType(), vscp->varp()->fileline(), - *DfgDataType::fromAst(vscp->varp()->dtypep())} + : DfgVertex{dfg, + dfgType(), + vscp->varp()->fileline(), + *DfgDataType::fromAst(vscp->varp()->dtypep()), + nullptr, + 0} , m_vscp{vscp} { UASSERT_OBJ(!DfgVertexVar::hasPrev(vscp), vscp, "Variable already has a DfgPrev"); m_vscp->user1(m_vscp->user1() | 0x20); // Mark having a DfgPrev @@ -209,8 +215,14 @@ class DfgVertexAst VL_NOT_FINAL : public DfgVertex { AstNodeExpr* m_exprp; // The AstNodeExpr representing this reference public: - DfgVertexAst(DfgGraph& dfg, VDfgType type, AstNodeExpr* exprp) - : DfgVertex{dfg, type, exprp->fileline(), *DfgDataType::fromAst(exprp->dtypep())} + DfgVertexAst(DfgGraph& dfg, VDfgType type, AstNodeExpr* exprp, DfgEdge* inlineInputsp, + uint32_t nInputs) + : DfgVertex{dfg, + type, + exprp->fileline(), + *DfgDataType::fromAst(exprp->dtypep()), + inlineInputsp, + nInputs} , m_exprp{exprp} {} ASTGEN_MEMBERS_DfgVertexAst; @@ -222,17 +234,15 @@ class DfgAstRd final : public DfgVertexAst { friend class DfgVertex; friend class DfgVisitor; + DfgInlineEdgeStorage<1> m_inputs{this}; // Input edges const bool m_inSenItem; // Reference is in a sensitivity list const bool m_inLoop; // Reference is in a loop public: DfgAstRd(DfgGraph& dfg, AstNodeExpr* exprp, bool inSenItem, bool inLoop) - : DfgVertexAst{dfg, dfgType(), exprp} + : DfgVertexAst{dfg, dfgType(), exprp, m_inputs.m_edges, 1} , m_inSenItem{inSenItem} - , m_inLoop{inLoop} { - // Allocate sources - newInput(); - } + , m_inLoop{inLoop} {} ASTGEN_MEMBERS_DfgAstRd; DfgVertex* srcp() const { return inputp(0); } @@ -248,7 +258,7 @@ public: class DfgVertexNullary VL_NOT_FINAL : public DfgVertex { protected: DfgVertexNullary(DfgGraph& dfg, VDfgType type, FileLine* flp, const DfgDataType& dtype) - : DfgVertex{dfg, type, flp, dtype} {} + : DfgVertex{dfg, type, flp, dtype, nullptr, 0} {} public: ASTGEN_MEMBERS_DfgVertexNullary; @@ -298,11 +308,11 @@ public: // Unary vertices - 1 inputs class DfgVertexUnary VL_NOT_FINAL : public DfgVertex { + DfgInlineEdgeStorage<1> m_inputs{this}; // Input edges + protected: DfgVertexUnary(DfgGraph& dfg, VDfgType type, FileLine* flp, const DfgDataType& dtype) - : DfgVertex{dfg, type, flp, dtype} { - newInput(); - } + : DfgVertex{dfg, type, flp, dtype, m_inputs.m_edges, 1} {} public: ASTGEN_MEMBERS_DfgVertexUnary; @@ -356,12 +366,11 @@ public: // Binary vertices - 2 inputs class DfgVertexBinary VL_NOT_FINAL : public DfgVertex { + DfgInlineEdgeStorage<2> m_inputs{this}; // Input edges + protected: DfgVertexBinary(DfgGraph& dfg, VDfgType type, FileLine* flp, const DfgDataType& dtype) - : DfgVertex{dfg, type, flp, dtype} { - newInput(); - newInput(); - } + : DfgVertex{dfg, type, flp, dtype, m_inputs.m_edges, 2} {} public: ASTGEN_MEMBERS_DfgVertexBinary; @@ -403,13 +412,11 @@ public: // Ternary vertices - 3 inputs class DfgVertexTernary VL_NOT_FINAL : public DfgVertex { + DfgInlineEdgeStorage<3> m_inputs{this}; // Input edges + protected: DfgVertexTernary(DfgGraph& dfg, VDfgType type, FileLine* flp, const DfgDataType& dtype) - : DfgVertex{dfg, type, flp, dtype} { - newInput(); - newInput(); - newInput(); - } + : DfgVertex{dfg, type, flp, dtype, m_inputs.m_edges, 3} {} public: ASTGEN_MEMBERS_DfgVertexTernary; @@ -419,9 +426,26 @@ public: // Variadic vertices - variable number of inputs class DfgVertexVariadic VL_NOT_FINAL : public DfgVertex { + friend class DfgVertex; // For 'DfgVertex::inputEdgep' to access 'm_edgeps' + + std::vector> m_edgeps; // Input edges + protected: DfgVertexVariadic(DfgGraph& dfg, VDfgType type, FileLine* flp, const DfgDataType& dtype) - : DfgVertex{dfg, type, flp, dtype} {} + : DfgVertex{dfg, type, flp, dtype, nullptr, 0} {} + + // Create a new input edge and return it + DfgEdge* newInput() { + m_edgeps.emplace_back(new DfgEdge{this}); + m_nInputs = m_edgeps.size(); + return m_edgeps.back().get(); + } + + // Unlink all inputs and reset to no inputs - use very carefully + void resetInputs() { + m_edgeps.clear(); + m_nInputs = 0; + } public: ASTGEN_MEMBERS_DfgVertexVariadic;