Optimize Dfg vertex input edge storage (#8313)

Store input edges of fixed arity vertices inline in the vertex class.
This reduces heap allocations, memory fragmentation, and pointer chasing
and speeds up Dfg passes. The extra branch introduced in inputEdgep() is
well predictable and profiling shows branchless alternatives are a loss.

src/V3Dfg.cpp

@@ -473,8 +473,11 @@ DfgGraph::neighborhood(const std::vector<const DfgVertex*>& vtxps, size_t n) con
 //------------------------------------------------------------------------------
 // DfgVertex
 
-DfgVertex::DfgVertex(DfgGraph& dfg, VDfgType type, FileLine* flp, const DfgDataType& dt)
-    : m_filelinep{flp}
+DfgVertex::DfgVertex(DfgGraph& dfg, VDfgType type, FileLine* flp, const DfgDataType& dt,
+                     DfgEdge* inlineInputsp, uint32_t nInputs)
+    : m_inlineInputsp{inlineInputsp}
+    , m_nInputs{nInputs}
+    , m_filelinep{flp}
     , m_dtype{dt}
     , m_type{type} {
     dfg.addVertex(*this);

src/V3Dfg.h

@@ -120,6 +120,28 @@ public:
     inline void relinkSrcp(DfgVertex* srcp);
 };
 
+//------------------------------------------------------------------------------
+// Input edge storage of a fixed arity vertices, embedded in the vertex itself.
+template <uint32_t N_Edges>
+class DfgInlineEdgeStorage final {
+    static_assert(N_Edges > 0, "'DfgInlineEdgeStorage' must hold at least one edge");
+    VL_UNCOPYABLE(DfgInlineEdgeStorage);
+    VL_UNMOVABLE(DfgInlineEdgeStorage);
+
+public:
+    union {  // Union, for manual memory management, so DfgEdge need no implicit constructor
+        DfgEdge m_edges[N_Edges];  // The input edges of the owning vertex
+    };
+
+    explicit DfgInlineEdgeStorage(DfgVertex* vtxp) {
+        for (uint32_t i = 0; i < N_Edges; ++i) new (&m_edges[i]) DfgEdge{vtxp};
+    }
+    ~DfgInlineEdgeStorage() {
+        for (uint32_t i = 0; i < N_Edges; ++i) m_edges[i].~DfgEdge();
+    }
+    DfgInlineEdgeStorage() = delete;
+};
+
 //------------------------------------------------------------------------------
 // Dataflow graph vertex
 class DfgVertex VL_NOT_FINAL {
@@ -128,10 +150,12 @@ class DfgVertex VL_NOT_FINAL {
     friend class DfgVisitor;
     template <typename, bool>
     friend class DfgUserMap;
+    friend class DfgVertexVariadic;
 
     // STATE
     V3ListLinks<DfgVertex> m_links;  // V3List links in the DfgGraph
-    std::vector<std::unique_ptr<DfgEdge>> m_inputps;  // Input edges, as vector, for fast indexing
+    DfgEdge* const m_inlineInputsp;  // Input edges stored inline in the vertex (iff fixed arity)
+    uint32_t m_nInputs;  // Number of input edges
     DfgEdge::List m_sinks;  // List of sink edges of this vertex
 
     FileLine* const m_filelinep;  // Source location
@@ -161,25 +185,22 @@ public:
 
 protected:
     // CONSTRUCTOR
-    DfgVertex(DfgGraph& dfg, VDfgType type, FileLine* flp, const DfgDataType& dt) VL_MT_DISABLED;
+    DfgVertex(DfgGraph& dfg, VDfgType type, FileLine* flp, const DfgDataType& dt,
+              DfgEdge* inlineInputsp, uint32_t nInputs) VL_MT_DISABLED;
     // Use unlinkDelete instead
     virtual ~DfgVertex() VL_MT_DISABLED = default;
 
-    // Create a new input edge and return it
-    DfgEdge* newInput() {
-        m_inputps.emplace_back(new DfgEdge{this});
-        return m_inputps.back().get();
-    }
+private:
+    // Get input edge 'i'
+    inline DfgEdge* inputEdgep(size_t i) const;
 
 public:
     // Get input 'i'
-    DfgVertex* inputp(size_t i) const { return m_inputps[i]->srcp(); }
+    DfgVertex* inputp(size_t i) const { return inputEdgep(i)->srcp(); }
     // Relink input 'i'
-    void inputp(size_t i, DfgVertex* vtxp) { m_inputps[i]->relinkSrcp(vtxp); }
+    void inputp(size_t i, DfgVertex* vtxp) { inputEdgep(i)->relinkSrcp(vtxp); }
     // The number of inputs this vertex has. Some might be unconnected.
-    size_t nInputs() const { return m_inputps.size(); }
-    // Unlink all inputs and reset to no inputs - use very carefully
-    void resetInputs() { m_inputps.clear(); }
+    size_t nInputs() const { return m_nInputs; }
 
     // The type of this vertex
     VDfgType type() const { return m_type; }
@@ -252,8 +273,8 @@ public:
     bool foreachSource(T_Callable&& f) {
         static_assert(vlstd::is_invocable_r<bool, T_Callable, DfgVertex&>::value,
                       "T_Callable 'f' must have a signature compatible with 'bool(DfgVertex&)'");
-        for (const std::unique_ptr<DfgEdge>& edgep : m_inputps) {
-            if (DfgVertex* const srcp = edgep->srcp()) {
+        for (size_t i = 0; i < m_nInputs; ++i) {
+            if (DfgVertex* const srcp = inputEdgep(i)->srcp()) {
                 if (f(*srcp)) return true;
             }
         }
@@ -268,8 +289,8 @@ public:
         static_assert(
             vlstd::is_invocable_r<bool, T_Callable, const DfgVertex&>::value,
             "T_Callable 'f' must have a signature compatible with 'bool(const DfgVertex&)'");
-        for (const std::unique_ptr<DfgEdge>& edgep : m_inputps) {
-            if (const DfgVertex* const srcp = edgep->srcp()) {
+        for (size_t i = 0; i < m_nInputs; ++i) {
+            if (const DfgVertex* const srcp = inputEdgep(i)->srcp()) {
                 if (f(*srcp)) return true;
             }
         }
@@ -826,6 +847,14 @@ void DfgEdge::relinkSrcp(DfgVertex* srcp) {
 
 // DfgVertex {{{
 
+DfgEdge* DfgVertex::inputEdgep(size_t i) const {
+    UDEBUGONLY(UASSERT_OBJ(i < m_nInputs, this, "Input index out of range"););
+    if (VL_LIKELY(m_inlineInputsp)) return m_inlineInputsp + i;
+    // 'm_inlineInputsp' is null exactly for a DfgVertexVariadic
+    UDEBUGONLY(UASSERT_OBJ(is<DfgVertexVariadic>(), this, "Vertex without input edge storage"););
+    return static_cast<const DfgVertexVariadic*>(this)->m_edgeps[i].get();
+}
+
 bool DfgVertex::isCheaperThanLoad() const {
     // Constants
     if (is<DfgConst>()) return true;

src/V3DfgVertices.h

@@ -48,6 +48,7 @@
 class DfgVertexVar VL_NOT_FINAL : public DfgVertex {
     // Represents a variable. It has 2 optional inputs, 'srcp' and 'defaultp'.
 
+    DfgInlineEdgeStorage<2> m_inputs{this};  // Input edges
     AstVarScope* const m_vscp;  // The AstVarScope associated with this vertex (not owned)
     // Location of driver of this variable. Only used for converting back to Ast. Might be nullptr.
     FileLine* m_driverFileLine = nullptr;
@@ -58,15 +59,16 @@ class DfgVertexVar VL_NOT_FINAL : public DfgVertex {
 
 protected:
     DfgVertexVar(DfgGraph& dfg, VDfgType type, AstVarScope* vscp)
-        : DfgVertex{dfg, type, vscp->varp()->fileline(),
-                    *DfgDataType::fromAst(vscp->varp()->dtypep())}
+        : DfgVertex{dfg,
+                    type,
+                    vscp->varp()->fileline(),
+                    *DfgDataType::fromAst(vscp->varp()->dtypep()),
+                    m_inputs.m_edges,
+                    2}
         , m_vscp{vscp} {
         // Increment reference count
         m_vscp->user1(m_vscp->user1() + 0x40);
         UASSERT_OBJ((m_vscp->user1() >> 6) > 0, m_vscp, "Reference count overflow");
-        // Allocate sources
-        newInput();
-        newInput();
     }
 
 public:
@@ -183,8 +185,12 @@ class DfgPrev final : public DfgVertex {
 
 public:
     DfgPrev(DfgGraph& dfg, AstVarScope* vscp)
-        : DfgVertex{dfg, dfgType(), vscp->varp()->fileline(),
-                    *DfgDataType::fromAst(vscp->varp()->dtypep())}
+        : DfgVertex{dfg,
+                    dfgType(),
+                    vscp->varp()->fileline(),
+                    *DfgDataType::fromAst(vscp->varp()->dtypep()),
+                    nullptr,
+                    0}
         , m_vscp{vscp} {
         UASSERT_OBJ(!DfgVertexVar::hasPrev(vscp), vscp, "Variable already has a DfgPrev");
         m_vscp->user1(m_vscp->user1() | 0x20);  // Mark having a DfgPrev
@@ -209,8 +215,14 @@ class DfgVertexAst VL_NOT_FINAL : public DfgVertex {
     AstNodeExpr* m_exprp;  // The AstNodeExpr representing this reference
 
 public:
-    DfgVertexAst(DfgGraph& dfg, VDfgType type, AstNodeExpr* exprp)
-        : DfgVertex{dfg, type, exprp->fileline(), *DfgDataType::fromAst(exprp->dtypep())}
+    DfgVertexAst(DfgGraph& dfg, VDfgType type, AstNodeExpr* exprp, DfgEdge* inlineInputsp,
+                 uint32_t nInputs)
+        : DfgVertex{dfg,
+                    type,
+                    exprp->fileline(),
+                    *DfgDataType::fromAst(exprp->dtypep()),
+                    inlineInputsp,
+                    nInputs}
         , m_exprp{exprp} {}
     ASTGEN_MEMBERS_DfgVertexAst;
 
@@ -222,17 +234,15 @@ class DfgAstRd final : public DfgVertexAst {
     friend class DfgVertex;
     friend class DfgVisitor;
 
+    DfgInlineEdgeStorage<1> m_inputs{this};  // Input edges
     const bool m_inSenItem;  // Reference is in a sensitivity list
     const bool m_inLoop;  // Reference is in a loop
 
 public:
     DfgAstRd(DfgGraph& dfg, AstNodeExpr* exprp, bool inSenItem, bool inLoop)
-        : DfgVertexAst{dfg, dfgType(), exprp}
+        : DfgVertexAst{dfg, dfgType(), exprp, m_inputs.m_edges, 1}
         , m_inSenItem{inSenItem}
-        , m_inLoop{inLoop} {
-        // Allocate sources
-        newInput();
-    }
+        , m_inLoop{inLoop} {}
     ASTGEN_MEMBERS_DfgAstRd;
 
     DfgVertex* srcp() const { return inputp(0); }
@@ -248,7 +258,7 @@ public:
 class DfgVertexNullary VL_NOT_FINAL : public DfgVertex {
 protected:
     DfgVertexNullary(DfgGraph& dfg, VDfgType type, FileLine* flp, const DfgDataType& dtype)
-        : DfgVertex{dfg, type, flp, dtype} {}
+        : DfgVertex{dfg, type, flp, dtype, nullptr, 0} {}
 
 public:
     ASTGEN_MEMBERS_DfgVertexNullary;
@@ -298,11 +308,11 @@ public:
 // Unary vertices - 1 inputs
 
 class DfgVertexUnary VL_NOT_FINAL : public DfgVertex {
+    DfgInlineEdgeStorage<1> m_inputs{this};  // Input edges
+
 protected:
     DfgVertexUnary(DfgGraph& dfg, VDfgType type, FileLine* flp, const DfgDataType& dtype)
-        : DfgVertex{dfg, type, flp, dtype} {
-        newInput();
-    }
+        : DfgVertex{dfg, type, flp, dtype, m_inputs.m_edges, 1} {}
 
 public:
     ASTGEN_MEMBERS_DfgVertexUnary;
@@ -356,12 +366,11 @@ public:
 // Binary vertices - 2 inputs
 
 class DfgVertexBinary VL_NOT_FINAL : public DfgVertex {
+    DfgInlineEdgeStorage<2> m_inputs{this};  // Input edges
+
 protected:
     DfgVertexBinary(DfgGraph& dfg, VDfgType type, FileLine* flp, const DfgDataType& dtype)
-        : DfgVertex{dfg, type, flp, dtype} {
-        newInput();
-        newInput();
-    }
+        : DfgVertex{dfg, type, flp, dtype, m_inputs.m_edges, 2} {}
 
 public:
     ASTGEN_MEMBERS_DfgVertexBinary;
@@ -403,13 +412,11 @@ public:
 // Ternary vertices - 3 inputs
 
 class DfgVertexTernary VL_NOT_FINAL : public DfgVertex {
+    DfgInlineEdgeStorage<3> m_inputs{this};  // Input edges
+
 protected:
     DfgVertexTernary(DfgGraph& dfg, VDfgType type, FileLine* flp, const DfgDataType& dtype)
-        : DfgVertex{dfg, type, flp, dtype} {
-        newInput();
-        newInput();
-        newInput();
-    }
+        : DfgVertex{dfg, type, flp, dtype, m_inputs.m_edges, 3} {}
 
 public:
     ASTGEN_MEMBERS_DfgVertexTernary;
@@ -419,9 +426,26 @@ public:
 // Variadic vertices - variable number of inputs
 
 class DfgVertexVariadic VL_NOT_FINAL : public DfgVertex {
+    friend class DfgVertex;  // For 'DfgVertex::inputEdgep' to access 'm_edgeps'
+
+    std::vector<std::unique_ptr<DfgEdge>> m_edgeps;  // Input edges
+
 protected:
     DfgVertexVariadic(DfgGraph& dfg, VDfgType type, FileLine* flp, const DfgDataType& dtype)
-        : DfgVertex{dfg, type, flp, dtype} {}
+        : DfgVertex{dfg, type, flp, dtype, nullptr, 0} {}
+
+    // Create a new input edge and return it
+    DfgEdge* newInput() {
+        m_edgeps.emplace_back(new DfgEdge{this});
+        m_nInputs = m_edgeps.size();
+        return m_edgeps.back().get();
+    }
+
+    // Unlink all inputs and reset to no inputs - use very carefully
+    void resetInputs() {
+        m_edgeps.clear();
+        m_nInputs = 0;
+    }
 
 public:
     ASTGEN_MEMBERS_DfgVertexVariadic;