2012-04-13 03:08:20 +02:00
|
|
|
// -*- mode: C++; c-file-style: "cc-mode" -*-
|
2006-08-26 13:35:28 +02:00
|
|
|
//*************************************************************************
|
|
|
|
|
// DESCRIPTION: Verilator: Add temporaries, such as for inline nodes
|
|
|
|
|
//
|
2019-11-08 04:33:59 +01:00
|
|
|
// Code available from: https://verilator.org
|
2006-08-26 13:35:28 +02:00
|
|
|
//
|
|
|
|
|
//*************************************************************************
|
|
|
|
|
//
|
2026-01-27 02:24:34 +01:00
|
|
|
// This program is free software; you can redistribute it and/or modify it
|
|
|
|
|
// under the terms of either the GNU Lesser General Public License Version 3
|
|
|
|
|
// or the Perl Artistic License Version 2.0.
|
|
|
|
|
// SPDX-FileCopyrightText: 2003-2026 Wilson Snyder
|
2020-03-21 16:24:24 +01:00
|
|
|
// SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0
|
2006-08-26 13:35:28 +02:00
|
|
|
//
|
|
|
|
|
//*************************************************************************
|
|
|
|
|
// V3Inline's Transformations:
|
2008-06-10 03:25:10 +02:00
|
|
|
//
|
2006-08-26 13:35:28 +02:00
|
|
|
// Each module:
|
2019-05-19 22:13:13 +02:00
|
|
|
// Look for CELL... PRAGMA INLINE_MODULE
|
|
|
|
|
// Replicate the cell's module
|
|
|
|
|
// Convert pins to wires that make assignments
|
|
|
|
|
// Rename vars to include cell name
|
|
|
|
|
// Insert cell's module statements into the upper module
|
2006-08-26 13:35:28 +02:00
|
|
|
//
|
|
|
|
|
//*************************************************************************
|
2019-10-05 02:17:11 +02:00
|
|
|
|
2023-10-18 12:37:46 +02:00
|
|
|
#include "V3PchAstNoMT.h" // VL_MT_DISABLED_CODE_UNIT
|
|
|
|
|
|
2006-08-26 13:35:28 +02:00
|
|
|
#include "V3Inline.h"
|
2022-08-05 11:56:57 +02:00
|
|
|
|
|
|
|
|
#include "V3AstUserAllocator.h"
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
#include "V3Graph.h"
|
2006-08-26 13:35:28 +02:00
|
|
|
#include "V3Inst.h"
|
|
|
|
|
#include "V3Stats.h"
|
|
|
|
|
|
2020-08-15 16:03:34 +02:00
|
|
|
#include <unordered_set>
|
2021-12-22 12:41:29 +01:00
|
|
|
#include <vector>
|
2018-10-14 19:43:24 +02:00
|
|
|
|
2022-09-18 21:53:42 +02:00
|
|
|
VL_DEFINE_DEBUG_FUNCTIONS;
|
|
|
|
|
|
2006-08-26 13:35:28 +02:00
|
|
|
// CONFIG
|
2019-05-19 22:13:13 +02:00
|
|
|
static const int INLINE_MODS_SMALLER = 100; // If a mod is < this # nodes, can always inline it
|
2006-08-26 13:35:28 +02:00
|
|
|
|
|
|
|
|
//######################################################################
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
// Bipartite module instantiation graph containing module and cell vertices
|
2021-12-22 12:41:29 +01:00
|
|
|
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
class InlineModModuleVertex;
|
|
|
|
|
class InlineModCellVertex;
|
2021-12-22 12:41:29 +01:00
|
|
|
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
class InlineModGraph final : public V3Graph {
|
|
|
|
|
// NODE STATE
|
|
|
|
|
// AstNodeModule::user4p() -> InlineModModuleVertex*, the module vertex
|
|
|
|
|
// AstCell::user4p() -> InlineModCellVertex*, the cell vertex
|
2021-12-22 12:41:29 +01:00
|
|
|
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
VNUser4InUse m_user4InUse;
|
2021-12-22 12:41:29 +01:00
|
|
|
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
public:
|
|
|
|
|
InlineModGraph()
|
|
|
|
|
: V3Graph{} {}
|
|
|
|
|
~InlineModGraph() override = default;
|
2006-08-26 13:35:28 +02:00
|
|
|
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
// METHODS
|
|
|
|
|
InlineModModuleVertex* getInlineModModuleVertexp(AstNodeModule* modp);
|
|
|
|
|
InlineModCellVertex* getInlineModCellVertexp(AstCell* cellp);
|
|
|
|
|
void addEdge(InlineModModuleVertex& from, InlineModCellVertex& to);
|
|
|
|
|
void addEdge(InlineModCellVertex& from, InlineModModuleVertex& to);
|
2013-05-26 17:17:42 +02:00
|
|
|
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
// debug
|
|
|
|
|
std::string dotRankDir() const override { return "LR"; }
|
|
|
|
|
};
|
2017-10-02 00:00:27 +02:00
|
|
|
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
class InlineModEitherVertex VL_NOT_FINAL : public V3GraphVertex {
|
|
|
|
|
VL_RTTI_IMPL(InlineModEitherVertex, V3GraphVertex)
|
|
|
|
|
protected:
|
|
|
|
|
explicit InlineModEitherVertex(InlineModGraph& graph)
|
|
|
|
|
: V3GraphVertex{&graph} {}
|
|
|
|
|
};
|
2017-10-02 00:00:27 +02:00
|
|
|
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
class InlineModModuleVertex final : public InlineModEitherVertex {
|
|
|
|
|
VL_RTTI_IMPL(InlineModModuleVertex, InlineModEitherVertex)
|
|
|
|
|
AstNodeModule* const m_modp; // The module
|
|
|
|
|
const char* m_noInlineHardWyp = nullptr; // First reason the module can never be inlined
|
|
|
|
|
const char* m_noInlineSoftWyp = nullptr; // First reason not to inline unless forced
|
|
|
|
|
const char* m_shouldInlineWhyp = nullptr; // First reason why this module should be inlined
|
|
|
|
|
size_t m_size = 0; // The size (statement count) of the module
|
|
|
|
|
size_t mutable m_flattenedSize = 0; // The size of the module if flattened
|
|
|
|
|
bool mutable m_flattenedSizeValid = false; // Whether the flattened size is valid
|
|
|
|
|
size_t mutable m_instanceCount = 0; // The number of total instances of this module
|
|
|
|
|
bool mutable m_instanceCountValid = false; // Whether the instance count is valid
|
2017-10-02 00:00:27 +02:00
|
|
|
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
public:
|
|
|
|
|
InlineModModuleVertex(InlineModGraph& graph, AstNodeModule* modp)
|
|
|
|
|
: InlineModEitherVertex{graph}
|
|
|
|
|
, m_modp{modp} {}
|
|
|
|
|
~InlineModModuleVertex() override = default;
|
|
|
|
|
|
|
|
|
|
// ACCESSORS
|
|
|
|
|
AstNodeModule* modp() const { return m_modp; }
|
|
|
|
|
size_t size() const { return m_size; }
|
|
|
|
|
void size(size_t value) { m_size = value; }
|
|
|
|
|
void sizeInc(size_t value = 1) { m_size += value; }
|
|
|
|
|
bool noInlineHard() const { return m_noInlineHardWyp; }
|
|
|
|
|
void setNoInlineHard(const char* whyp) {
|
|
|
|
|
if (!m_noInlineHardWyp) m_noInlineHardWyp = whyp;
|
|
|
|
|
}
|
|
|
|
|
bool noInlineSoft() const { return m_noInlineSoftWyp; }
|
|
|
|
|
void setNoInlineSoft(const char* whyp) {
|
|
|
|
|
if (!m_noInlineSoftWyp) m_noInlineSoftWyp = whyp;
|
|
|
|
|
}
|
|
|
|
|
bool shouldInline() const { return m_shouldInlineWhyp; }
|
|
|
|
|
void setShouldInline(const char* whyp) {
|
|
|
|
|
if (!m_shouldInlineWhyp) m_shouldInlineWhyp = whyp;
|
|
|
|
|
}
|
|
|
|
|
// Mark every instance below this module for inlining
|
|
|
|
|
void setFlatten();
|
|
|
|
|
|
|
|
|
|
// Total size of module, with all hierarchy below flattened
|
|
|
|
|
size_t flattenedSize() const {
|
|
|
|
|
if (!m_flattenedSizeValid) {
|
|
|
|
|
m_flattenedSizeValid = true;
|
|
|
|
|
m_flattenedSize = m_size;
|
|
|
|
|
for (const V3GraphEdge& e1 : outEdges()) {
|
|
|
|
|
for (const V3GraphEdge& e2 : e1.top()->outEdges()) {
|
|
|
|
|
InlineModModuleVertex* const mVtxp = e2.top()->as<InlineModModuleVertex>();
|
|
|
|
|
m_flattenedSize += mVtxp->flattenedSize();
|
|
|
|
|
}
|
2019-05-19 22:13:13 +02:00
|
|
|
}
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
}
|
|
|
|
|
return m_flattenedSize;
|
|
|
|
|
}
|
|
|
|
|
// Total number of instances of this module in the whole hierarchy of the design
|
|
|
|
|
size_t instanceCount() const {
|
|
|
|
|
if (!m_instanceCountValid) {
|
|
|
|
|
m_instanceCountValid = true;
|
|
|
|
|
m_instanceCount = 0;
|
|
|
|
|
for (const V3GraphEdge& e1 : inEdges()) {
|
|
|
|
|
for (const V3GraphEdge& e2 : e1.fromp()->inEdges()) {
|
|
|
|
|
InlineModModuleVertex* const mVtxp = e2.fromp()->as<InlineModModuleVertex>();
|
|
|
|
|
m_instanceCount += mVtxp->instanceCount();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (!m_instanceCount) {
|
|
|
|
|
UASSERT_OBJ(m_modp->isTop(), m_modp, "non-top level module should have instances");
|
|
|
|
|
m_instanceCount = 1;
|
2019-05-19 22:13:13 +02:00
|
|
|
}
|
|
|
|
|
}
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
return m_instanceCount;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// debug
|
|
|
|
|
FileLine* fileline() const override { return m_modp->fileline(); }
|
|
|
|
|
std::string dotShape() const override { return "box"; }
|
|
|
|
|
std::string dotColor() const override {
|
|
|
|
|
return m_noInlineHardWyp ? "red"
|
|
|
|
|
: m_shouldInlineWhyp ? "blue"
|
|
|
|
|
: m_noInlineSoftWyp ? "orange"
|
|
|
|
|
: "black";
|
|
|
|
|
}
|
|
|
|
|
std::string name() const override VL_MT_STABLE {
|
|
|
|
|
std::string str = m_modp->typeName() + " "s + cvtToHex(m_modp);
|
|
|
|
|
str += "\n" + m_modp->name() + " @ " + fileline()->ascii();
|
|
|
|
|
str += "\ninstanceCount: " + std::to_string(instanceCount());
|
|
|
|
|
str += "\nsize: " + std::to_string(m_size);
|
|
|
|
|
str += "\nflattenedSize: " + std::to_string(flattenedSize());
|
|
|
|
|
if (m_shouldInlineWhyp) str += "\nShouldInline: "s + m_shouldInlineWhyp;
|
|
|
|
|
if (m_noInlineHardWyp) str += "\nNoInlineHard: "s + m_noInlineHardWyp;
|
|
|
|
|
if (m_noInlineSoftWyp) str += "\nNoInlineSoft: "s + m_noInlineSoftWyp;
|
|
|
|
|
str += "\n";
|
|
|
|
|
return str;
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
class InlineModCellVertex final : public InlineModEitherVertex {
|
|
|
|
|
VL_RTTI_IMPL(InlineModCellVertex, InlineModEitherVertex)
|
|
|
|
|
AstCell* const m_cellp; // The cell (instance)
|
|
|
|
|
const char* m_doInlineWyp = nullptr; // First reason this instance should be inlined
|
|
|
|
|
bool m_flatten = false; // Whether this cell and below already flattened (avoid O(n^2))
|
|
|
|
|
|
|
|
|
|
public:
|
|
|
|
|
InlineModCellVertex(InlineModGraph& graph, AstCell* cellp)
|
|
|
|
|
: InlineModEitherVertex{graph}
|
|
|
|
|
, m_cellp{cellp} {}
|
|
|
|
|
~InlineModCellVertex() override = default;
|
|
|
|
|
|
|
|
|
|
// ACCESSORS
|
|
|
|
|
AstCell* cellp() const { return m_cellp; }
|
|
|
|
|
bool doInline() const { return m_doInlineWyp; }
|
|
|
|
|
void setDoInline(const char* whyp) {
|
|
|
|
|
if (!m_doInlineWyp) m_doInlineWyp = whyp;
|
|
|
|
|
}
|
|
|
|
|
bool flatten() const { return m_flatten; }
|
|
|
|
|
void setFlatten() { m_flatten = true; }
|
|
|
|
|
|
|
|
|
|
// The module vertx this cell is instantiating
|
|
|
|
|
InlineModModuleVertex& instanceOf() const {
|
|
|
|
|
UASSERT_OBJ(outSize1(), this, "Cell should have exactly one outgoing edge");
|
|
|
|
|
return *outEdges().frontp()->top()->as<InlineModModuleVertex>();
|
|
|
|
|
}
|
|
|
|
|
// The module vertex this cell is instantiated in
|
|
|
|
|
InlineModModuleVertex& instanceIn() const {
|
|
|
|
|
UASSERT_OBJ(inSize1(), this, "Cell should have exactly one incoming edge");
|
|
|
|
|
return *inEdges().frontp()->fromp()->as<InlineModModuleVertex>();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// debug
|
|
|
|
|
FileLine* fileline() const override { return m_cellp->fileline(); }
|
|
|
|
|
std::string dotColor() const override { return m_doInlineWyp ? "green" : "black"; }
|
|
|
|
|
std::string dotShape() const override { return "ellipse"; }
|
|
|
|
|
std::string name() const override VL_MT_STABLE {
|
|
|
|
|
std::string str = m_cellp->typeName() + " "s + cvtToHex(m_cellp);
|
|
|
|
|
str += "\n" + m_cellp->name() + " @ " + fileline()->ascii();
|
|
|
|
|
if (m_doInlineWyp) str += "\nDoInline: "s + m_doInlineWyp;
|
|
|
|
|
str += "\n";
|
|
|
|
|
return str;
|
2006-08-26 13:35:28 +02:00
|
|
|
}
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
};
|
|
|
|
|
|
|
|
|
|
InlineModModuleVertex* InlineModGraph::getInlineModModuleVertexp(AstNodeModule* modp) {
|
|
|
|
|
if (!modp->user4p()) modp->user4p(new InlineModModuleVertex{*this, modp});
|
|
|
|
|
return modp->user4u().to<InlineModModuleVertex*>();
|
|
|
|
|
}
|
|
|
|
|
InlineModCellVertex* InlineModGraph::getInlineModCellVertexp(AstCell* cellp) {
|
|
|
|
|
if (!cellp->user4p()) cellp->user4p(new InlineModCellVertex{*this, cellp});
|
|
|
|
|
return cellp->user4u().to<InlineModCellVertex*>();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void InlineModGraph::addEdge(InlineModModuleVertex& parent, InlineModCellVertex& cell) {
|
|
|
|
|
UASSERT_OBJ(cell.inEmpty(), &cell, "Cell should have at most one incoming edge");
|
|
|
|
|
new V3GraphEdge{this, &parent, &cell, 1, /* cutable: */ false};
|
|
|
|
|
}
|
|
|
|
|
void InlineModGraph::addEdge(InlineModCellVertex& cell, InlineModModuleVertex& submodule) {
|
|
|
|
|
UASSERT_OBJ(cell.outEmpty(), &cell, "Cell should have at most one outgoing edge");
|
|
|
|
|
new V3GraphEdge{this, &cell, &submodule, 1, /* cutable: */ false};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void InlineModModuleVertex::setFlatten() {
|
|
|
|
|
for (V3GraphEdge& edge : outEdges()) {
|
|
|
|
|
InlineModCellVertex& cVtx = *edge.top()->as<InlineModCellVertex>();
|
|
|
|
|
if (cVtx.flatten()) continue;
|
|
|
|
|
cVtx.setFlatten();
|
|
|
|
|
InlineModModuleVertex& iVtx = cVtx.instanceOf();
|
|
|
|
|
if (!iVtx.noInlineHard() && !iVtx.noInlineSoft()) cVtx.setDoInline("flatten parent");
|
|
|
|
|
iVtx.setFlatten();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
//######################################################################
|
|
|
|
|
// Visitor that builds the bipartite module instantiation graph
|
|
|
|
|
|
|
|
|
|
class InlineModGraphBuilder final : public VNVisitor {
|
|
|
|
|
// STATE
|
|
|
|
|
std::unique_ptr<InlineModGraph> m_graphp{new InlineModGraph}; // The graph being built
|
|
|
|
|
InlineModModuleVertex* m_modVtxp = nullptr; // Vertex of module currently being iterated
|
2006-08-26 13:35:28 +02:00
|
|
|
|
|
|
|
|
// VISITORS
|
2022-09-16 12:22:11 +02:00
|
|
|
void visit(AstNodeModule* nodep) override {
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
if (nodep == v3Global.rootp()->constPoolp()->modp()) return; // Ignore const pool module
|
|
|
|
|
|
|
|
|
|
UASSERT_OBJ(!m_modVtxp, nodep, "Unsupported: Nested modules");
|
|
|
|
|
|
|
|
|
|
// Create the module vertex
|
|
|
|
|
InlineModModuleVertex* const vtxp = m_graphp->getInlineModModuleVertexp(nodep);
|
|
|
|
|
|
|
|
|
|
// Check if the module itself is not inlineable
|
|
|
|
|
|
|
|
|
|
// Inlining an interface means we no longer have a cell handle to resolve to.
|
|
|
|
|
// If inlining moves post-scope this can perhaps be relaxed.
|
|
|
|
|
if (VN_IS(nodep, Iface)) vtxp->setNoInlineHard("Interface");
|
|
|
|
|
// Never inline packages - TODO: conceptually fine, but why not?
|
|
|
|
|
if (VN_IS(nodep, Package)) vtxp->setNoInlineHard("Package");
|
|
|
|
|
// A --lib-create library stub instance that needs tracing must not be
|
|
|
|
|
// inlined, so we still know it is a lib stub in V3TraceDecl (see #7001)
|
|
|
|
|
if (nodep->verilatorLib() && v3Global.opt.trace()) {
|
|
|
|
|
vtxp->setNoInlineHard("verilatorLib with --trace");
|
2026-02-13 02:54:03 +01:00
|
|
|
}
|
2017-10-02 00:00:27 +02:00
|
|
|
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
// Don't inline public modules by default
|
|
|
|
|
if (nodep->modPublic()) vtxp->setNoInlineSoft("Public module");
|
|
|
|
|
|
|
|
|
|
// Iterate children
|
|
|
|
|
VL_RESTORER(m_modVtxp);
|
|
|
|
|
m_modVtxp = vtxp;
|
|
|
|
|
iterateChildrenConst(nodep);
|
2006-08-26 13:35:28 +02:00
|
|
|
}
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
|
2022-09-16 12:22:11 +02:00
|
|
|
void visit(AstClass* nodep) override {
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
// TODO allow inlining of modules that contain classes
|
|
|
|
|
if (m_modVtxp) m_modVtxp->setNoInlineHard("Contains class");
|
|
|
|
|
iterateChildrenConst(nodep); // TODO: this is only needed or FTaskRef cleanup
|
2020-04-05 15:30:23 +02:00
|
|
|
}
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
|
|
|
|
|
// Cells instantiate modules
|
2022-09-16 12:22:11 +02:00
|
|
|
void visit(AstCell* nodep) override {
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
UASSERT_OBJ(m_modVtxp, nodep, "Cell should be under a module");
|
|
|
|
|
|
|
|
|
|
// Create the cell vertex
|
|
|
|
|
InlineModCellVertex* const vtxp = m_graphp->getInlineModCellVertexp(nodep);
|
|
|
|
|
|
|
|
|
|
// Add containing-module/instantiated-module edges
|
|
|
|
|
m_graphp->addEdge(*m_modVtxp, *vtxp);
|
|
|
|
|
m_graphp->addEdge(*vtxp, *m_graphp->getInlineModModuleVertexp(nodep->modp()));
|
|
|
|
|
|
|
|
|
|
// Iterate children
|
|
|
|
|
iterateChildrenConst(nodep);
|
2006-08-26 13:35:28 +02:00
|
|
|
}
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
|
2022-09-16 12:22:11 +02:00
|
|
|
void visit(AstPragma* nodep) override {
|
2022-01-02 19:56:40 +01:00
|
|
|
if (nodep->pragType() == VPragmaType::INLINE_MODULE) {
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
if (!m_modVtxp) {
|
2019-07-01 04:37:03 +02:00
|
|
|
nodep->v3error("Inline pragma not under a module"); // LCOV_EXCL_LINE
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
} else {
|
|
|
|
|
m_modVtxp->setShouldInline("Pragma INLINE_MODULE");
|
2019-05-19 22:13:13 +02:00
|
|
|
}
|
2020-01-17 02:17:11 +01:00
|
|
|
VL_DO_DANGLING(nodep->unlinkFrBack()->deleteTree(), nodep);
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (nodep->pragType() == VPragmaType::NO_INLINE_MODULE) {
|
|
|
|
|
if (!m_modVtxp) {
|
2019-07-01 04:37:03 +02:00
|
|
|
nodep->v3error("Inline pragma not under a module"); // LCOV_EXCL_LINE
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
} else {
|
|
|
|
|
m_modVtxp->setNoInlineSoft("Pragma NO_INLINE_MODULE");
|
2019-05-19 22:13:13 +02:00
|
|
|
}
|
2020-01-17 02:17:11 +01:00
|
|
|
VL_DO_DANGLING(nodep->unlinkFrBack()->deleteTree(), nodep);
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
return;
|
2019-05-19 22:13:13 +02:00
|
|
|
}
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
|
|
|
|
|
iterateChildrenConst(nodep);
|
2006-08-26 13:35:28 +02:00
|
|
|
}
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
|
|
|
|
|
// TODO: Bit nasty to do this here, but historically present, and still necessary
|
2022-09-16 12:22:11 +02:00
|
|
|
void visit(AstNodeFTaskRef* nodep) override {
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
if (m_modVtxp) m_modVtxp->sizeInc();
|
2021-12-21 17:21:27 +01:00
|
|
|
// Remove link. V3LinkDot will reestablish it after inlining.
|
2020-03-07 18:52:11 +01:00
|
|
|
// MethodCalls not currently supported by inliner, so keep linked
|
2023-09-18 15:21:30 +02:00
|
|
|
if (!nodep->classOrPackagep() && !VN_IS(nodep, MethodCall)) {
|
|
|
|
|
nodep->taskp(nullptr);
|
2023-10-15 12:25:42 +02:00
|
|
|
VIsCached::clearCacheTree();
|
2023-09-18 15:21:30 +02:00
|
|
|
}
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
iterateChildrenConst(nodep);
|
2006-08-26 13:35:28 +02:00
|
|
|
}
|
2017-10-02 00:00:27 +02:00
|
|
|
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
// Base node
|
2022-09-16 12:22:11 +02:00
|
|
|
void visit(AstNode* nodep) override {
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
if (m_modVtxp) m_modVtxp->sizeInc();
|
|
|
|
|
iterateChildrenConst(nodep);
|
2006-08-26 13:35:28 +02:00
|
|
|
}
|
|
|
|
|
|
2019-09-12 13:22:22 +02:00
|
|
|
// CONSTRUCTORS
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
explicit InlineModGraphBuilder(AstNetlist* nodep) {
|
|
|
|
|
// Build the module instantiation graph
|
|
|
|
|
iterateConst(nodep);
|
|
|
|
|
// Order vertices (any topological order is fine), can't be cyclic at this point
|
|
|
|
|
m_graphp->order();
|
|
|
|
|
// Check that the first vertex is the top level module (everything, including packages,
|
|
|
|
|
// have a corresponding AstCell under the top level at this point).
|
|
|
|
|
UASSERT_OBJ(m_graphp->vertices().frontp()->as<InlineModModuleVertex>()->modp()->isTop(),
|
|
|
|
|
nodep, "First vertex should be top level module");
|
|
|
|
|
#ifdef VL_DEBUG
|
|
|
|
|
for (const V3GraphVertex& vtx : m_graphp->vertices()) {
|
|
|
|
|
// First vertex is the top levelmodule, we checked above
|
|
|
|
|
if (&vtx == m_graphp->vertices().frontp()) continue;
|
|
|
|
|
// Otherwise it should have instantiations
|
|
|
|
|
UASSERT_OBJ(!vtx.inEmpty(), &vtx, "Should have edges from root");
|
|
|
|
|
}
|
|
|
|
|
#endif
|
2021-12-22 12:41:29 +01:00
|
|
|
}
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
~InlineModGraphBuilder() override = default;
|
|
|
|
|
|
|
|
|
|
public:
|
|
|
|
|
static std::unique_ptr<InlineModGraph> apply(AstNetlist* nodep) {
|
|
|
|
|
return std::move(InlineModGraphBuilder{nodep}.m_graphp);
|
2006-08-26 13:35:28 +02:00
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
2013-05-25 16:42:44 +02:00
|
|
|
//######################################################################
|
|
|
|
|
// After cell is cloned, relink the new module's contents
|
|
|
|
|
|
2022-01-02 19:56:40 +01:00
|
|
|
class InlineRelinkVisitor final : public VNVisitor {
|
2013-05-25 16:42:44 +02:00
|
|
|
// NODE STATE
|
|
|
|
|
// Input:
|
|
|
|
|
// See InlineVisitor
|
|
|
|
|
|
|
|
|
|
// STATE
|
2021-03-12 23:26:53 +01:00
|
|
|
std::unordered_set<std::string> m_renamedInterfaces; // Name of renamed interface variables
|
2026-05-15 13:52:26 +02:00
|
|
|
std::unordered_set<std::string>
|
|
|
|
|
m_priorInlinedCells; // Cells previously inlined into the module being inlined here.
|
|
|
|
|
// Used to recognize VarXRefs whose inlinedDots was stamped by a
|
|
|
|
|
// prior V3Inline pass (vs. by V3Begin generate-block unrolling).
|
|
|
|
|
std::unordered_set<const AstVarXRef*>
|
|
|
|
|
m_pinSubstitutedXRefs; // VarXRefs created by pin substitution in this relink pass.
|
|
|
|
|
// Their dotted/inlinedDots already represent the parent's scope
|
|
|
|
|
// and must not be rewritten on the immediate visit.
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
InlineModGraph& m_graph; // The instance graph
|
2025-08-22 22:43:49 +02:00
|
|
|
AstNodeModule* const m_modp; // The module we are inlining into
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
// The vertex of the module we are inlining into, for updating the graph
|
|
|
|
|
InlineModModuleVertex* const m_mVtxp = m_graph.getInlineModModuleVertexp(m_modp);
|
2025-08-22 22:43:49 +02:00
|
|
|
const AstCell* const m_cellp; // The cell being inlined
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
|
2025-08-22 22:43:49 +02:00
|
|
|
size_t m_nPlaceholders = 0; // Unique identifier sequence number for placeholder variables
|
2013-05-25 16:42:44 +02:00
|
|
|
|
|
|
|
|
// VISITORS
|
2022-09-16 12:22:11 +02:00
|
|
|
void visit(AstCellInline* nodep) override {
|
2019-05-19 22:13:13 +02:00
|
|
|
// Inlined cell under the inline cell, need to move to avoid conflicts
|
|
|
|
|
nodep->unlinkFrBack();
|
|
|
|
|
m_modp->addInlinesp(nodep);
|
|
|
|
|
// Rename
|
2021-12-21 17:21:27 +01:00
|
|
|
nodep->name(m_cellp->name() + "__DOT__" + nodep->name());
|
2025-05-23 02:29:32 +02:00
|
|
|
UINFO(6, " Inline " << nodep);
|
2019-05-19 22:13:13 +02:00
|
|
|
// Do CellInlines under this, but don't move them
|
2018-05-11 02:55:37 +02:00
|
|
|
iterateChildren(nodep);
|
2013-05-25 16:42:44 +02:00
|
|
|
}
|
2022-09-16 12:22:11 +02:00
|
|
|
void visit(AstCell* nodep) override {
|
2019-05-19 22:13:13 +02:00
|
|
|
// Cell under the inline cell, need to rename to avoid conflicts
|
2021-12-21 17:21:27 +01:00
|
|
|
nodep->name(m_cellp->name() + "__DOT__" + nodep->name());
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
// Need to update graph
|
|
|
|
|
nodep->user4p(nullptr); // clone copied user4p, reset to make new vertex
|
|
|
|
|
InlineModCellVertex* const vtxp = m_graph.getInlineModCellVertexp(nodep);
|
|
|
|
|
m_graph.addEdge(*m_mVtxp, *vtxp);
|
|
|
|
|
m_graph.addEdge(*vtxp, *m_graph.getInlineModModuleVertexp(nodep->modp()));
|
2018-05-11 02:55:37 +02:00
|
|
|
iterateChildren(nodep);
|
2013-05-25 16:42:44 +02:00
|
|
|
}
|
2022-09-16 12:22:11 +02:00
|
|
|
void visit(AstClass* nodep) override {
|
2021-12-21 17:21:27 +01:00
|
|
|
nodep->name(m_cellp->name() + "__DOT__" + nodep->name());
|
2020-04-05 15:30:23 +02:00
|
|
|
iterateChildren(nodep);
|
|
|
|
|
}
|
2022-09-16 12:22:11 +02:00
|
|
|
void visit(AstModule* nodep) override {
|
2019-05-19 22:13:13 +02:00
|
|
|
m_renamedInterfaces.clear();
|
2018-05-11 02:55:37 +02:00
|
|
|
iterateChildren(nodep);
|
2016-01-22 01:11:53 +01:00
|
|
|
}
|
2022-09-16 12:22:11 +02:00
|
|
|
void visit(AstVar* nodep) override {
|
2019-05-19 22:13:13 +02:00
|
|
|
// Iterate won't hit AstIfaceRefDType directly as it is no longer underneath the module
|
2021-11-13 19:50:44 +01:00
|
|
|
if (AstIfaceRefDType* const ifacerefp = VN_CAST(nodep->dtypep(), IfaceRefDType)) {
|
2019-05-19 22:13:13 +02:00
|
|
|
m_renamedInterfaces.insert(nodep->name());
|
|
|
|
|
// Each inlined cell that contain an interface variable need to
|
|
|
|
|
// copy the IfaceRefDType and point it to the newly cloned
|
|
|
|
|
// interface cell.
|
2021-11-13 19:50:44 +01:00
|
|
|
AstIfaceRefDType* const newdp = ifacerefp->cloneTree(false);
|
2019-05-19 22:13:13 +02:00
|
|
|
nodep->dtypep(newdp);
|
|
|
|
|
ifacerefp->addNextHere(newdp);
|
|
|
|
|
// Relink to point to newly cloned cell
|
|
|
|
|
if (newdp->cellp()) {
|
2025-08-22 22:43:49 +02:00
|
|
|
if (AstCell* const newcellp = VN_CAST(newdp->cellp()->user3p(), Cell)) {
|
2019-05-19 22:13:13 +02:00
|
|
|
newdp->cellp(newcellp);
|
|
|
|
|
newdp->cellName(newcellp->name());
|
|
|
|
|
// Tag the old ifacerefp to ensure it leaves no stale
|
|
|
|
|
// reference to the inlined cell.
|
2023-10-29 02:12:27 +02:00
|
|
|
newdp->user1(false);
|
|
|
|
|
ifacerefp->user1(true);
|
2019-05-19 22:13:13 +02:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
// Variable under the inline cell, need to rename to avoid conflicts
|
|
|
|
|
// Also clear I/O bits, as it is now local.
|
2021-06-21 00:32:57 +02:00
|
|
|
const string name = m_cellp->name() + "__DOT__" + nodep->name();
|
2020-04-05 15:30:23 +02:00
|
|
|
if (!nodep->isFuncLocal() && !nodep->isClassMember()) nodep->inlineAttrReset(name);
|
2019-05-19 22:13:13 +02:00
|
|
|
if (!m_cellp->isTrace()) nodep->trace(false);
|
2025-08-02 19:44:40 +02:00
|
|
|
UINFOTREE(9, nodep, "", "varchanged");
|
2013-05-25 16:42:44 +02:00
|
|
|
}
|
2022-09-16 12:22:11 +02:00
|
|
|
void visit(AstNodeFTask* nodep) override {
|
2019-05-19 22:13:13 +02:00
|
|
|
// Function under the inline cell, need to rename to avoid conflicts
|
|
|
|
|
nodep->name(m_cellp->name() + "__DOT__" + nodep->name());
|
2018-05-11 02:55:37 +02:00
|
|
|
iterateChildren(nodep);
|
2013-05-25 16:42:44 +02:00
|
|
|
}
|
2022-09-16 12:22:11 +02:00
|
|
|
void visit(AstTypedef* nodep) override {
|
2019-05-19 22:13:13 +02:00
|
|
|
// Typedef under the inline cell, need to rename to avoid conflicts
|
|
|
|
|
nodep->name(m_cellp->name() + "__DOT__" + nodep->name());
|
2018-05-11 02:55:37 +02:00
|
|
|
iterateChildren(nodep);
|
2013-05-25 16:42:44 +02:00
|
|
|
}
|
2025-09-22 22:30:26 +02:00
|
|
|
void visit(AstAlias* nodep) override {
|
2025-08-22 22:43:49 +02:00
|
|
|
// Don't replace port variable in the alias
|
2025-04-09 02:48:57 +02:00
|
|
|
}
|
2022-09-16 12:22:11 +02:00
|
|
|
void visit(AstVarRef* nodep) override {
|
2025-08-22 22:43:49 +02:00
|
|
|
// If the target port is being inlined, replace reference with the
|
2026-02-04 22:26:20 +01:00
|
|
|
// connected expression (a Const, VarRef, or VarXRef).
|
2025-08-22 22:43:49 +02:00
|
|
|
AstNode* const pinExpr = nodep->varp()->user2p();
|
|
|
|
|
if (!pinExpr) return;
|
|
|
|
|
|
|
|
|
|
// If it's a constant, inline it
|
|
|
|
|
if (AstConst* const constp = VN_CAST(pinExpr, Const)) {
|
|
|
|
|
// You might think we would not try to substitute a constant for
|
|
|
|
|
// a written variable, but we might need to do this if for example
|
|
|
|
|
// there is an assignment to an input port, and that input port
|
|
|
|
|
// is tied to a constant on the cell we are inlining. This does
|
|
|
|
|
// generate an ASSIGNIN warning, but that can be downgraded to
|
|
|
|
|
// a warning. (Also assigning to an input can has valid uses if
|
|
|
|
|
// e.g. done via a hierarchical reference from outside to an input
|
|
|
|
|
// unconnected on the instance, so we don't want ASSIGNIN fatal.)
|
|
|
|
|
// Same applies when there is a static initialzier for an input.
|
|
|
|
|
// To avoid having to special case malformed assignment, or worse
|
|
|
|
|
// yet emiting code like 0 = 0, we instead substitute a placeholder
|
|
|
|
|
// variable that will later be pruned (it will otherwise be unreferenced).
|
|
|
|
|
if (!nodep->access().isReadOnly()) {
|
|
|
|
|
AstVar* const varp = nodep->varp();
|
2026-02-08 00:11:55 +01:00
|
|
|
const std::string name
|
|
|
|
|
= m_cellp->name() + "__vInlPlaceholder_" + std::to_string(++m_nPlaceholders);
|
2025-08-22 22:43:49 +02:00
|
|
|
AstVar* const holdep = new AstVar{varp->fileline(), VVarType::VAR, name, varp};
|
|
|
|
|
m_modp->addStmtsp(holdep);
|
|
|
|
|
AstVarRef* const newp = new AstVarRef{nodep->fileline(), holdep, nodep->access()};
|
|
|
|
|
nodep->replaceWith(newp);
|
2020-03-16 03:30:20 +01:00
|
|
|
} else {
|
2025-08-22 22:43:49 +02:00
|
|
|
nodep->replaceWith(constp->cloneTree(false));
|
2019-05-19 22:13:13 +02:00
|
|
|
}
|
2025-08-22 22:43:49 +02:00
|
|
|
VL_DO_DANGLING(nodep->deleteTree(), nodep);
|
|
|
|
|
return;
|
2019-05-19 22:13:13 +02:00
|
|
|
}
|
2025-08-22 22:43:49 +02:00
|
|
|
|
2026-02-04 22:26:20 +01:00
|
|
|
// Handle VarRef: simple retarget
|
|
|
|
|
if (const AstVarRef* const vrefp = VN_CAST(pinExpr, VarRef)) {
|
|
|
|
|
nodep->varp(vrefp->varp());
|
|
|
|
|
nodep->classOrPackagep(vrefp->classOrPackagep());
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Handle VarXRef: replace VarRef with VarXRef (e.g., nested interface port)
|
|
|
|
|
const AstVarXRef* const xrefp = VN_AS(pinExpr, VarXRef);
|
2026-02-04 22:27:14 +01:00
|
|
|
AstVarXRef* const newp
|
|
|
|
|
= new AstVarXRef{nodep->fileline(), xrefp->name(), xrefp->dotted(), nodep->access()};
|
2026-02-04 22:26:20 +01:00
|
|
|
newp->varp(xrefp->varp());
|
2026-05-15 13:52:26 +02:00
|
|
|
// The pin expression came from m_modp (the parent we are inlining into), so its
|
|
|
|
|
// dotted/inlinedDots already describe a path in m_modp's scope. Record this xref
|
|
|
|
|
// so visit(AstVarXRef) leaves it alone on the immediate visit; later inline
|
|
|
|
|
// passes will prepend their cell names normally.
|
2026-02-04 22:26:20 +01:00
|
|
|
newp->inlinedDots(xrefp->inlinedDots());
|
2026-05-15 13:52:26 +02:00
|
|
|
m_pinSubstitutedXRefs.insert(newp);
|
2026-02-04 22:26:20 +01:00
|
|
|
nodep->replaceWith(newp);
|
|
|
|
|
VL_DO_DANGLING(nodep->deleteTree(), nodep);
|
2013-05-25 16:42:44 +02:00
|
|
|
}
|
2022-09-16 12:22:11 +02:00
|
|
|
void visit(AstVarXRef* nodep) override {
|
2026-05-15 13:52:26 +02:00
|
|
|
// VarXRefs just created by pin substitution in this pass already describe a path
|
|
|
|
|
// in m_modp's scope (the parent we are inlining into). Leave them untouched on
|
|
|
|
|
// this immediate visit; subsequent inline passes will prepend their cell names.
|
|
|
|
|
if (m_pinSubstitutedXRefs.erase(nodep)) {
|
|
|
|
|
iterateChildren(nodep);
|
|
|
|
|
return;
|
|
|
|
|
}
|
2019-05-19 22:13:13 +02:00
|
|
|
// Track what scope it was originally under so V3LinkDot can resolve it
|
2026-05-15 13:52:26 +02:00
|
|
|
const string origInlinedDots = nodep->inlinedDots();
|
|
|
|
|
nodep->inlinedDots(VString::dot(m_cellp->name(), ".", origInlinedDots));
|
|
|
|
|
// If origInlinedDots starts with the name of a previously-inlined cell, this
|
|
|
|
|
// VarXRef came from that cell's body and its dotted refers to that child's
|
|
|
|
|
// local scope; renaming it against m_renamedInterfaces would wrongly alias it
|
|
|
|
|
// to a coincidentally-named var in the current module (#5120). VarXRefs whose
|
|
|
|
|
// inlinedDots was stamped by V3Begin generate-block unrolling are unaffected,
|
|
|
|
|
// since V3Begin's CellInlines have origModName "__BEGIN__" and don't appear in
|
|
|
|
|
// m_priorInlinedCells.
|
|
|
|
|
const string::size_type firstDot = origInlinedDots.find('.');
|
|
|
|
|
const string firstSeg
|
|
|
|
|
= firstDot == string::npos ? origInlinedDots : origInlinedDots.substr(0, firstDot);
|
|
|
|
|
const bool fromPriorInline = m_priorInlinedCells.count(firstSeg);
|
2020-11-11 03:40:14 +01:00
|
|
|
for (string tryname = nodep->dotted(); true;) {
|
2017-12-14 01:42:49 +01:00
|
|
|
if (m_renamedInterfaces.count(tryname)) {
|
2026-05-15 13:52:26 +02:00
|
|
|
// matchIsRenamed: the matched name itself was created by a prior V3Inline
|
|
|
|
|
// rename (contains "__DOT__"). When true, we are following the chain of
|
|
|
|
|
// renames for the same var across nested inlines, so apply the rename
|
|
|
|
|
// even if the VarXRef came from a prior-inlined child.
|
|
|
|
|
const bool matchIsRenamed = tryname.find("__DOT__") != string::npos;
|
|
|
|
|
if (!fromPriorInline || matchIsRenamed) {
|
|
|
|
|
nodep->dotted(m_cellp->name() + "__DOT__" + nodep->dotted());
|
|
|
|
|
}
|
2017-12-14 01:42:49 +01:00
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
// If foo.bar, and foo is an interface, then need to search again for foo
|
2021-11-26 23:55:36 +01:00
|
|
|
const string::size_type pos = tryname.rfind('.');
|
2020-03-16 03:30:20 +01:00
|
|
|
if (pos == string::npos || pos == 0) {
|
2017-12-14 01:42:49 +01:00
|
|
|
break;
|
|
|
|
|
} else {
|
2025-08-21 10:43:37 +02:00
|
|
|
tryname.resize(pos);
|
2017-12-14 01:42:49 +01:00
|
|
|
}
|
|
|
|
|
}
|
2018-05-11 02:55:37 +02:00
|
|
|
iterateChildren(nodep);
|
2013-05-25 16:42:44 +02:00
|
|
|
}
|
2022-09-16 12:22:11 +02:00
|
|
|
void visit(AstNodeFTaskRef* nodep) override {
|
2019-05-19 22:13:13 +02:00
|
|
|
// Track what scope it was originally under so V3LinkDot can resolve it
|
2021-12-21 17:21:27 +01:00
|
|
|
nodep->inlinedDots(VString::dot(m_cellp->name(), ".", nodep->inlinedDots()));
|
2019-05-19 22:13:13 +02:00
|
|
|
if (m_renamedInterfaces.count(nodep->dotted())) {
|
|
|
|
|
nodep->dotted(m_cellp->name() + "__DOT__" + nodep->dotted());
|
|
|
|
|
}
|
2025-05-23 02:29:32 +02:00
|
|
|
UINFO(8, " " << nodep);
|
2018-05-11 02:55:37 +02:00
|
|
|
iterateChildren(nodep);
|
2013-05-25 16:42:44 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Not needed, as V3LinkDot doesn't care about typedefs
|
2022-09-16 12:22:11 +02:00
|
|
|
// void visit(AstRefDType* nodep) override {}
|
2013-05-25 16:42:44 +02:00
|
|
|
|
2022-09-16 12:22:11 +02:00
|
|
|
void visit(AstScopeName* nodep) override {
|
2019-05-19 22:13:13 +02:00
|
|
|
// If there's a %m in the display text, we add a special node that will contain the name()
|
|
|
|
|
// Similar code in V3Begin
|
2025-10-14 13:08:35 +02:00
|
|
|
// To keep correct visual order, must add before exising
|
|
|
|
|
nodep->scopeAttr("__DOT__" + m_cellp->name() + nodep->scopeAttr());
|
|
|
|
|
nodep->scopeEntr("__DOT__" + m_cellp->name() + nodep->scopeEntr());
|
2018-05-11 02:55:37 +02:00
|
|
|
iterateChildren(nodep);
|
2013-05-25 16:42:44 +02:00
|
|
|
}
|
2025-08-04 14:29:56 +02:00
|
|
|
void visit(AstNodeCoverDecl* nodep) override {
|
2019-05-19 22:13:13 +02:00
|
|
|
// Fix path in coverage statements
|
2018-03-16 00:46:05 +01:00
|
|
|
nodep->hier(VString::dot(m_cellp->prettyName(), ".", nodep->hier()));
|
2018-05-11 02:55:37 +02:00
|
|
|
iterateChildren(nodep);
|
2013-05-25 16:42:44 +02:00
|
|
|
}
|
2022-09-16 12:22:11 +02:00
|
|
|
void visit(AstNode* nodep) override { iterateChildren(nodep); }
|
2013-05-25 16:42:44 +02:00
|
|
|
|
|
|
|
|
public:
|
2019-09-12 13:22:22 +02:00
|
|
|
// CONSTRUCTORS
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
InlineRelinkVisitor(AstNodeModule* cloneModp, AstNodeModule* oldModp, AstCell* cellp,
|
|
|
|
|
InlineModGraph& graph)
|
|
|
|
|
: m_graph{graph}
|
|
|
|
|
, m_modp{oldModp}
|
2020-08-16 15:55:36 +02:00
|
|
|
, m_cellp{cellp} {
|
2026-05-15 13:52:26 +02:00
|
|
|
// CellInlines added by V3Begin for generate/named blocks have origModName
|
|
|
|
|
// "__BEGIN__"; only those added by prior V3Inline passes carry a real module
|
|
|
|
|
// name. Track the latter so visit(AstVarXRef) can distinguish VarXRefs
|
|
|
|
|
// originating from previously-inlined children.
|
|
|
|
|
for (AstNode* nodep = cloneModp->inlinesp(); nodep; nodep = nodep->nextp()) {
|
|
|
|
|
const AstCellInline* const cip = VN_CAST(nodep, CellInline);
|
|
|
|
|
if (cip && cip->origModName() != "__BEGIN__") {
|
|
|
|
|
m_priorInlinedCells.insert(cip->name());
|
|
|
|
|
}
|
|
|
|
|
}
|
2018-05-11 02:55:37 +02:00
|
|
|
iterate(cloneModp);
|
2013-05-25 16:42:44 +02:00
|
|
|
}
|
2022-09-16 12:22:11 +02:00
|
|
|
~InlineRelinkVisitor() override = default;
|
2013-05-25 16:42:44 +02:00
|
|
|
};
|
|
|
|
|
|
|
|
|
|
//######################################################################
|
2025-08-22 22:43:49 +02:00
|
|
|
// Module inliner
|
|
|
|
|
|
|
|
|
|
namespace ModuleInliner {
|
|
|
|
|
|
|
|
|
|
// A port variable in an inlined module can be connected 2 ways.
|
|
|
|
|
// Either add a continuous assignment between the pin expression from
|
|
|
|
|
// the instance and the port variable, or simply inline the pin expression
|
|
|
|
|
// in place of the port variable. We will prefer to do the later whenever
|
|
|
|
|
// possible (and sometimes required). When inlining, we need to create an
|
2025-08-26 00:47:08 +02:00
|
|
|
// alias for the inlined variable, in order to resovle hierarchical references
|
2025-08-22 22:43:49 +02:00
|
|
|
// against it later in V3Scope (and also for tracing, which is inserted
|
|
|
|
|
//later). Returns ture iff the given port variable should be inlined,
|
|
|
|
|
// and false if a continuous assignment should be used.
|
2025-11-13 00:54:22 +01:00
|
|
|
bool inlinePort(const AstVar* nodep) {
|
2025-08-22 22:43:49 +02:00
|
|
|
// Interface references are always inlined
|
|
|
|
|
if (nodep->isIfaceRef()) return true;
|
|
|
|
|
// Ref ports must be always inlined
|
|
|
|
|
if (nodep->direction() == VDirection::REF) return true;
|
|
|
|
|
// Forced signals must not be inlined. The port signal can be
|
|
|
|
|
// forced separately from the connected signals.
|
|
|
|
|
if (nodep->isForced()) return false;
|
|
|
|
|
|
|
|
|
|
// Note: For singls marked 'public' (and not 'public_flat') inlining
|
2025-08-26 00:47:08 +02:00
|
|
|
// of their containing modules is disabled so they wont reach here.
|
2025-08-22 22:43:49 +02:00
|
|
|
|
|
|
|
|
// TODO: For now, writable public signals inside the cell cannot be
|
|
|
|
|
// eliminated as they are entered into the VerilatedScope, and
|
|
|
|
|
// changes would not propagate to it when assigned. (The alias created
|
|
|
|
|
// for them ensures they would be read correctly, but would not
|
|
|
|
|
// propagate any changes.) This can be removed when the VerialtedScope
|
|
|
|
|
// construction in V3EmitCSyms understands aliases.
|
|
|
|
|
if (nodep->isSigUserRWPublic()) return false;
|
|
|
|
|
|
|
|
|
|
// Otherwise we can repalce the variable
|
|
|
|
|
return true;
|
|
|
|
|
}
|
2013-05-25 16:42:44 +02:00
|
|
|
|
2025-08-22 22:43:49 +02:00
|
|
|
// Connect the given port 'nodep' (being inlined into 'modp') to the given
|
|
|
|
|
// expression (from the Cell Pin)
|
|
|
|
|
void connectPort(AstNodeModule* modp, AstVar* nodep, AstNodeExpr* pinExprp) {
|
|
|
|
|
UINFO(6, "Connecting " << pinExprp);
|
|
|
|
|
UINFO(6, " to " << nodep);
|
2013-05-25 16:42:44 +02:00
|
|
|
|
2025-08-22 22:43:49 +02:00
|
|
|
// Decide whether to inline the port variable or use continuous assignments
|
|
|
|
|
const bool inlineIt = inlinePort(nodep);
|
2021-12-22 12:41:29 +01:00
|
|
|
|
2025-08-22 22:43:49 +02:00
|
|
|
// If we deccided to inline it, record the expression to substitute this variable with
|
|
|
|
|
if (inlineIt) nodep->user2p(pinExprp);
|
2013-05-25 16:42:44 +02:00
|
|
|
|
2025-08-22 22:43:49 +02:00
|
|
|
FileLine* const flp = nodep->fileline();
|
2021-12-22 12:41:29 +01:00
|
|
|
|
2025-08-22 22:43:49 +02:00
|
|
|
// Helper to creates an AstVarRef reference to the port variable
|
|
|
|
|
const auto portRef = [&](VAccess access) { return new AstVarRef{flp, nodep, access}; };
|
2021-12-22 12:41:29 +01:00
|
|
|
|
2025-08-22 22:43:49 +02:00
|
|
|
// If the connected expression is a constant, add an assignment to set
|
|
|
|
|
// the port variable. The constant can still be inlined, in which case
|
|
|
|
|
// this is needed for tracing the inlined port variable.
|
|
|
|
|
if (AstConst* const pinp = VN_CAST(pinExprp, Const)) {
|
Internals: Make AstAssignW a procedural statement (#6280) (#6556)
Initial idea was to remodel AssignW as Assign under Alway. Trying that
uncovered some issues, the most difficult of them was that a delay
attached to a continuous assignment behaves differently from a delay
attached to a blocking assignment statement, so we need to keep the
knowledge of which flavour an assignment was until V3Timing.
So instead of removing AstAssignW, we always wrap it in an AstAlways,
with a special `keyword()` type. This makes it into a proper procedural
statement, which is almost equivalent to AstAssign, except for the case
when they contain a delay. We still gain the benefits of #6280 and can
simplify some code. Every AstNodeStmt should now be under an
AstNodeProcedure - which we should rename to AstProcess, or an
AstNodeFTask). As a result, V3Table can now handle AssignW for free.
Also uncovered and fixed a bug in handling intra-assignment delays if
a function is present on the RHS of an AssignW.
There is more work to be done towards #6280, and potentially simplifying
AssignW handing, but this is the minimal change required to tick it off
the TODO list for #6280.
2025-10-14 10:05:19 +02:00
|
|
|
AstAssignW* const ap
|
|
|
|
|
= new AstAssignW{flp, portRef(VAccess::WRITE), pinp->cloneTree(false)};
|
|
|
|
|
modp->addStmtsp(new AstAlways{ap});
|
2025-08-22 22:43:49 +02:00
|
|
|
return;
|
|
|
|
|
}
|
2021-12-22 12:41:29 +01:00
|
|
|
|
2025-08-22 22:43:49 +02:00
|
|
|
// Otherwise it must be a variable reference due to having called pinReconnectSimple
|
2026-02-04 22:26:20 +01:00
|
|
|
const AstNodeVarRef* const pinRefp = VN_AS(pinExprp, NodeVarRef);
|
|
|
|
|
|
|
|
|
|
const auto pinRefAsVarRef = [&](VAccess access) -> AstVarRef* {
|
|
|
|
|
const AstVarRef* const vrp = VN_AS(pinRefp, VarRef);
|
|
|
|
|
AstVarRef* const newp = new AstVarRef{vrp->fileline(), vrp->varp(), access};
|
|
|
|
|
newp->classOrPackagep(vrp->classOrPackagep());
|
|
|
|
|
return newp;
|
|
|
|
|
};
|
2021-12-22 12:41:29 +01:00
|
|
|
|
2026-02-04 22:26:20 +01:00
|
|
|
const auto pinRefAsExpr = [&](VAccess access) -> AstNodeExpr* {
|
|
|
|
|
if (const AstVarRef* const vrp = VN_CAST(pinRefp, VarRef)) {
|
|
|
|
|
AstVarRef* const newp = new AstVarRef{vrp->fileline(), vrp->varp(), access};
|
|
|
|
|
newp->classOrPackagep(vrp->classOrPackagep());
|
|
|
|
|
return newp;
|
|
|
|
|
} else {
|
|
|
|
|
const AstVarXRef* const xrp = VN_AS(pinRefp, VarXRef);
|
2026-02-04 22:27:14 +01:00
|
|
|
AstVarXRef* const newp
|
|
|
|
|
= new AstVarXRef{xrp->fileline(), xrp->name(), xrp->dotted(), access};
|
2026-02-04 22:26:20 +01:00
|
|
|
newp->varp(xrp->varp());
|
|
|
|
|
newp->inlinedDots(xrp->inlinedDots());
|
|
|
|
|
return newp;
|
|
|
|
|
}
|
2025-08-22 22:43:49 +02:00
|
|
|
};
|
2021-12-22 12:41:29 +01:00
|
|
|
|
2025-08-22 22:43:49 +02:00
|
|
|
// If it is being inlined, create the alias for it
|
|
|
|
|
if (inlineIt) {
|
2025-12-29 03:30:16 +01:00
|
|
|
UINFO(6, "Inlining port variable: " << nodep);
|
2025-08-22 22:43:49 +02:00
|
|
|
if (nodep->isIfaceRef()) {
|
|
|
|
|
modp->addStmtsp(
|
2026-02-04 22:26:20 +01:00
|
|
|
new AstAliasScope{flp, portRef(VAccess::WRITE), pinRefAsExpr(VAccess::READ)});
|
2025-08-22 22:43:49 +02:00
|
|
|
} else {
|
2025-09-29 19:23:51 +02:00
|
|
|
AstVarRef* const aliasArgsp = portRef(VAccess::WRITE);
|
2026-02-04 22:26:20 +01:00
|
|
|
aliasArgsp->addNext(pinRefAsVarRef(VAccess::READ));
|
2025-09-29 19:23:51 +02:00
|
|
|
modp->addStmtsp(new AstAlias{flp, aliasArgsp});
|
2021-12-22 12:41:29 +01:00
|
|
|
}
|
2025-08-22 22:43:49 +02:00
|
|
|
// They will become the same variable, so propagate file-line and variable attributes
|
|
|
|
|
pinRefp->varp()->fileline()->modifyStateInherit(flp);
|
|
|
|
|
flp->modifyStateInherit(pinRefp->varp()->fileline());
|
|
|
|
|
pinRefp->varp()->propagateAttrFrom(nodep);
|
|
|
|
|
nodep->propagateAttrFrom(pinRefp->varp());
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Otherwise create the continuous assignment between the port var and the pin expression
|
2025-12-29 03:30:16 +01:00
|
|
|
UINFO(6, "Not inlining port variable: " << nodep);
|
2025-08-22 22:43:49 +02:00
|
|
|
if (nodep->direction() == VDirection::INPUT) {
|
2026-02-04 22:26:20 +01:00
|
|
|
AstAssignW* const ap
|
|
|
|
|
= new AstAssignW{flp, portRef(VAccess::WRITE), pinRefAsExpr(VAccess::READ)};
|
Internals: Make AstAssignW a procedural statement (#6280) (#6556)
Initial idea was to remodel AssignW as Assign under Alway. Trying that
uncovered some issues, the most difficult of them was that a delay
attached to a continuous assignment behaves differently from a delay
attached to a blocking assignment statement, so we need to keep the
knowledge of which flavour an assignment was until V3Timing.
So instead of removing AstAssignW, we always wrap it in an AstAlways,
with a special `keyword()` type. This makes it into a proper procedural
statement, which is almost equivalent to AstAssign, except for the case
when they contain a delay. We still gain the benefits of #6280 and can
simplify some code. Every AstNodeStmt should now be under an
AstNodeProcedure - which we should rename to AstProcess, or an
AstNodeFTask). As a result, V3Table can now handle AssignW for free.
Also uncovered and fixed a bug in handling intra-assignment delays if
a function is present on the RHS of an AssignW.
There is more work to be done towards #6280, and potentially simplifying
AssignW handing, but this is the minimal change required to tick it off
the TODO list for #6280.
2025-10-14 10:05:19 +02:00
|
|
|
modp->addStmtsp(new AstAlways{ap});
|
2025-08-22 22:43:49 +02:00
|
|
|
} else if (nodep->direction() == VDirection::OUTPUT) {
|
2026-02-04 22:26:20 +01:00
|
|
|
AstAssignW* const ap
|
|
|
|
|
= new AstAssignW{flp, pinRefAsExpr(VAccess::WRITE), portRef(VAccess::READ)};
|
Internals: Make AstAssignW a procedural statement (#6280) (#6556)
Initial idea was to remodel AssignW as Assign under Alway. Trying that
uncovered some issues, the most difficult of them was that a delay
attached to a continuous assignment behaves differently from a delay
attached to a blocking assignment statement, so we need to keep the
knowledge of which flavour an assignment was until V3Timing.
So instead of removing AstAssignW, we always wrap it in an AstAlways,
with a special `keyword()` type. This makes it into a proper procedural
statement, which is almost equivalent to AstAssign, except for the case
when they contain a delay. We still gain the benefits of #6280 and can
simplify some code. Every AstNodeStmt should now be under an
AstNodeProcedure - which we should rename to AstProcess, or an
AstNodeFTask). As a result, V3Table can now handle AssignW for free.
Also uncovered and fixed a bug in handling intra-assignment delays if
a function is present on the RHS of an AssignW.
There is more work to be done towards #6280, and potentially simplifying
AssignW handing, but this is the minimal change required to tick it off
the TODO list for #6280.
2025-10-14 10:05:19 +02:00
|
|
|
modp->addStmtsp(new AstAlways{ap});
|
2025-08-22 22:43:49 +02:00
|
|
|
} else {
|
2025-10-03 12:49:13 +02:00
|
|
|
pinExprp->v3fatalSrc("V3Tristate left INOUT port");
|
2021-12-22 12:41:29 +01:00
|
|
|
}
|
2025-08-22 22:43:49 +02:00
|
|
|
}
|
2021-12-22 12:41:29 +01:00
|
|
|
|
2025-08-22 22:43:49 +02:00
|
|
|
// Inline 'cellp' into 'modp'. 'last' indicatest this is tha last instance of the inlined module
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
void inlineCell(AstNodeModule* modp, AstCell* cellp, bool last, InlineModGraph& graph) {
|
2025-08-22 22:43:49 +02:00
|
|
|
UINFO(5, " Inline Cell " << cellp);
|
|
|
|
|
UINFO(5, " into Module " << modp);
|
|
|
|
|
|
|
|
|
|
const VNUser2InUse user2InUse;
|
|
|
|
|
|
|
|
|
|
// Important: If this is the last cell, then don't clone the instantiated module but
|
|
|
|
|
// inline the original directly. While this requires some special casing, doing so
|
|
|
|
|
// saves us having to temporarily clone the module for the last cell, which
|
|
|
|
|
// significantly reduces Verilator memory usage. This is especially true as often the
|
|
|
|
|
// top few levels of the hierarchy are singleton wrapper modules, which we always
|
|
|
|
|
// inline. In this case this special casing saves us from having to clone essentially
|
|
|
|
|
// the entire netlist, which would in effect double Verilator memory consumption, or
|
|
|
|
|
// worse if we put off deleting the inlined modules until the end. Not having to clone
|
|
|
|
|
// large trees also improves speed.
|
|
|
|
|
|
|
|
|
|
// The module we will yank the contents out of and put into 'modp'
|
|
|
|
|
AstNodeModule* const inlinedp = last ? cellp->modp()->unlinkFrBack() //
|
|
|
|
|
: cellp->modp()->cloneTree(false);
|
|
|
|
|
|
|
|
|
|
// Compute map from original port variables and cells to their clones
|
|
|
|
|
for (AstNode *ap = cellp->modp()->stmtsp(), *bp = inlinedp->stmtsp(); ap || bp;
|
|
|
|
|
ap = ap->nextp(), bp = bp->nextp()) {
|
|
|
|
|
UASSERT_OBJ(ap && bp, ap ? ap : bp, "Clone has different number of children");
|
|
|
|
|
// We only care about AstVar and AstCell, but faster to just set them all
|
|
|
|
|
ap->user3p(bp);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Create data for resolving hierarchical references later.
|
2025-10-03 12:49:13 +02:00
|
|
|
modp->addInlinesp(
|
|
|
|
|
new AstCellInline{cellp->fileline(), cellp->name(), cellp->modp()->origName()});
|
2025-08-22 22:43:49 +02:00
|
|
|
|
|
|
|
|
// Connect the pins on the instance
|
|
|
|
|
for (AstPin* pinp = cellp->pinsp(); pinp; pinp = VN_AS(pinp->nextp(), Pin)) {
|
|
|
|
|
if (!pinp->exprp()) continue;
|
2026-05-22 23:57:11 +02:00
|
|
|
UINFO(6, "Connecting port " << pinp->modVarp());
|
2025-08-22 22:43:49 +02:00
|
|
|
UINFO(6, " of instance " << cellp);
|
|
|
|
|
|
|
|
|
|
// Make sure the conneccted pin expression is always a VarRef or a Const
|
|
|
|
|
V3Inst::pinReconnectSimple(pinp, cellp, false);
|
|
|
|
|
|
|
|
|
|
// Warn
|
|
|
|
|
V3Inst::checkOutputShort(pinp);
|
2025-12-05 16:07:53 +01:00
|
|
|
if (!pinp->exprp()) continue;
|
2025-08-22 22:43:49 +02:00
|
|
|
|
|
|
|
|
// Pick up the old and new port variables signal (new is the same on last instance)
|
|
|
|
|
const AstVar* const oldModVarp = pinp->modVarp();
|
|
|
|
|
AstVar* const newModVarp = VN_AS(oldModVarp->user3p(), Var);
|
|
|
|
|
// Pick up the connected expression (a VarRef or Const due to pinReconnectSimple)
|
|
|
|
|
AstNodeExpr* const pinExprp = VN_AS(pinp->exprp(), NodeExpr);
|
|
|
|
|
|
|
|
|
|
// Connect up the port
|
|
|
|
|
connectPort(modp, newModVarp, pinExprp);
|
|
|
|
|
}
|
|
|
|
|
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
// Cleanup var names, etc, to not conflict, relink replaced variables, adjust graph
|
|
|
|
|
{ InlineRelinkVisitor{inlinedp, modp, cellp, graph}; }
|
2025-08-22 22:43:49 +02:00
|
|
|
// Move statements from the inlined module into the module we are inlining into
|
|
|
|
|
if (AstNode* const stmtsp = inlinedp->stmtsp()) {
|
|
|
|
|
modp->addStmtsp(stmtsp->unlinkFrBackWithNext());
|
|
|
|
|
}
|
|
|
|
|
// Delete the empty shell of the inlined module
|
|
|
|
|
VL_DO_DANGLING(inlinedp->deleteTree(), inlinedp);
|
|
|
|
|
// Remove the cell we just inlined
|
2025-09-09 23:39:44 +02:00
|
|
|
VL_DO_DANGLING(cellp->unlinkFrBack()->deleteTree(), cellp);
|
2025-08-22 22:43:49 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Apply all inlining decisions
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
void process(AstNetlist* netlistp, InlineModGraph& graph) {
|
2025-08-22 22:43:49 +02:00
|
|
|
// NODE STATE
|
|
|
|
|
// Cleared entire netlist
|
|
|
|
|
// AstIfaceRefDType::user1() // Whether the cell pointed to by this
|
|
|
|
|
// // AstIfaceRefDType has been inlined
|
|
|
|
|
// AstCell::user3p() // AstCell*, the clone
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
// AstVar::user3p() // AstVar*, the clone
|
2025-08-22 22:43:49 +02:00
|
|
|
// Cleared each cell
|
|
|
|
|
// AstVar::user2p() // AstVarRef*/AstConst* This port is connected to (AstPin::expr())
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
const VNUser1InUse user1InUse;
|
|
|
|
|
const VNUser3InUse user3InUse;
|
2025-08-22 22:43:49 +02:00
|
|
|
|
|
|
|
|
// Number of inlined instances, for statistics
|
|
|
|
|
VDouble0 m_nInlined;
|
|
|
|
|
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
// Gather all cells that need to be inlined (this is in topological order)
|
|
|
|
|
std::vector<InlineModCellVertex*> cVtxps;
|
|
|
|
|
for (V3GraphVertex& vtx : graph.vertices()) {
|
|
|
|
|
InlineModCellVertex* const cVtxp = vtx.cast<InlineModCellVertex>();
|
|
|
|
|
if (!cVtxp) continue;
|
|
|
|
|
if (!cVtxp->doInline()) continue;
|
|
|
|
|
cVtxps.push_back(cVtxp);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Inline cells bottom up (leaves into roots)
|
|
|
|
|
for (InlineModCellVertex* const cVtxp : vlstd::reverse_view(cVtxps)) {
|
|
|
|
|
// Pick up parts before deleting
|
|
|
|
|
InlineModModuleVertex& mVtx = cVtxp->instanceIn();
|
|
|
|
|
InlineModModuleVertex* const iVtxp = &cVtxp->instanceOf();
|
|
|
|
|
AstCell* const cellp = cVtxp->cellp();
|
|
|
|
|
const bool last = iVtxp->inSize1();
|
|
|
|
|
UASSERT_OBJ(!iVtxp->noInlineHard(), cellp, "Should not be inlining if not possible");
|
|
|
|
|
|
|
|
|
|
// Update
|
|
|
|
|
++m_nInlined;
|
|
|
|
|
mVtx.sizeInc(iVtxp->size()); // For debug dump only
|
|
|
|
|
|
|
|
|
|
// Delete the cell we are inlining
|
|
|
|
|
VL_DO_DANGLING(cVtxp->unlinkDelete(&graph), cVtxp);
|
|
|
|
|
// Delete the module we are inlining if this is the last instance
|
|
|
|
|
if (last) {
|
|
|
|
|
while (!iVtxp->outEmpty()) {
|
|
|
|
|
InlineModCellVertex* const tVtxp
|
|
|
|
|
= iVtxp->outEdges().frontp()->top()->as<InlineModCellVertex>();
|
|
|
|
|
// Bottom up ordering ensures this
|
|
|
|
|
UASSERT_OBJ(!tVtxp->doInline(), tVtxp, "Should have been inlined");
|
|
|
|
|
VL_DO_DANGLING(tVtxp->unlinkDelete(&graph), tVtxp);
|
|
|
|
|
}
|
|
|
|
|
VL_DO_DANGLING(iVtxp->unlinkDelete(&graph), iVtxp);
|
2019-05-19 22:13:13 +02:00
|
|
|
}
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
|
|
|
|
|
// Do it
|
|
|
|
|
inlineCell(mVtx.modp(), cellp, last, graph);
|
|
|
|
|
if (dumpGraphLevel() >= 9) graph.dumpDotFilePrefixed("inlinemod-cell");
|
2016-01-22 01:11:53 +01:00
|
|
|
}
|
2013-05-25 16:42:44 +02:00
|
|
|
|
2025-08-22 22:43:49 +02:00
|
|
|
V3Stats::addStat("Optimizations, Inlined instances", m_nInlined);
|
2013-05-25 16:42:44 +02:00
|
|
|
|
2025-08-22 22:43:49 +02:00
|
|
|
// Clean up AstIfaceRefDType references
|
|
|
|
|
// If the cell has been removed let's make sure we don't leave a
|
|
|
|
|
// reference to it. This dtype may still be in use by the
|
2025-09-30 07:40:17 +02:00
|
|
|
// AstAliasScope created earlier but that'll get cleared up later
|
2025-08-22 22:43:49 +02:00
|
|
|
netlistp->typeTablep()->foreach([](AstIfaceRefDType* nodep) {
|
|
|
|
|
if (nodep->user1()) nodep->cellp(nullptr);
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
} //namespace ModuleInliner
|
2013-05-25 16:42:44 +02:00
|
|
|
|
2006-08-26 13:35:28 +02:00
|
|
|
//######################################################################
|
2025-08-22 22:43:49 +02:00
|
|
|
// V3Inline class functions
|
2006-08-26 13:35:28 +02:00
|
|
|
|
|
|
|
|
void V3Inline::inlineAll(AstNetlist* nodep) {
|
2025-05-23 02:29:32 +02:00
|
|
|
UINFO(2, __FUNCTION__ << ":");
|
2021-12-22 12:41:29 +01:00
|
|
|
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
// Build the bipartite module instantiation graph
|
|
|
|
|
std::unique_ptr<InlineModGraph> graphp = InlineModGraphBuilder::apply(nodep);
|
|
|
|
|
if (dumpGraphLevel() >= 6) graphp->dumpDotFilePrefixed("inlinemod-graph");
|
|
|
|
|
|
|
|
|
|
// Decide which instances to inline
|
|
|
|
|
const size_t designSize
|
|
|
|
|
= graphp->vertices().frontp()->as<InlineModModuleVertex>()->flattenedSize();
|
|
|
|
|
for (V3GraphVertex& vtx : graphp->vertices()) {
|
|
|
|
|
if (InlineModModuleVertex* const mVtxp = vtx.cast<InlineModModuleVertex>()) {
|
|
|
|
|
// If this module is less than 10% of the design, flatten this module
|
|
|
|
|
if (mVtxp->flattenedSize() * 10 < designSize) mVtxp->setFlatten();
|
|
|
|
|
// Don't inline if can't inline
|
|
|
|
|
if (mVtxp->noInlineHard()) continue;
|
|
|
|
|
// Don't inline if soft off
|
|
|
|
|
if (mVtxp->noInlineSoft()) continue;
|
|
|
|
|
// If all instances of this module combined are less than 20% of the design, inline all
|
|
|
|
|
size_t totalSize = mVtxp->flattenedSize() * mVtxp->instanceCount();
|
|
|
|
|
if (totalSize * 5 < designSize) {
|
|
|
|
|
for (V3GraphEdge& edge : mVtxp->inEdges()) {
|
|
|
|
|
InlineModCellVertex* const cVtxp = edge.fromp()->as<InlineModCellVertex>();
|
|
|
|
|
cVtxp->setDoInline("< 20% of design");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
// No more decisions based on module vertex
|
|
|
|
|
continue;
|
2019-05-19 22:13:13 +02:00
|
|
|
}
|
Optimize module inlining heuristic (#7837)
Rewrite module inlining decision to be based on a bipartite Module/Cell
graph, similar to V3InlineCFuncs. Preserved all old heuristics, but
added 2 new ones:
- If a module, and all the sub-hierarchy below it, is less than 10% the
total flattened size of the design, then flatten the contents of that
module (but the module itself is not necessarily inlined).
- If the flattened size of all instances of a module is less than 20% of
the total flattened size of the design, then inline all instances of
that module.
These are both relative to the total size of the design, so they
auto-scale with complexity. The net effect is that large shared
instances are preserved, but their contents are flattened out. E.g. in a
multi-core CPU this would keep the cores non-inlined but flatten out
most everything else. This still enables V3Combining and sharing those
later, but avoids potentially big overheads e.g. with small widely used
library modules.
Empirically this yields less generated C++ than the previous version
(due to removing lots of small functions), and can improve performance
10-20% while still having meaningful combining relative to the size of
the design.
2026-06-25 15:14:15 +02:00
|
|
|
|
|
|
|
|
// The instantiation
|
|
|
|
|
InlineModCellVertex& cVtx = *vtx.as<InlineModCellVertex>();
|
|
|
|
|
// The module instantiated by this cell
|
|
|
|
|
InlineModModuleVertex& mVtx = cVtx.instanceOf();
|
|
|
|
|
|
|
|
|
|
// Don't inline if can't inline, duh!
|
|
|
|
|
if (mVtx.noInlineHard()) continue;
|
|
|
|
|
|
|
|
|
|
// If it should be inlined, inlined it
|
|
|
|
|
if (mVtx.shouldInline()) cVtx.setDoInline("should inline");
|
|
|
|
|
// If --flatten, inline it
|
|
|
|
|
if (v3Global.opt.flatten()) cVtx.setDoInline("--flatten");
|
|
|
|
|
|
|
|
|
|
// Don't inline for other reasons if soft off
|
|
|
|
|
if (mVtx.noInlineSoft()) continue;
|
|
|
|
|
|
|
|
|
|
// If instatiated in exactly one static site, inline it
|
|
|
|
|
if (mVtx.inSize1()) cVtx.setDoInline("Single static instance");
|
|
|
|
|
// If small, inline it
|
|
|
|
|
if (mVtx.size() < INLINE_MODS_SMALLER) cVtx.setDoInline("Small");
|
|
|
|
|
// If inlineMult is 0, inline it
|
|
|
|
|
if (v3Global.opt.inlineMult() < 1) cVtx.setDoInline("inlineMult < 1");
|
|
|
|
|
// If it would yield less than the given number of ops, inline it
|
|
|
|
|
const size_t inlinedSize = mVtx.inEdges().size() * mVtx.size();
|
|
|
|
|
const size_t limit = v3Global.opt.inlineMult();
|
|
|
|
|
if (inlinedSize < limit) cVtx.setDoInline("inlinedSize < inlineMult");
|
|
|
|
|
}
|
|
|
|
|
if (dumpGraphLevel() >= 6) graphp->dumpDotFilePrefixed("inlinemod-decision");
|
|
|
|
|
|
|
|
|
|
// Inline the modles we decided to inline
|
|
|
|
|
ModuleInliner::process(nodep, *graphp);
|
|
|
|
|
if (dumpGraphLevel() >= 6) graphp->dumpDotFilePrefixed("inlinemod-inlined");
|
2021-12-22 12:41:29 +01:00
|
|
|
|
2024-01-09 16:35:13 +01:00
|
|
|
V3Global::dumpCheckGlobalTree("inline", 0, dumpTreeEitherLevel() >= 3);
|
2006-08-26 13:35:28 +02:00
|
|
|
}
|
