luke/verilator
1
0
Fork 0
mirror of https://github.com/verilator/verilator.git synced 2026-10-02 16:13:34 +02:00
Code Issues Packages Projects Releases Wiki Activity

Optimize Dfg algorithms with an open addressing hash table (#8307)

Browse Source 
This patch introduces V3HashTable.h, which defines an open addressing,
linear probing hash table. The table implement the public V3HashSet and
V3HashMap templates, which are generic containers. The benefit of this
over std::unordered_map and std::unordered_set is far better memory
locality during lookup. (The STL containers use chaining and require a
new heap allocation for every insertion, similarly probing involves
pointer chasing on collisions).

The new data structure is use in V3DfgCache, and V3DfgCse and yields a
significant speed improvement of those passes on large designs.
This commit is contained in:
Geza Lore
2026-09-08 23:21:13 +01:00
committed by GitHub
parent 5bb0e9b216
commit e4f210eb38
8 changed files with 1882 additions and 278 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/verilatedos.h
+9
View File
@@ -318,6 +318,15 @@
# define VL_CONSTEXPR_CXX17
#endif
//=========================================================================
// C++-2020
#if __cplusplus >= 202002L
# define VL_NO_UNIQUE_ADDRESS_CXX20 [[no_unique_address]]
#else
# define VL_NO_UNIQUE_ADDRESS_CXX20
#endif
//=========================================================================
// Optimization
src/CMakeLists.txt
+2
View File
@@ -114,6 +114,7 @@ set(HEADERS
V3GraphPathChecker.h
V3GraphStream.h
V3Hash.h
V3HashTable.h
V3Hasher.h
V3HierBlock.h
V3Inline.h
@@ -292,6 +293,7 @@ set(COMMON_SOURCES
V3GraphPathChecker.cpp
V3GraphTest.cpp
V3Hash.cpp
V3HashTable.cpp
V3Hasher.cpp
V3HierBlock.cpp
V3Inline.cpp
src/Makefile_obj.in
+1
View File
@@ -196,6 +196,7 @@ RAW_OBJS = \
V3GraphPathChecker.o \
V3GraphTest.o \
V3Hash.o \
V3HashTable.o \
V3OptionParser.o \
V3Os.o \
V3ParseGrammar.o \
src/V3DfgCache.h
+129 -174
View File
@@ -25,8 +25,11 @@
#ifndef VERILATOR_V3DFGCACHE_H_
#define VERILATOR_V3DFGCACHE_H_
#include "verilatedos.h"
#include "V3Dfg.h"
#include "V3DfgDataType.h"
#include "V3HashTable.h"
#include <type_traits>
@@ -52,137 +55,107 @@ struct V3DfgCacheType<Vertex, CacheBase, VertexBase, Cache, Pairs...> final {
class V3DfgCache final {
// TYPES
class KeySel final {
const DfgDataType& m_dtype;
const DfgVertex* const m_fromp;
const uint32_t m_lsb;
// Hashing and comparison of the cached vertices. Each takes either a vertex, or the
// parts a vertex would be created from, so a lookup needs no vertex and no key object.
public:
KeySel(const DfgDataType& dtype, DfgVertex* fromp, uint32_t lsb)
: m_dtype{dtype}
, m_fromp{fromp}
, m_lsb{lsb} {}
explicit KeySel(const DfgSel* vtxp)
: m_dtype{vtxp->dtype()}
, m_fromp{vtxp->fromp()}
, m_lsb{vtxp->lsb()} {}
struct Hash final {
size_t operator()(const KeySel& key) const {
// cppcheck-suppress unreadVariable // cppcheck bug
V3Hash hash = key.m_dtype.hash();
hash += vertexHash(key.m_fromp);
hash += key.m_lsb;
return hash.value();
}
};
struct Equal final {
bool operator()(const KeySel& a, const KeySel& b) const {
return a.m_lsb == b.m_lsb && a.m_dtype == b.m_dtype
&& vertexEqual(a.m_fromp, b.m_fromp);
}
};
// DfgSel
struct HashSel final {
size_t operator()(const DfgSel* vtxp) const {
return operator()(vtxp->dtype(), vtxp->fromp(), vtxp->lsb());
}
size_t operator()(const DfgDataType& dtype, const DfgVertex* fromp, uint32_t lsb) const {
// cppcheck-suppress unreadVariable // cppcheck bug
V3Hash hash = dtype.hash();
hash += vertexHash(fromp);
hash += lsb;
return hash.value();
}
};
struct EqualSel final {
bool operator()(const DfgSel* ap, const DfgSel* bp) const {
return operator()(ap, bp->dtype(), bp->fromp(), bp->lsb());
}
bool operator()(const DfgSel* vtxp, const DfgDataType& dtype, const DfgVertex* fromp,
uint32_t lsb) const {
return vtxp->lsb() == lsb && vtxp->dtype() == dtype
&& vertexEqual(vtxp->fromp(), fromp);
}
};
class KeyUnary final {
const DfgDataType& m_dtype;
const DfgVertex* const m_source0p;
public:
// cppcheck-suppress noExplicitConstructor
KeyUnary(const DfgDataType& dtype, DfgVertex* source0p)
: m_dtype{dtype}
, m_source0p{source0p} {}
explicit KeyUnary(const DfgVertexUnary* vtxp)
: m_dtype{vtxp->dtype()}
, m_source0p{vtxp->inputp(0)} {}
struct Hash final {
size_t operator()(const KeyUnary& key) const { //
V3Hash hash = key.m_dtype.hash();
hash += vertexHash(key.m_source0p);
return hash.value();
}
};
struct Equal final {
bool operator()(const KeyUnary& a, const KeyUnary& b) const {
return a.m_dtype == b.m_dtype && vertexEqual(a.m_source0p, b.m_source0p);
}
};
// DfgVertexUnary
struct HashUnary final {
size_t operator()(const DfgVertexUnary* vtxp) const {
return operator()(vtxp->dtype(), vtxp->inputp(0));
}
size_t operator()(const DfgDataType& dtype, const DfgVertex* source0p) const {
V3Hash hash = dtype.hash();
hash += vertexHash(source0p);
return hash.value();
}
};
struct EqualUnary final {
bool operator()(const DfgVertexUnary* ap, const DfgVertexUnary* bp) const {
return operator()(ap, bp->dtype(), bp->inputp(0));
}
bool operator()(const DfgVertexUnary* vtxp, const DfgDataType& dtype,
const DfgVertex* source0p) const {
return vtxp->dtype() == dtype && vertexEqual(vtxp->inputp(0), source0p);
}
};
class KeyBinary final {
const DfgDataType& m_dtype;
const DfgVertex* const m_source0p;
const DfgVertex* const m_source1p;
public:
KeyBinary(const DfgDataType& dtype, DfgVertex* source0p, DfgVertex* source1p)
: m_dtype{dtype}
, m_source0p{source0p}
, m_source1p{source1p} {}
explicit KeyBinary(const DfgVertexBinary* vtxp)
: m_dtype{vtxp->dtype()}
, m_source0p{vtxp->inputp(0)}
, m_source1p{vtxp->inputp(1)} {}
struct Hash final {
size_t operator()(const KeyBinary& key) const {
V3Hash hash = key.m_dtype.hash();
hash += vertexHash(key.m_source0p);
hash += vertexHash(key.m_source1p);
return hash.value();
}
};
struct Equal final {
bool operator()(const KeyBinary& a, const KeyBinary& b) const {
return a.m_dtype == b.m_dtype && vertexEqual(a.m_source0p, b.m_source0p)
&& vertexEqual(a.m_source1p, b.m_source1p);
}
};
// DfgVertexBinary
struct HashBinary final {
size_t operator()(const DfgVertexBinary* vtxp) const {
return operator()(vtxp->dtype(), vtxp->inputp(0), vtxp->inputp(1));
}
size_t operator()(const DfgDataType& dtype, const DfgVertex* source0p,
const DfgVertex* source1p) const {
V3Hash hash = dtype.hash();
hash += vertexHash(source0p);
hash += vertexHash(source1p);
return hash.value();
}
};
struct EqualBinary final {
bool operator()(const DfgVertexBinary* ap, const DfgVertexBinary* bp) const {
return operator()(ap, bp->dtype(), bp->inputp(0), bp->inputp(1));
}
bool operator()(const DfgVertexBinary* vtxp, const DfgDataType& dtype,
const DfgVertex* source0p, const DfgVertex* source1p) const {
return vtxp->dtype() == dtype && vertexEqual(vtxp->inputp(0), source0p)
&& vertexEqual(vtxp->inputp(1), source1p);
}
};
class KeyTernary final {
const DfgDataType& m_dtype;
const DfgVertex* const m_source0p;
const DfgVertex* const m_source1p;
const DfgVertex* const m_source2p;
public:
KeyTernary(const DfgDataType& dtype, DfgVertex* source0p, DfgVertex* source1p,
DfgVertex* source2p)
: m_dtype{dtype}
, m_source0p{source0p}
, m_source1p{source1p}
, m_source2p{source2p} {}
explicit KeyTernary(const DfgVertexTernary* vtxp)
: m_dtype{vtxp->dtype()}
, m_source0p{vtxp->inputp(0)}
, m_source1p{vtxp->inputp(1)}
, m_source2p{vtxp->inputp(2)} {}
struct Hash final {
size_t operator()(const KeyTernary& key) const {
V3Hash hash = key.m_dtype.hash();
hash += vertexHash(key.m_source0p);
hash += vertexHash(key.m_source1p);
hash += vertexHash(key.m_source2p);
return hash.value();
}
};
struct Equal final {
bool operator()(const KeyTernary& a, const KeyTernary& b) const {
return a.m_dtype == b.m_dtype && vertexEqual(a.m_source0p, b.m_source0p)
&& vertexEqual(a.m_source1p, b.m_source1p)
&& vertexEqual(a.m_source2p, b.m_source2p);
}
};
// DfgVertexTernary
struct HashTernary final {
size_t operator()(const DfgVertexTernary* vtxp) const {
return operator()(vtxp->dtype(), vtxp->inputp(0), vtxp->inputp(1), vtxp->inputp(2));
}
size_t operator()(const DfgDataType& dtype, const DfgVertex* source0p,
const DfgVertex* source1p, const DfgVertex* source2p) const {
V3Hash hash = dtype.hash();
hash += vertexHash(source0p);
hash += vertexHash(source1p);
hash += vertexHash(source2p);
return hash.value();
}
};
struct EqualTernary final {
bool operator()(const DfgVertexTernary* ap, const DfgVertexTernary* bp) const {
return operator()(ap, bp->dtype(), bp->inputp(0), bp->inputp(1), bp->inputp(2));
}
bool operator()(const DfgVertexTernary* vtxp, const DfgDataType& dtype,
const DfgVertex* source0p, const DfgVertex* source1p,
const DfgVertex* source2p) const {
return vtxp->dtype() == dtype && vertexEqual(vtxp->inputp(0), source0p)
&& vertexEqual(vtxp->inputp(1), source1p)
&& vertexEqual(vtxp->inputp(2), source2p);
}
};
// Base class of vertex caches
class CacheBase VL_NOT_FINAL {
protected:
// These set the operands of a new vertex
@@ -210,86 +183,68 @@ class V3DfgCache final {
public:
// CacheBase does not cache anything
virtual DfgVertex* cache(DfgVertex*) { return nullptr; }
virtual void invalidate(const DfgVertex*) {}
virtual void invalidate(DfgVertex*) {}
};
template <typename T_Key, typename T_Vertex>
template <typename T_Vertex, typename T_Hash, typename T_Equal>
class Cache final : public CacheBase {
static_assert(std::is_base_of<DfgVertex, T_Vertex>::value, "T_Vertex must be a DfgVertex");
// TYPES
using Hash = typename T_Key::Hash;
using Equal = typename T_Key::Equal;
using Map = std::unordered_map<T_Key, T_Vertex*, Hash, Equal>;
// STATE
Map m_map;
// METHODS
// These return a reference to the mapped entry, inserting a nullptr if not yet exists
template <typename... T_Args>
T_Vertex*& entry(T_Args&&... args) {
const T_Key key{std::forward<T_Args>(args)...};
return m_map[key];
}
template <typename... T_Args>
typename Map::iterator find(T_Args&&... args) {
const T_Key key{std::forward<T_Args>(args)...};
return m_map.find(key);
}
V3HashSet<T_Vertex*, T_Hash, T_Equal> m_set;
public:
// Add an existing vertex to the cache. If an equivalent exists,
// it is returned and the cache is not updated.
// Add an existing vertex to the cache. If an equivalent but different vertex exists,
// it is returned and the cache is not updated. Returns nullptr if the vertex is inserted.
DfgVertex* cache(DfgVertex* vtxp) override {
UASSERT_OBJ(vtxp->is<T_Vertex>(), vtxp, "Vertex is wrong type");
T_Vertex*& entrypr = entry(static_cast<const T_Vertex*>(vtxp));
if (entrypr && entrypr != vtxp) return entrypr;
entrypr = static_cast<T_Vertex*>(vtxp);
return nullptr;
UDEBUGONLY(UASSERT_OBJ(vtxp->is<T_Vertex>(), vtxp, "Vertex is wrong type"););
T_Vertex* const typedp = static_cast<T_Vertex*>(vtxp);
T_Vertex* const cachedp = *m_set.insert(typedp).first;
return cachedp != vtxp ? cachedp : nullptr;
}
// Remove an existing vertex from the cache, if it is the cached vertex, otherwise no-op
void invalidate(const DfgVertex* vtxp) override {
UASSERT_OBJ(vtxp->is<T_Vertex>(), vtxp, "Vertex is wrong type");
const auto it = find(static_cast<const T_Vertex*>(vtxp));
if (it != m_map.end() && it->second == vtxp) m_map.erase(it);
void invalidate(DfgVertex* vtxp) override {
UDEBUGONLY(UASSERT_OBJ(vtxp->is<T_Vertex>(), vtxp, "Vertex is wrong type"););
T_Vertex* const typedp = static_cast<T_Vertex*>(vtxp);
const auto it = m_set.find(typedp);
if (it != m_set.end() && *it == typedp) m_set.erase(it);
}
// Get vertex with given operands, return nullptr if not in cache
template <typename Vertex, typename... Operands>
Vertex* get(const DfgDataType& dtype, Operands... operands) {
const auto it = find(dtype, operands...);
return it != m_map.end() ? static_cast<Vertex*>(it->second) : nullptr;
const auto it = m_set.find(dtype, operands...);
return it != m_set.end() ? static_cast<Vertex*>(*it) : nullptr;
}
// Get or create (and insert) vertex with given operands
// Get vertex with given operands, if does not exist, create it
template <typename Vertex, typename... Operands>
Vertex* getOrCreate(DfgGraph& dfg, FileLine* flp, const DfgDataType& dtype,
Operands... operands) {
T_Vertex*& entryr = entry(dtype, operands...);
if (!entryr) {
T_Vertex* const newp = new Vertex{dfg, flp, dtype};
const auto pair = m_set.insertLazy(dtype, operands..., [&]() -> T_Vertex* {
Vertex* const newp = new Vertex{dfg, flp, dtype};
setOperands(newp, operands...);
entryr = newp;
}
return static_cast<Vertex*>(entryr);
return newp;
});
T_Vertex* const vtxp = *pair.first;
UDEBUGONLY(UASSERT_OBJ(vtxp->template is<Vertex>(), vtxp, "Vertex is wrong type"););
return static_cast<Vertex*>(vtxp);
}
};
// Map from Vertex type to cache type
// clang-format off
template <typename Vertex>
using CacheType =
typename V3DfgCacheType<Vertex, CacheBase, //
DfgSel, Cache<KeySel, DfgSel>, //
DfgVertexUnary, Cache<KeyUnary, DfgVertexUnary>, //
DfgVertexBinary, Cache<KeyBinary, DfgVertexBinary>, //
DfgVertexTernary, Cache<KeyTernary, DfgVertexTernary> //
>::Type;
using CacheType = typename V3DfgCacheType<Vertex, CacheBase,
DfgSel, /* -> */ Cache<DfgSel, HashSel, EqualSel>,
DfgVertexUnary, /* -> */ Cache<DfgVertexUnary, HashUnary, EqualUnary>,
DfgVertexBinary, /* -> */ Cache<DfgVertexBinary, HashBinary, EqualBinary>,
DfgVertexTernary, /* -> */ Cache<DfgVertexTernary, HashTernary, EqualTernary>
>::Type;
// clang-format on
// STATE
DfgGraph& m_dfg; // The DfgGraph we are caching the vertices of
// The per type caches
// The per type caches
#define VERTEX_CACHE_DECLARE_CACHE(t) CacheType<t> m_cache##t;
FOREACH_DFG_VERTEX_TYPE(VERTEX_CACHE_DECLARE_CACHE)
#undef VERTEX_CACHE_DECLARE_CACHE
src/V3DfgCse.cpp
+123 -104
View File
@@ -18,30 +18,43 @@
#include "V3Dfg.h"
#include "V3DfgPasses.h"
#include "V3HashTable.h"
VL_DEFINE_DEBUG_FUNCTIONS;
class V3DfgCse final {
// TYPES
using VertexPair = std::pair<const DfgVertex*, const DfgVertex*>;
struct VertexPairHash final {
size_t operator()(const VertexPair& pair) const {
V3Hash hash;
hash += pair.first;
hash += pair.second;
return hash.value();
}
};
// Hash functor for V3HashSet - depends on vertex and all its inputs
class DfgCseHash final {
// STATE
// The graph being processed
DfgGraph& m_dfg;
// Cache for vertex hashes
DfgUserMap<V3Hash> m_hashCache = m_dfg.makeUserMap<V3Hash>();
// Cache for vertex equality
std::unordered_map<VertexPair, uint8_t, VertexPairHash> m_equivalentCache;
mutable DfgUserMap<V3Hash> m_cache; // Cache for vertex hashes
public:
// CONSTRUCTOR
explicit DfgCseHash(DfgGraph& dfg)
: m_cache{dfg.makeUserMap<V3Hash>()} {
// Pre-hash variables, these are all unique, so just set their hash to a unique value
uint32_t fixedHash = 0;
for (const DfgVertexVar& vtx : dfg.varVertices()) m_cache[vtx] = V3Hash{++fixedHash};
// Pre-hash Ast references, these are all unique like variables
for (const DfgVertexAst& vtx : dfg.astVertices()) m_cache[vtx] = V3Hash{++fixedHash};
// Pre-hash CReset and Prev vertices, these are all unique
for (const DfgVertex& vtx : dfg.opVertices()) {
if (vtx.is<DfgCReset>() || vtx.is<DfgPrev>()) m_cache[vtx] = V3Hash{++fixedHash};
}
// Similarly pre-hash constants for speed. While we don't combine constants, we do want
// expressions using the same constants to be combined, so we do need to hash equal
// constants to equal values.
++fixedHash;
for (const DfgConst& vtx : dfg.constVertices()) {
const V3Hash hash = vtx.num().toHash() + fixedHash;
// Technically possible for a hash to be zero, 'vertexSelfHash' assumes it isn't
m_cache[vtx] = VL_LIKELY(hash.value()) ? hash : V3Hash{1};
}
}
// METHODS
size_t operator()(DfgVertex* vtxp) const { return vertexHash(*vtxp).value(); }
private:
// Returns hash of vertex dependent on information internal to the vertex
static V3Hash vertexSelfHash(const DfgVertex& vtx) {
switch (vtx.type()) {
@@ -135,29 +148,53 @@ class V3DfgCse final {
VL_UNREACHABLE;
}
// Returns hash of vertex dependent on and all its input
V3Hash vertexHash(DfgVertex& vtx) {
V3Hash& result = m_hashCache[vtx];
// Returns hash of vertex dependent on itself and all its inputs - memoized
V3Hash vertexHash(DfgVertex& vtx) const {
V3Hash& result = m_cache[vtx];
// Technically possible for a hash to be zero, but rare, so assume 0 means uninitialized
if (!result.value()) {
V3Hash hash{vertexSelfHash(vtx)};
// Variables are defined by themselves, so there is no need to hash them further
// (especially the sources). This enables sound hashing of graphs circular only through
// variables, which we rely on.
if (!vtx.is<DfgVertexVar>()) {
hash += vtx.type();
hash += vtx.size();
vtx.foreachSource([&](DfgVertex& src) {
hash += vertexHash(src);
return false;
});
}
hash += vtx.type();
hash += vtx.size();
vtx.foreachSource([&](DfgVertex& src) {
hash += vertexHash(src); // Graph is acyclic, so this terminates
return false;
});
result = hash;
}
return result;
}
};
// Equal functor for V3HashSet - depends on vertex and all its inputs
class DfgCseEqual final {
// TYPES
using VertexPair = std::pair<const DfgVertex*, const DfgVertex*>;
struct VertexPairHash final {
size_t operator()(const VertexPair& pair) const {
V3Hash hash;
hash += pair.first;
hash += pair.second;
return hash.value();
}
};
// STATE
mutable V3HashMap<VertexPair, bool, VertexPairHash> m_cache; // Cache for vertex equality
mutable std::vector<uint32_t> m_driverLo; // Low indices of drivers
const size_t m_size; // Size of the graph
public:
// CONSTRUCTORS
explicit DfgCseEqual(const DfgGraph& dfg)
: m_size{dfg.size()} {}
// METHODS
bool operator()(DfgVertex* ap, DfgVertex* bp) const { return vertexEquivalent(*ap, *bp); }
private:
// Compare 'a' and 'b' for equivalence based on their internal information only
bool vertexSelfEquivalent(const DfgVertex& a, const DfgVertex& b) {
bool vertexSelfEquivalent(const DfgVertex& a, const DfgVertex& b) const {
// Note: 'a' and 'b' are of the same Vertex type, data type, and have
// the same number of inputs with matching types. This is established
// by 'vertexEquivalent'.
@@ -187,16 +224,17 @@ class V3DfgCse final {
case VDfgType::SplicePacked: {
const DfgVertexSplice* const ap = a.as<DfgVertexSplice>();
// Gather indices of drivers of 'a'
std::vector<uint32_t> aLo;
aLo.reserve(ap->nInputs());
m_driverLo.clear();
m_driverLo.reserve(ap->nInputs());
ap->foreachDriver([&](const DfgVertex&, uint32_t lo) {
aLo.push_back(lo);
m_driverLo.push_back(lo);
return false;
});
// Compare indices of drivers of 'b'
uint32_t* aLop = aLo.data();
return !b.as<DfgVertexSplice>()->foreachDriver(
[&](const DfgVertex&, uint32_t lo) { return *aLop++ != lo; });
// Compare indices of drivers of 'b', equal if all match
uint32_t* aLop = m_driverLo.data();
return !b.as<DfgVertexSplice>()->foreachDriver([&](const DfgVertex&, uint32_t lo) { //
return *aLop++ != lo;
});
}
// Vertices with no internal information
@@ -263,19 +301,19 @@ class V3DfgCse final {
}
// Compares the sources of 'a' and 'b' for equivalence
bool sourcesEquivalent(const DfgVertex& a, const DfgVertex& b) {
bool sourcesEquivalent(const DfgVertex& a, const DfgVertex& b) const {
for (size_t i = 0; i < a.nInputs(); ++i) {
const DfgVertex* const ap = a.inputp(i);
const DfgVertex* const bp = b.inputp(i);
if (!ap && !bp) continue;
if (!ap || !bp) return false;
if (!vertexEquivalent(*ap, *bp)) return false;
if (!vertexEquivalent(*ap, *bp)) return false; // Graph is acyclic, so this terminates
}
return true;
}
// Compares 'a' and 'b' for equivalence
bool vertexEquivalent(const DfgVertex& a, const DfgVertex& b) {
bool vertexEquivalent(const DfgVertex& a, const DfgVertex& b) const {
// If same vertex, then equal
if (&a == &b) return true;
@@ -297,70 +335,51 @@ class V3DfgCse final {
// be looked up again through multiple paths.
if (!a.hasMultipleSinks() && !b.hasMultipleSinks()) return sourcesEquivalent(a, b);
// Check sources
// Need to compare the source vertices, check memo
const VertexPair key = (&a < &b) ? std::make_pair(&a, &b) : std::make_pair(&b, &a);
// The recursive invocation can cause a re-hash but that will not invalidate references
uint8_t& result = m_equivalentCache[key];
if (!result) result = (static_cast<uint8_t>(sourcesEquivalent(a, b)) << 1) | 1;
return result >> 1;
}
const auto it = m_cache.find(key);
if (it != m_cache.end()) return it->second;
V3DfgCse(DfgGraph& dfg, V3DfgCseContext& ctx)
: m_dfg{dfg} {
std::unordered_map<V3Hash, std::vector<DfgVertex*>> verticesWithEqualHashes;
verticesWithEqualHashes.reserve(dfg.size());
// Not memoized yet, so compute and memoize, reserve table on first insert
const bool equal = sourcesEquivalent(a, b);
if (VL_UNLIKELY(m_cache.empty())) m_cache.reserve(m_size / 4);
m_cache.insert({key, equal});
// Pre-hash variables, these are all unique, so just set their hash to a unique value
uint32_t varHash = 0;
for (const DfgVertexVar& vtx : dfg.varVertices()) m_hashCache[vtx] = V3Hash{++varHash};
// Pre-hash Ast references, these are all unique like variables
for (const DfgVertexAst& vtx : dfg.astVertices()) m_hashCache[vtx] = V3Hash{++varHash};
// Pre-hash CReset and Prev vertices, these are all unique
for (const DfgVertex& vtx : dfg.opVertices()) {
if (vtx.is<DfgCReset>() || vtx.is<DfgPrev>()) m_hashCache[vtx] = V3Hash{++varHash};
}
// Similarly pre-hash constants for speed. While we don't combine constants, we do want
// expressions using the same constants to be combined, so we do need to hash equal
// constants to equal values.
for (DfgConst* const vtxp : dfg.constVertices().unlinkable()) {
// Delete unused constants while we are at it.
if (!vtxp->hasSinks()) {
VL_DO_DANGLING(vtxp->unlinkDelete(dfg), vtxp);
continue;
}
m_hashCache[vtxp] = vtxp->num().toHash() + varHash;
}
// Combine operation vertices
for (DfgVertex* const vtxp : dfg.opVertices().unlinkable()) {
// Delete unused nodes while we are at it.
if (!vtxp->hasSinks()) {
vtxp->unlinkDelete(dfg);
continue;
}
std::vector<DfgVertex*>& vec = verticesWithEqualHashes[vertexHash(*vtxp)];
bool replaced = false;
for (DfgVertex* const candidatep : vec) {
if (vertexEquivalent(*candidatep, *vtxp)) {
++ctx.m_eliminated;
vtxp->replaceWith(candidatep);
VL_DO_DANGLING(vtxp->unlinkDelete(dfg), vtxp);
replaced = true;
break;
}
}
if (replaced) continue;
vec.push_back(vtxp);
}
}
public:
static void apply(DfgGraph& dfg, V3DfgCseContext& ctx) {
{ V3DfgCse{dfg, ctx}; }
// Prune unused nodes
V3DfgPasses::removeUnused(dfg);
// The predicate result
return equal;
}
};
void V3DfgPasses::cse(DfgGraph& dfg, V3DfgCseContext& ctx) { V3DfgCse::apply(dfg, ctx); }
// Combine equivalent operation vertices
void dfgCseCombineEquivalent(DfgGraph& dfg, V3DfgCseContext& ctx) {
// Delete unused constants, so the pre-hashing below need not consider them
for (DfgConst* const vtxp : dfg.constVertices().unlinkable()) {
if (!vtxp->hasSinks()) VL_DO_DANGLING(vtxp->unlinkDelete(dfg), vtxp);
}
// Set of unique vertices. This set does all the work identifying equivalent vertices.
V3HashSet<DfgVertex*, DfgCseHash, DfgCseEqual> uniqueVtxps{DfgCseHash{dfg}, DfgCseEqual{dfg}};
// There is at most one entry per vertex
uniqueVtxps.reserve(dfg.size());
// Combine operation vertices
for (DfgVertex* const vtxp : dfg.opVertices().unlinkable()) {
// Delete unused nodes while we are at it.
if (!vtxp->hasSinks()) {
vtxp->unlinkDelete(dfg);
continue;
}
// Insert the vertex into the set, if an equivalent is found, replace the vertex with it
const auto pair = uniqueVtxps.insert(vtxp);
if (!pair.second) {
++ctx.m_eliminated;
vtxp->replaceWith(*pair.first);
VL_DO_DANGLING(vtxp->unlinkDelete(dfg), vtxp);
}
}
}
void V3DfgPasses::cse(DfgGraph& dfg, V3DfgCseContext& ctx) {
dfgCseCombineEquivalent(dfg, ctx);
V3DfgPasses::removeUnused(dfg);
}
src/V3HashTable.cpp
+1144
View File
File diff suppressed because it is too large Load Diff
src/V3HashTable.h
+472
View File
@@ -0,0 +1,472 @@
// -*- mode: C++; c-file-style: "cc-mode" -*-
//*************************************************************************
// DESCRIPTION: Verilator: Open addressing hash set and hash map
//
// Code available from: https://verilator.org
//
//*************************************************************************
//
// This program is free software; you can redistribute it and/or modify it
// under the terms of either the GNU Lesser General Public License Version 3
// or the Perl Artistic License Version 2.0.
// SPDX-FileCopyrightText: 2003-2026 Wilson Snyder
// SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0
//
//*************************************************************************
//
// An open addressing, linear probing hash table, with backward shift deletion.
// Usable as V3HashSet or V3HashMap. The benefit of these over
// std::unordered_set and std::unordered_map is far better memory locality
// during lookup, and fewer dynamic memory allocations (which also means less
// heap fragmentation). Consider using these if profiling shows that the
// unordered STL collections contribute a significant cost to an algorithm.
//
// Four types tell the table what it holds: the entry, a hash, an equality, and
// a key extractor yielding the lookup key of an entry. The extractor is what
// lets one table serve both roles: a set's entry is its own key, a map's is a
// pair keyed by its first. The hash and equality hence only ever see keys,
// never entries. V3HashSet and V3HashMap at the bottom of this file derive
// from the table, pairing it with the extractor that suits each.
//
// Those two work on keys via Hash and Equal functors as in std::unordered_set
// or std::unordered_map, but lookup is always heterogeneous, with no
// is_transparent to opt in like in the STL, and a lookup key can be spelled as
// several arguments, being the parts a key is made of. An entry can hence be
// looked up without one at hand, as when it is only created on a miss. The
// functors must provide call operators as const members, for the key of an
// entry and for every lookup key spelling used:
//
// size_t Hash::operator()(const T_Key&) const
// size_t Hash::operator()(<lookup keys>...) const
// bool Equal::operator()(const T_Key&, const T_Key&) const
// bool Equal::operator()(const T_Key&, <lookup keys>...) const
//
// with equal keys hashing equal, as usual, and consistently across the
// spellings.
//
// As only entries are stored, a slot is just a hash and an entry, so probing
// touches few cache lines. The table is doubled when an insertion would take
// it over the maximum load factor, or sized up front with 'reserve', to keep
// the probe runs short.
//
// Entries are referred to by iterators, as in the STL containers, but unlike
// STL containers, the mapped value in a V3HashMap is not mutable through an
// iterator. Iterators and entry addresses stay valid until the table grows or
// an entry is erased; either invalidates all of them.
//
// Erasure uses backward shift deletion: entries following the hole are moved
// back over it where their probe run ran through it (no tombstones).
//
//*************************************************************************
#ifndef VERILATOR_V3HASHTABLE_H_
#define VERILATOR_V3HASHTABLE_H_
#include "config_build.h"
#include "verilatedos.h"
#include "V3Error.h"
#include "V3StdFuture.h"
#include <functional>
#include <memory>
#include <new>
#include <tuple>
#include <type_traits>
#include <utility>
namespace V3HashTableInternals {
constexpr size_t MIN_CAPACITY = 16; // Smallest table allocated
constexpr size_t LOAD_FACTOR_NUM = 3; // Numerator of the maximum load factor
constexpr size_t LOAD_FACTOR_DEN = 4; // Denominator of the maximum load factor
// Key extractor for a table whose entries are their own keys, that is, a set
template <typename T_Key>
struct V3HashTableKeyIsEntry final {
using Key = T_Key; // What it yields, so the table need not deduce it
const T_Key& operator()(const T_Key& entry) const { return entry; }
};
// Key extractor for a table whose entries are pairs keyed by the first, that is, a map
template <typename T_Key, typename T_Val>
struct V3HashTableKeyIsFirst final {
using Key = T_Key; // What it yields, so the table need not deduce it
const T_Key& operator()(const std::pair<T_Key, T_Val>& entry) const { return entry.first; }
};
void selfTest();
} // namespace V3HashTableInternals
// V3HashTable, see the file header
// T_Entry The entries (STL calls this value_type)
// T_Hash Hashes a lookup key
// T_Equal Compares a key to a lookup key
// T_KeyOf Yields the key of an entry
template <typename T_Entry, typename T_Hash, typename T_Equal, typename T_KeyOf>
class V3HashTable VL_NOT_FINAL {
public:
// TYPES
using Entry = T_Entry; // What is stored
using Key = typename T_KeyOf::Key; // What entries are looked up by
private:
// TYPES
// Holds if the hash accepts a lookup key spelled as the given arguments
template <typename... T_Args>
using ValidHash = vlstd::is_invocable_r<size_t, const T_Hash&, const T_Args&...>;
// Holds if the equality accepts a key and such a lookup key
template <typename... T_Args>
using ValidEqual = vlstd::is_invocable_r<bool, const T_Equal&, const Key&, const T_Args&...>;
// The Key must itself be a valid lookup key, as every lookup ends in comparing one
// against a stored entry. Asserted separately, so the failure names the functor.
static_assert(ValidHash<Key>::value, "The 'Hash' functor must accept the 'Key'");
static_assert(ValidEqual<Key>::value, "The 'Equal' functor must accept two 'Key's");
// A table slot
struct Slot final {
// The entry comes first, so it starts the slot whatever its alignment.
// It is a union so it is alive only while the slot is occupied.
union {
Entry m_entry;
};
size_t m_hash = 0; // Hash of the entry, or zero when the slot is free
Slot() {} // Leaves 'm_entry' uninitialized, as the slot is free
~Slot() {
if (!isFree()) destroy();
}
Slot(const Slot&) = delete;
Slot(Slot&&) = delete;
const Slot& operator=(const Slot&) = delete;
Slot& operator=(Slot&& that) {
UDEBUGONLY(UASSERT(this != &that, "Moving a slot onto itself"););
UDEBUGONLY(UASSERT(!that.isFree(), "Moving from a free slot"););
UDEBUGONLY(UASSERT(isFree(), "Moving into an occupied slot"););
new (&m_entry) Entry{std::move(that.m_entry)};
m_hash = that.m_hash;
that.destroy();
return *this;
}
bool isFree() const { return !m_hash; }
// Construct the entry of this free slot from the given entry
void construct(size_t hash, Entry&& entry) {
UDEBUGONLY(UASSERT(isFree(), "Constructing the entry of an occupied slot"););
new (&m_entry) Entry{std::move(entry)};
m_hash = hash;
}
// Destroy the entry of this occupied slot, leaving it free
void destroy() {
UDEBUGONLY(UASSERT(!isFree(), "Destroying the entry of a free slot"););
m_entry.~Entry();
m_hash = 0;
}
};
public:
// Iterator over the entries, see the file header on invalidation
class iterator final {
friend class V3HashTable;
Slot* m_slotp = nullptr; // The slot iterated, or the end of the table
Slot* m_endp = nullptr; // One past the last slot
iterator(Slot* slotp, Slot* endp)
: m_slotp{slotp}
, m_endp{endp} {}
public:
iterator() = default;
// As opposed to the STL, this always returns a const reference so the
// collection is not mutable through an iterator alone. This is
// required because entries must be movable, hence can't be const, but
// the key of a map must not be modified.
const Entry& operator*() const { return m_slotp->m_entry; }
const Entry* operator->() const { return &m_slotp->m_entry; }
// Pre-increment, skipping the free slots
iterator& operator++() {
while (++m_slotp != m_endp && m_slotp->isFree()) {}
return *this;
}
bool operator==(const iterator& that) const { return m_slotp == that.m_slotp; }
bool operator!=(const iterator& that) const { return m_slotp != that.m_slotp; }
};
private:
// STATE
std::unique_ptr<Slot[]> m_table; // The table, null when unallocated
size_t m_capacity = 0; // Number of slots in the table, a power of two, or zero
size_t m_size = 0; // Number of occupied slots
VL_NO_UNIQUE_ADDRESS_CXX20 T_Hash m_hash; // Hashes a lookup key
VL_NO_UNIQUE_ADDRESS_CXX20 T_Equal m_equal; // Compares a key to a lookup key
VL_NO_UNIQUE_ADDRESS_CXX20 T_KeyOf m_keyOf; // Yields the lookup key of an entry
// METHODS
// The hash of the given entry or lookup key, as stored in a slot
template <typename... T_Args>
size_t hashOf(const T_Args&... args) const {
// A free slot is one with a zero hash, so force the high bit into every hash.
constexpr size_t USED_BIT = size_t{1} << (sizeof(size_t) * 8 - 1);
return static_cast<size_t>(m_hash(args...)) | USED_BIT;
}
// Index of the free slot the given hash probes to. There must always be one.
size_t freeSlot(size_t hash) const {
const size_t mask = m_capacity - 1;
size_t i = hash & mask;
while (!m_table[i].isFree()) i = (i + 1) & mask;
return i;
}
// Resize to the given number of slots, which must fit all entries
void resize(size_t count) {
UDEBUGONLY(UASSERT(count && !(count & (count - 1)), "Capacity not a power of 2"););
const std::unique_ptr<Slot[]> oldTable{std::move(m_table)};
const size_t oldCapacity = m_capacity;
m_table = std::make_unique<Slot[]>(count);
m_capacity = count;
// Reinsert the entries. 'freeSlot' appends to the probe run of each, so the runs
// come out contiguous whatever order this visits the old slots in.
for (size_t i = 0; i < oldCapacity; ++i) {
Slot& slot = oldTable[i];
if (!slot.isFree()) m_table[freeSlot(slot.m_hash)] = std::move(slot);
}
}
// Index of the slot holding the entry equal to the given key, or of the free slot its
// probe sequence ends at. The table must not be empty.
template <typename... T_Args>
size_t probe(size_t hash, const T_Args&... args) const {
UDEBUGONLY(UASSERT(m_table, "Table must be allocated"););
const size_t mask = m_capacity - 1;
size_t i = hash & mask;
while (!m_table[i].isFree()) {
const Slot& slot = m_table[i];
if (slot.m_hash == hash && m_equal(m_keyOf(slot.m_entry), args...)) break;
i = (i + 1) & mask;
}
return i;
}
// Implementation of 'insertLazy' below. 'all' holds the key arguments, followed by
// the callable that creates the entry, so 'N_Key' indexes the key.
template <size_t... N_Key, typename T_All>
std::pair<iterator, bool> insertLazyImpl(std::index_sequence<N_Key...>, T_All&& all) {
static_assert(ValidHash<std::tuple_element_t<N_Key, T_All>...>::value,
"The 'Hash' functor does not accept a lookup key spelled like this");
static_assert(ValidEqual<std::tuple_element_t<N_Key, T_All>...>::value,
"The 'Equal' functor does not accept a lookup key spelled like this");
const size_t hash = hashOf(std::get<N_Key>(all)...);
// Allocate on the first insertion
if (VL_UNLIKELY(!m_capacity)) resize(V3HashTableInternals::MIN_CAPACITY);
// Find the slot for the entry
Slot* slotp = m_table.get() + probe(hash, std::get<N_Key>(all)...);
// If occupied, it's the equivalent, and we are done
if (!slotp->isFree()) return {iterator{slotp, m_table.get() + m_capacity}, false};
// Table is growing
++m_size;
// Increase if necessary by load factor
if (VL_UNLIKELY(m_size * V3HashTableInternals::LOAD_FACTOR_DEN
> m_capacity * V3HashTableInternals::LOAD_FACTOR_NUM)) {
resize(m_capacity * 2);
slotp = m_table.get() + freeSlot(hash);
}
// Construct the entry via the user provided callable (last item in 'all')
slotp->construct(hash, std::get<sizeof...(N_Key)>(all)());
// The key of the created entry must both hash and compare as the key looked up
#ifdef VL_DEBUG
const Key& key = m_keyOf(slotp->m_entry);
UASSERT(hashOf(key) == hash,
"Created entry does not hash as the key it was looked up with");
UASSERT(m_equal(key, std::get<N_Key>(all)...),
"Created entry does not match the key it was looked up with");
#endif
// Return newly create entry
return {iterator{slotp, m_table.get() + m_capacity}, true};
}
protected:
// CONSTRUCTORS
V3HashTable() = default;
V3HashTable(T_Hash hash, T_Equal equal)
: m_hash{std::move(hash)}
, m_equal{std::move(equal)} {}
~V3HashTable() = default;
VL_UNCOPYABLE(V3HashTable);
// Movable, as the table is just a pointer. The source is left empty rather than
// merely unspecified, so it remains a usable, empty table.
V3HashTable(V3HashTable&& that)
: m_table{std::move(that.m_table)}
, m_capacity{that.m_capacity}
, m_size{that.m_size}
, m_hash{std::move(that.m_hash)}
, m_equal{std::move(that.m_equal)}
, m_keyOf{std::move(that.m_keyOf)} {
that.m_capacity = 0;
that.m_size = 0;
}
V3HashTable& operator=(V3HashTable&& that) {
m_table = std::move(that.m_table); // Frees the table this held, if any
m_capacity = that.m_capacity;
m_size = that.m_size;
m_hash = std::move(that.m_hash);
m_equal = std::move(that.m_equal);
m_keyOf = std::move(that.m_keyOf);
that.m_capacity = 0;
that.m_size = 0;
return *this;
}
public:
// METHODS
size_t size() const { return m_size; }
bool empty() const { return !m_size; }
iterator begin() const {
Slot* const endp = m_table.get() + m_capacity;
Slot* slotp = m_table.get();
while (slotp != endp && slotp->isFree()) ++slotp;
return iterator{slotp, endp};
}
iterator end() const {
Slot* const endp = m_table.get() + m_capacity;
return iterator{endp, endp};
}
// Make room for the given number of entries, so inserting that many will not resize
void reserve(size_t count) {
size_t capacity = V3HashTableInternals::MIN_CAPACITY;
while (capacity * V3HashTableInternals::LOAD_FACTOR_NUM
< count * V3HashTableInternals::LOAD_FACTOR_DEN)
capacity *= 2;
if (capacity > m_capacity) resize(capacity);
}
// Return iterator to the entry equal to the given key, or 'end()' if there
// is none. The key is whatever T_Hash and T_Equal accept, spelled as any
// number of arguments. Same as STL containers.
template <typename... T_Args>
iterator find(const T_Args&... args) const {
static_assert(ValidHash<T_Args...>::value,
"The 'Hash' functor does not accept a lookup key spelled like this");
static_assert(ValidEqual<T_Args...>::value,
"The 'Equal' functor does not accept a lookup key spelled like this");
if (!m_size) return end(); // Nothing to find, and this also covers there being no table
Slot* const slotp = m_table.get() + probe(hashOf(args...), args...);
return slotp->isFree() ? end() : iterator{slotp, m_table.get() + m_capacity};
}
// Add the given entry, unless an equal one is in the table already. Return
// iterator to the entry and true if insertion happened. Same as STL containers.
std::pair<iterator, bool> insert(const Entry& entry) {
static_assert(std::is_copy_constructible<Entry>::value,
"'Entry' must be copy constructible to use 'insert'");
return insertLazy(m_keyOf(entry), [&entry]() -> Entry { return entry; });
}
// As 'insert', but the entry is only made when needed: all but the last argument spell
// the key, and the last is a callable to create the entry on a miss. Note the created entry
// must hash and compare equal to the key, and the call must not touch the container, as this
// holds the slot the entry will go in.
template <typename... T_Args>
std::pair<iterator, bool> insertLazy(T_Args&&... args) {
static_assert(sizeof...(T_Args) >= 2,
"'insertLazy' needs a lookup key, then a callable to create the entry");
using Callable = std::tuple_element_t<sizeof...(T_Args) - 1, std::tuple<T_Args...>>;
static_assert(vlstd::is_invocable_r<Entry, Callable>::value,
"The last argument of 'insertLazy' must be a callable that takes no "
"arguments and returns an 'Entry'");
return insertLazyImpl(std::make_index_sequence<sizeof...(T_Args) - 1>{},
std::forward_as_tuple(std::forward<T_Args>(args)...));
}
// Whether an entry equal to the given key is in the table. The key is spelled as for 'find'.
template <typename... T_Args>
bool contains(const T_Args&... args) const {
return find(args...) != end();
}
// Remove the entry equal to the given key, and return whether there was one.
template <typename... T_Args>
bool erase(const T_Args&... args) {
const iterator it = find(args...);
if (it == end()) return false;
erase(it);
return true;
}
// Remove the entry the given iterator refers to, which must not be 'end()'. Note that
// unlike STL erase this returns nothing, as every iterator is invalidated on deletion.
void erase(iterator it) {
UDEBUGONLY(UASSERT(it != end() && !it.m_slotp->isFree(), "Erasing a bad iterator"););
const size_t mask = m_capacity - 1;
size_t i = static_cast<size_t>(it.m_slotp - m_table.get());
// Destroy the entry
m_table[i].destroy();
// The entry is gone, so slot 'i' is now a hole
--m_size;
// Backward shift deletion: move back the entries whose probing the hole breaks
size_t j = i;
while (true) {
j = (j + 1) & mask;
Slot& slot = m_table[j];
if (slot.isFree()) break;
// Move back if its home position does not lie in the cyclic range (i, j]
if (((j - (slot.m_hash & mask)) & mask) >= ((j - i) & mask)) {
m_table[i] = std::move(slot); // Frees 'slot', which is then the hole
i = j;
}
}
}
};
template <typename T_Key, typename T_Hash = std::hash<T_Key>,
typename T_Equal = std::equal_to<T_Key>>
class V3HashSet final : public V3HashTable<T_Key, T_Hash, T_Equal,
V3HashTableInternals::V3HashTableKeyIsEntry<T_Key>> {
using Super
= V3HashTable<T_Key, T_Hash, T_Equal, V3HashTableInternals::V3HashTableKeyIsEntry<T_Key>>;
// Entries are only ever moved. Note 'insert' additionally needs copy construction.
static_assert(std::is_move_constructible<T_Key>::value, "'T_Key' must be move constructible");
static_assert(std::is_destructible<T_Key>::value, "'T_Key' must be destructible");
public:
// CONSTRUCTORS
V3HashSet() = default;
V3HashSet(T_Hash hash, T_Equal equal)
: Super{std::move(hash), std::move(equal)} {}
};
template <typename T_Key, typename T_Val, typename T_Hash = std::hash<T_Key>,
typename T_Equal = std::equal_to<T_Key>>
class V3HashMap final
: public V3HashTable<std::pair<T_Key, T_Val>, T_Hash, T_Equal,
V3HashTableInternals::V3HashTableKeyIsFirst<T_Key, T_Val>> {
using Super = V3HashTable<std::pair<T_Key, T_Val>, T_Hash, T_Equal,
V3HashTableInternals::V3HashTableKeyIsFirst<T_Key, T_Val>>;
// Entries are only ever moved. Note 'insert' additionally needs copy construction.
// Asserted separately, so the failure names the one at fault.
static_assert(std::is_move_constructible<T_Key>::value, "'T_Key' must be move constructible");
static_assert(std::is_destructible<T_Key>::value, "'T_Key' must be destructible");
static_assert(std::is_move_constructible<T_Val>::value, "'T_Val' must be move constructible");
static_assert(std::is_destructible<T_Val>::value, "'T_Val' must be destructible");
public:
// TYPES
using Value = T_Val; // What a key maps to
// CONSTRUCTORS
V3HashMap() = default;
V3HashMap(T_Hash hash, T_Equal equal)
: Super{std::move(hash), std::move(equal)} {}
};
#endif // Guard
src/Verilator.cpp
+2
View File
@@ -62,6 +62,7 @@
#include "V3Gate.h"
#include "V3Global.h"
#include "V3Graph.h"
#include "V3HashTable.h"
#include "V3HierBlock.h"
#include "V3Inline.h"
#include "V3InlineCFuncs.h"
@@ -741,6 +742,7 @@ static bool verilate(const string& argString) {
V3PreShell::selfTest();
V3Broken::selfTest();
V3Control::selfTest();
V3HashTableInternals::selfTest();
V3ThreadPool::selfTest();
UINFO(2, "selfTest done");
}