From e4f210eb38d6bcc0536a588aed8e84799ddebbec Mon Sep 17 00:00:00 2001 From: Geza Lore Date: Tue, 8 Sep 2026 23:21:13 +0100 Subject: [PATCH] Optimize Dfg algorithms with an open addressing hash table (#8307) This patch introduces V3HashTable.h, which defines an open addressing, linear probing hash table. The table implement the public V3HashSet and V3HashMap templates, which are generic containers. The benefit of this over std::unordered_map and std::unordered_set is far better memory locality during lookup. (The STL containers use chaining and require a new heap allocation for every insertion, similarly probing involves pointer chasing on collisions). The new data structure is use in V3DfgCache, and V3DfgCse and yields a significant speed improvement of those passes on large designs. --- include/verilatedos.h | 9 + src/CMakeLists.txt | 2 + src/Makefile_obj.in | 1 + src/V3DfgCache.h | 303 +++++------ src/V3DfgCse.cpp | 227 ++++---- src/V3HashTable.cpp | 1144 +++++++++++++++++++++++++++++++++++++++++ src/V3HashTable.h | 472 +++++++++++++++++ src/Verilator.cpp | 2 + 8 files changed, 1882 insertions(+), 278 deletions(-) create mode 100644 src/V3HashTable.cpp create mode 100644 src/V3HashTable.h diff --git a/include/verilatedos.h b/include/verilatedos.h index 0d173cb2f..21ea88e11 100644 --- a/include/verilatedos.h +++ b/include/verilatedos.h @@ -318,6 +318,15 @@ # define VL_CONSTEXPR_CXX17 #endif +//========================================================================= +// C++-2020 + +#if __cplusplus >= 202002L +# define VL_NO_UNIQUE_ADDRESS_CXX20 [[no_unique_address]] +#else +# define VL_NO_UNIQUE_ADDRESS_CXX20 +#endif + //========================================================================= // Optimization diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5b3ee4a1c..000e63e1d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -114,6 +114,7 @@ set(HEADERS V3GraphPathChecker.h V3GraphStream.h V3Hash.h + V3HashTable.h V3Hasher.h V3HierBlock.h V3Inline.h @@ -292,6 +293,7 @@ set(COMMON_SOURCES V3GraphPathChecker.cpp V3GraphTest.cpp V3Hash.cpp + V3HashTable.cpp V3Hasher.cpp V3HierBlock.cpp V3Inline.cpp diff --git a/src/Makefile_obj.in b/src/Makefile_obj.in index f076e1f8e..755b42450 100644 --- a/src/Makefile_obj.in +++ b/src/Makefile_obj.in @@ -196,6 +196,7 @@ RAW_OBJS = \ V3GraphPathChecker.o \ V3GraphTest.o \ V3Hash.o \ + V3HashTable.o \ V3OptionParser.o \ V3Os.o \ V3ParseGrammar.o \ diff --git a/src/V3DfgCache.h b/src/V3DfgCache.h index d3cce4c85..b0c7302aa 100644 --- a/src/V3DfgCache.h +++ b/src/V3DfgCache.h @@ -25,8 +25,11 @@ #ifndef VERILATOR_V3DFGCACHE_H_ #define VERILATOR_V3DFGCACHE_H_ +#include "verilatedos.h" + #include "V3Dfg.h" #include "V3DfgDataType.h" +#include "V3HashTable.h" #include @@ -52,137 +55,107 @@ struct V3DfgCacheType final { class V3DfgCache final { // TYPES - class KeySel final { - const DfgDataType& m_dtype; - const DfgVertex* const m_fromp; - const uint32_t m_lsb; + // Hashing and comparison of the cached vertices. Each takes either a vertex, or the + // parts a vertex would be created from, so a lookup needs no vertex and no key object. - public: - KeySel(const DfgDataType& dtype, DfgVertex* fromp, uint32_t lsb) - : m_dtype{dtype} - , m_fromp{fromp} - , m_lsb{lsb} {} - explicit KeySel(const DfgSel* vtxp) - : m_dtype{vtxp->dtype()} - , m_fromp{vtxp->fromp()} - , m_lsb{vtxp->lsb()} {} - - struct Hash final { - size_t operator()(const KeySel& key) const { - // cppcheck-suppress unreadVariable // cppcheck bug - V3Hash hash = key.m_dtype.hash(); - hash += vertexHash(key.m_fromp); - hash += key.m_lsb; - return hash.value(); - } - }; - - struct Equal final { - bool operator()(const KeySel& a, const KeySel& b) const { - return a.m_lsb == b.m_lsb && a.m_dtype == b.m_dtype - && vertexEqual(a.m_fromp, b.m_fromp); - } - }; + // DfgSel + struct HashSel final { + size_t operator()(const DfgSel* vtxp) const { + return operator()(vtxp->dtype(), vtxp->fromp(), vtxp->lsb()); + } + size_t operator()(const DfgDataType& dtype, const DfgVertex* fromp, uint32_t lsb) const { + // cppcheck-suppress unreadVariable // cppcheck bug + V3Hash hash = dtype.hash(); + hash += vertexHash(fromp); + hash += lsb; + return hash.value(); + } + }; + struct EqualSel final { + bool operator()(const DfgSel* ap, const DfgSel* bp) const { + return operator()(ap, bp->dtype(), bp->fromp(), bp->lsb()); + } + bool operator()(const DfgSel* vtxp, const DfgDataType& dtype, const DfgVertex* fromp, + uint32_t lsb) const { + return vtxp->lsb() == lsb && vtxp->dtype() == dtype + && vertexEqual(vtxp->fromp(), fromp); + } }; - class KeyUnary final { - const DfgDataType& m_dtype; - const DfgVertex* const m_source0p; - - public: - // cppcheck-suppress noExplicitConstructor - KeyUnary(const DfgDataType& dtype, DfgVertex* source0p) - : m_dtype{dtype} - , m_source0p{source0p} {} - explicit KeyUnary(const DfgVertexUnary* vtxp) - : m_dtype{vtxp->dtype()} - , m_source0p{vtxp->inputp(0)} {} - - struct Hash final { - size_t operator()(const KeyUnary& key) const { // - V3Hash hash = key.m_dtype.hash(); - hash += vertexHash(key.m_source0p); - return hash.value(); - } - }; - - struct Equal final { - bool operator()(const KeyUnary& a, const KeyUnary& b) const { - return a.m_dtype == b.m_dtype && vertexEqual(a.m_source0p, b.m_source0p); - } - }; + // DfgVertexUnary + struct HashUnary final { + size_t operator()(const DfgVertexUnary* vtxp) const { + return operator()(vtxp->dtype(), vtxp->inputp(0)); + } + size_t operator()(const DfgDataType& dtype, const DfgVertex* source0p) const { + V3Hash hash = dtype.hash(); + hash += vertexHash(source0p); + return hash.value(); + } + }; + struct EqualUnary final { + bool operator()(const DfgVertexUnary* ap, const DfgVertexUnary* bp) const { + return operator()(ap, bp->dtype(), bp->inputp(0)); + } + bool operator()(const DfgVertexUnary* vtxp, const DfgDataType& dtype, + const DfgVertex* source0p) const { + return vtxp->dtype() == dtype && vertexEqual(vtxp->inputp(0), source0p); + } }; - class KeyBinary final { - const DfgDataType& m_dtype; - const DfgVertex* const m_source0p; - const DfgVertex* const m_source1p; - - public: - KeyBinary(const DfgDataType& dtype, DfgVertex* source0p, DfgVertex* source1p) - : m_dtype{dtype} - , m_source0p{source0p} - , m_source1p{source1p} {} - explicit KeyBinary(const DfgVertexBinary* vtxp) - : m_dtype{vtxp->dtype()} - , m_source0p{vtxp->inputp(0)} - , m_source1p{vtxp->inputp(1)} {} - - struct Hash final { - size_t operator()(const KeyBinary& key) const { - V3Hash hash = key.m_dtype.hash(); - hash += vertexHash(key.m_source0p); - hash += vertexHash(key.m_source1p); - return hash.value(); - } - }; - - struct Equal final { - bool operator()(const KeyBinary& a, const KeyBinary& b) const { - return a.m_dtype == b.m_dtype && vertexEqual(a.m_source0p, b.m_source0p) - && vertexEqual(a.m_source1p, b.m_source1p); - } - }; + // DfgVertexBinary + struct HashBinary final { + size_t operator()(const DfgVertexBinary* vtxp) const { + return operator()(vtxp->dtype(), vtxp->inputp(0), vtxp->inputp(1)); + } + size_t operator()(const DfgDataType& dtype, const DfgVertex* source0p, + const DfgVertex* source1p) const { + V3Hash hash = dtype.hash(); + hash += vertexHash(source0p); + hash += vertexHash(source1p); + return hash.value(); + } + }; + struct EqualBinary final { + bool operator()(const DfgVertexBinary* ap, const DfgVertexBinary* bp) const { + return operator()(ap, bp->dtype(), bp->inputp(0), bp->inputp(1)); + } + bool operator()(const DfgVertexBinary* vtxp, const DfgDataType& dtype, + const DfgVertex* source0p, const DfgVertex* source1p) const { + return vtxp->dtype() == dtype && vertexEqual(vtxp->inputp(0), source0p) + && vertexEqual(vtxp->inputp(1), source1p); + } }; - class KeyTernary final { - const DfgDataType& m_dtype; - const DfgVertex* const m_source0p; - const DfgVertex* const m_source1p; - const DfgVertex* const m_source2p; - - public: - KeyTernary(const DfgDataType& dtype, DfgVertex* source0p, DfgVertex* source1p, - DfgVertex* source2p) - : m_dtype{dtype} - , m_source0p{source0p} - , m_source1p{source1p} - , m_source2p{source2p} {} - explicit KeyTernary(const DfgVertexTernary* vtxp) - : m_dtype{vtxp->dtype()} - , m_source0p{vtxp->inputp(0)} - , m_source1p{vtxp->inputp(1)} - , m_source2p{vtxp->inputp(2)} {} - - struct Hash final { - size_t operator()(const KeyTernary& key) const { - V3Hash hash = key.m_dtype.hash(); - hash += vertexHash(key.m_source0p); - hash += vertexHash(key.m_source1p); - hash += vertexHash(key.m_source2p); - return hash.value(); - } - }; - - struct Equal final { - bool operator()(const KeyTernary& a, const KeyTernary& b) const { - return a.m_dtype == b.m_dtype && vertexEqual(a.m_source0p, b.m_source0p) - && vertexEqual(a.m_source1p, b.m_source1p) - && vertexEqual(a.m_source2p, b.m_source2p); - } - }; + // DfgVertexTernary + struct HashTernary final { + size_t operator()(const DfgVertexTernary* vtxp) const { + return operator()(vtxp->dtype(), vtxp->inputp(0), vtxp->inputp(1), vtxp->inputp(2)); + } + size_t operator()(const DfgDataType& dtype, const DfgVertex* source0p, + const DfgVertex* source1p, const DfgVertex* source2p) const { + V3Hash hash = dtype.hash(); + hash += vertexHash(source0p); + hash += vertexHash(source1p); + hash += vertexHash(source2p); + return hash.value(); + } }; + struct EqualTernary final { + bool operator()(const DfgVertexTernary* ap, const DfgVertexTernary* bp) const { + return operator()(ap, bp->dtype(), bp->inputp(0), bp->inputp(1), bp->inputp(2)); + } + bool operator()(const DfgVertexTernary* vtxp, const DfgDataType& dtype, + const DfgVertex* source0p, const DfgVertex* source1p, + const DfgVertex* source2p) const { + return vtxp->dtype() == dtype && vertexEqual(vtxp->inputp(0), source0p) + && vertexEqual(vtxp->inputp(1), source1p) + && vertexEqual(vtxp->inputp(2), source2p); + } + }; + + // Base class of vertex caches class CacheBase VL_NOT_FINAL { protected: // These set the operands of a new vertex @@ -210,86 +183,68 @@ class V3DfgCache final { public: // CacheBase does not cache anything virtual DfgVertex* cache(DfgVertex*) { return nullptr; } - virtual void invalidate(const DfgVertex*) {} + virtual void invalidate(DfgVertex*) {} }; - template + template class Cache final : public CacheBase { static_assert(std::is_base_of::value, "T_Vertex must be a DfgVertex"); - // TYPES - using Hash = typename T_Key::Hash; - using Equal = typename T_Key::Equal; - using Map = std::unordered_map; // STATE - Map m_map; - - // METHODS - - // These return a reference to the mapped entry, inserting a nullptr if not yet exists - - template - T_Vertex*& entry(T_Args&&... args) { - const T_Key key{std::forward(args)...}; - return m_map[key]; - } - template - typename Map::iterator find(T_Args&&... args) { - const T_Key key{std::forward(args)...}; - return m_map.find(key); - } + V3HashSet m_set; public: - // Add an existing vertex to the cache. If an equivalent exists, - // it is returned and the cache is not updated. + // Add an existing vertex to the cache. If an equivalent but different vertex exists, + // it is returned and the cache is not updated. Returns nullptr if the vertex is inserted. DfgVertex* cache(DfgVertex* vtxp) override { - UASSERT_OBJ(vtxp->is(), vtxp, "Vertex is wrong type"); - T_Vertex*& entrypr = entry(static_cast(vtxp)); - if (entrypr && entrypr != vtxp) return entrypr; - entrypr = static_cast(vtxp); - return nullptr; + UDEBUGONLY(UASSERT_OBJ(vtxp->is(), vtxp, "Vertex is wrong type");); + T_Vertex* const typedp = static_cast(vtxp); + T_Vertex* const cachedp = *m_set.insert(typedp).first; + return cachedp != vtxp ? cachedp : nullptr; } // Remove an existing vertex from the cache, if it is the cached vertex, otherwise no-op - void invalidate(const DfgVertex* vtxp) override { - UASSERT_OBJ(vtxp->is(), vtxp, "Vertex is wrong type"); - const auto it = find(static_cast(vtxp)); - if (it != m_map.end() && it->second == vtxp) m_map.erase(it); + void invalidate(DfgVertex* vtxp) override { + UDEBUGONLY(UASSERT_OBJ(vtxp->is(), vtxp, "Vertex is wrong type");); + T_Vertex* const typedp = static_cast(vtxp); + const auto it = m_set.find(typedp); + if (it != m_set.end() && *it == typedp) m_set.erase(it); } - // Get vertex with given operands, return nullptr if not in cache template Vertex* get(const DfgDataType& dtype, Operands... operands) { - const auto it = find(dtype, operands...); - return it != m_map.end() ? static_cast(it->second) : nullptr; + const auto it = m_set.find(dtype, operands...); + return it != m_set.end() ? static_cast(*it) : nullptr; } - - // Get or create (and insert) vertex with given operands + // Get vertex with given operands, if does not exist, create it template Vertex* getOrCreate(DfgGraph& dfg, FileLine* flp, const DfgDataType& dtype, Operands... operands) { - T_Vertex*& entryr = entry(dtype, operands...); - if (!entryr) { - T_Vertex* const newp = new Vertex{dfg, flp, dtype}; + const auto pair = m_set.insertLazy(dtype, operands..., [&]() -> T_Vertex* { + Vertex* const newp = new Vertex{dfg, flp, dtype}; setOperands(newp, operands...); - entryr = newp; - } - return static_cast(entryr); + return newp; + }); + T_Vertex* const vtxp = *pair.first; + UDEBUGONLY(UASSERT_OBJ(vtxp->template is(), vtxp, "Vertex is wrong type");); + return static_cast(vtxp); } }; // Map from Vertex type to cache type + // clang-format off template - using CacheType = - typename V3DfgCacheType, // - DfgVertexUnary, Cache, // - DfgVertexBinary, Cache, // - DfgVertexTernary, Cache // - >::Type; + using CacheType = typename V3DfgCacheType */ Cache, + DfgVertexUnary, /* -> */ Cache, + DfgVertexBinary, /* -> */ Cache, + DfgVertexTernary, /* -> */ Cache + >::Type; + // clang-format on + // STATE DfgGraph& m_dfg; // The DfgGraph we are caching the vertices of - // The per type caches +// The per type caches #define VERTEX_CACHE_DECLARE_CACHE(t) CacheType m_cache##t; FOREACH_DFG_VERTEX_TYPE(VERTEX_CACHE_DECLARE_CACHE) #undef VERTEX_CACHE_DECLARE_CACHE diff --git a/src/V3DfgCse.cpp b/src/V3DfgCse.cpp index 8f05a6030..bfdc5b886 100644 --- a/src/V3DfgCse.cpp +++ b/src/V3DfgCse.cpp @@ -18,30 +18,43 @@ #include "V3Dfg.h" #include "V3DfgPasses.h" +#include "V3HashTable.h" VL_DEFINE_DEBUG_FUNCTIONS; -class V3DfgCse final { - // TYPES - using VertexPair = std::pair; - struct VertexPairHash final { - size_t operator()(const VertexPair& pair) const { - V3Hash hash; - hash += pair.first; - hash += pair.second; - return hash.value(); - } - }; - +// Hash functor for V3HashSet - depends on vertex and all its inputs +class DfgCseHash final { // STATE - // The graph being processed - DfgGraph& m_dfg; - // Cache for vertex hashes - DfgUserMap m_hashCache = m_dfg.makeUserMap(); - // Cache for vertex equality - std::unordered_map m_equivalentCache; + mutable DfgUserMap m_cache; // Cache for vertex hashes + +public: + // CONSTRUCTOR + explicit DfgCseHash(DfgGraph& dfg) + : m_cache{dfg.makeUserMap()} { + // Pre-hash variables, these are all unique, so just set their hash to a unique value + uint32_t fixedHash = 0; + for (const DfgVertexVar& vtx : dfg.varVertices()) m_cache[vtx] = V3Hash{++fixedHash}; + // Pre-hash Ast references, these are all unique like variables + for (const DfgVertexAst& vtx : dfg.astVertices()) m_cache[vtx] = V3Hash{++fixedHash}; + // Pre-hash CReset and Prev vertices, these are all unique + for (const DfgVertex& vtx : dfg.opVertices()) { + if (vtx.is() || vtx.is()) m_cache[vtx] = V3Hash{++fixedHash}; + } + // Similarly pre-hash constants for speed. While we don't combine constants, we do want + // expressions using the same constants to be combined, so we do need to hash equal + // constants to equal values. + ++fixedHash; + for (const DfgConst& vtx : dfg.constVertices()) { + const V3Hash hash = vtx.num().toHash() + fixedHash; + // Technically possible for a hash to be zero, 'vertexSelfHash' assumes it isn't + m_cache[vtx] = VL_LIKELY(hash.value()) ? hash : V3Hash{1}; + } + } // METHODS + size_t operator()(DfgVertex* vtxp) const { return vertexHash(*vtxp).value(); } + +private: // Returns hash of vertex dependent on information internal to the vertex static V3Hash vertexSelfHash(const DfgVertex& vtx) { switch (vtx.type()) { @@ -135,29 +148,53 @@ class V3DfgCse final { VL_UNREACHABLE; } - // Returns hash of vertex dependent on and all its input - V3Hash vertexHash(DfgVertex& vtx) { - V3Hash& result = m_hashCache[vtx]; + // Returns hash of vertex dependent on itself and all its inputs - memoized + V3Hash vertexHash(DfgVertex& vtx) const { + V3Hash& result = m_cache[vtx]; + // Technically possible for a hash to be zero, but rare, so assume 0 means uninitialized if (!result.value()) { V3Hash hash{vertexSelfHash(vtx)}; - // Variables are defined by themselves, so there is no need to hash them further - // (especially the sources). This enables sound hashing of graphs circular only through - // variables, which we rely on. - if (!vtx.is()) { - hash += vtx.type(); - hash += vtx.size(); - vtx.foreachSource([&](DfgVertex& src) { - hash += vertexHash(src); - return false; - }); - } + hash += vtx.type(); + hash += vtx.size(); + vtx.foreachSource([&](DfgVertex& src) { + hash += vertexHash(src); // Graph is acyclic, so this terminates + return false; + }); result = hash; } return result; } +}; +// Equal functor for V3HashSet - depends on vertex and all its inputs +class DfgCseEqual final { + // TYPES + using VertexPair = std::pair; + struct VertexPairHash final { + size_t operator()(const VertexPair& pair) const { + V3Hash hash; + hash += pair.first; + hash += pair.second; + return hash.value(); + } + }; + + // STATE + mutable V3HashMap m_cache; // Cache for vertex equality + mutable std::vector m_driverLo; // Low indices of drivers + const size_t m_size; // Size of the graph + +public: + // CONSTRUCTORS + explicit DfgCseEqual(const DfgGraph& dfg) + : m_size{dfg.size()} {} + + // METHODS + bool operator()(DfgVertex* ap, DfgVertex* bp) const { return vertexEquivalent(*ap, *bp); } + +private: // Compare 'a' and 'b' for equivalence based on their internal information only - bool vertexSelfEquivalent(const DfgVertex& a, const DfgVertex& b) { + bool vertexSelfEquivalent(const DfgVertex& a, const DfgVertex& b) const { // Note: 'a' and 'b' are of the same Vertex type, data type, and have // the same number of inputs with matching types. This is established // by 'vertexEquivalent'. @@ -187,16 +224,17 @@ class V3DfgCse final { case VDfgType::SplicePacked: { const DfgVertexSplice* const ap = a.as(); // Gather indices of drivers of 'a' - std::vector aLo; - aLo.reserve(ap->nInputs()); + m_driverLo.clear(); + m_driverLo.reserve(ap->nInputs()); ap->foreachDriver([&](const DfgVertex&, uint32_t lo) { - aLo.push_back(lo); + m_driverLo.push_back(lo); return false; }); - // Compare indices of drivers of 'b' - uint32_t* aLop = aLo.data(); - return !b.as()->foreachDriver( - [&](const DfgVertex&, uint32_t lo) { return *aLop++ != lo; }); + // Compare indices of drivers of 'b', equal if all match + uint32_t* aLop = m_driverLo.data(); + return !b.as()->foreachDriver([&](const DfgVertex&, uint32_t lo) { // + return *aLop++ != lo; + }); } // Vertices with no internal information @@ -263,19 +301,19 @@ class V3DfgCse final { } // Compares the sources of 'a' and 'b' for equivalence - bool sourcesEquivalent(const DfgVertex& a, const DfgVertex& b) { + bool sourcesEquivalent(const DfgVertex& a, const DfgVertex& b) const { for (size_t i = 0; i < a.nInputs(); ++i) { const DfgVertex* const ap = a.inputp(i); const DfgVertex* const bp = b.inputp(i); if (!ap && !bp) continue; if (!ap || !bp) return false; - if (!vertexEquivalent(*ap, *bp)) return false; + if (!vertexEquivalent(*ap, *bp)) return false; // Graph is acyclic, so this terminates } return true; } // Compares 'a' and 'b' for equivalence - bool vertexEquivalent(const DfgVertex& a, const DfgVertex& b) { + bool vertexEquivalent(const DfgVertex& a, const DfgVertex& b) const { // If same vertex, then equal if (&a == &b) return true; @@ -297,70 +335,51 @@ class V3DfgCse final { // be looked up again through multiple paths. if (!a.hasMultipleSinks() && !b.hasMultipleSinks()) return sourcesEquivalent(a, b); - // Check sources + // Need to compare the source vertices, check memo const VertexPair key = (&a < &b) ? std::make_pair(&a, &b) : std::make_pair(&b, &a); - // The recursive invocation can cause a re-hash but that will not invalidate references - uint8_t& result = m_equivalentCache[key]; - if (!result) result = (static_cast(sourcesEquivalent(a, b)) << 1) | 1; - return result >> 1; - } + const auto it = m_cache.find(key); + if (it != m_cache.end()) return it->second; - V3DfgCse(DfgGraph& dfg, V3DfgCseContext& ctx) - : m_dfg{dfg} { - std::unordered_map> verticesWithEqualHashes; - verticesWithEqualHashes.reserve(dfg.size()); + // Not memoized yet, so compute and memoize, reserve table on first insert + const bool equal = sourcesEquivalent(a, b); + if (VL_UNLIKELY(m_cache.empty())) m_cache.reserve(m_size / 4); + m_cache.insert({key, equal}); - // Pre-hash variables, these are all unique, so just set their hash to a unique value - uint32_t varHash = 0; - for (const DfgVertexVar& vtx : dfg.varVertices()) m_hashCache[vtx] = V3Hash{++varHash}; - // Pre-hash Ast references, these are all unique like variables - for (const DfgVertexAst& vtx : dfg.astVertices()) m_hashCache[vtx] = V3Hash{++varHash}; - // Pre-hash CReset and Prev vertices, these are all unique - for (const DfgVertex& vtx : dfg.opVertices()) { - if (vtx.is() || vtx.is()) m_hashCache[vtx] = V3Hash{++varHash}; - } - - // Similarly pre-hash constants for speed. While we don't combine constants, we do want - // expressions using the same constants to be combined, so we do need to hash equal - // constants to equal values. - for (DfgConst* const vtxp : dfg.constVertices().unlinkable()) { - // Delete unused constants while we are at it. - if (!vtxp->hasSinks()) { - VL_DO_DANGLING(vtxp->unlinkDelete(dfg), vtxp); - continue; - } - m_hashCache[vtxp] = vtxp->num().toHash() + varHash; - } - - // Combine operation vertices - for (DfgVertex* const vtxp : dfg.opVertices().unlinkable()) { - // Delete unused nodes while we are at it. - if (!vtxp->hasSinks()) { - vtxp->unlinkDelete(dfg); - continue; - } - std::vector& vec = verticesWithEqualHashes[vertexHash(*vtxp)]; - bool replaced = false; - for (DfgVertex* const candidatep : vec) { - if (vertexEquivalent(*candidatep, *vtxp)) { - ++ctx.m_eliminated; - vtxp->replaceWith(candidatep); - VL_DO_DANGLING(vtxp->unlinkDelete(dfg), vtxp); - replaced = true; - break; - } - } - if (replaced) continue; - vec.push_back(vtxp); - } - } - -public: - static void apply(DfgGraph& dfg, V3DfgCseContext& ctx) { - { V3DfgCse{dfg, ctx}; } - // Prune unused nodes - V3DfgPasses::removeUnused(dfg); + // The predicate result + return equal; } }; -void V3DfgPasses::cse(DfgGraph& dfg, V3DfgCseContext& ctx) { V3DfgCse::apply(dfg, ctx); } +// Combine equivalent operation vertices +void dfgCseCombineEquivalent(DfgGraph& dfg, V3DfgCseContext& ctx) { + // Delete unused constants, so the pre-hashing below need not consider them + for (DfgConst* const vtxp : dfg.constVertices().unlinkable()) { + if (!vtxp->hasSinks()) VL_DO_DANGLING(vtxp->unlinkDelete(dfg), vtxp); + } + + // Set of unique vertices. This set does all the work identifying equivalent vertices. + V3HashSet uniqueVtxps{DfgCseHash{dfg}, DfgCseEqual{dfg}}; + // There is at most one entry per vertex + uniqueVtxps.reserve(dfg.size()); + + // Combine operation vertices + for (DfgVertex* const vtxp : dfg.opVertices().unlinkable()) { + // Delete unused nodes while we are at it. + if (!vtxp->hasSinks()) { + vtxp->unlinkDelete(dfg); + continue; + } + // Insert the vertex into the set, if an equivalent is found, replace the vertex with it + const auto pair = uniqueVtxps.insert(vtxp); + if (!pair.second) { + ++ctx.m_eliminated; + vtxp->replaceWith(*pair.first); + VL_DO_DANGLING(vtxp->unlinkDelete(dfg), vtxp); + } + } +} + +void V3DfgPasses::cse(DfgGraph& dfg, V3DfgCseContext& ctx) { + dfgCseCombineEquivalent(dfg, ctx); + V3DfgPasses::removeUnused(dfg); +} diff --git a/src/V3HashTable.cpp b/src/V3HashTable.cpp new file mode 100644 index 000000000..b6592d408 --- /dev/null +++ b/src/V3HashTable.cpp @@ -0,0 +1,1144 @@ +// -*- mode: C++; c-file-style: "cc-mode" -*- +//************************************************************************* +// DESCRIPTION: Verilator: Tests for V3HashTable.h +// +// Code available from: https://verilator.org +// +//************************************************************************* +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of either the GNU Lesser General Public License Version 3 +// or the Perl Artistic License Version 2.0. +// SPDX-FileCopyrightText: 2003-2026 Wilson Snyder +// SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0 +// +//************************************************************************* + +#include "V3HashTable.h" + +#include "V3Error.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace V3HashTableInternals { + +// Entries that fill a table of the given capacity to its maximum load, so one more grows it +constexpr size_t maxLoad(size_t capacity) { return capacity * LOAD_FACTOR_NUM / LOAD_FACTOR_DEN; } + +// Enough entries to grow the smallest table twice, that is, to fill four times the minimum +constexpr size_t GROWS_TWICE = maxLoad(4 * MIN_CAPACITY); +static_assert(GROWS_TWICE > maxLoad(2 * MIN_CAPACITY), + "SelfTest: 'GROWS_TWICE' must overflow a table twice the minimum"); + +//###################################################################### +// Set key by value + +void testValueKeys() { + struct Value final { + size_t m_hash = 0; // Hash of this entry + size_t m_id = 0; // Id of this entry + }; + + struct Hash final { + size_t operator()(const Value& value) const { return operator()(value.m_hash, 0); } + size_t operator()(size_t hash, size_t) const { return hash; } + }; + + struct Equal final { + bool operator()(const Value& a, const Value& b) const { + return operator()(a, b.m_hash, b.m_id); + } + bool operator()(const Value& value, size_t hash, size_t id) const { + return value.m_hash == hash && value.m_id == id; + } + }; + + using Set = V3HashSet; + + // Entries that are added are found, entries that are not are not + { + const Value value{1, 0}; + const Value equal{1, 0}; // Equal to 'value', and hashes the same + Set set; + UASSERT_SELFTEST(set.empty(), true); // Starts empty + UASSERT_SELFTEST(set.begin() == set.end(), true); // So iterates nothing + UASSERT_SELFTEST(set.find(value) == set.end(), true); // And finds nothing + const std::pair added = set.insert(value); + UASSERT_SELFTEST(added.second, true); // Added, as it was absent + UASSERT_SELFTEST(added.first->m_id, value.m_id); // The iterator is at the entry + UASSERT_SELFTEST(&*added.first != &value, true); // Which is a copy of the argument + UASSERT_SELFTEST(set.size(), 1); // And is the only one + UASSERT_SELFTEST(set.empty(), false); // So the set is no longer empty + { + const Set::iterator it = set.find(value); + UASSERT_SELFTEST(it != set.end(), true); // Found by itself + UASSERT_SELFTEST(&*it, &*added.first); // At the entry held + } + { + const Set::iterator it = set.find(equal); + UASSERT_SELFTEST(it != set.end(), true); // And by an equal entry + UASSERT_SELFTEST(&*it, &*added.first); // At that same entry + } + { + const Set::iterator it = set.find(1, size_t{0}); + UASSERT_SELFTEST(it != set.end(), true); // And by its parts + UASSERT_SELFTEST(&*it, &*added.first); // At that same entry again + } + UASSERT_SELFTEST(set.find(1, size_t{1}) == set.end(), true); // Hash same, id not + UASSERT_SELFTEST(set.find(2, size_t{0}) == set.end(), true); // Id same, hash not + // Adding an equal entry returns the one in the set, and does not add + const std::pair again = set.insert(equal); + UASSERT_SELFTEST(again.second, false); // Not added, as an equal is present + UASSERT_SELFTEST(&*again.first, &*added.first); // At the entry already held + UASSERT_SELFTEST(set.size(), 1); // Still just the one entry + // Erasing through an iterator removes the entry it is at + { + const Set::iterator it = set.find(value); + UASSERT_SELFTEST(it != set.end(), true); // Present before the erase + set.erase(it); + } + UASSERT_SELFTEST(set.empty(), true); // Empty again + UASSERT_SELFTEST(set.find(value) == set.end(), true); // So the entry is gone + UASSERT_SELFTEST(set.contains(value), false); // As 'contains' agrees + // Erase by key removes whatever that key finds + UASSERT_SELFTEST(set.erase(value), false); // Nothing to erase, so says so + set.insert(value); + UASSERT_SELFTEST(set.erase(equal), true); // An equal key erases the entry held + UASSERT_SELFTEST(set.empty(), true); // Which leaves the set empty + // And by a key spelled as the parts of an entry + set.insert(value); + UASSERT_SELFTEST(set.erase(1, size_t{0}), true); // Those parts find and erase it + UASSERT_SELFTEST(set.empty(), true); // Empty once more + } + + // Colliding entries are all kept and stay reachable, including after the table grows + { + constexpr size_t N = GROWS_TWICE; + Set set; + for (size_t i = 0; i < N; ++i) { + const std::pair added = set.insert(Value{7, i}); + UASSERT_SELFTEST(added.second, true); // Ids differ, so each is added + } + UASSERT_SELFTEST(set.size(), N); // All of them, in the one probe run + for (size_t i = 0; i < N; ++i) { + const Set::iterator it = set.find(size_t{7}, i); + UASSERT_SELFTEST(it != set.end(), true); // The growth lost nothing + UASSERT_SELFTEST(it->m_id, i); // And is the entry asked for + } + // Iterating visits every entry exactly once + std::array seen{}; + size_t n = 0; + for (const Value& entry : set) { + UASSERT_SELFTEST(entry.m_hash, size_t{7}); // Only entries added + UASSERT_SELFTEST(seen[entry.m_id], false); // Each of them exactly once + seen[entry.m_id] = true; + ++n; + } + UASSERT_SELFTEST(n, N); // And every one of them + } +} + +//###################################################################### +// Set key by pointer + +void testPointerKeys() { + struct Value final { + size_t m_hash = 0; // Hash of this entry + size_t m_id = 0; // Id of this entry + }; + + struct Hash final { + size_t operator()(const Value* valuep) const { return operator()(valuep->m_hash, 0); } + size_t operator()(size_t hash, size_t) const { return hash; } + }; + + struct Equal final { + bool operator()(const Value* ap, const Value* bp) const { + return operator()(ap, bp->m_hash, bp->m_id); + } + bool operator()(const Value* valuep, size_t hash, size_t id) const { + return valuep->m_hash == hash && valuep->m_id == id; + } + }; + + using Set = V3HashSet; + + // Entries that are added are found, entries that are not are not + { + Value value{1, 0}; + Value equal{1, 0}; // Equal to 'value', and hashes the same + Set set; + UASSERT_SELFTEST(set.empty(), true); // Starts empty + UASSERT_SELFTEST(set.begin() == set.end(), true); // So iterates nothing + UASSERT_SELFTEST(set.find(&value) == set.end(), true); // And finds nothing + const std::pair added = set.insert(&value); + UASSERT_SELFTEST(added.second, true); // Added, as it was absent + UASSERT_SELFTEST(*added.first, &value); // The iterator is at the new entry + UASSERT_SELFTEST(set.size(), 1); // Which is the only one + UASSERT_SELFTEST(set.empty(), false); // So the set is no longer empty + { + const Set::iterator it = set.find(&value); + UASSERT_SELFTEST(it != set.end(), true); // Found by itself + UASSERT_SELFTEST(*it, &value); // At the entry held + } + { + const Set::iterator it = set.find(&equal); + UASSERT_SELFTEST(it != set.end(), true); // And by an equal entry + UASSERT_SELFTEST(*it, &value); // At that same entry + } + { + const Set::iterator it = set.find(1, size_t{0}); + UASSERT_SELFTEST(it != set.end(), true); // And by its parts + UASSERT_SELFTEST(*it, &value); // At that same entry again + } + UASSERT_SELFTEST(set.find(1, size_t{1}) == set.end(), true); // Hash same, id not + UASSERT_SELFTEST(set.find(2, size_t{0}) == set.end(), true); // Id same, hash not + // Adding an equal entry returns the one in the set, and does not add + const std::pair again = set.insert(&equal); + UASSERT_SELFTEST(again.second, false); // Not added, as an equal is present + UASSERT_SELFTEST(*again.first, &value); // The iterator is at the stored one + UASSERT_SELFTEST(set.size(), 1); // Still just the one entry + // Given the distinct but equal '&equal', 'find' still yields the stored '&value' + { + const Set::iterator it = set.find(&equal); + UASSERT_SELFTEST(it != set.end(), true); // Found, as they compare equal + UASSERT_SELFTEST(*it, &value); // But it is the stored one + UASSERT_SELFTEST(*it == &equal, false); // Not the one asked for + } + { + const Set::iterator it = set.find(&value); + UASSERT_SELFTEST(it != set.end(), true); // The entry is still there + UASSERT_SELFTEST(*it, &value); // And is the one stored + } + // Erasing the very entry held does remove it + { + const Set::iterator it = set.find(&value); + UASSERT_SELFTEST(it != set.end(), true); // Present before the erase + UASSERT_SELFTEST(*it, &value); // And is the object asked for + set.erase(it); + } + UASSERT_SELFTEST(set.empty(), true); // Empty again + UASSERT_SELFTEST(set.find(&value) == set.end(), true); // So the entry is gone + UASSERT_SELFTEST(set.contains(&value), false); // As 'contains' agrees + UASSERT_SELFTEST(set.empty(), true); // And no lookup added anything + // Erase by key makes no such check, so it removes whatever the key finds, here + // the stored '&value' when given the equal '&equal' + UASSERT_SELFTEST(set.erase(&value), false); // Nothing to erase, so says so + set.insert(&value); + UASSERT_SELFTEST(set.erase(&equal), true); // The equal key erases the stored one + UASSERT_SELFTEST(set.empty(), true); // Which leaves the set empty + UASSERT_SELFTEST(set.find(&value) == set.end(), true); // And unreachable + // And by a key spelled as the parts of an entry + set.insert(&value); + UASSERT_SELFTEST(set.erase(1, size_t{0}), true); // Those parts find and erase it + UASSERT_SELFTEST(set.empty(), true); // Empty once more + } + + // Erasing leaves the other entries reachable, whichever is erased, including when + // entries with equal hashes are all in the one probe run + for (size_t erase = 0; erase < 4; ++erase) { + for (const size_t hash : {size_t{0}, size_t{7}, ~size_t{0}}) { // Wraps too + std::array values{Value{hash, 0}, Value{hash, 1}, // + Value{hash, 2}, Value{hash, 3}}; + Set set; + for (Value& value : values) set.insert(&value); + UASSERT_SELFTEST(set.size(), 4); // All distinct, so all added + { + const Set::iterator it = set.find(&values[erase]); + UASSERT_SELFTEST(it != set.end(), true); // The one to erase is present + UASSERT_SELFTEST(*it, &values[erase]); // And is the object held + set.erase(it); + } + UASSERT_SELFTEST(set.size(), 3); // Exactly one was erased + UASSERT_SELFTEST(set.find(&values[erase]) == set.end(), true); // That one + for (size_t i = 0; i < values.size(); ++i) { + if (i == erase) continue; + const Set::iterator it = set.find(hash, i); + UASSERT_SELFTEST(it != set.end(), true); // The others are all there + UASSERT_SELFTEST(*it, &values[i]); // Each at the entry inserted + } + } + } + + // Growing leaves all entries reachable, including when their run wraps around the end + // of the table. Note this needs enough entries to actually grow, so do not reserve here. + for (const size_t hash : {size_t{0}, size_t{7}, ~size_t{0}}) { + std::array many{}; + Set set; + for (size_t i = 0; i < many.size(); ++i) { + many[i] = Value{hash, i}; + set.insert(&many[i]); + } + UASSERT_SELFTEST(set.size(), many.size()); // All of them were added + for (size_t i = 0; i < many.size(); ++i) { + const Set::iterator it = set.find(hash, i); + UASSERT_SELFTEST(it != set.end(), true); // And all survived the growth + UASSERT_SELFTEST(*it, &many[i]); // Each at the entry inserted + } + } + + // Entries that collide but are not equal are both kept, and erase removes only the one + { + Value value{1, 0}; + Value other{1, 1}; // Not equal to 'value', but lands on the same slot + Set set; + set.reserve(2); + set.insert(&value); + set.insert(&other); + UASSERT_SELFTEST(set.size(), 2); // Both kept, despite the collision + { + const Set::iterator it = set.find(&value); + UASSERT_SELFTEST(it != set.end(), true); // The first of the run is there + UASSERT_SELFTEST(*it, &value); // And is that entry + } + { + const Set::iterator it = set.find(&other); + UASSERT_SELFTEST(it != set.end(), true); // As is the one behind it + UASSERT_SELFTEST(*it, &other); // Which probing reached past the first + } + { + const Set::iterator it = set.find(&value); + UASSERT_SELFTEST(it != set.end(), true); // Present before the erase + UASSERT_SELFTEST(*it, &value); // And is the object held + set.erase(it); + } + UASSERT_SELFTEST(set.size(), 1); // Only one was erased + UASSERT_SELFTEST(set.find(&value) == set.end(), true); // Namely that one + { + const Set::iterator it = set.find(&other); + UASSERT_SELFTEST(it != set.end(), true); // The collider stays + UASSERT_SELFTEST(*it, &other); // Shifted back over the hole left + } + } + + // Reserving avoids growing, and all entries survive either way + for (const bool doReserve : {false, true}) { + std::array values{}; + Set set; + if (doReserve) set.reserve(values.size()); + for (size_t i = 0; i < values.size(); ++i) { + values[i] = Value{static_cast(i * 1234567), i}; + set.insert(&values[i]); + } + for (Value& value : values) { + const Set::iterator it = set.find(&value); + UASSERT_SELFTEST(it != set.end(), true); // Each entry survives + UASSERT_SELFTEST(*it, &value); // And is the one inserted + } + UASSERT_SELFTEST(set.size(), values.size()); // And none was added twice + } + + // 'insertLazy' creates only on a miss, including when that grows the table + { + constexpr size_t N = GROWS_TWICE; + std::array values{}; + Set set; + for (size_t i = 0; i < N; ++i) { + values[i] = Value{7, i}; // All of them hash the same + const std::pair pair + = set.insertLazy(size_t{7}, i, [&]() -> Value* { return &values[i]; }); + UASSERT_SELFTEST(pair.second, true); // Created, as it was absent + UASSERT_SELFTEST(*pair.first, &values[i]); // And is what the factory made + } + UASSERT_SELFTEST(set.size(), N); // All of them were added + // They are all present now, so nothing is created + for (size_t i = 0; i < N; ++i) { + bool created = false; + const std::pair pair + = set.insertLazy(size_t{7}, i, [&]() -> Value* { + created = true; // LCOV_EXCL_START + return &values[i]; // LCOV_EXCL_STOP + }); + UASSERT_SELFTEST(created, false); // The factory was never called + UASSERT_SELFTEST(pair.second, false); // As the lookup hit + UASSERT_SELFTEST(*pair.first, &values[i]); // On the stored entry + } + UASSERT_SELFTEST(set.size(), N); // And the set is unchanged + } + + // Iterating visits every entry exactly once + { + std::array values{}; + std::array seen{}; + Set set; + for (size_t i = 0; i < values.size(); ++i) { + values[i] = Value{i / 2, i}; // Pairs of them collide + set.insert(&values[i]); + } + size_t n = 0; + for (Value* const valuep : set) { + UASSERT_SELFTEST(valuep->m_id < seen.size(), true); // Only entries added + UASSERT_SELFTEST(seen[valuep->m_id], false); // Each of them exactly once + seen[valuep->m_id] = true; + ++n; + } + UASSERT_SELFTEST(n, values.size()); // And every one of them + } +} + +//###################################################################### +// Entries held by value, which are constructed and destroyed in step with the slots + +void testEntryLifetime() { + size_t alive = 0; // Number of live entries, which the entries themselves count + + // A test entry that is not default constructible, nor assignable, and that counts how + // many are alive. A move makes another live entry, so the count tracks the occupied + // slots however the container shuffles them about. + class NoDefault final { + size_t* m_alivep; // Where the live entries are counted + size_t m_hash; // Hash of this entry + size_t m_id; // Entries with equal ids are equal + + public: + NoDefault(size_t* alivep, size_t hash, size_t id) + : m_alivep{alivep} + , m_hash{hash} + , m_id{id} { + ++*m_alivep; + } + NoDefault(const NoDefault& that) + : m_alivep{that.m_alivep} + , m_hash{that.m_hash} + , m_id{that.m_id} { + ++*m_alivep; + } + NoDefault(NoDefault&& that) + : m_alivep{that.m_alivep} + , m_hash{that.m_hash} + , m_id{that.m_id} { + ++*m_alivep; + } + ~NoDefault() { --*m_alivep; } + NoDefault& operator=(const NoDefault&) = delete; + NoDefault& operator=(NoDefault&&) = delete; + + size_t hash() const { return m_hash; } + size_t id() const { return m_id; } + bool operator==(const NoDefault& that) const { return m_id == that.m_id; } + }; + + static_assert(!std::is_default_constructible::value, + "SelfTest: 'NoDefault' must not be default constructible"); + static_assert(!std::is_copy_assignable::value, + "SelfTest: 'NoDefault' must not be copy assignable"); + static_assert(!std::is_move_assignable::value, + "SelfTest: 'NoDefault' must not be move assignable"); + static_assert(std::is_move_constructible::value, + "SelfTest: 'NoDefault' must be move constructible, to exercise moving"); + + struct Hash final { + size_t operator()(const NoDefault& value) const { return value.hash(); } + size_t operator()(size_t hash, size_t) const { return hash; } + }; + + struct Equal final { + bool operator()(const NoDefault& a, const NoDefault& b) const { + return operator()(a, b.hash(), b.id()); + } + bool operator()(const NoDefault& a, size_t, size_t id) const { return a.id() == id; } + }; + + using Set = V3HashSet; + + constexpr size_t N = GROWS_TWICE; + UASSERT_SELFTEST(alive, 0); // Nothing built yet + { + Set set; + for (size_t i = 0; i < N; ++i) { + const std::pair added = set.insert(NoDefault{&alive, 7, i}); + UASSERT_SELFTEST(added.second, true); // Ids differ, so each is added + } + UASSERT_SELFTEST(set.size(), N); // All of them are in + UASSERT_SELFTEST(alive, N); // Held by exactly that many slots + // All of them hash the same, so they are all in the one probe run + for (size_t i = 0; i < N; ++i) { + const Set::iterator it = set.find(size_t{7}, i); + UASSERT_SELFTEST(it != set.end(), true); // Reachable through the run + UASSERT_SELFTEST(it->id(), i); // And is the entry asked for + } + // A rejected insert constructs no entry: the argument is only copied on a miss + { + const std::pair dup = set.insert(NoDefault{&alive, 7, 0}); + UASSERT_SELFTEST(dup.second, false); // Not added, as an equal is present + UASSERT_SELFTEST(dup.first->id(), 0); // The iterator is at the stored one + } + UASSERT_SELFTEST(set.size(), N); // Nothing was added + UASSERT_SELFTEST(alive, N); // And no copy of the argument was kept + { + const Set::iterator it = set.find(size_t{7}, size_t{0}); + UASSERT_SELFTEST(it != set.end(), true); // Present before the erase + UASSERT_SELFTEST(it->id(), size_t{0}); // And is the entry asked for + set.erase(it); + } + UASSERT_SELFTEST(set.size(), N - 1); // One fewer entry + UASSERT_SELFTEST(alive, N - 1); // And one fewer live object + UASSERT_SELFTEST(set.contains(size_t{7}, size_t{0}), false); // Namely that one + // Erasing the rest keeps the live entries in step with the slots, all the way down. + // They all collide, so every erase shifts entries back over the hole. + for (size_t i = 1; i < N; ++i) { + { + const Set::iterator it = set.find(size_t{7}, i); + UASSERT_SELFTEST(it != set.end(), true); // Still reachable + UASSERT_SELFTEST(it->id(), i); // And is the entry asked for + set.erase(it); + } + UASSERT_SELFTEST(set.size(), N - 1 - i); // The count follows the erases + UASSERT_SELFTEST(alive, N - 1 - i); // As do the live entries + for (size_t j = i + 1; j < N; ++j) { + UASSERT_SELFTEST(set.contains(size_t{7}, j), true); // Shifted, not lost + } + } + UASSERT_SELFTEST(set.begin() == set.end(), true); // Erased down to empty + UASSERT_SELFTEST(alive, 0); // With every entry destroyed + // Leave entries in the set, so its destructor has some to destroy + for (size_t i = 0; i < N; ++i) set.insert(NoDefault{&alive, 7, i}); + UASSERT_SELFTEST(alive, N); // Live as the set goes out of scope + } + // The set is gone, so every entry it still held has been destroyed + UASSERT_SELFTEST(alive, 0); // Leaking none of them +} + +//###################################################################### +// A table can be moved, handing over the entries and leaving an empty table behind + +void testMove() { + size_t alive = 0; // Number of live entries, which the entries themselves count + + // A test entry that counts the live ones, so a move that copied an entry, dropped + // one, or destroyed one twice, shows up in the count + struct Counted final { + size_t* m_alivep; // Where the live entries are counted + size_t m_id; // Entries with equal ids are equal + + Counted(size_t* alivep, size_t id) + : m_alivep{alivep} + , m_id{id} { + ++*m_alivep; + } + Counted(const Counted& that) + : Counted{that.m_alivep, that.m_id} {} + Counted(Counted&& that) + : Counted{that.m_alivep, that.m_id} {} + ~Counted() { --*m_alivep; } + Counted& operator=(const Counted&) = delete; + Counted& operator=(Counted&&) = delete; + }; + + struct Hash final { + size_t operator()(const Counted& entry) const { return operator()(entry.m_id); } + size_t operator()(size_t id) const { return id; } + }; + + struct Equal final { + bool operator()(const Counted& a, const Counted& b) const { return operator()(a, b.m_id); } + bool operator()(const Counted& a, size_t id) const { return a.m_id == id; } + }; + + using Set = V3HashSet; + + constexpr size_t N = GROWS_TWICE; + { + Set set; + for (size_t i = 0; i < N; ++i) set.insert(Counted{&alive, i}); + UASSERT_SELFTEST(set.size(), N); // All of them are in + UASSERT_SELFTEST(alive, N); // Held by exactly that many slots + + // Move construction takes the entries, making and destroying none + Set moved{std::move(set)}; + UASSERT_SELFTEST(moved.size(), N); // Which the target now holds + UASSERT_SELFTEST(alive, N); // With no entry made or destroyed + UASSERT_SELFTEST(set.empty(), true); // And the source no longer holds them + UASSERT_SELFTEST(set.begin() == set.end(), true); // So it iterates nothing + for (size_t i = 0; i < N; ++i) { + UASSERT_SELFTEST(moved.contains(i), true); // Every entry came across + UASSERT_SELFTEST(set.contains(i), false); // And none stayed behind + } + + // The moved from table is empty rather than broken, so it can be filled again + UASSERT_SELFTEST(set.insert(Counted{&alive, N}).second, true); // It took an entry + UASSERT_SELFTEST(set.size(), 1); // Which is all it holds + UASSERT_SELFTEST(set.empty(), false); // So it is no longer empty + UASSERT_SELFTEST(alive, N + 1); // And is one more live entry + + // Move assignment destroys what the target held, then takes the source's + set = std::move(moved); + UASSERT_SELFTEST(set.size(), N); // The target holds the moved entries + UASSERT_SELFTEST(alive, N); // The entry it held itself was destroyed + UASSERT_SELFTEST(set.contains(N), false); // Namely that one + UASSERT_SELFTEST(moved.empty(), true); // And the source is empty again + for (size_t i = 0; i < N; ++i) UASSERT_SELFTEST(set.contains(i), true); // The rest moved + } + // Both tables are gone, so every entry either still held has been destroyed + UASSERT_SELFTEST(alive, 0); // Leaking none of them +} + +//###################################################################### +// Entries that cannot be copied, only moved + +void testMoveOnlyEntries() { + // Only 'insertLazy' can add one of these, as 'insert' would copy it, and any copy the + // container made of an entry would stop this compiling. + class MoveOnly final { + size_t m_id; // Entries with equal ids are equal + + public: + explicit MoveOnly(size_t id) + : m_id{id} {} + MoveOnly(MoveOnly&&) = default; + MoveOnly(const MoveOnly&) = delete; + MoveOnly& operator=(const MoveOnly&) = delete; + MoveOnly& operator=(MoveOnly&&) = delete; + ~MoveOnly() = default; + + size_t id() const { return m_id; } + }; + + static_assert(!std::is_copy_constructible::value, + "SelfTest: 'MoveOnly' must not be copy constructible"); + static_assert(std::is_move_constructible::value, + "SelfTest: 'MoveOnly' must be move constructible"); + + struct Hash final { + // Every entry hashes the same, so they all end up in the one probe run + size_t operator()(const MoveOnly& value) const { return operator()(value.id()); } + size_t operator()(size_t) const { return 7; } + }; + + struct Equal final { + bool operator()(const MoveOnly& a, const MoveOnly& b) const { + return operator()(a, b.id()); + } + bool operator()(const MoveOnly& a, size_t id) const { return a.id() == id; } + }; + + using Set = V3HashSet; + + constexpr size_t N = GROWS_TWICE; + Set set; + for (size_t i = 0; i < N; ++i) { + const std::pair added + = set.insertLazy(i, [i] { return MoveOnly{i}; }); + UASSERT_SELFTEST(added.second, true); // Ids differ, so each is added + UASSERT_SELFTEST(added.first->id(), i); // Moved into the slot, never copied + } + UASSERT_SELFTEST(set.size(), N); // All of them are in + // They all collide, so erasing every other one shifts the rest back over the holes + for (size_t i = 0; i < N; i += 2) { + const Set::iterator it = set.find(i); + UASSERT_SELFTEST(it != set.end(), true); // Present before the erase + set.erase(it); + } + UASSERT_SELFTEST(set.size(), N / 2); // Half of them are gone + for (size_t i = 0; i < N; ++i) { + const bool erased = (i % 2) == 0; + UASSERT_SELFTEST(set.contains(i), !erased); // And it is the right half + } +} + +//###################################################################### +// Stateful functors, which the two argument constructor moves in + +void testStatefulFunctors() { + // Hashing and comparison that both depend on a mask the functor holds, so the set only + // works if it keeps the instances it was handed + class Hash final { + size_t m_mask; // Only these bits of an entry matter + + public: + explicit Hash(size_t mask) + : m_mask{mask} {} + size_t operator()(size_t value) const { return value & m_mask; } + }; + + class Equal final { + size_t m_mask; // Only these bits of an entry matter + + public: + explicit Equal(size_t mask) + : m_mask{mask} {} + bool operator()(size_t a, size_t b) const { return (a & m_mask) == (b & m_mask); } + }; + + // Neither is default constructible, so the set cannot make its own + static_assert(!std::is_default_constructible::value, + "SelfTest: 'Hash' must not be default constructible"); + static_assert(!std::is_default_constructible::value, + "SelfTest: 'Equal' must not be default constructible"); + + // Only the low two bits matter, so the 16 entries fall into 4 classes + V3HashSet set{Hash{3}, Equal{3}}; + for (size_t i = 0; i < 16; ++i) set.insert(i); + UASSERT_SELFTEST(set.size(), 4); // So the set kept the functors it was given + // Every entry finds the first one added of its class, which is the class itself + for (size_t i = 0; i < 16; ++i) { + const V3HashSet::iterator it = set.find(i); + UASSERT_SELFTEST(it != set.end(), true); // Its class is present + UASSERT_SELFTEST(*it, (i & 3)); // Represented by the first one added + } +} + +//###################################################################### +// Backward shift deletion moves back exactly the entries whose probe run crosses the +// hole: an entry standing at its home position past the hole must stay put + +void testBackwardShiftDeletion() { + // A test entry with an explicit hash, so probe runs can be laid out at will + struct Value final { + size_t m_hash = 0; // Hash of this entry + size_t m_id = 0; // Entries with equal ids are equal + }; + + struct Hash final { + size_t operator()(const Value* valuep) const { return operator()(valuep->m_hash, 0); } + size_t operator()(size_t hash, size_t) const { return hash; } + }; + + struct Equal final { + bool operator()(const Value* ap, const Value* bp) const { + return operator()(ap, bp->m_hash, bp->m_id); + } + bool operator()(const Value* valuep, size_t hash, size_t id) const { + return valuep->m_hash == hash && valuep->m_id == id; + } + }; + + using Set = V3HashSet; + + // One probe run of four entries with alternating home positions 'h' and 'h + 1', + // occupying four adjacent slots. Erasing the first leaves a hole: the second sits + // at its own home and must not be moved into it, while the third and fourth have + // their runs broken by the hole and must be moved back. + for (const size_t h : {size_t{0}, size_t{5}, ~size_t{0}}) { // Wraps too + std::array values{Value{h, 0}, Value{h + 1, 1}, // + Value{h, 2}, Value{h + 1, 3}}; + Set set; + for (Value& value : values) set.insert(&value); + UASSERT_SELFTEST(set.size(), 4); // The run holds all four + + // Erase the entry at the head of the run + { + const Set::iterator it = set.find(h, size_t{0}); + UASSERT_SELFTEST(it != set.end(), true); // Present before the erase + set.erase(it); + } + UASSERT_SELFTEST(set.size(), 3); // One fewer entry + UASSERT_SELFTEST(set.contains(h, size_t{0}), false); // Namely that one + // Whether moved back or left in place, every entry must remain reachable + for (size_t i = 1; i < values.size(); ++i) { + const Set::iterator it = set.find(values[i].m_hash, i); + UASSERT_SELFTEST(it != set.end(), true); // The shift lost nothing + UASSERT_SELFTEST(*it, &values[i]); // And moved back the right entries + } + + // Erase the entry that stayed at its home position, shifting the last one again + { + const Set::iterator it = set.find(h + 1, size_t{1}); + UASSERT_SELFTEST(it != set.end(), true); // Left where the first erase found it + set.erase(it); + } + UASSERT_SELFTEST(set.size(), 2); // Two are left + for (size_t i = 2; i < values.size(); ++i) { + const Set::iterator it = set.find(values[i].m_hash, i); + UASSERT_SELFTEST(it != set.end(), true); // Both still reachable + UASSERT_SELFTEST(*it, &values[i]); // And are the entries they were + } + // Iterating visits exactly the remaining entries + std::array seen{}; + for (const Value* const valuep : set) { + UASSERT_SELFTEST(seen[valuep->m_id], false); // Each entry once + seen[valuep->m_id] = true; + } + UASSERT_SELFTEST(seen[0], false); // Erased first + UASSERT_SELFTEST(seen[1], false); // Erased second + UASSERT_SELFTEST(seen[2], true); // Moved back over the first hole + UASSERT_SELFTEST(seen[3], true); // And back again over the second + } +} + +//###################################################################### +// Entries stay in place unless the table grows or an entry is erased, as only those +// two invalidate iterators + +void testReferenceStability() { + struct Value final { + size_t m_hash = 0; // Hash of this entry + size_t m_id = 0; // Id of this entry + }; + + struct Hash final { + size_t operator()(const Value* valuep) const { return operator()(valuep->m_hash, 0); } + size_t operator()(size_t hash, size_t) const { return hash; } + }; + + struct Equal final { + bool operator()(const Value* ap, const Value* bp) const { + return operator()(ap, bp->m_hash, bp->m_id); + } + bool operator()(const Value* valuep, size_t hash, size_t id) const { + return valuep->m_hash == hash && valuep->m_id == id; + } + }; + + using Set = V3HashSet; + + // A reserved but still empty set finds nothing and iterates nothing + { + Set set; + set.reserve(8); + UASSERT_SELFTEST(set.contains(size_t{7}, size_t{0}), false); // Room, but no entry + UASSERT_SELFTEST(set.begin() == set.end(), true); // So iterates nothing + UASSERT_SELFTEST(set.empty(), true); // And holds nothing + } + + // 'N' is exactly what the reservation must hold, so this also checks the boundary + // arithmetic of 'reserve' against that of the growth check. It is more than an + // unreserved table holds without growing, so the reservation is doing the work. + constexpr size_t N = maxLoad(2 * MIN_CAPACITY); + static_assert(N > maxLoad(MIN_CAPACITY), "SelfTest: 'N' must need more than a new table"); + std::array values{}; + std::array entrypps{}; // Where each entry is stored, as inserted + Set set; + set.reserve(N); + // All entries collide, so every insertion probes through the whole existing run + for (size_t i = 0; i < N; ++i) { + values[i] = Value{7, i}; + const std::pair pair = set.insert(&values[i]); + UASSERT_SELFTEST(pair.second, true); // Ids differ, so each is added + entrypps[i] = &*pair.first; + } + // The set was reserved, so no insertion grew the table, and no entry has moved + for (size_t i = 0; i < N; ++i) { + const Set::iterator it = set.find(size_t{7}, i); + UASSERT_SELFTEST(it != set.end(), true); // Every entry is still there + UASSERT_SELFTEST(&*it, entrypps[i]); // In the slot it was put in + } + // Inserting entries that are present adds nothing and moves nothing + for (size_t i = 0; i < N; ++i) { + const std::pair pair = set.insert(&values[i]); + UASSERT_SELFTEST(pair.second, false); // Rejected, as it is present + UASSERT_SELFTEST(&*pair.first, entrypps[i]); // And nothing moved + } + UASSERT_SELFTEST(set.size(), N); // No duplicate was added + // Growing an occupied set through 'reserve' keeps every entry + set.reserve(8 * N); + UASSERT_SELFTEST(set.size(), N); // Rehashing dropped nothing + std::array grownpps{}; // Where each entry is after the growth + for (size_t i = 0; i < N; ++i) { + const Set::iterator it = set.find(size_t{7}, i); + UASSERT_SELFTEST(it != set.end(), true); // And left every entry reachable + grownpps[i] = &*it; + } + // Reserving room that is there already leaves the table alone, so nothing moves + set.reserve(N); + UASSERT_SELFTEST(set.size(), N); // The smaller request changed nothing + for (size_t i = 0; i < N; ++i) { + const Set::iterator it = set.find(size_t{7}, i); + UASSERT_SELFTEST(it != set.end(), true); // Every entry is still there + UASSERT_SELFTEST(&*it, grownpps[i]); // In the slot the growth left it in + } +} + +//###################################################################### +// A pseudo random workload checked against std::set or std::map as the reference model + +// Only a few distinct hashes, so probe runs are long and every erase shifts entries +struct ClusteredHash final { + size_t operator()(size_t value) const { return value & 0x7; } +}; + +// Hashes spread by a large odd multiplier, so most probe runs are short +struct SpreadHash final { + size_t operator()(size_t value) const { + return static_cast(value * 0x9e3779b97f4a7c15ULL); + } +}; + +// Drive a set through a deterministic pseudo random workload of insertions, erasures, +// and lookups, checking every step against a std::set holding the same entries +template +void testSetAgainstModel() { + struct Equal final { + bool operator()(size_t a, size_t b) const { return a == b; } + }; + + using Set = V3HashSet; + + constexpr size_t UNIVERSE = 64; // Entries drawn from a small range, so lookups hit + constexpr size_t STEPS = 10000; // Number of operations applied + + // A simple linear congruential generator, with a fixed seed so failures reproduce + uint64_t state = 0x123456789abcdef0ULL; + const auto nextRand = [&state]() -> size_t { + state = state * 6364136223846793005ULL + 1442695040888963407ULL; + return static_cast(state >> 32); + }; + + Set set; + std::set model; + + for (size_t step = 0; step < STEPS; ++step) { + const size_t id = nextRand() % UNIVERSE; + switch (nextRand() % 4) { + case 0: { // Insert a copy + const std::pair pair = set.insert(id); + UASSERT_SELFTEST(pair.second, model.insert(id).second); // As the model + UASSERT_SELFTEST(*pair.first, id); // At the entry asked for + break; + } + case 1: { // Insert lazily, which must create only on a miss + bool created = false; + const std::pair pair + = set.insertLazy(id, [&]() -> size_t { + created = true; + return id; + }); + UASSERT_SELFTEST(pair.second, created); // Created only on a miss + UASSERT_SELFTEST(pair.second, model.insert(id).second); // As the model + UASSERT_SELFTEST(*pair.first, id); // At the entry asked for + break; + } + case 2: { // Erase, if present + const typename Set::iterator it = set.find(id); + UASSERT_SELFTEST(it != set.end(), model.count(id) != 0); // As the model + if (it != set.end()) { + set.erase(it); + model.erase(id); + } + break; + } + default: { // Look up only + const typename Set::iterator it = set.find(id); + UASSERT_SELFTEST(it != set.end(), model.count(id) != 0); // As the model + if (it != set.end()) UASSERT_SELFTEST(*it, id); // And is the entry + break; + } + } + UASSERT_SELFTEST(set.size(), model.size()); // Every step, not just at the end + } + + // Check the final content exhaustively + for (size_t id = 0; id < UNIVERSE; ++id) { + UASSERT_SELFTEST(set.contains(id), model.count(id) != 0); // Present iff modelled + } + // And that iterating visits exactly the model content, each entry once + std::array seen{}; + size_t n = 0; + for (const size_t entry : set) { + UASSERT_SELFTEST(entry < UNIVERSE, true); // Nothing out of thin air + UASSERT_SELFTEST(model.count(entry) != 0, true); // Only what was inserted + UASSERT_SELFTEST(seen[entry], false); // Each entry once + seen[entry] = true; + ++n; + } + UASSERT_SELFTEST(n, model.size()); // And all of them +} + +//###################################################################### +// A hash table used as a map, whose entries are plain key and value pairs + +void testMap() { + // The key extractor means the hash and equality see only keys, never entries, so + // the standard functors do, and they are what V3HashMap defaults to + using Map = V3HashMap; + using Entry = Map::Entry; + + // A map entry is a plain pair, so a caller never meets an internal type + static_assert(std::is_same>::value, + "SelfTest: a map entry must be a plain pair"); + + constexpr size_t N = GROWS_TWICE; + Map map; + for (size_t i = 0; i < N; ++i) { + const std::string key = "key" + std::to_string(i); + // 'insertLazy' calls the factory only on a miss, so a hit builds no entry + const std::pair pair + = map.insertLazy(key, [&]() -> Entry { return {key, static_cast(i)}; }); + UASSERT_SELFTEST(pair.second, true); // Keys differ, so each is added + } + UASSERT_SELFTEST(map.size(), N); // All of them are in + + // A bare key finds the entry, with no Entry built to look it up + for (size_t i = 0; i < N; ++i) { + const Map::iterator it = map.find("key" + std::to_string(i)); + UASSERT_SELFTEST(it != map.end(), true); // The key alone finds it + UASSERT_SELFTEST(it->second, static_cast(i)); // With the value given + } + + // A value is changed by erasing the entry and inserting it again, as an iterator + // yields a const entry + for (size_t i = 0; i < N; ++i) { + const std::string key = "key" + std::to_string(i); + map.erase(map.find(key)); + UASSERT_SELFTEST(map.insert({key, static_cast(i) + 100}).second, true); // Anew + } + UASSERT_SELFTEST(map.size(), N); // The same keys, so the map has not grown + for (size_t i = 0; i < N; ++i) { + const Map::iterator it = map.find("key" + std::to_string(i)); + UASSERT_SELFTEST(it != map.end(), true); // Each key is still there + UASSERT_SELFTEST(it->second, static_cast(i) + 100); // With its new value + } + + // Adding a key that is present changes nothing + { + const std::string key = "key0"; + bool created = false; + const std::pair pair = map.insertLazy(key, [&]() -> Entry { + created = true; // LCOV_EXCL_START + return {key, 0}; // LCOV_EXCL_STOP + }); + UASSERT_SELFTEST(created, false); // The factory was never called + UASSERT_SELFTEST(pair.second, false); // As the key is present + UASSERT_SELFTEST(pair.first->second, 100); // Holding its old value + } + UASSERT_SELFTEST(map.size(), N); // And nothing was added + + // 'insert' adds a key and a value as a plain pair, as std::unordered_map::insert does + { + const std::pair added = map.insert({std::string{"fresh"}, 5}); + UASSERT_SELFTEST(added.second, true); // Added, as the key is new + UASSERT_SELFTEST(added.first->second, 5); // With the value given + const std::pair again = map.insert({std::string{"fresh"}, 6}); + UASSERT_SELFTEST(again.second, false); // Rejected, as the key is present + UASSERT_SELFTEST(again.first->second, 5); // And the value is untouched + } + UASSERT_SELFTEST(map.size(), N + 1); // Just the one entry was added + { + const Map::iterator it = map.find(std::string{"fresh"}); + UASSERT_SELFTEST(it != map.end(), true); // Present before the erase + UASSERT_SELFTEST(it->second, 5); // Still holding the first value + map.erase(it); + } + UASSERT_SELFTEST(map.size(), N); // Back to the earlier content + + // Erasing by key removes just that entry, and says whether it did + UASSERT_SELFTEST(map.erase(std::string{"absent"}), false); // No such key + UASSERT_SELFTEST(map.size(), N); // So nothing was erased + UASSERT_SELFTEST(map.erase(std::string{"key0"}), true); // That key is present + UASSERT_SELFTEST(map.size(), N - 1); // And just the one entry went + UASSERT_SELFTEST(map.contains(std::string{"key0"}), false); // Namely that one + for (size_t i = 1; i < N; ++i) { + UASSERT_SELFTEST(map.contains("key" + std::to_string(i)), true); // Others intact + } +} + +//###################################################################### +// The pseudo random workload again, run against a map and checked against std::map + +// Drive a map through a deterministic pseudo random workload of insertions, erasures, +// value changes and lookups, checking every step against a std::map holding the same +template +void testMapAgainstModel() { + struct Equal final { + bool operator()(size_t a, size_t b) const { return a == b; } + }; + + using Map = V3HashMap; + using Entry = typename Map::Entry; + + constexpr size_t UNIVERSE = 64; // Keys drawn from a small range, so lookups hit + constexpr size_t STEPS = 10000; // Number of operations applied + + // A simple linear congruential generator, with a fixed seed so failures reproduce + uint64_t state = 0x0fedcba987654321ULL; + const auto nextRand = [&state]() -> size_t { + state = state * 6364136223846793005ULL + 1442695040888963407ULL; + return static_cast(state >> 32); + }; + + Map map; + std::map model; + + for (size_t step = 0; step < STEPS; ++step) { + const size_t key = nextRand() % UNIVERSE; + const size_t val = step; // Distinct every step, so a stale value is visible + switch (nextRand() % 5) { + case 0: { // Insert a whole entry + const std::pair pair = map.insert({key, val}); + UASSERT_SELFTEST(pair.second, model.insert({key, val}).second); // Ditto + UASSERT_SELFTEST(pair.first->first, key); // At the key asked for + break; + } + case 1: { // Insert lazily, which must create only on a miss + bool created = false; + const std::pair pair + = map.insertLazy(key, [&]() -> Entry { + created = true; + return {key, val}; + }); + UASSERT_SELFTEST(pair.second, created); // Created only on a miss + UASSERT_SELFTEST(pair.second, model.insert({key, val}).second); // Ditto + UASSERT_SELFTEST(pair.first->first, key); // At the key asked for + break; + } + case 2: { // Erase by key, if present + UASSERT_SELFTEST(map.erase(key), model.erase(key) != 0); // As model + break; + } + case 3: { // Change a value, which is erasing the entry and inserting it again + const typename Map::iterator it = map.find(key); + if (it != map.end()) { + map.erase(it); + UASSERT_SELFTEST(map.insert({key, val}).second, true); // The key is free + model[key] = val; + } + break; + } + default: { // Look up only + const typename Map::iterator it = map.find(key); + UASSERT_SELFTEST(it != map.end(), model.count(key) != 0); // As the model + if (it != map.end()) { + UASSERT_SELFTEST(it->first, key); // At the key asked for + UASSERT_SELFTEST(it->second, model.at(key)); // With its value + } + break; + } + } + UASSERT_SELFTEST(map.size(), model.size()); // Every step, not just at the end + } + + // Check the final content exhaustively, values included + for (size_t key = 0; key < UNIVERSE; ++key) { + const typename Map::iterator it = map.find(key); + UASSERT_SELFTEST(it != map.end(), model.count(key) != 0); // Present iff modelled + if (it != map.end()) { + UASSERT_SELFTEST(it->second, model.at(key)); // With the right value + } + } + // And that iterating visits exactly the model content, each entry once + std::array seen{}; + size_t n = 0; + for (const Entry& entry : map) { + UASSERT_SELFTEST(entry.first < UNIVERSE, true); // Nothing out of thin air + UASSERT_SELFTEST(model.count(entry.first) != 0, true); // Only what was inserted + UASSERT_SELFTEST(entry.second, model.at(entry.first)); // With the right value + UASSERT_SELFTEST(seen[entry.first], false); // Each entry once + seen[entry.first] = true; + ++n; + } + UASSERT_SELFTEST(n, model.size()); // And all of them +} + +void selfTest() { + testValueKeys(); + testPointerKeys(); + testEntryLifetime(); + testMove(); + testMoveOnlyEntries(); + testStatefulFunctors(); + testBackwardShiftDeletion(); + testReferenceStability(); + testSetAgainstModel(); + testSetAgainstModel(); + testMap(); + testMapAgainstModel(); + testMapAgainstModel(); +} + +} // namespace V3HashTableInternals diff --git a/src/V3HashTable.h b/src/V3HashTable.h new file mode 100644 index 000000000..ae87f1322 --- /dev/null +++ b/src/V3HashTable.h @@ -0,0 +1,472 @@ +// -*- mode: C++; c-file-style: "cc-mode" -*- +//************************************************************************* +// DESCRIPTION: Verilator: Open addressing hash set and hash map +// +// Code available from: https://verilator.org +// +//************************************************************************* +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of either the GNU Lesser General Public License Version 3 +// or the Perl Artistic License Version 2.0. +// SPDX-FileCopyrightText: 2003-2026 Wilson Snyder +// SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0 +// +//************************************************************************* +// +// An open addressing, linear probing hash table, with backward shift deletion. +// Usable as V3HashSet or V3HashMap. The benefit of these over +// std::unordered_set and std::unordered_map is far better memory locality +// during lookup, and fewer dynamic memory allocations (which also means less +// heap fragmentation). Consider using these if profiling shows that the +// unordered STL collections contribute a significant cost to an algorithm. +// +// Four types tell the table what it holds: the entry, a hash, an equality, and +// a key extractor yielding the lookup key of an entry. The extractor is what +// lets one table serve both roles: a set's entry is its own key, a map's is a +// pair keyed by its first. The hash and equality hence only ever see keys, +// never entries. V3HashSet and V3HashMap at the bottom of this file derive +// from the table, pairing it with the extractor that suits each. +// +// Those two work on keys via Hash and Equal functors as in std::unordered_set +// or std::unordered_map, but lookup is always heterogeneous, with no +// is_transparent to opt in like in the STL, and a lookup key can be spelled as +// several arguments, being the parts a key is made of. An entry can hence be +// looked up without one at hand, as when it is only created on a miss. The +// functors must provide call operators as const members, for the key of an +// entry and for every lookup key spelling used: +// +// size_t Hash::operator()(const T_Key&) const +// size_t Hash::operator()(...) const +// bool Equal::operator()(const T_Key&, const T_Key&) const +// bool Equal::operator()(const T_Key&, ...) const +// +// with equal keys hashing equal, as usual, and consistently across the +// spellings. +// +// As only entries are stored, a slot is just a hash and an entry, so probing +// touches few cache lines. The table is doubled when an insertion would take +// it over the maximum load factor, or sized up front with 'reserve', to keep +// the probe runs short. +// +// Entries are referred to by iterators, as in the STL containers, but unlike +// STL containers, the mapped value in a V3HashMap is not mutable through an +// iterator. Iterators and entry addresses stay valid until the table grows or +// an entry is erased; either invalidates all of them. +// +// Erasure uses backward shift deletion: entries following the hole are moved +// back over it where their probe run ran through it (no tombstones). +// +//************************************************************************* + +#ifndef VERILATOR_V3HASHTABLE_H_ +#define VERILATOR_V3HASHTABLE_H_ + +#include "config_build.h" +#include "verilatedos.h" + +#include "V3Error.h" +#include "V3StdFuture.h" + +#include +#include +#include +#include +#include +#include + +namespace V3HashTableInternals { + +constexpr size_t MIN_CAPACITY = 16; // Smallest table allocated +constexpr size_t LOAD_FACTOR_NUM = 3; // Numerator of the maximum load factor +constexpr size_t LOAD_FACTOR_DEN = 4; // Denominator of the maximum load factor + +// Key extractor for a table whose entries are their own keys, that is, a set +template +struct V3HashTableKeyIsEntry final { + using Key = T_Key; // What it yields, so the table need not deduce it + const T_Key& operator()(const T_Key& entry) const { return entry; } +}; + +// Key extractor for a table whose entries are pairs keyed by the first, that is, a map +template +struct V3HashTableKeyIsFirst final { + using Key = T_Key; // What it yields, so the table need not deduce it + const T_Key& operator()(const std::pair& entry) const { return entry.first; } +}; + +void selfTest(); + +} // namespace V3HashTableInternals + +// V3HashTable, see the file header +// T_Entry The entries (STL calls this value_type) +// T_Hash Hashes a lookup key +// T_Equal Compares a key to a lookup key +// T_KeyOf Yields the key of an entry +template +class V3HashTable VL_NOT_FINAL { +public: + // TYPES + using Entry = T_Entry; // What is stored + using Key = typename T_KeyOf::Key; // What entries are looked up by + +private: + // TYPES + // Holds if the hash accepts a lookup key spelled as the given arguments + template + using ValidHash = vlstd::is_invocable_r; + + // Holds if the equality accepts a key and such a lookup key + template + using ValidEqual = vlstd::is_invocable_r; + + // The Key must itself be a valid lookup key, as every lookup ends in comparing one + // against a stored entry. Asserted separately, so the failure names the functor. + static_assert(ValidHash::value, "The 'Hash' functor must accept the 'Key'"); + static_assert(ValidEqual::value, "The 'Equal' functor must accept two 'Key's"); + + // A table slot + struct Slot final { + // The entry comes first, so it starts the slot whatever its alignment. + // It is a union so it is alive only while the slot is occupied. + union { + Entry m_entry; + }; + size_t m_hash = 0; // Hash of the entry, or zero when the slot is free + + Slot() {} // Leaves 'm_entry' uninitialized, as the slot is free + ~Slot() { + if (!isFree()) destroy(); + } + Slot(const Slot&) = delete; + Slot(Slot&&) = delete; + const Slot& operator=(const Slot&) = delete; + Slot& operator=(Slot&& that) { + UDEBUGONLY(UASSERT(this != &that, "Moving a slot onto itself");); + UDEBUGONLY(UASSERT(!that.isFree(), "Moving from a free slot");); + UDEBUGONLY(UASSERT(isFree(), "Moving into an occupied slot");); + new (&m_entry) Entry{std::move(that.m_entry)}; + m_hash = that.m_hash; + that.destroy(); + return *this; + } + + bool isFree() const { return !m_hash; } + + // Construct the entry of this free slot from the given entry + void construct(size_t hash, Entry&& entry) { + UDEBUGONLY(UASSERT(isFree(), "Constructing the entry of an occupied slot");); + new (&m_entry) Entry{std::move(entry)}; + m_hash = hash; + } + // Destroy the entry of this occupied slot, leaving it free + void destroy() { + UDEBUGONLY(UASSERT(!isFree(), "Destroying the entry of a free slot");); + m_entry.~Entry(); + m_hash = 0; + } + }; + +public: + // Iterator over the entries, see the file header on invalidation + class iterator final { + friend class V3HashTable; + + Slot* m_slotp = nullptr; // The slot iterated, or the end of the table + Slot* m_endp = nullptr; // One past the last slot + + iterator(Slot* slotp, Slot* endp) + : m_slotp{slotp} + , m_endp{endp} {} + + public: + iterator() = default; + // As opposed to the STL, this always returns a const reference so the + // collection is not mutable through an iterator alone. This is + // required because entries must be movable, hence can't be const, but + // the key of a map must not be modified. + const Entry& operator*() const { return m_slotp->m_entry; } + const Entry* operator->() const { return &m_slotp->m_entry; } + // Pre-increment, skipping the free slots + iterator& operator++() { + while (++m_slotp != m_endp && m_slotp->isFree()) {} + return *this; + } + bool operator==(const iterator& that) const { return m_slotp == that.m_slotp; } + bool operator!=(const iterator& that) const { return m_slotp != that.m_slotp; } + }; + +private: + // STATE + std::unique_ptr m_table; // The table, null when unallocated + size_t m_capacity = 0; // Number of slots in the table, a power of two, or zero + size_t m_size = 0; // Number of occupied slots + VL_NO_UNIQUE_ADDRESS_CXX20 T_Hash m_hash; // Hashes a lookup key + VL_NO_UNIQUE_ADDRESS_CXX20 T_Equal m_equal; // Compares a key to a lookup key + VL_NO_UNIQUE_ADDRESS_CXX20 T_KeyOf m_keyOf; // Yields the lookup key of an entry + + // METHODS + + // The hash of the given entry or lookup key, as stored in a slot + template + size_t hashOf(const T_Args&... args) const { + // A free slot is one with a zero hash, so force the high bit into every hash. + constexpr size_t USED_BIT = size_t{1} << (sizeof(size_t) * 8 - 1); + return static_cast(m_hash(args...)) | USED_BIT; + } + + // Index of the free slot the given hash probes to. There must always be one. + size_t freeSlot(size_t hash) const { + const size_t mask = m_capacity - 1; + size_t i = hash & mask; + while (!m_table[i].isFree()) i = (i + 1) & mask; + return i; + } + + // Resize to the given number of slots, which must fit all entries + void resize(size_t count) { + UDEBUGONLY(UASSERT(count && !(count & (count - 1)), "Capacity not a power of 2");); + const std::unique_ptr oldTable{std::move(m_table)}; + const size_t oldCapacity = m_capacity; + m_table = std::make_unique(count); + m_capacity = count; + // Reinsert the entries. 'freeSlot' appends to the probe run of each, so the runs + // come out contiguous whatever order this visits the old slots in. + for (size_t i = 0; i < oldCapacity; ++i) { + Slot& slot = oldTable[i]; + if (!slot.isFree()) m_table[freeSlot(slot.m_hash)] = std::move(slot); + } + } + + // Index of the slot holding the entry equal to the given key, or of the free slot its + // probe sequence ends at. The table must not be empty. + template + size_t probe(size_t hash, const T_Args&... args) const { + UDEBUGONLY(UASSERT(m_table, "Table must be allocated");); + const size_t mask = m_capacity - 1; + size_t i = hash & mask; + while (!m_table[i].isFree()) { + const Slot& slot = m_table[i]; + if (slot.m_hash == hash && m_equal(m_keyOf(slot.m_entry), args...)) break; + i = (i + 1) & mask; + } + return i; + } + + // Implementation of 'insertLazy' below. 'all' holds the key arguments, followed by + // the callable that creates the entry, so 'N_Key' indexes the key. + template + std::pair insertLazyImpl(std::index_sequence, T_All&& all) { + static_assert(ValidHash...>::value, + "The 'Hash' functor does not accept a lookup key spelled like this"); + static_assert(ValidEqual...>::value, + "The 'Equal' functor does not accept a lookup key spelled like this"); + const size_t hash = hashOf(std::get(all)...); + // Allocate on the first insertion + if (VL_UNLIKELY(!m_capacity)) resize(V3HashTableInternals::MIN_CAPACITY); + // Find the slot for the entry + Slot* slotp = m_table.get() + probe(hash, std::get(all)...); + // If occupied, it's the equivalent, and we are done + if (!slotp->isFree()) return {iterator{slotp, m_table.get() + m_capacity}, false}; + // Table is growing + ++m_size; + // Increase if necessary by load factor + if (VL_UNLIKELY(m_size * V3HashTableInternals::LOAD_FACTOR_DEN + > m_capacity * V3HashTableInternals::LOAD_FACTOR_NUM)) { + resize(m_capacity * 2); + slotp = m_table.get() + freeSlot(hash); + } + // Construct the entry via the user provided callable (last item in 'all') + slotp->construct(hash, std::get(all)()); + // The key of the created entry must both hash and compare as the key looked up +#ifdef VL_DEBUG + const Key& key = m_keyOf(slotp->m_entry); + UASSERT(hashOf(key) == hash, + "Created entry does not hash as the key it was looked up with"); + UASSERT(m_equal(key, std::get(all)...), + "Created entry does not match the key it was looked up with"); +#endif + // Return newly create entry + return {iterator{slotp, m_table.get() + m_capacity}, true}; + } + +protected: + // CONSTRUCTORS + V3HashTable() = default; + V3HashTable(T_Hash hash, T_Equal equal) + : m_hash{std::move(hash)} + , m_equal{std::move(equal)} {} + ~V3HashTable() = default; + VL_UNCOPYABLE(V3HashTable); + // Movable, as the table is just a pointer. The source is left empty rather than + // merely unspecified, so it remains a usable, empty table. + V3HashTable(V3HashTable&& that) + : m_table{std::move(that.m_table)} + , m_capacity{that.m_capacity} + , m_size{that.m_size} + , m_hash{std::move(that.m_hash)} + , m_equal{std::move(that.m_equal)} + , m_keyOf{std::move(that.m_keyOf)} { + that.m_capacity = 0; + that.m_size = 0; + } + V3HashTable& operator=(V3HashTable&& that) { + m_table = std::move(that.m_table); // Frees the table this held, if any + m_capacity = that.m_capacity; + m_size = that.m_size; + m_hash = std::move(that.m_hash); + m_equal = std::move(that.m_equal); + m_keyOf = std::move(that.m_keyOf); + that.m_capacity = 0; + that.m_size = 0; + return *this; + } + +public: + // METHODS + size_t size() const { return m_size; } + bool empty() const { return !m_size; } + + iterator begin() const { + Slot* const endp = m_table.get() + m_capacity; + Slot* slotp = m_table.get(); + while (slotp != endp && slotp->isFree()) ++slotp; + return iterator{slotp, endp}; + } + iterator end() const { + Slot* const endp = m_table.get() + m_capacity; + return iterator{endp, endp}; + } + + // Make room for the given number of entries, so inserting that many will not resize + void reserve(size_t count) { + size_t capacity = V3HashTableInternals::MIN_CAPACITY; + while (capacity * V3HashTableInternals::LOAD_FACTOR_NUM + < count * V3HashTableInternals::LOAD_FACTOR_DEN) + capacity *= 2; + if (capacity > m_capacity) resize(capacity); + } + + // Return iterator to the entry equal to the given key, or 'end()' if there + // is none. The key is whatever T_Hash and T_Equal accept, spelled as any + // number of arguments. Same as STL containers. + template + iterator find(const T_Args&... args) const { + static_assert(ValidHash::value, + "The 'Hash' functor does not accept a lookup key spelled like this"); + static_assert(ValidEqual::value, + "The 'Equal' functor does not accept a lookup key spelled like this"); + if (!m_size) return end(); // Nothing to find, and this also covers there being no table + Slot* const slotp = m_table.get() + probe(hashOf(args...), args...); + return slotp->isFree() ? end() : iterator{slotp, m_table.get() + m_capacity}; + } + + // Add the given entry, unless an equal one is in the table already. Return + // iterator to the entry and true if insertion happened. Same as STL containers. + std::pair insert(const Entry& entry) { + static_assert(std::is_copy_constructible::value, + "'Entry' must be copy constructible to use 'insert'"); + return insertLazy(m_keyOf(entry), [&entry]() -> Entry { return entry; }); + } + + // As 'insert', but the entry is only made when needed: all but the last argument spell + // the key, and the last is a callable to create the entry on a miss. Note the created entry + // must hash and compare equal to the key, and the call must not touch the container, as this + // holds the slot the entry will go in. + template + std::pair insertLazy(T_Args&&... args) { + static_assert(sizeof...(T_Args) >= 2, + "'insertLazy' needs a lookup key, then a callable to create the entry"); + using Callable = std::tuple_element_t>; + static_assert(vlstd::is_invocable_r::value, + "The last argument of 'insertLazy' must be a callable that takes no " + "arguments and returns an 'Entry'"); + return insertLazyImpl(std::make_index_sequence{}, + std::forward_as_tuple(std::forward(args)...)); + } + + // Whether an entry equal to the given key is in the table. The key is spelled as for 'find'. + template + bool contains(const T_Args&... args) const { + return find(args...) != end(); + } + + // Remove the entry equal to the given key, and return whether there was one. + template + bool erase(const T_Args&... args) { + const iterator it = find(args...); + if (it == end()) return false; + erase(it); + return true; + } + + // Remove the entry the given iterator refers to, which must not be 'end()'. Note that + // unlike STL erase this returns nothing, as every iterator is invalidated on deletion. + void erase(iterator it) { + UDEBUGONLY(UASSERT(it != end() && !it.m_slotp->isFree(), "Erasing a bad iterator");); + const size_t mask = m_capacity - 1; + size_t i = static_cast(it.m_slotp - m_table.get()); + // Destroy the entry + m_table[i].destroy(); + // The entry is gone, so slot 'i' is now a hole + --m_size; + // Backward shift deletion: move back the entries whose probing the hole breaks + size_t j = i; + while (true) { + j = (j + 1) & mask; + Slot& slot = m_table[j]; + if (slot.isFree()) break; + // Move back if its home position does not lie in the cyclic range (i, j] + if (((j - (slot.m_hash & mask)) & mask) >= ((j - i) & mask)) { + m_table[i] = std::move(slot); // Frees 'slot', which is then the hole + i = j; + } + } + } +}; + +template , + typename T_Equal = std::equal_to> +class V3HashSet final : public V3HashTable> { + using Super + = V3HashTable>; + + // Entries are only ever moved. Note 'insert' additionally needs copy construction. + static_assert(std::is_move_constructible::value, "'T_Key' must be move constructible"); + static_assert(std::is_destructible::value, "'T_Key' must be destructible"); + +public: + // CONSTRUCTORS + V3HashSet() = default; + V3HashSet(T_Hash hash, T_Equal equal) + : Super{std::move(hash), std::move(equal)} {} +}; + +template , + typename T_Equal = std::equal_to> +class V3HashMap final + : public V3HashTable, T_Hash, T_Equal, + V3HashTableInternals::V3HashTableKeyIsFirst> { + using Super = V3HashTable, T_Hash, T_Equal, + V3HashTableInternals::V3HashTableKeyIsFirst>; + + // Entries are only ever moved. Note 'insert' additionally needs copy construction. + // Asserted separately, so the failure names the one at fault. + static_assert(std::is_move_constructible::value, "'T_Key' must be move constructible"); + static_assert(std::is_destructible::value, "'T_Key' must be destructible"); + static_assert(std::is_move_constructible::value, "'T_Val' must be move constructible"); + static_assert(std::is_destructible::value, "'T_Val' must be destructible"); + +public: + // TYPES + using Value = T_Val; // What a key maps to + + // CONSTRUCTORS + V3HashMap() = default; + V3HashMap(T_Hash hash, T_Equal equal) + : Super{std::move(hash), std::move(equal)} {} +}; + +#endif // Guard diff --git a/src/Verilator.cpp b/src/Verilator.cpp index dca85ae99..524e2b77a 100644 --- a/src/Verilator.cpp +++ b/src/Verilator.cpp @@ -62,6 +62,7 @@ #include "V3Gate.h" #include "V3Global.h" #include "V3Graph.h" +#include "V3HashTable.h" #include "V3HierBlock.h" #include "V3Inline.h" #include "V3InlineCFuncs.h" @@ -741,6 +742,7 @@ static bool verilate(const string& argString) { V3PreShell::selfTest(); V3Broken::selfTest(); V3Control::selfTest(); + V3HashTableInternals::selfTest(); V3ThreadPool::selfTest(); UINFO(2, "selfTest done"); }