diff --git a/include/verilatedos.h b/include/verilatedos.h index 0d173cb2f..21ea88e11 100644 --- a/include/verilatedos.h +++ b/include/verilatedos.h @@ -318,6 +318,15 @@ # define VL_CONSTEXPR_CXX17 #endif +//========================================================================= +// C++-2020 + +#if __cplusplus >= 202002L +# define VL_NO_UNIQUE_ADDRESS_CXX20 [[no_unique_address]] +#else +# define VL_NO_UNIQUE_ADDRESS_CXX20 +#endif + //========================================================================= // Optimization diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5b3ee4a1c..000e63e1d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -114,6 +114,7 @@ set(HEADERS V3GraphPathChecker.h V3GraphStream.h V3Hash.h + V3HashTable.h V3Hasher.h V3HierBlock.h V3Inline.h @@ -292,6 +293,7 @@ set(COMMON_SOURCES V3GraphPathChecker.cpp V3GraphTest.cpp V3Hash.cpp + V3HashTable.cpp V3Hasher.cpp V3HierBlock.cpp V3Inline.cpp diff --git a/src/Makefile_obj.in b/src/Makefile_obj.in index f076e1f8e..755b42450 100644 --- a/src/Makefile_obj.in +++ b/src/Makefile_obj.in @@ -196,6 +196,7 @@ RAW_OBJS = \ V3GraphPathChecker.o \ V3GraphTest.o \ V3Hash.o \ + V3HashTable.o \ V3OptionParser.o \ V3Os.o \ V3ParseGrammar.o \ diff --git a/src/V3DfgCache.h b/src/V3DfgCache.h index d3cce4c85..b0c7302aa 100644 --- a/src/V3DfgCache.h +++ b/src/V3DfgCache.h @@ -25,8 +25,11 @@ #ifndef VERILATOR_V3DFGCACHE_H_ #define VERILATOR_V3DFGCACHE_H_ +#include "verilatedos.h" + #include "V3Dfg.h" #include "V3DfgDataType.h" +#include "V3HashTable.h" #include @@ -52,137 +55,107 @@ struct V3DfgCacheType final { class V3DfgCache final { // TYPES - class KeySel final { - const DfgDataType& m_dtype; - const DfgVertex* const m_fromp; - const uint32_t m_lsb; + // Hashing and comparison of the cached vertices. Each takes either a vertex, or the + // parts a vertex would be created from, so a lookup needs no vertex and no key object. - public: - KeySel(const DfgDataType& dtype, DfgVertex* fromp, uint32_t lsb) - : m_dtype{dtype} - , m_fromp{fromp} - , m_lsb{lsb} {} - explicit KeySel(const DfgSel* vtxp) - : m_dtype{vtxp->dtype()} - , m_fromp{vtxp->fromp()} - , m_lsb{vtxp->lsb()} {} - - struct Hash final { - size_t operator()(const KeySel& key) const { - // cppcheck-suppress unreadVariable // cppcheck bug - V3Hash hash = key.m_dtype.hash(); - hash += vertexHash(key.m_fromp); - hash += key.m_lsb; - return hash.value(); - } - }; - - struct Equal final { - bool operator()(const KeySel& a, const KeySel& b) const { - return a.m_lsb == b.m_lsb && a.m_dtype == b.m_dtype - && vertexEqual(a.m_fromp, b.m_fromp); - } - }; + // DfgSel + struct HashSel final { + size_t operator()(const DfgSel* vtxp) const { + return operator()(vtxp->dtype(), vtxp->fromp(), vtxp->lsb()); + } + size_t operator()(const DfgDataType& dtype, const DfgVertex* fromp, uint32_t lsb) const { + // cppcheck-suppress unreadVariable // cppcheck bug + V3Hash hash = dtype.hash(); + hash += vertexHash(fromp); + hash += lsb; + return hash.value(); + } + }; + struct EqualSel final { + bool operator()(const DfgSel* ap, const DfgSel* bp) const { + return operator()(ap, bp->dtype(), bp->fromp(), bp->lsb()); + } + bool operator()(const DfgSel* vtxp, const DfgDataType& dtype, const DfgVertex* fromp, + uint32_t lsb) const { + return vtxp->lsb() == lsb && vtxp->dtype() == dtype + && vertexEqual(vtxp->fromp(), fromp); + } }; - class KeyUnary final { - const DfgDataType& m_dtype; - const DfgVertex* const m_source0p; - - public: - // cppcheck-suppress noExplicitConstructor - KeyUnary(const DfgDataType& dtype, DfgVertex* source0p) - : m_dtype{dtype} - , m_source0p{source0p} {} - explicit KeyUnary(const DfgVertexUnary* vtxp) - : m_dtype{vtxp->dtype()} - , m_source0p{vtxp->inputp(0)} {} - - struct Hash final { - size_t operator()(const KeyUnary& key) const { // - V3Hash hash = key.m_dtype.hash(); - hash += vertexHash(key.m_source0p); - return hash.value(); - } - }; - - struct Equal final { - bool operator()(const KeyUnary& a, const KeyUnary& b) const { - return a.m_dtype == b.m_dtype && vertexEqual(a.m_source0p, b.m_source0p); - } - }; + // DfgVertexUnary + struct HashUnary final { + size_t operator()(const DfgVertexUnary* vtxp) const { + return operator()(vtxp->dtype(), vtxp->inputp(0)); + } + size_t operator()(const DfgDataType& dtype, const DfgVertex* source0p) const { + V3Hash hash = dtype.hash(); + hash += vertexHash(source0p); + return hash.value(); + } + }; + struct EqualUnary final { + bool operator()(const DfgVertexUnary* ap, const DfgVertexUnary* bp) const { + return operator()(ap, bp->dtype(), bp->inputp(0)); + } + bool operator()(const DfgVertexUnary* vtxp, const DfgDataType& dtype, + const DfgVertex* source0p) const { + return vtxp->dtype() == dtype && vertexEqual(vtxp->inputp(0), source0p); + } }; - class KeyBinary final { - const DfgDataType& m_dtype; - const DfgVertex* const m_source0p; - const DfgVertex* const m_source1p; - - public: - KeyBinary(const DfgDataType& dtype, DfgVertex* source0p, DfgVertex* source1p) - : m_dtype{dtype} - , m_source0p{source0p} - , m_source1p{source1p} {} - explicit KeyBinary(const DfgVertexBinary* vtxp) - : m_dtype{vtxp->dtype()} - , m_source0p{vtxp->inputp(0)} - , m_source1p{vtxp->inputp(1)} {} - - struct Hash final { - size_t operator()(const KeyBinary& key) const { - V3Hash hash = key.m_dtype.hash(); - hash += vertexHash(key.m_source0p); - hash += vertexHash(key.m_source1p); - return hash.value(); - } - }; - - struct Equal final { - bool operator()(const KeyBinary& a, const KeyBinary& b) const { - return a.m_dtype == b.m_dtype && vertexEqual(a.m_source0p, b.m_source0p) - && vertexEqual(a.m_source1p, b.m_source1p); - } - }; + // DfgVertexBinary + struct HashBinary final { + size_t operator()(const DfgVertexBinary* vtxp) const { + return operator()(vtxp->dtype(), vtxp->inputp(0), vtxp->inputp(1)); + } + size_t operator()(const DfgDataType& dtype, const DfgVertex* source0p, + const DfgVertex* source1p) const { + V3Hash hash = dtype.hash(); + hash += vertexHash(source0p); + hash += vertexHash(source1p); + return hash.value(); + } + }; + struct EqualBinary final { + bool operator()(const DfgVertexBinary* ap, const DfgVertexBinary* bp) const { + return operator()(ap, bp->dtype(), bp->inputp(0), bp->inputp(1)); + } + bool operator()(const DfgVertexBinary* vtxp, const DfgDataType& dtype, + const DfgVertex* source0p, const DfgVertex* source1p) const { + return vtxp->dtype() == dtype && vertexEqual(vtxp->inputp(0), source0p) + && vertexEqual(vtxp->inputp(1), source1p); + } }; - class KeyTernary final { - const DfgDataType& m_dtype; - const DfgVertex* const m_source0p; - const DfgVertex* const m_source1p; - const DfgVertex* const m_source2p; - - public: - KeyTernary(const DfgDataType& dtype, DfgVertex* source0p, DfgVertex* source1p, - DfgVertex* source2p) - : m_dtype{dtype} - , m_source0p{source0p} - , m_source1p{source1p} - , m_source2p{source2p} {} - explicit KeyTernary(const DfgVertexTernary* vtxp) - : m_dtype{vtxp->dtype()} - , m_source0p{vtxp->inputp(0)} - , m_source1p{vtxp->inputp(1)} - , m_source2p{vtxp->inputp(2)} {} - - struct Hash final { - size_t operator()(const KeyTernary& key) const { - V3Hash hash = key.m_dtype.hash(); - hash += vertexHash(key.m_source0p); - hash += vertexHash(key.m_source1p); - hash += vertexHash(key.m_source2p); - return hash.value(); - } - }; - - struct Equal final { - bool operator()(const KeyTernary& a, const KeyTernary& b) const { - return a.m_dtype == b.m_dtype && vertexEqual(a.m_source0p, b.m_source0p) - && vertexEqual(a.m_source1p, b.m_source1p) - && vertexEqual(a.m_source2p, b.m_source2p); - } - }; + // DfgVertexTernary + struct HashTernary final { + size_t operator()(const DfgVertexTernary* vtxp) const { + return operator()(vtxp->dtype(), vtxp->inputp(0), vtxp->inputp(1), vtxp->inputp(2)); + } + size_t operator()(const DfgDataType& dtype, const DfgVertex* source0p, + const DfgVertex* source1p, const DfgVertex* source2p) const { + V3Hash hash = dtype.hash(); + hash += vertexHash(source0p); + hash += vertexHash(source1p); + hash += vertexHash(source2p); + return hash.value(); + } }; + struct EqualTernary final { + bool operator()(const DfgVertexTernary* ap, const DfgVertexTernary* bp) const { + return operator()(ap, bp->dtype(), bp->inputp(0), bp->inputp(1), bp->inputp(2)); + } + bool operator()(const DfgVertexTernary* vtxp, const DfgDataType& dtype, + const DfgVertex* source0p, const DfgVertex* source1p, + const DfgVertex* source2p) const { + return vtxp->dtype() == dtype && vertexEqual(vtxp->inputp(0), source0p) + && vertexEqual(vtxp->inputp(1), source1p) + && vertexEqual(vtxp->inputp(2), source2p); + } + }; + + // Base class of vertex caches class CacheBase VL_NOT_FINAL { protected: // These set the operands of a new vertex @@ -210,86 +183,68 @@ class V3DfgCache final { public: // CacheBase does not cache anything virtual DfgVertex* cache(DfgVertex*) { return nullptr; } - virtual void invalidate(const DfgVertex*) {} + virtual void invalidate(DfgVertex*) {} }; - template + template class Cache final : public CacheBase { static_assert(std::is_base_of::value, "T_Vertex must be a DfgVertex"); - // TYPES - using Hash = typename T_Key::Hash; - using Equal = typename T_Key::Equal; - using Map = std::unordered_map; // STATE - Map m_map; - - // METHODS - - // These return a reference to the mapped entry, inserting a nullptr if not yet exists - - template - T_Vertex*& entry(T_Args&&... args) { - const T_Key key{std::forward(args)...}; - return m_map[key]; - } - template - typename Map::iterator find(T_Args&&... args) { - const T_Key key{std::forward(args)...}; - return m_map.find(key); - } + V3HashSet m_set; public: - // Add an existing vertex to the cache. If an equivalent exists, - // it is returned and the cache is not updated. + // Add an existing vertex to the cache. If an equivalent but different vertex exists, + // it is returned and the cache is not updated. Returns nullptr if the vertex is inserted. DfgVertex* cache(DfgVertex* vtxp) override { - UASSERT_OBJ(vtxp->is(), vtxp, "Vertex is wrong type"); - T_Vertex*& entrypr = entry(static_cast(vtxp)); - if (entrypr && entrypr != vtxp) return entrypr; - entrypr = static_cast(vtxp); - return nullptr; + UDEBUGONLY(UASSERT_OBJ(vtxp->is(), vtxp, "Vertex is wrong type");); + T_Vertex* const typedp = static_cast(vtxp); + T_Vertex* const cachedp = *m_set.insert(typedp).first; + return cachedp != vtxp ? cachedp : nullptr; } // Remove an existing vertex from the cache, if it is the cached vertex, otherwise no-op - void invalidate(const DfgVertex* vtxp) override { - UASSERT_OBJ(vtxp->is(), vtxp, "Vertex is wrong type"); - const auto it = find(static_cast(vtxp)); - if (it != m_map.end() && it->second == vtxp) m_map.erase(it); + void invalidate(DfgVertex* vtxp) override { + UDEBUGONLY(UASSERT_OBJ(vtxp->is(), vtxp, "Vertex is wrong type");); + T_Vertex* const typedp = static_cast(vtxp); + const auto it = m_set.find(typedp); + if (it != m_set.end() && *it == typedp) m_set.erase(it); } - // Get vertex with given operands, return nullptr if not in cache template Vertex* get(const DfgDataType& dtype, Operands... operands) { - const auto it = find(dtype, operands...); - return it != m_map.end() ? static_cast(it->second) : nullptr; + const auto it = m_set.find(dtype, operands...); + return it != m_set.end() ? static_cast(*it) : nullptr; } - - // Get or create (and insert) vertex with given operands + // Get vertex with given operands, if does not exist, create it template Vertex* getOrCreate(DfgGraph& dfg, FileLine* flp, const DfgDataType& dtype, Operands... operands) { - T_Vertex*& entryr = entry(dtype, operands...); - if (!entryr) { - T_Vertex* const newp = new Vertex{dfg, flp, dtype}; + const auto pair = m_set.insertLazy(dtype, operands..., [&]() -> T_Vertex* { + Vertex* const newp = new Vertex{dfg, flp, dtype}; setOperands(newp, operands...); - entryr = newp; - } - return static_cast(entryr); + return newp; + }); + T_Vertex* const vtxp = *pair.first; + UDEBUGONLY(UASSERT_OBJ(vtxp->template is(), vtxp, "Vertex is wrong type");); + return static_cast(vtxp); } }; // Map from Vertex type to cache type + // clang-format off template - using CacheType = - typename V3DfgCacheType, // - DfgVertexUnary, Cache, // - DfgVertexBinary, Cache, // - DfgVertexTernary, Cache // - >::Type; + using CacheType = typename V3DfgCacheType */ Cache, + DfgVertexUnary, /* -> */ Cache, + DfgVertexBinary, /* -> */ Cache, + DfgVertexTernary, /* -> */ Cache + >::Type; + // clang-format on + // STATE DfgGraph& m_dfg; // The DfgGraph we are caching the vertices of - // The per type caches +// The per type caches #define VERTEX_CACHE_DECLARE_CACHE(t) CacheType m_cache##t; FOREACH_DFG_VERTEX_TYPE(VERTEX_CACHE_DECLARE_CACHE) #undef VERTEX_CACHE_DECLARE_CACHE diff --git a/src/V3DfgCse.cpp b/src/V3DfgCse.cpp index 8f05a6030..bfdc5b886 100644 --- a/src/V3DfgCse.cpp +++ b/src/V3DfgCse.cpp @@ -18,30 +18,43 @@ #include "V3Dfg.h" #include "V3DfgPasses.h" +#include "V3HashTable.h" VL_DEFINE_DEBUG_FUNCTIONS; -class V3DfgCse final { - // TYPES - using VertexPair = std::pair; - struct VertexPairHash final { - size_t operator()(const VertexPair& pair) const { - V3Hash hash; - hash += pair.first; - hash += pair.second; - return hash.value(); - } - }; - +// Hash functor for V3HashSet - depends on vertex and all its inputs +class DfgCseHash final { // STATE - // The graph being processed - DfgGraph& m_dfg; - // Cache for vertex hashes - DfgUserMap m_hashCache = m_dfg.makeUserMap(); - // Cache for vertex equality - std::unordered_map m_equivalentCache; + mutable DfgUserMap m_cache; // Cache for vertex hashes + +public: + // CONSTRUCTOR + explicit DfgCseHash(DfgGraph& dfg) + : m_cache{dfg.makeUserMap()} { + // Pre-hash variables, these are all unique, so just set their hash to a unique value + uint32_t fixedHash = 0; + for (const DfgVertexVar& vtx : dfg.varVertices()) m_cache[vtx] = V3Hash{++fixedHash}; + // Pre-hash Ast references, these are all unique like variables + for (const DfgVertexAst& vtx : dfg.astVertices()) m_cache[vtx] = V3Hash{++fixedHash}; + // Pre-hash CReset and Prev vertices, these are all unique + for (const DfgVertex& vtx : dfg.opVertices()) { + if (vtx.is() || vtx.is()) m_cache[vtx] = V3Hash{++fixedHash}; + } + // Similarly pre-hash constants for speed. While we don't combine constants, we do want + // expressions using the same constants to be combined, so we do need to hash equal + // constants to equal values. + ++fixedHash; + for (const DfgConst& vtx : dfg.constVertices()) { + const V3Hash hash = vtx.num().toHash() + fixedHash; + // Technically possible for a hash to be zero, 'vertexSelfHash' assumes it isn't + m_cache[vtx] = VL_LIKELY(hash.value()) ? hash : V3Hash{1}; + } + } // METHODS + size_t operator()(DfgVertex* vtxp) const { return vertexHash(*vtxp).value(); } + +private: // Returns hash of vertex dependent on information internal to the vertex static V3Hash vertexSelfHash(const DfgVertex& vtx) { switch (vtx.type()) { @@ -135,29 +148,53 @@ class V3DfgCse final { VL_UNREACHABLE; } - // Returns hash of vertex dependent on and all its input - V3Hash vertexHash(DfgVertex& vtx) { - V3Hash& result = m_hashCache[vtx]; + // Returns hash of vertex dependent on itself and all its inputs - memoized + V3Hash vertexHash(DfgVertex& vtx) const { + V3Hash& result = m_cache[vtx]; + // Technically possible for a hash to be zero, but rare, so assume 0 means uninitialized if (!result.value()) { V3Hash hash{vertexSelfHash(vtx)}; - // Variables are defined by themselves, so there is no need to hash them further - // (especially the sources). This enables sound hashing of graphs circular only through - // variables, which we rely on. - if (!vtx.is()) { - hash += vtx.type(); - hash += vtx.size(); - vtx.foreachSource([&](DfgVertex& src) { - hash += vertexHash(src); - return false; - }); - } + hash += vtx.type(); + hash += vtx.size(); + vtx.foreachSource([&](DfgVertex& src) { + hash += vertexHash(src); // Graph is acyclic, so this terminates + return false; + }); result = hash; } return result; } +}; +// Equal functor for V3HashSet - depends on vertex and all its inputs +class DfgCseEqual final { + // TYPES + using VertexPair = std::pair; + struct VertexPairHash final { + size_t operator()(const VertexPair& pair) const { + V3Hash hash; + hash += pair.first; + hash += pair.second; + return hash.value(); + } + }; + + // STATE + mutable V3HashMap m_cache; // Cache for vertex equality + mutable std::vector m_driverLo; // Low indices of drivers + const size_t m_size; // Size of the graph + +public: + // CONSTRUCTORS + explicit DfgCseEqual(const DfgGraph& dfg) + : m_size{dfg.size()} {} + + // METHODS + bool operator()(DfgVertex* ap, DfgVertex* bp) const { return vertexEquivalent(*ap, *bp); } + +private: // Compare 'a' and 'b' for equivalence based on their internal information only - bool vertexSelfEquivalent(const DfgVertex& a, const DfgVertex& b) { + bool vertexSelfEquivalent(const DfgVertex& a, const DfgVertex& b) const { // Note: 'a' and 'b' are of the same Vertex type, data type, and have // the same number of inputs with matching types. This is established // by 'vertexEquivalent'. @@ -187,16 +224,17 @@ class V3DfgCse final { case VDfgType::SplicePacked: { const DfgVertexSplice* const ap = a.as(); // Gather indices of drivers of 'a' - std::vector aLo; - aLo.reserve(ap->nInputs()); + m_driverLo.clear(); + m_driverLo.reserve(ap->nInputs()); ap->foreachDriver([&](const DfgVertex&, uint32_t lo) { - aLo.push_back(lo); + m_driverLo.push_back(lo); return false; }); - // Compare indices of drivers of 'b' - uint32_t* aLop = aLo.data(); - return !b.as()->foreachDriver( - [&](const DfgVertex&, uint32_t lo) { return *aLop++ != lo; }); + // Compare indices of drivers of 'b', equal if all match + uint32_t* aLop = m_driverLo.data(); + return !b.as()->foreachDriver([&](const DfgVertex&, uint32_t lo) { // + return *aLop++ != lo; + }); } // Vertices with no internal information @@ -263,19 +301,19 @@ class V3DfgCse final { } // Compares the sources of 'a' and 'b' for equivalence - bool sourcesEquivalent(const DfgVertex& a, const DfgVertex& b) { + bool sourcesEquivalent(const DfgVertex& a, const DfgVertex& b) const { for (size_t i = 0; i < a.nInputs(); ++i) { const DfgVertex* const ap = a.inputp(i); const DfgVertex* const bp = b.inputp(i); if (!ap && !bp) continue; if (!ap || !bp) return false; - if (!vertexEquivalent(*ap, *bp)) return false; + if (!vertexEquivalent(*ap, *bp)) return false; // Graph is acyclic, so this terminates } return true; } // Compares 'a' and 'b' for equivalence - bool vertexEquivalent(const DfgVertex& a, const DfgVertex& b) { + bool vertexEquivalent(const DfgVertex& a, const DfgVertex& b) const { // If same vertex, then equal if (&a == &b) return true; @@ -297,70 +335,51 @@ class V3DfgCse final { // be looked up again through multiple paths. if (!a.hasMultipleSinks() && !b.hasMultipleSinks()) return sourcesEquivalent(a, b); - // Check sources + // Need to compare the source vertices, check memo const VertexPair key = (&a < &b) ? std::make_pair(&a, &b) : std::make_pair(&b, &a); - // The recursive invocation can cause a re-hash but that will not invalidate references - uint8_t& result = m_equivalentCache[key]; - if (!result) result = (static_cast(sourcesEquivalent(a, b)) << 1) | 1; - return result >> 1; - } + const auto it = m_cache.find(key); + if (it != m_cache.end()) return it->second; - V3DfgCse(DfgGraph& dfg, V3DfgCseContext& ctx) - : m_dfg{dfg} { - std::unordered_map> verticesWithEqualHashes; - verticesWithEqualHashes.reserve(dfg.size()); + // Not memoized yet, so compute and memoize, reserve table on first insert + const bool equal = sourcesEquivalent(a, b); + if (VL_UNLIKELY(m_cache.empty())) m_cache.reserve(m_size / 4); + m_cache.insert({key, equal}); - // Pre-hash variables, these are all unique, so just set their hash to a unique value - uint32_t varHash = 0; - for (const DfgVertexVar& vtx : dfg.varVertices()) m_hashCache[vtx] = V3Hash{++varHash}; - // Pre-hash Ast references, these are all unique like variables - for (const DfgVertexAst& vtx : dfg.astVertices()) m_hashCache[vtx] = V3Hash{++varHash}; - // Pre-hash CReset and Prev vertices, these are all unique - for (const DfgVertex& vtx : dfg.opVertices()) { - if (vtx.is() || vtx.is()) m_hashCache[vtx] = V3Hash{++varHash}; - } - - // Similarly pre-hash constants for speed. While we don't combine constants, we do want - // expressions using the same constants to be combined, so we do need to hash equal - // constants to equal values. - for (DfgConst* const vtxp : dfg.constVertices().unlinkable()) { - // Delete unused constants while we are at it. - if (!vtxp->hasSinks()) { - VL_DO_DANGLING(vtxp->unlinkDelete(dfg), vtxp); - continue; - } - m_hashCache[vtxp] = vtxp->num().toHash() + varHash; - } - - // Combine operation vertices - for (DfgVertex* const vtxp : dfg.opVertices().unlinkable()) { - // Delete unused nodes while we are at it. - if (!vtxp->hasSinks()) { - vtxp->unlinkDelete(dfg); - continue; - } - std::vector& vec = verticesWithEqualHashes[vertexHash(*vtxp)]; - bool replaced = false; - for (DfgVertex* const candidatep : vec) { - if (vertexEquivalent(*candidatep, *vtxp)) { - ++ctx.m_eliminated; - vtxp->replaceWith(candidatep); - VL_DO_DANGLING(vtxp->unlinkDelete(dfg), vtxp); - replaced = true; - break; - } - } - if (replaced) continue; - vec.push_back(vtxp); - } - } - -public: - static void apply(DfgGraph& dfg, V3DfgCseContext& ctx) { - { V3DfgCse{dfg, ctx}; } - // Prune unused nodes - V3DfgPasses::removeUnused(dfg); + // The predicate result + return equal; } }; -void V3DfgPasses::cse(DfgGraph& dfg, V3DfgCseContext& ctx) { V3DfgCse::apply(dfg, ctx); } +// Combine equivalent operation vertices +void dfgCseCombineEquivalent(DfgGraph& dfg, V3DfgCseContext& ctx) { + // Delete unused constants, so the pre-hashing below need not consider them + for (DfgConst* const vtxp : dfg.constVertices().unlinkable()) { + if (!vtxp->hasSinks()) VL_DO_DANGLING(vtxp->unlinkDelete(dfg), vtxp); + } + + // Set of unique vertices. This set does all the work identifying equivalent vertices. + V3HashSet uniqueVtxps{DfgCseHash{dfg}, DfgCseEqual{dfg}}; + // There is at most one entry per vertex + uniqueVtxps.reserve(dfg.size()); + + // Combine operation vertices + for (DfgVertex* const vtxp : dfg.opVertices().unlinkable()) { + // Delete unused nodes while we are at it. + if (!vtxp->hasSinks()) { + vtxp->unlinkDelete(dfg); + continue; + } + // Insert the vertex into the set, if an equivalent is found, replace the vertex with it + const auto pair = uniqueVtxps.insert(vtxp); + if (!pair.second) { + ++ctx.m_eliminated; + vtxp->replaceWith(*pair.first); + VL_DO_DANGLING(vtxp->unlinkDelete(dfg), vtxp); + } + } +} + +void V3DfgPasses::cse(DfgGraph& dfg, V3DfgCseContext& ctx) { + dfgCseCombineEquivalent(dfg, ctx); + V3DfgPasses::removeUnused(dfg); +} diff --git a/src/V3HashTable.cpp b/src/V3HashTable.cpp new file mode 100644 index 000000000..b6592d408 --- /dev/null +++ b/src/V3HashTable.cpp @@ -0,0 +1,1144 @@ +// -*- mode: C++; c-file-style: "cc-mode" -*- +//************************************************************************* +// DESCRIPTION: Verilator: Tests for V3HashTable.h +// +// Code available from: https://verilator.org +// +//************************************************************************* +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of either the GNU Lesser General Public License Version 3 +// or the Perl Artistic License Version 2.0. +// SPDX-FileCopyrightText: 2003-2026 Wilson Snyder +// SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0 +// +//************************************************************************* + +#include "V3HashTable.h" + +#include "V3Error.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace V3HashTableInternals { + +// Entries that fill a table of the given capacity to its maximum load, so one more grows it +constexpr size_t maxLoad(size_t capacity) { return capacity * LOAD_FACTOR_NUM / LOAD_FACTOR_DEN; } + +// Enough entries to grow the smallest table twice, that is, to fill four times the minimum +constexpr size_t GROWS_TWICE = maxLoad(4 * MIN_CAPACITY); +static_assert(GROWS_TWICE > maxLoad(2 * MIN_CAPACITY), + "SelfTest: 'GROWS_TWICE' must overflow a table twice the minimum"); + +//###################################################################### +// Set key by value + +void testValueKeys() { + struct Value final { + size_t m_hash = 0; // Hash of this entry + size_t m_id = 0; // Id of this entry + }; + + struct Hash final { + size_t operator()(const Value& value) const { return operator()(value.m_hash, 0); } + size_t operator()(size_t hash, size_t) const { return hash; } + }; + + struct Equal final { + bool operator()(const Value& a, const Value& b) const { + return operator()(a, b.m_hash, b.m_id); + } + bool operator()(const Value& value, size_t hash, size_t id) const { + return value.m_hash == hash && value.m_id == id; + } + }; + + using Set = V3HashSet; + + // Entries that are added are found, entries that are not are not + { + const Value value{1, 0}; + const Value equal{1, 0}; // Equal to 'value', and hashes the same + Set set; + UASSERT_SELFTEST(set.empty(), true); // Starts empty + UASSERT_SELFTEST(set.begin() == set.end(), true); // So iterates nothing + UASSERT_SELFTEST(set.find(value) == set.end(), true); // And finds nothing + const std::pair added = set.insert(value); + UASSERT_SELFTEST(added.second, true); // Added, as it was absent + UASSERT_SELFTEST(added.first->m_id, value.m_id); // The iterator is at the entry + UASSERT_SELFTEST(&*added.first != &value, true); // Which is a copy of the argument + UASSERT_SELFTEST(set.size(), 1); // And is the only one + UASSERT_SELFTEST(set.empty(), false); // So the set is no longer empty + { + const Set::iterator it = set.find(value); + UASSERT_SELFTEST(it != set.end(), true); // Found by itself + UASSERT_SELFTEST(&*it, &*added.first); // At the entry held + } + { + const Set::iterator it = set.find(equal); + UASSERT_SELFTEST(it != set.end(), true); // And by an equal entry + UASSERT_SELFTEST(&*it, &*added.first); // At that same entry + } + { + const Set::iterator it = set.find(1, size_t{0}); + UASSERT_SELFTEST(it != set.end(), true); // And by its parts + UASSERT_SELFTEST(&*it, &*added.first); // At that same entry again + } + UASSERT_SELFTEST(set.find(1, size_t{1}) == set.end(), true); // Hash same, id not + UASSERT_SELFTEST(set.find(2, size_t{0}) == set.end(), true); // Id same, hash not + // Adding an equal entry returns the one in the set, and does not add + const std::pair again = set.insert(equal); + UASSERT_SELFTEST(again.second, false); // Not added, as an equal is present + UASSERT_SELFTEST(&*again.first, &*added.first); // At the entry already held + UASSERT_SELFTEST(set.size(), 1); // Still just the one entry + // Erasing through an iterator removes the entry it is at + { + const Set::iterator it = set.find(value); + UASSERT_SELFTEST(it != set.end(), true); // Present before the erase + set.erase(it); + } + UASSERT_SELFTEST(set.empty(), true); // Empty again + UASSERT_SELFTEST(set.find(value) == set.end(), true); // So the entry is gone + UASSERT_SELFTEST(set.contains(value), false); // As 'contains' agrees + // Erase by key removes whatever that key finds + UASSERT_SELFTEST(set.erase(value), false); // Nothing to erase, so says so + set.insert(value); + UASSERT_SELFTEST(set.erase(equal), true); // An equal key erases the entry held + UASSERT_SELFTEST(set.empty(), true); // Which leaves the set empty + // And by a key spelled as the parts of an entry + set.insert(value); + UASSERT_SELFTEST(set.erase(1, size_t{0}), true); // Those parts find and erase it + UASSERT_SELFTEST(set.empty(), true); // Empty once more + } + + // Colliding entries are all kept and stay reachable, including after the table grows + { + constexpr size_t N = GROWS_TWICE; + Set set; + for (size_t i = 0; i < N; ++i) { + const std::pair added = set.insert(Value{7, i}); + UASSERT_SELFTEST(added.second, true); // Ids differ, so each is added + } + UASSERT_SELFTEST(set.size(), N); // All of them, in the one probe run + for (size_t i = 0; i < N; ++i) { + const Set::iterator it = set.find(size_t{7}, i); + UASSERT_SELFTEST(it != set.end(), true); // The growth lost nothing + UASSERT_SELFTEST(it->m_id, i); // And is the entry asked for + } + // Iterating visits every entry exactly once + std::array seen{}; + size_t n = 0; + for (const Value& entry : set) { + UASSERT_SELFTEST(entry.m_hash, size_t{7}); // Only entries added + UASSERT_SELFTEST(seen[entry.m_id], false); // Each of them exactly once + seen[entry.m_id] = true; + ++n; + } + UASSERT_SELFTEST(n, N); // And every one of them + } +} + +//###################################################################### +// Set key by pointer + +void testPointerKeys() { + struct Value final { + size_t m_hash = 0; // Hash of this entry + size_t m_id = 0; // Id of this entry + }; + + struct Hash final { + size_t operator()(const Value* valuep) const { return operator()(valuep->m_hash, 0); } + size_t operator()(size_t hash, size_t) const { return hash; } + }; + + struct Equal final { + bool operator()(const Value* ap, const Value* bp) const { + return operator()(ap, bp->m_hash, bp->m_id); + } + bool operator()(const Value* valuep, size_t hash, size_t id) const { + return valuep->m_hash == hash && valuep->m_id == id; + } + }; + + using Set = V3HashSet; + + // Entries that are added are found, entries that are not are not + { + Value value{1, 0}; + Value equal{1, 0}; // Equal to 'value', and hashes the same + Set set; + UASSERT_SELFTEST(set.empty(), true); // Starts empty + UASSERT_SELFTEST(set.begin() == set.end(), true); // So iterates nothing + UASSERT_SELFTEST(set.find(&value) == set.end(), true); // And finds nothing + const std::pair added = set.insert(&value); + UASSERT_SELFTEST(added.second, true); // Added, as it was absent + UASSERT_SELFTEST(*added.first, &value); // The iterator is at the new entry + UASSERT_SELFTEST(set.size(), 1); // Which is the only one + UASSERT_SELFTEST(set.empty(), false); // So the set is no longer empty + { + const Set::iterator it = set.find(&value); + UASSERT_SELFTEST(it != set.end(), true); // Found by itself + UASSERT_SELFTEST(*it, &value); // At the entry held + } + { + const Set::iterator it = set.find(&equal); + UASSERT_SELFTEST(it != set.end(), true); // And by an equal entry + UASSERT_SELFTEST(*it, &value); // At that same entry + } + { + const Set::iterator it = set.find(1, size_t{0}); + UASSERT_SELFTEST(it != set.end(), true); // And by its parts + UASSERT_SELFTEST(*it, &value); // At that same entry again + } + UASSERT_SELFTEST(set.find(1, size_t{1}) == set.end(), true); // Hash same, id not + UASSERT_SELFTEST(set.find(2, size_t{0}) == set.end(), true); // Id same, hash not + // Adding an equal entry returns the one in the set, and does not add + const std::pair again = set.insert(&equal); + UASSERT_SELFTEST(again.second, false); // Not added, as an equal is present + UASSERT_SELFTEST(*again.first, &value); // The iterator is at the stored one + UASSERT_SELFTEST(set.size(), 1); // Still just the one entry + // Given the distinct but equal '&equal', 'find' still yields the stored '&value' + { + const Set::iterator it = set.find(&equal); + UASSERT_SELFTEST(it != set.end(), true); // Found, as they compare equal + UASSERT_SELFTEST(*it, &value); // But it is the stored one + UASSERT_SELFTEST(*it == &equal, false); // Not the one asked for + } + { + const Set::iterator it = set.find(&value); + UASSERT_SELFTEST(it != set.end(), true); // The entry is still there + UASSERT_SELFTEST(*it, &value); // And is the one stored + } + // Erasing the very entry held does remove it + { + const Set::iterator it = set.find(&value); + UASSERT_SELFTEST(it != set.end(), true); // Present before the erase + UASSERT_SELFTEST(*it, &value); // And is the object asked for + set.erase(it); + } + UASSERT_SELFTEST(set.empty(), true); // Empty again + UASSERT_SELFTEST(set.find(&value) == set.end(), true); // So the entry is gone + UASSERT_SELFTEST(set.contains(&value), false); // As 'contains' agrees + UASSERT_SELFTEST(set.empty(), true); // And no lookup added anything + // Erase by key makes no such check, so it removes whatever the key finds, here + // the stored '&value' when given the equal '&equal' + UASSERT_SELFTEST(set.erase(&value), false); // Nothing to erase, so says so + set.insert(&value); + UASSERT_SELFTEST(set.erase(&equal), true); // The equal key erases the stored one + UASSERT_SELFTEST(set.empty(), true); // Which leaves the set empty + UASSERT_SELFTEST(set.find(&value) == set.end(), true); // And unreachable + // And by a key spelled as the parts of an entry + set.insert(&value); + UASSERT_SELFTEST(set.erase(1, size_t{0}), true); // Those parts find and erase it + UASSERT_SELFTEST(set.empty(), true); // Empty once more + } + + // Erasing leaves the other entries reachable, whichever is erased, including when + // entries with equal hashes are all in the one probe run + for (size_t erase = 0; erase < 4; ++erase) { + for (const size_t hash : {size_t{0}, size_t{7}, ~size_t{0}}) { // Wraps too + std::array values{Value{hash, 0}, Value{hash, 1}, // + Value{hash, 2}, Value{hash, 3}}; + Set set; + for (Value& value : values) set.insert(&value); + UASSERT_SELFTEST(set.size(), 4); // All distinct, so all added + { + const Set::iterator it = set.find(&values[erase]); + UASSERT_SELFTEST(it != set.end(), true); // The one to erase is present + UASSERT_SELFTEST(*it, &values[erase]); // And is the object held + set.erase(it); + } + UASSERT_SELFTEST(set.size(), 3); // Exactly one was erased + UASSERT_SELFTEST(set.find(&values[erase]) == set.end(), true); // That one + for (size_t i = 0; i < values.size(); ++i) { + if (i == erase) continue; + const Set::iterator it = set.find(hash, i); + UASSERT_SELFTEST(it != set.end(), true); // The others are all there + UASSERT_SELFTEST(*it, &values[i]); // Each at the entry inserted + } + } + } + + // Growing leaves all entries reachable, including when their run wraps around the end + // of the table. Note this needs enough entries to actually grow, so do not reserve here. + for (const size_t hash : {size_t{0}, size_t{7}, ~size_t{0}}) { + std::array many{}; + Set set; + for (size_t i = 0; i < many.size(); ++i) { + many[i] = Value{hash, i}; + set.insert(&many[i]); + } + UASSERT_SELFTEST(set.size(), many.size()); // All of them were added + for (size_t i = 0; i < many.size(); ++i) { + const Set::iterator it = set.find(hash, i); + UASSERT_SELFTEST(it != set.end(), true); // And all survived the growth + UASSERT_SELFTEST(*it, &many[i]); // Each at the entry inserted + } + } + + // Entries that collide but are not equal are both kept, and erase removes only the one + { + Value value{1, 0}; + Value other{1, 1}; // Not equal to 'value', but lands on the same slot + Set set; + set.reserve(2); + set.insert(&value); + set.insert(&other); + UASSERT_SELFTEST(set.size(), 2); // Both kept, despite the collision + { + const Set::iterator it = set.find(&value); + UASSERT_SELFTEST(it != set.end(), true); // The first of the run is there + UASSERT_SELFTEST(*it, &value); // And is that entry + } + { + const Set::iterator it = set.find(&other); + UASSERT_SELFTEST(it != set.end(), true); // As is the one behind it + UASSERT_SELFTEST(*it, &other); // Which probing reached past the first + } + { + const Set::iterator it = set.find(&value); + UASSERT_SELFTEST(it != set.end(), true); // Present before the erase + UASSERT_SELFTEST(*it, &value); // And is the object held + set.erase(it); + } + UASSERT_SELFTEST(set.size(), 1); // Only one was erased + UASSERT_SELFTEST(set.find(&value) == set.end(), true); // Namely that one + { + const Set::iterator it = set.find(&other); + UASSERT_SELFTEST(it != set.end(), true); // The collider stays + UASSERT_SELFTEST(*it, &other); // Shifted back over the hole left + } + } + + // Reserving avoids growing, and all entries survive either way + for (const bool doReserve : {false, true}) { + std::array values{}; + Set set; + if (doReserve) set.reserve(values.size()); + for (size_t i = 0; i < values.size(); ++i) { + values[i] = Value{static_cast(i * 1234567), i}; + set.insert(&values[i]); + } + for (Value& value : values) { + const Set::iterator it = set.find(&value); + UASSERT_SELFTEST(it != set.end(), true); // Each entry survives + UASSERT_SELFTEST(*it, &value); // And is the one inserted + } + UASSERT_SELFTEST(set.size(), values.size()); // And none was added twice + } + + // 'insertLazy' creates only on a miss, including when that grows the table + { + constexpr size_t N = GROWS_TWICE; + std::array values{}; + Set set; + for (size_t i = 0; i < N; ++i) { + values[i] = Value{7, i}; // All of them hash the same + const std::pair pair + = set.insertLazy(size_t{7}, i, [&]() -> Value* { return &values[i]; }); + UASSERT_SELFTEST(pair.second, true); // Created, as it was absent + UASSERT_SELFTEST(*pair.first, &values[i]); // And is what the factory made + } + UASSERT_SELFTEST(set.size(), N); // All of them were added + // They are all present now, so nothing is created + for (size_t i = 0; i < N; ++i) { + bool created = false; + const std::pair pair + = set.insertLazy(size_t{7}, i, [&]() -> Value* { + created = true; // LCOV_EXCL_START + return &values[i]; // LCOV_EXCL_STOP + }); + UASSERT_SELFTEST(created, false); // The factory was never called + UASSERT_SELFTEST(pair.second, false); // As the lookup hit + UASSERT_SELFTEST(*pair.first, &values[i]); // On the stored entry + } + UASSERT_SELFTEST(set.size(), N); // And the set is unchanged + } + + // Iterating visits every entry exactly once + { + std::array values{}; + std::array seen{}; + Set set; + for (size_t i = 0; i < values.size(); ++i) { + values[i] = Value{i / 2, i}; // Pairs of them collide + set.insert(&values[i]); + } + size_t n = 0; + for (Value* const valuep : set) { + UASSERT_SELFTEST(valuep->m_id < seen.size(), true); // Only entries added + UASSERT_SELFTEST(seen[valuep->m_id], false); // Each of them exactly once + seen[valuep->m_id] = true; + ++n; + } + UASSERT_SELFTEST(n, values.size()); // And every one of them + } +} + +//###################################################################### +// Entries held by value, which are constructed and destroyed in step with the slots + +void testEntryLifetime() { + size_t alive = 0; // Number of live entries, which the entries themselves count + + // A test entry that is not default constructible, nor assignable, and that counts how + // many are alive. A move makes another live entry, so the count tracks the occupied + // slots however the container shuffles them about. + class NoDefault final { + size_t* m_alivep; // Where the live entries are counted + size_t m_hash; // Hash of this entry + size_t m_id; // Entries with equal ids are equal + + public: + NoDefault(size_t* alivep, size_t hash, size_t id) + : m_alivep{alivep} + , m_hash{hash} + , m_id{id} { + ++*m_alivep; + } + NoDefault(const NoDefault& that) + : m_alivep{that.m_alivep} + , m_hash{that.m_hash} + , m_id{that.m_id} { + ++*m_alivep; + } + NoDefault(NoDefault&& that) + : m_alivep{that.m_alivep} + , m_hash{that.m_hash} + , m_id{that.m_id} { + ++*m_alivep; + } + ~NoDefault() { --*m_alivep; } + NoDefault& operator=(const NoDefault&) = delete; + NoDefault& operator=(NoDefault&&) = delete; + + size_t hash() const { return m_hash; } + size_t id() const { return m_id; } + bool operator==(const NoDefault& that) const { return m_id == that.m_id; } + }; + + static_assert(!std::is_default_constructible::value, + "SelfTest: 'NoDefault' must not be default constructible"); + static_assert(!std::is_copy_assignable::value, + "SelfTest: 'NoDefault' must not be copy assignable"); + static_assert(!std::is_move_assignable::value, + "SelfTest: 'NoDefault' must not be move assignable"); + static_assert(std::is_move_constructible::value, + "SelfTest: 'NoDefault' must be move constructible, to exercise moving"); + + struct Hash final { + size_t operator()(const NoDefault& value) const { return value.hash(); } + size_t operator()(size_t hash, size_t) const { return hash; } + }; + + struct Equal final { + bool operator()(const NoDefault& a, const NoDefault& b) const { + return operator()(a, b.hash(), b.id()); + } + bool operator()(const NoDefault& a, size_t, size_t id) const { return a.id() == id; } + }; + + using Set = V3HashSet; + + constexpr size_t N = GROWS_TWICE; + UASSERT_SELFTEST(alive, 0); // Nothing built yet + { + Set set; + for (size_t i = 0; i < N; ++i) { + const std::pair added = set.insert(NoDefault{&alive, 7, i}); + UASSERT_SELFTEST(added.second, true); // Ids differ, so each is added + } + UASSERT_SELFTEST(set.size(), N); // All of them are in + UASSERT_SELFTEST(alive, N); // Held by exactly that many slots + // All of them hash the same, so they are all in the one probe run + for (size_t i = 0; i < N; ++i) { + const Set::iterator it = set.find(size_t{7}, i); + UASSERT_SELFTEST(it != set.end(), true); // Reachable through the run + UASSERT_SELFTEST(it->id(), i); // And is the entry asked for + } + // A rejected insert constructs no entry: the argument is only copied on a miss + { + const std::pair dup = set.insert(NoDefault{&alive, 7, 0}); + UASSERT_SELFTEST(dup.second, false); // Not added, as an equal is present + UASSERT_SELFTEST(dup.first->id(), 0); // The iterator is at the stored one + } + UASSERT_SELFTEST(set.size(), N); // Nothing was added + UASSERT_SELFTEST(alive, N); // And no copy of the argument was kept + { + const Set::iterator it = set.find(size_t{7}, size_t{0}); + UASSERT_SELFTEST(it != set.end(), true); // Present before the erase + UASSERT_SELFTEST(it->id(), size_t{0}); // And is the entry asked for + set.erase(it); + } + UASSERT_SELFTEST(set.size(), N - 1); // One fewer entry + UASSERT_SELFTEST(alive, N - 1); // And one fewer live object + UASSERT_SELFTEST(set.contains(size_t{7}, size_t{0}), false); // Namely that one + // Erasing the rest keeps the live entries in step with the slots, all the way down. + // They all collide, so every erase shifts entries back over the hole. + for (size_t i = 1; i < N; ++i) { + { + const Set::iterator it = set.find(size_t{7}, i); + UASSERT_SELFTEST(it != set.end(), true); // Still reachable + UASSERT_SELFTEST(it->id(), i); // And is the entry asked for + set.erase(it); + } + UASSERT_SELFTEST(set.size(), N - 1 - i); // The count follows the erases + UASSERT_SELFTEST(alive, N - 1 - i); // As do the live entries + for (size_t j = i + 1; j < N; ++j) { + UASSERT_SELFTEST(set.contains(size_t{7}, j), true); // Shifted, not lost + } + } + UASSERT_SELFTEST(set.begin() == set.end(), true); // Erased down to empty + UASSERT_SELFTEST(alive, 0); // With every entry destroyed + // Leave entries in the set, so its destructor has some to destroy + for (size_t i = 0; i < N; ++i) set.insert(NoDefault{&alive, 7, i}); + UASSERT_SELFTEST(alive, N); // Live as the set goes out of scope + } + // The set is gone, so every entry it still held has been destroyed + UASSERT_SELFTEST(alive, 0); // Leaking none of them +} + +//###################################################################### +// A table can be moved, handing over the entries and leaving an empty table behind + +void testMove() { + size_t alive = 0; // Number of live entries, which the entries themselves count + + // A test entry that counts the live ones, so a move that copied an entry, dropped + // one, or destroyed one twice, shows up in the count + struct Counted final { + size_t* m_alivep; // Where the live entries are counted + size_t m_id; // Entries with equal ids are equal + + Counted(size_t* alivep, size_t id) + : m_alivep{alivep} + , m_id{id} { + ++*m_alivep; + } + Counted(const Counted& that) + : Counted{that.m_alivep, that.m_id} {} + Counted(Counted&& that) + : Counted{that.m_alivep, that.m_id} {} + ~Counted() { --*m_alivep; } + Counted& operator=(const Counted&) = delete; + Counted& operator=(Counted&&) = delete; + }; + + struct Hash final { + size_t operator()(const Counted& entry) const { return operator()(entry.m_id); } + size_t operator()(size_t id) const { return id; } + }; + + struct Equal final { + bool operator()(const Counted& a, const Counted& b) const { return operator()(a, b.m_id); } + bool operator()(const Counted& a, size_t id) const { return a.m_id == id; } + }; + + using Set = V3HashSet; + + constexpr size_t N = GROWS_TWICE; + { + Set set; + for (size_t i = 0; i < N; ++i) set.insert(Counted{&alive, i}); + UASSERT_SELFTEST(set.size(), N); // All of them are in + UASSERT_SELFTEST(alive, N); // Held by exactly that many slots + + // Move construction takes the entries, making and destroying none + Set moved{std::move(set)}; + UASSERT_SELFTEST(moved.size(), N); // Which the target now holds + UASSERT_SELFTEST(alive, N); // With no entry made or destroyed + UASSERT_SELFTEST(set.empty(), true); // And the source no longer holds them + UASSERT_SELFTEST(set.begin() == set.end(), true); // So it iterates nothing + for (size_t i = 0; i < N; ++i) { + UASSERT_SELFTEST(moved.contains(i), true); // Every entry came across + UASSERT_SELFTEST(set.contains(i), false); // And none stayed behind + } + + // The moved from table is empty rather than broken, so it can be filled again + UASSERT_SELFTEST(set.insert(Counted{&alive, N}).second, true); // It took an entry + UASSERT_SELFTEST(set.size(), 1); // Which is all it holds + UASSERT_SELFTEST(set.empty(), false); // So it is no longer empty + UASSERT_SELFTEST(alive, N + 1); // And is one more live entry + + // Move assignment destroys what the target held, then takes the source's + set = std::move(moved); + UASSERT_SELFTEST(set.size(), N); // The target holds the moved entries + UASSERT_SELFTEST(alive, N); // The entry it held itself was destroyed + UASSERT_SELFTEST(set.contains(N), false); // Namely that one + UASSERT_SELFTEST(moved.empty(), true); // And the source is empty again + for (size_t i = 0; i < N; ++i) UASSERT_SELFTEST(set.contains(i), true); // The rest moved + } + // Both tables are gone, so every entry either still held has been destroyed + UASSERT_SELFTEST(alive, 0); // Leaking none of them +} + +//###################################################################### +// Entries that cannot be copied, only moved + +void testMoveOnlyEntries() { + // Only 'insertLazy' can add one of these, as 'insert' would copy it, and any copy the + // container made of an entry would stop this compiling. + class MoveOnly final { + size_t m_id; // Entries with equal ids are equal + + public: + explicit MoveOnly(size_t id) + : m_id{id} {} + MoveOnly(MoveOnly&&) = default; + MoveOnly(const MoveOnly&) = delete; + MoveOnly& operator=(const MoveOnly&) = delete; + MoveOnly& operator=(MoveOnly&&) = delete; + ~MoveOnly() = default; + + size_t id() const { return m_id; } + }; + + static_assert(!std::is_copy_constructible::value, + "SelfTest: 'MoveOnly' must not be copy constructible"); + static_assert(std::is_move_constructible::value, + "SelfTest: 'MoveOnly' must be move constructible"); + + struct Hash final { + // Every entry hashes the same, so they all end up in the one probe run + size_t operator()(const MoveOnly& value) const { return operator()(value.id()); } + size_t operator()(size_t) const { return 7; } + }; + + struct Equal final { + bool operator()(const MoveOnly& a, const MoveOnly& b) const { + return operator()(a, b.id()); + } + bool operator()(const MoveOnly& a, size_t id) const { return a.id() == id; } + }; + + using Set = V3HashSet; + + constexpr size_t N = GROWS_TWICE; + Set set; + for (size_t i = 0; i < N; ++i) { + const std::pair added + = set.insertLazy(i, [i] { return MoveOnly{i}; }); + UASSERT_SELFTEST(added.second, true); // Ids differ, so each is added + UASSERT_SELFTEST(added.first->id(), i); // Moved into the slot, never copied + } + UASSERT_SELFTEST(set.size(), N); // All of them are in + // They all collide, so erasing every other one shifts the rest back over the holes + for (size_t i = 0; i < N; i += 2) { + const Set::iterator it = set.find(i); + UASSERT_SELFTEST(it != set.end(), true); // Present before the erase + set.erase(it); + } + UASSERT_SELFTEST(set.size(), N / 2); // Half of them are gone + for (size_t i = 0; i < N; ++i) { + const bool erased = (i % 2) == 0; + UASSERT_SELFTEST(set.contains(i), !erased); // And it is the right half + } +} + +//###################################################################### +// Stateful functors, which the two argument constructor moves in + +void testStatefulFunctors() { + // Hashing and comparison that both depend on a mask the functor holds, so the set only + // works if it keeps the instances it was handed + class Hash final { + size_t m_mask; // Only these bits of an entry matter + + public: + explicit Hash(size_t mask) + : m_mask{mask} {} + size_t operator()(size_t value) const { return value & m_mask; } + }; + + class Equal final { + size_t m_mask; // Only these bits of an entry matter + + public: + explicit Equal(size_t mask) + : m_mask{mask} {} + bool operator()(size_t a, size_t b) const { return (a & m_mask) == (b & m_mask); } + }; + + // Neither is default constructible, so the set cannot make its own + static_assert(!std::is_default_constructible::value, + "SelfTest: 'Hash' must not be default constructible"); + static_assert(!std::is_default_constructible::value, + "SelfTest: 'Equal' must not be default constructible"); + + // Only the low two bits matter, so the 16 entries fall into 4 classes + V3HashSet set{Hash{3}, Equal{3}}; + for (size_t i = 0; i < 16; ++i) set.insert(i); + UASSERT_SELFTEST(set.size(), 4); // So the set kept the functors it was given + // Every entry finds the first one added of its class, which is the class itself + for (size_t i = 0; i < 16; ++i) { + const V3HashSet::iterator it = set.find(i); + UASSERT_SELFTEST(it != set.end(), true); // Its class is present + UASSERT_SELFTEST(*it, (i & 3)); // Represented by the first one added + } +} + +//###################################################################### +// Backward shift deletion moves back exactly the entries whose probe run crosses the +// hole: an entry standing at its home position past the hole must stay put + +void testBackwardShiftDeletion() { + // A test entry with an explicit hash, so probe runs can be laid out at will + struct Value final { + size_t m_hash = 0; // Hash of this entry + size_t m_id = 0; // Entries with equal ids are equal + }; + + struct Hash final { + size_t operator()(const Value* valuep) const { return operator()(valuep->m_hash, 0); } + size_t operator()(size_t hash, size_t) const { return hash; } + }; + + struct Equal final { + bool operator()(const Value* ap, const Value* bp) const { + return operator()(ap, bp->m_hash, bp->m_id); + } + bool operator()(const Value* valuep, size_t hash, size_t id) const { + return valuep->m_hash == hash && valuep->m_id == id; + } + }; + + using Set = V3HashSet; + + // One probe run of four entries with alternating home positions 'h' and 'h + 1', + // occupying four adjacent slots. Erasing the first leaves a hole: the second sits + // at its own home and must not be moved into it, while the third and fourth have + // their runs broken by the hole and must be moved back. + for (const size_t h : {size_t{0}, size_t{5}, ~size_t{0}}) { // Wraps too + std::array values{Value{h, 0}, Value{h + 1, 1}, // + Value{h, 2}, Value{h + 1, 3}}; + Set set; + for (Value& value : values) set.insert(&value); + UASSERT_SELFTEST(set.size(), 4); // The run holds all four + + // Erase the entry at the head of the run + { + const Set::iterator it = set.find(h, size_t{0}); + UASSERT_SELFTEST(it != set.end(), true); // Present before the erase + set.erase(it); + } + UASSERT_SELFTEST(set.size(), 3); // One fewer entry + UASSERT_SELFTEST(set.contains(h, size_t{0}), false); // Namely that one + // Whether moved back or left in place, every entry must remain reachable + for (size_t i = 1; i < values.size(); ++i) { + const Set::iterator it = set.find(values[i].m_hash, i); + UASSERT_SELFTEST(it != set.end(), true); // The shift lost nothing + UASSERT_SELFTEST(*it, &values[i]); // And moved back the right entries + } + + // Erase the entry that stayed at its home position, shifting the last one again + { + const Set::iterator it = set.find(h + 1, size_t{1}); + UASSERT_SELFTEST(it != set.end(), true); // Left where the first erase found it + set.erase(it); + } + UASSERT_SELFTEST(set.size(), 2); // Two are left + for (size_t i = 2; i < values.size(); ++i) { + const Set::iterator it = set.find(values[i].m_hash, i); + UASSERT_SELFTEST(it != set.end(), true); // Both still reachable + UASSERT_SELFTEST(*it, &values[i]); // And are the entries they were + } + // Iterating visits exactly the remaining entries + std::array seen{}; + for (const Value* const valuep : set) { + UASSERT_SELFTEST(seen[valuep->m_id], false); // Each entry once + seen[valuep->m_id] = true; + } + UASSERT_SELFTEST(seen[0], false); // Erased first + UASSERT_SELFTEST(seen[1], false); // Erased second + UASSERT_SELFTEST(seen[2], true); // Moved back over the first hole + UASSERT_SELFTEST(seen[3], true); // And back again over the second + } +} + +//###################################################################### +// Entries stay in place unless the table grows or an entry is erased, as only those +// two invalidate iterators + +void testReferenceStability() { + struct Value final { + size_t m_hash = 0; // Hash of this entry + size_t m_id = 0; // Id of this entry + }; + + struct Hash final { + size_t operator()(const Value* valuep) const { return operator()(valuep->m_hash, 0); } + size_t operator()(size_t hash, size_t) const { return hash; } + }; + + struct Equal final { + bool operator()(const Value* ap, const Value* bp) const { + return operator()(ap, bp->m_hash, bp->m_id); + } + bool operator()(const Value* valuep, size_t hash, size_t id) const { + return valuep->m_hash == hash && valuep->m_id == id; + } + }; + + using Set = V3HashSet; + + // A reserved but still empty set finds nothing and iterates nothing + { + Set set; + set.reserve(8); + UASSERT_SELFTEST(set.contains(size_t{7}, size_t{0}), false); // Room, but no entry + UASSERT_SELFTEST(set.begin() == set.end(), true); // So iterates nothing + UASSERT_SELFTEST(set.empty(), true); // And holds nothing + } + + // 'N' is exactly what the reservation must hold, so this also checks the boundary + // arithmetic of 'reserve' against that of the growth check. It is more than an + // unreserved table holds without growing, so the reservation is doing the work. + constexpr size_t N = maxLoad(2 * MIN_CAPACITY); + static_assert(N > maxLoad(MIN_CAPACITY), "SelfTest: 'N' must need more than a new table"); + std::array values{}; + std::array entrypps{}; // Where each entry is stored, as inserted + Set set; + set.reserve(N); + // All entries collide, so every insertion probes through the whole existing run + for (size_t i = 0; i < N; ++i) { + values[i] = Value{7, i}; + const std::pair pair = set.insert(&values[i]); + UASSERT_SELFTEST(pair.second, true); // Ids differ, so each is added + entrypps[i] = &*pair.first; + } + // The set was reserved, so no insertion grew the table, and no entry has moved + for (size_t i = 0; i < N; ++i) { + const Set::iterator it = set.find(size_t{7}, i); + UASSERT_SELFTEST(it != set.end(), true); // Every entry is still there + UASSERT_SELFTEST(&*it, entrypps[i]); // In the slot it was put in + } + // Inserting entries that are present adds nothing and moves nothing + for (size_t i = 0; i < N; ++i) { + const std::pair pair = set.insert(&values[i]); + UASSERT_SELFTEST(pair.second, false); // Rejected, as it is present + UASSERT_SELFTEST(&*pair.first, entrypps[i]); // And nothing moved + } + UASSERT_SELFTEST(set.size(), N); // No duplicate was added + // Growing an occupied set through 'reserve' keeps every entry + set.reserve(8 * N); + UASSERT_SELFTEST(set.size(), N); // Rehashing dropped nothing + std::array grownpps{}; // Where each entry is after the growth + for (size_t i = 0; i < N; ++i) { + const Set::iterator it = set.find(size_t{7}, i); + UASSERT_SELFTEST(it != set.end(), true); // And left every entry reachable + grownpps[i] = &*it; + } + // Reserving room that is there already leaves the table alone, so nothing moves + set.reserve(N); + UASSERT_SELFTEST(set.size(), N); // The smaller request changed nothing + for (size_t i = 0; i < N; ++i) { + const Set::iterator it = set.find(size_t{7}, i); + UASSERT_SELFTEST(it != set.end(), true); // Every entry is still there + UASSERT_SELFTEST(&*it, grownpps[i]); // In the slot the growth left it in + } +} + +//###################################################################### +// A pseudo random workload checked against std::set or std::map as the reference model + +// Only a few distinct hashes, so probe runs are long and every erase shifts entries +struct ClusteredHash final { + size_t operator()(size_t value) const { return value & 0x7; } +}; + +// Hashes spread by a large odd multiplier, so most probe runs are short +struct SpreadHash final { + size_t operator()(size_t value) const { + return static_cast(value * 0x9e3779b97f4a7c15ULL); + } +}; + +// Drive a set through a deterministic pseudo random workload of insertions, erasures, +// and lookups, checking every step against a std::set holding the same entries +template +void testSetAgainstModel() { + struct Equal final { + bool operator()(size_t a, size_t b) const { return a == b; } + }; + + using Set = V3HashSet; + + constexpr size_t UNIVERSE = 64; // Entries drawn from a small range, so lookups hit + constexpr size_t STEPS = 10000; // Number of operations applied + + // A simple linear congruential generator, with a fixed seed so failures reproduce + uint64_t state = 0x123456789abcdef0ULL; + const auto nextRand = [&state]() -> size_t { + state = state * 6364136223846793005ULL + 1442695040888963407ULL; + return static_cast(state >> 32); + }; + + Set set; + std::set model; + + for (size_t step = 0; step < STEPS; ++step) { + const size_t id = nextRand() % UNIVERSE; + switch (nextRand() % 4) { + case 0: { // Insert a copy + const std::pair pair = set.insert(id); + UASSERT_SELFTEST(pair.second, model.insert(id).second); // As the model + UASSERT_SELFTEST(*pair.first, id); // At the entry asked for + break; + } + case 1: { // Insert lazily, which must create only on a miss + bool created = false; + const std::pair pair + = set.insertLazy(id, [&]() -> size_t { + created = true; + return id; + }); + UASSERT_SELFTEST(pair.second, created); // Created only on a miss + UASSERT_SELFTEST(pair.second, model.insert(id).second); // As the model + UASSERT_SELFTEST(*pair.first, id); // At the entry asked for + break; + } + case 2: { // Erase, if present + const typename Set::iterator it = set.find(id); + UASSERT_SELFTEST(it != set.end(), model.count(id) != 0); // As the model + if (it != set.end()) { + set.erase(it); + model.erase(id); + } + break; + } + default: { // Look up only + const typename Set::iterator it = set.find(id); + UASSERT_SELFTEST(it != set.end(), model.count(id) != 0); // As the model + if (it != set.end()) UASSERT_SELFTEST(*it, id); // And is the entry + break; + } + } + UASSERT_SELFTEST(set.size(), model.size()); // Every step, not just at the end + } + + // Check the final content exhaustively + for (size_t id = 0; id < UNIVERSE; ++id) { + UASSERT_SELFTEST(set.contains(id), model.count(id) != 0); // Present iff modelled + } + // And that iterating visits exactly the model content, each entry once + std::array seen{}; + size_t n = 0; + for (const size_t entry : set) { + UASSERT_SELFTEST(entry < UNIVERSE, true); // Nothing out of thin air + UASSERT_SELFTEST(model.count(entry) != 0, true); // Only what was inserted + UASSERT_SELFTEST(seen[entry], false); // Each entry once + seen[entry] = true; + ++n; + } + UASSERT_SELFTEST(n, model.size()); // And all of them +} + +//###################################################################### +// A hash table used as a map, whose entries are plain key and value pairs + +void testMap() { + // The key extractor means the hash and equality see only keys, never entries, so + // the standard functors do, and they are what V3HashMap defaults to + using Map = V3HashMap; + using Entry = Map::Entry; + + // A map entry is a plain pair, so a caller never meets an internal type + static_assert(std::is_same>::value, + "SelfTest: a map entry must be a plain pair"); + + constexpr size_t N = GROWS_TWICE; + Map map; + for (size_t i = 0; i < N; ++i) { + const std::string key = "key" + std::to_string(i); + // 'insertLazy' calls the factory only on a miss, so a hit builds no entry + const std::pair pair + = map.insertLazy(key, [&]() -> Entry { return {key, static_cast(i)}; }); + UASSERT_SELFTEST(pair.second, true); // Keys differ, so each is added + } + UASSERT_SELFTEST(map.size(), N); // All of them are in + + // A bare key finds the entry, with no Entry built to look it up + for (size_t i = 0; i < N; ++i) { + const Map::iterator it = map.find("key" + std::to_string(i)); + UASSERT_SELFTEST(it != map.end(), true); // The key alone finds it + UASSERT_SELFTEST(it->second, static_cast(i)); // With the value given + } + + // A value is changed by erasing the entry and inserting it again, as an iterator + // yields a const entry + for (size_t i = 0; i < N; ++i) { + const std::string key = "key" + std::to_string(i); + map.erase(map.find(key)); + UASSERT_SELFTEST(map.insert({key, static_cast(i) + 100}).second, true); // Anew + } + UASSERT_SELFTEST(map.size(), N); // The same keys, so the map has not grown + for (size_t i = 0; i < N; ++i) { + const Map::iterator it = map.find("key" + std::to_string(i)); + UASSERT_SELFTEST(it != map.end(), true); // Each key is still there + UASSERT_SELFTEST(it->second, static_cast(i) + 100); // With its new value + } + + // Adding a key that is present changes nothing + { + const std::string key = "key0"; + bool created = false; + const std::pair pair = map.insertLazy(key, [&]() -> Entry { + created = true; // LCOV_EXCL_START + return {key, 0}; // LCOV_EXCL_STOP + }); + UASSERT_SELFTEST(created, false); // The factory was never called + UASSERT_SELFTEST(pair.second, false); // As the key is present + UASSERT_SELFTEST(pair.first->second, 100); // Holding its old value + } + UASSERT_SELFTEST(map.size(), N); // And nothing was added + + // 'insert' adds a key and a value as a plain pair, as std::unordered_map::insert does + { + const std::pair added = map.insert({std::string{"fresh"}, 5}); + UASSERT_SELFTEST(added.second, true); // Added, as the key is new + UASSERT_SELFTEST(added.first->second, 5); // With the value given + const std::pair again = map.insert({std::string{"fresh"}, 6}); + UASSERT_SELFTEST(again.second, false); // Rejected, as the key is present + UASSERT_SELFTEST(again.first->second, 5); // And the value is untouched + } + UASSERT_SELFTEST(map.size(), N + 1); // Just the one entry was added + { + const Map::iterator it = map.find(std::string{"fresh"}); + UASSERT_SELFTEST(it != map.end(), true); // Present before the erase + UASSERT_SELFTEST(it->second, 5); // Still holding the first value + map.erase(it); + } + UASSERT_SELFTEST(map.size(), N); // Back to the earlier content + + // Erasing by key removes just that entry, and says whether it did + UASSERT_SELFTEST(map.erase(std::string{"absent"}), false); // No such key + UASSERT_SELFTEST(map.size(), N); // So nothing was erased + UASSERT_SELFTEST(map.erase(std::string{"key0"}), true); // That key is present + UASSERT_SELFTEST(map.size(), N - 1); // And just the one entry went + UASSERT_SELFTEST(map.contains(std::string{"key0"}), false); // Namely that one + for (size_t i = 1; i < N; ++i) { + UASSERT_SELFTEST(map.contains("key" + std::to_string(i)), true); // Others intact + } +} + +//###################################################################### +// The pseudo random workload again, run against a map and checked against std::map + +// Drive a map through a deterministic pseudo random workload of insertions, erasures, +// value changes and lookups, checking every step against a std::map holding the same +template +void testMapAgainstModel() { + struct Equal final { + bool operator()(size_t a, size_t b) const { return a == b; } + }; + + using Map = V3HashMap; + using Entry = typename Map::Entry; + + constexpr size_t UNIVERSE = 64; // Keys drawn from a small range, so lookups hit + constexpr size_t STEPS = 10000; // Number of operations applied + + // A simple linear congruential generator, with a fixed seed so failures reproduce + uint64_t state = 0x0fedcba987654321ULL; + const auto nextRand = [&state]() -> size_t { + state = state * 6364136223846793005ULL + 1442695040888963407ULL; + return static_cast(state >> 32); + }; + + Map map; + std::map model; + + for (size_t step = 0; step < STEPS; ++step) { + const size_t key = nextRand() % UNIVERSE; + const size_t val = step; // Distinct every step, so a stale value is visible + switch (nextRand() % 5) { + case 0: { // Insert a whole entry + const std::pair pair = map.insert({key, val}); + UASSERT_SELFTEST(pair.second, model.insert({key, val}).second); // Ditto + UASSERT_SELFTEST(pair.first->first, key); // At the key asked for + break; + } + case 1: { // Insert lazily, which must create only on a miss + bool created = false; + const std::pair pair + = map.insertLazy(key, [&]() -> Entry { + created = true; + return {key, val}; + }); + UASSERT_SELFTEST(pair.second, created); // Created only on a miss + UASSERT_SELFTEST(pair.second, model.insert({key, val}).second); // Ditto + UASSERT_SELFTEST(pair.first->first, key); // At the key asked for + break; + } + case 2: { // Erase by key, if present + UASSERT_SELFTEST(map.erase(key), model.erase(key) != 0); // As model + break; + } + case 3: { // Change a value, which is erasing the entry and inserting it again + const typename Map::iterator it = map.find(key); + if (it != map.end()) { + map.erase(it); + UASSERT_SELFTEST(map.insert({key, val}).second, true); // The key is free + model[key] = val; + } + break; + } + default: { // Look up only + const typename Map::iterator it = map.find(key); + UASSERT_SELFTEST(it != map.end(), model.count(key) != 0); // As the model + if (it != map.end()) { + UASSERT_SELFTEST(it->first, key); // At the key asked for + UASSERT_SELFTEST(it->second, model.at(key)); // With its value + } + break; + } + } + UASSERT_SELFTEST(map.size(), model.size()); // Every step, not just at the end + } + + // Check the final content exhaustively, values included + for (size_t key = 0; key < UNIVERSE; ++key) { + const typename Map::iterator it = map.find(key); + UASSERT_SELFTEST(it != map.end(), model.count(key) != 0); // Present iff modelled + if (it != map.end()) { + UASSERT_SELFTEST(it->second, model.at(key)); // With the right value + } + } + // And that iterating visits exactly the model content, each entry once + std::array seen{}; + size_t n = 0; + for (const Entry& entry : map) { + UASSERT_SELFTEST(entry.first < UNIVERSE, true); // Nothing out of thin air + UASSERT_SELFTEST(model.count(entry.first) != 0, true); // Only what was inserted + UASSERT_SELFTEST(entry.second, model.at(entry.first)); // With the right value + UASSERT_SELFTEST(seen[entry.first], false); // Each entry once + seen[entry.first] = true; + ++n; + } + UASSERT_SELFTEST(n, model.size()); // And all of them +} + +void selfTest() { + testValueKeys(); + testPointerKeys(); + testEntryLifetime(); + testMove(); + testMoveOnlyEntries(); + testStatefulFunctors(); + testBackwardShiftDeletion(); + testReferenceStability(); + testSetAgainstModel(); + testSetAgainstModel(); + testMap(); + testMapAgainstModel(); + testMapAgainstModel(); +} + +} // namespace V3HashTableInternals diff --git a/src/V3HashTable.h b/src/V3HashTable.h new file mode 100644 index 000000000..ae87f1322 --- /dev/null +++ b/src/V3HashTable.h @@ -0,0 +1,472 @@ +// -*- mode: C++; c-file-style: "cc-mode" -*- +//************************************************************************* +// DESCRIPTION: Verilator: Open addressing hash set and hash map +// +// Code available from: https://verilator.org +// +//************************************************************************* +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of either the GNU Lesser General Public License Version 3 +// or the Perl Artistic License Version 2.0. +// SPDX-FileCopyrightText: 2003-2026 Wilson Snyder +// SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0 +// +//************************************************************************* +// +// An open addressing, linear probing hash table, with backward shift deletion. +// Usable as V3HashSet or V3HashMap. The benefit of these over +// std::unordered_set and std::unordered_map is far better memory locality +// during lookup, and fewer dynamic memory allocations (which also means less +// heap fragmentation). Consider using these if profiling shows that the +// unordered STL collections contribute a significant cost to an algorithm. +// +// Four types tell the table what it holds: the entry, a hash, an equality, and +// a key extractor yielding the lookup key of an entry. The extractor is what +// lets one table serve both roles: a set's entry is its own key, a map's is a +// pair keyed by its first. The hash and equality hence only ever see keys, +// never entries. V3HashSet and V3HashMap at the bottom of this file derive +// from the table, pairing it with the extractor that suits each. +// +// Those two work on keys via Hash and Equal functors as in std::unordered_set +// or std::unordered_map, but lookup is always heterogeneous, with no +// is_transparent to opt in like in the STL, and a lookup key can be spelled as +// several arguments, being the parts a key is made of. An entry can hence be +// looked up without one at hand, as when it is only created on a miss. The +// functors must provide call operators as const members, for the key of an +// entry and for every lookup key spelling used: +// +// size_t Hash::operator()(const T_Key&) const +// size_t Hash::operator()(...) const +// bool Equal::operator()(const T_Key&, const T_Key&) const +// bool Equal::operator()(const T_Key&, ...) const +// +// with equal keys hashing equal, as usual, and consistently across the +// spellings. +// +// As only entries are stored, a slot is just a hash and an entry, so probing +// touches few cache lines. The table is doubled when an insertion would take +// it over the maximum load factor, or sized up front with 'reserve', to keep +// the probe runs short. +// +// Entries are referred to by iterators, as in the STL containers, but unlike +// STL containers, the mapped value in a V3HashMap is not mutable through an +// iterator. Iterators and entry addresses stay valid until the table grows or +// an entry is erased; either invalidates all of them. +// +// Erasure uses backward shift deletion: entries following the hole are moved +// back over it where their probe run ran through it (no tombstones). +// +//************************************************************************* + +#ifndef VERILATOR_V3HASHTABLE_H_ +#define VERILATOR_V3HASHTABLE_H_ + +#include "config_build.h" +#include "verilatedos.h" + +#include "V3Error.h" +#include "V3StdFuture.h" + +#include +#include +#include +#include +#include +#include + +namespace V3HashTableInternals { + +constexpr size_t MIN_CAPACITY = 16; // Smallest table allocated +constexpr size_t LOAD_FACTOR_NUM = 3; // Numerator of the maximum load factor +constexpr size_t LOAD_FACTOR_DEN = 4; // Denominator of the maximum load factor + +// Key extractor for a table whose entries are their own keys, that is, a set +template +struct V3HashTableKeyIsEntry final { + using Key = T_Key; // What it yields, so the table need not deduce it + const T_Key& operator()(const T_Key& entry) const { return entry; } +}; + +// Key extractor for a table whose entries are pairs keyed by the first, that is, a map +template +struct V3HashTableKeyIsFirst final { + using Key = T_Key; // What it yields, so the table need not deduce it + const T_Key& operator()(const std::pair& entry) const { return entry.first; } +}; + +void selfTest(); + +} // namespace V3HashTableInternals + +// V3HashTable, see the file header +// T_Entry The entries (STL calls this value_type) +// T_Hash Hashes a lookup key +// T_Equal Compares a key to a lookup key +// T_KeyOf Yields the key of an entry +template +class V3HashTable VL_NOT_FINAL { +public: + // TYPES + using Entry = T_Entry; // What is stored + using Key = typename T_KeyOf::Key; // What entries are looked up by + +private: + // TYPES + // Holds if the hash accepts a lookup key spelled as the given arguments + template + using ValidHash = vlstd::is_invocable_r; + + // Holds if the equality accepts a key and such a lookup key + template + using ValidEqual = vlstd::is_invocable_r; + + // The Key must itself be a valid lookup key, as every lookup ends in comparing one + // against a stored entry. Asserted separately, so the failure names the functor. + static_assert(ValidHash::value, "The 'Hash' functor must accept the 'Key'"); + static_assert(ValidEqual::value, "The 'Equal' functor must accept two 'Key's"); + + // A table slot + struct Slot final { + // The entry comes first, so it starts the slot whatever its alignment. + // It is a union so it is alive only while the slot is occupied. + union { + Entry m_entry; + }; + size_t m_hash = 0; // Hash of the entry, or zero when the slot is free + + Slot() {} // Leaves 'm_entry' uninitialized, as the slot is free + ~Slot() { + if (!isFree()) destroy(); + } + Slot(const Slot&) = delete; + Slot(Slot&&) = delete; + const Slot& operator=(const Slot&) = delete; + Slot& operator=(Slot&& that) { + UDEBUGONLY(UASSERT(this != &that, "Moving a slot onto itself");); + UDEBUGONLY(UASSERT(!that.isFree(), "Moving from a free slot");); + UDEBUGONLY(UASSERT(isFree(), "Moving into an occupied slot");); + new (&m_entry) Entry{std::move(that.m_entry)}; + m_hash = that.m_hash; + that.destroy(); + return *this; + } + + bool isFree() const { return !m_hash; } + + // Construct the entry of this free slot from the given entry + void construct(size_t hash, Entry&& entry) { + UDEBUGONLY(UASSERT(isFree(), "Constructing the entry of an occupied slot");); + new (&m_entry) Entry{std::move(entry)}; + m_hash = hash; + } + // Destroy the entry of this occupied slot, leaving it free + void destroy() { + UDEBUGONLY(UASSERT(!isFree(), "Destroying the entry of a free slot");); + m_entry.~Entry(); + m_hash = 0; + } + }; + +public: + // Iterator over the entries, see the file header on invalidation + class iterator final { + friend class V3HashTable; + + Slot* m_slotp = nullptr; // The slot iterated, or the end of the table + Slot* m_endp = nullptr; // One past the last slot + + iterator(Slot* slotp, Slot* endp) + : m_slotp{slotp} + , m_endp{endp} {} + + public: + iterator() = default; + // As opposed to the STL, this always returns a const reference so the + // collection is not mutable through an iterator alone. This is + // required because entries must be movable, hence can't be const, but + // the key of a map must not be modified. + const Entry& operator*() const { return m_slotp->m_entry; } + const Entry* operator->() const { return &m_slotp->m_entry; } + // Pre-increment, skipping the free slots + iterator& operator++() { + while (++m_slotp != m_endp && m_slotp->isFree()) {} + return *this; + } + bool operator==(const iterator& that) const { return m_slotp == that.m_slotp; } + bool operator!=(const iterator& that) const { return m_slotp != that.m_slotp; } + }; + +private: + // STATE + std::unique_ptr m_table; // The table, null when unallocated + size_t m_capacity = 0; // Number of slots in the table, a power of two, or zero + size_t m_size = 0; // Number of occupied slots + VL_NO_UNIQUE_ADDRESS_CXX20 T_Hash m_hash; // Hashes a lookup key + VL_NO_UNIQUE_ADDRESS_CXX20 T_Equal m_equal; // Compares a key to a lookup key + VL_NO_UNIQUE_ADDRESS_CXX20 T_KeyOf m_keyOf; // Yields the lookup key of an entry + + // METHODS + + // The hash of the given entry or lookup key, as stored in a slot + template + size_t hashOf(const T_Args&... args) const { + // A free slot is one with a zero hash, so force the high bit into every hash. + constexpr size_t USED_BIT = size_t{1} << (sizeof(size_t) * 8 - 1); + return static_cast(m_hash(args...)) | USED_BIT; + } + + // Index of the free slot the given hash probes to. There must always be one. + size_t freeSlot(size_t hash) const { + const size_t mask = m_capacity - 1; + size_t i = hash & mask; + while (!m_table[i].isFree()) i = (i + 1) & mask; + return i; + } + + // Resize to the given number of slots, which must fit all entries + void resize(size_t count) { + UDEBUGONLY(UASSERT(count && !(count & (count - 1)), "Capacity not a power of 2");); + const std::unique_ptr oldTable{std::move(m_table)}; + const size_t oldCapacity = m_capacity; + m_table = std::make_unique(count); + m_capacity = count; + // Reinsert the entries. 'freeSlot' appends to the probe run of each, so the runs + // come out contiguous whatever order this visits the old slots in. + for (size_t i = 0; i < oldCapacity; ++i) { + Slot& slot = oldTable[i]; + if (!slot.isFree()) m_table[freeSlot(slot.m_hash)] = std::move(slot); + } + } + + // Index of the slot holding the entry equal to the given key, or of the free slot its + // probe sequence ends at. The table must not be empty. + template + size_t probe(size_t hash, const T_Args&... args) const { + UDEBUGONLY(UASSERT(m_table, "Table must be allocated");); + const size_t mask = m_capacity - 1; + size_t i = hash & mask; + while (!m_table[i].isFree()) { + const Slot& slot = m_table[i]; + if (slot.m_hash == hash && m_equal(m_keyOf(slot.m_entry), args...)) break; + i = (i + 1) & mask; + } + return i; + } + + // Implementation of 'insertLazy' below. 'all' holds the key arguments, followed by + // the callable that creates the entry, so 'N_Key' indexes the key. + template + std::pair insertLazyImpl(std::index_sequence, T_All&& all) { + static_assert(ValidHash...>::value, + "The 'Hash' functor does not accept a lookup key spelled like this"); + static_assert(ValidEqual...>::value, + "The 'Equal' functor does not accept a lookup key spelled like this"); + const size_t hash = hashOf(std::get(all)...); + // Allocate on the first insertion + if (VL_UNLIKELY(!m_capacity)) resize(V3HashTableInternals::MIN_CAPACITY); + // Find the slot for the entry + Slot* slotp = m_table.get() + probe(hash, std::get(all)...); + // If occupied, it's the equivalent, and we are done + if (!slotp->isFree()) return {iterator{slotp, m_table.get() + m_capacity}, false}; + // Table is growing + ++m_size; + // Increase if necessary by load factor + if (VL_UNLIKELY(m_size * V3HashTableInternals::LOAD_FACTOR_DEN + > m_capacity * V3HashTableInternals::LOAD_FACTOR_NUM)) { + resize(m_capacity * 2); + slotp = m_table.get() + freeSlot(hash); + } + // Construct the entry via the user provided callable (last item in 'all') + slotp->construct(hash, std::get(all)()); + // The key of the created entry must both hash and compare as the key looked up +#ifdef VL_DEBUG + const Key& key = m_keyOf(slotp->m_entry); + UASSERT(hashOf(key) == hash, + "Created entry does not hash as the key it was looked up with"); + UASSERT(m_equal(key, std::get(all)...), + "Created entry does not match the key it was looked up with"); +#endif + // Return newly create entry + return {iterator{slotp, m_table.get() + m_capacity}, true}; + } + +protected: + // CONSTRUCTORS + V3HashTable() = default; + V3HashTable(T_Hash hash, T_Equal equal) + : m_hash{std::move(hash)} + , m_equal{std::move(equal)} {} + ~V3HashTable() = default; + VL_UNCOPYABLE(V3HashTable); + // Movable, as the table is just a pointer. The source is left empty rather than + // merely unspecified, so it remains a usable, empty table. + V3HashTable(V3HashTable&& that) + : m_table{std::move(that.m_table)} + , m_capacity{that.m_capacity} + , m_size{that.m_size} + , m_hash{std::move(that.m_hash)} + , m_equal{std::move(that.m_equal)} + , m_keyOf{std::move(that.m_keyOf)} { + that.m_capacity = 0; + that.m_size = 0; + } + V3HashTable& operator=(V3HashTable&& that) { + m_table = std::move(that.m_table); // Frees the table this held, if any + m_capacity = that.m_capacity; + m_size = that.m_size; + m_hash = std::move(that.m_hash); + m_equal = std::move(that.m_equal); + m_keyOf = std::move(that.m_keyOf); + that.m_capacity = 0; + that.m_size = 0; + return *this; + } + +public: + // METHODS + size_t size() const { return m_size; } + bool empty() const { return !m_size; } + + iterator begin() const { + Slot* const endp = m_table.get() + m_capacity; + Slot* slotp = m_table.get(); + while (slotp != endp && slotp->isFree()) ++slotp; + return iterator{slotp, endp}; + } + iterator end() const { + Slot* const endp = m_table.get() + m_capacity; + return iterator{endp, endp}; + } + + // Make room for the given number of entries, so inserting that many will not resize + void reserve(size_t count) { + size_t capacity = V3HashTableInternals::MIN_CAPACITY; + while (capacity * V3HashTableInternals::LOAD_FACTOR_NUM + < count * V3HashTableInternals::LOAD_FACTOR_DEN) + capacity *= 2; + if (capacity > m_capacity) resize(capacity); + } + + // Return iterator to the entry equal to the given key, or 'end()' if there + // is none. The key is whatever T_Hash and T_Equal accept, spelled as any + // number of arguments. Same as STL containers. + template + iterator find(const T_Args&... args) const { + static_assert(ValidHash::value, + "The 'Hash' functor does not accept a lookup key spelled like this"); + static_assert(ValidEqual::value, + "The 'Equal' functor does not accept a lookup key spelled like this"); + if (!m_size) return end(); // Nothing to find, and this also covers there being no table + Slot* const slotp = m_table.get() + probe(hashOf(args...), args...); + return slotp->isFree() ? end() : iterator{slotp, m_table.get() + m_capacity}; + } + + // Add the given entry, unless an equal one is in the table already. Return + // iterator to the entry and true if insertion happened. Same as STL containers. + std::pair insert(const Entry& entry) { + static_assert(std::is_copy_constructible::value, + "'Entry' must be copy constructible to use 'insert'"); + return insertLazy(m_keyOf(entry), [&entry]() -> Entry { return entry; }); + } + + // As 'insert', but the entry is only made when needed: all but the last argument spell + // the key, and the last is a callable to create the entry on a miss. Note the created entry + // must hash and compare equal to the key, and the call must not touch the container, as this + // holds the slot the entry will go in. + template + std::pair insertLazy(T_Args&&... args) { + static_assert(sizeof...(T_Args) >= 2, + "'insertLazy' needs a lookup key, then a callable to create the entry"); + using Callable = std::tuple_element_t>; + static_assert(vlstd::is_invocable_r::value, + "The last argument of 'insertLazy' must be a callable that takes no " + "arguments and returns an 'Entry'"); + return insertLazyImpl(std::make_index_sequence{}, + std::forward_as_tuple(std::forward(args)...)); + } + + // Whether an entry equal to the given key is in the table. The key is spelled as for 'find'. + template + bool contains(const T_Args&... args) const { + return find(args...) != end(); + } + + // Remove the entry equal to the given key, and return whether there was one. + template + bool erase(const T_Args&... args) { + const iterator it = find(args...); + if (it == end()) return false; + erase(it); + return true; + } + + // Remove the entry the given iterator refers to, which must not be 'end()'. Note that + // unlike STL erase this returns nothing, as every iterator is invalidated on deletion. + void erase(iterator it) { + UDEBUGONLY(UASSERT(it != end() && !it.m_slotp->isFree(), "Erasing a bad iterator");); + const size_t mask = m_capacity - 1; + size_t i = static_cast(it.m_slotp - m_table.get()); + // Destroy the entry + m_table[i].destroy(); + // The entry is gone, so slot 'i' is now a hole + --m_size; + // Backward shift deletion: move back the entries whose probing the hole breaks + size_t j = i; + while (true) { + j = (j + 1) & mask; + Slot& slot = m_table[j]; + if (slot.isFree()) break; + // Move back if its home position does not lie in the cyclic range (i, j] + if (((j - (slot.m_hash & mask)) & mask) >= ((j - i) & mask)) { + m_table[i] = std::move(slot); // Frees 'slot', which is then the hole + i = j; + } + } + } +}; + +template , + typename T_Equal = std::equal_to> +class V3HashSet final : public V3HashTable> { + using Super + = V3HashTable>; + + // Entries are only ever moved. Note 'insert' additionally needs copy construction. + static_assert(std::is_move_constructible::value, "'T_Key' must be move constructible"); + static_assert(std::is_destructible::value, "'T_Key' must be destructible"); + +public: + // CONSTRUCTORS + V3HashSet() = default; + V3HashSet(T_Hash hash, T_Equal equal) + : Super{std::move(hash), std::move(equal)} {} +}; + +template , + typename T_Equal = std::equal_to> +class V3HashMap final + : public V3HashTable, T_Hash, T_Equal, + V3HashTableInternals::V3HashTableKeyIsFirst> { + using Super = V3HashTable, T_Hash, T_Equal, + V3HashTableInternals::V3HashTableKeyIsFirst>; + + // Entries are only ever moved. Note 'insert' additionally needs copy construction. + // Asserted separately, so the failure names the one at fault. + static_assert(std::is_move_constructible::value, "'T_Key' must be move constructible"); + static_assert(std::is_destructible::value, "'T_Key' must be destructible"); + static_assert(std::is_move_constructible::value, "'T_Val' must be move constructible"); + static_assert(std::is_destructible::value, "'T_Val' must be destructible"); + +public: + // TYPES + using Value = T_Val; // What a key maps to + + // CONSTRUCTORS + V3HashMap() = default; + V3HashMap(T_Hash hash, T_Equal equal) + : Super{std::move(hash), std::move(equal)} {} +}; + +#endif // Guard diff --git a/src/Verilator.cpp b/src/Verilator.cpp index dca85ae99..524e2b77a 100644 --- a/src/Verilator.cpp +++ b/src/Verilator.cpp @@ -62,6 +62,7 @@ #include "V3Gate.h" #include "V3Global.h" #include "V3Graph.h" +#include "V3HashTable.h" #include "V3HierBlock.h" #include "V3Inline.h" #include "V3InlineCFuncs.h" @@ -741,6 +742,7 @@ static bool verilate(const string& argString) { V3PreShell::selfTest(); V3Broken::selfTest(); V3Control::selfTest(); + V3HashTableInternals::selfTest(); V3ThreadPool::selfTest(); UINFO(2, "selfTest done"); }