From 0fed5f8b3e02088ee22df4c8b27e6e5e386ce227 Mon Sep 17 00:00:00 2001 From: Geza Lore Date: Wed, 6 Mar 2024 18:01:52 +0000 Subject: [PATCH] Avoid creating redundant vertices in V3DfgPeephole (#4944) Add a new data-structure V3DfgCache, which can be used to retrieve existing vertices with some given inputs vertices. Use this in V3DfgPeephole to eliminate the creation of redundant vertices. Overall this is performance neutral, but is in prep for some future work. --- include/verilatedos.h | 6 + src/CMakeLists.txt | 2 + src/Makefile_obj.in | 1 + src/V3DfgCache.cpp | 51 +++ src/V3DfgCache.h | 402 ++++++++++++++++++++++ src/V3DfgPeephole.cpp | 575 ++++++++++++++------------------ src/astgen | 4 +- test_regress/t/t_dfg_peephole.v | 2 +- 8 files changed, 715 insertions(+), 328 deletions(-) create mode 100644 src/V3DfgCache.cpp create mode 100644 src/V3DfgCache.h diff --git a/include/verilatedos.h b/include/verilatedos.h index 17a49c263..2a0739ff5 100644 --- a/include/verilatedos.h +++ b/include/verilatedos.h @@ -58,6 +58,9 @@ # define VL_ATTR_PRINTF(fmtArgNum) __attribute__((format(printf, (fmtArgNum), (fmtArgNum) + 1))) # define VL_ATTR_PURE __attribute__((pure)) # define VL_ATTR_UNUSED __attribute__((unused)) +#ifndef VL_ATTR_WARN_UNUSED_RESULT +# define VL_ATTR_WARN_UNUSED_RESULT __attribute__((warn_unused_result)) +#endif # if !defined(_WIN32) && !defined(__MINGW32__) // All VL_ATTR_WEAK symbols must be marked with the macOS -U linker flag in verilated.mk.in # define VL_ATTR_WEAK __attribute__((weak)) @@ -164,6 +167,9 @@ #ifndef VL_ATTR_UNUSED # define VL_ATTR_UNUSED ///< Attribute that function that may be never used #endif +#ifndef VL_ATTR_WARN_UNUSED_RESULT +# define VL_ATTR_WARN_UNUSED_RESULT ///< Attribute that return value of function must be used +#endif #ifndef VL_ATTR_WEAK # define VL_ATTR_WEAK ///< Attribute that function external that is optionally defined #endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4e1c9b8ae..eb0fcdcfc 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -62,6 +62,7 @@ set(HEADERS V3DepthBlock.h V3Descope.h V3Dfg.h + V3DfgCache.h V3DfgOptimizer.h V3DfgPasses.h V3DfgPatternStats.h @@ -214,6 +215,7 @@ set(COMMON_SOURCES V3Descope.cpp V3Dfg.cpp V3DfgAstToDfg.cpp + V3DfgCache.cpp V3DfgDecomposition.cpp V3DfgDfgToAst.cpp V3DfgOptimizer.cpp diff --git a/src/Makefile_obj.in b/src/Makefile_obj.in index 50bf1f9da..37874cd4b 100644 --- a/src/Makefile_obj.in +++ b/src/Makefile_obj.in @@ -228,6 +228,7 @@ RAW_OBJS_PCH_ASTNOMT = \ V3Descope.o \ V3Dfg.o \ V3DfgAstToDfg.o \ + V3DfgCache.o \ V3DfgDecomposition.o \ V3DfgDfgToAst.o \ V3DfgOptimizer.o \ diff --git a/src/V3DfgCache.cpp b/src/V3DfgCache.cpp new file mode 100644 index 000000000..036578e2b --- /dev/null +++ b/src/V3DfgCache.cpp @@ -0,0 +1,51 @@ +// -*- mode: C++; c-file-style: "cc-mode" -*- +//************************************************************************* +// DESCRIPTION: Verilator: Dfg vertex cache to find existing vertices +// +// Code available from: https://verilator.org +// +//************************************************************************* +// +// Copyright 2003-2024 by Wilson Snyder. This program is free software; you +// can redistribute it and/or modify it under the terms of either the GNU +// Lesser General Public License Version 3 or the Perl Artistic License +// Version 2.0. +// SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0 +// +//************************************************************************* +// +// A cache for DfgGraph, to find existing vertices with identical inputs. +// +// Beware that if you use data-structure, you must invalidate the cache any +// time you change the inputs of an existing vertex, otherwise you will +// have a very bad day. +// +//************************************************************************* + +#include "V3DfgCache.h" + +namespace V3DfgCacheInternal { + +void V3DfgCache::cache(DfgVertex* vtxp) { + switch (vtxp->type()) { +#define VERTEX_CACHE_ADD_CASE(t) \ + case t::dfgType(): V3DfgCacheInternal::cache(m_cache##t, reinterpret_cast(vtxp)); break; + FOREACH_CACHED_VERTEX_TYPE(VERTEX_CACHE_ADD_CASE) +#undef VERTEX_CACHE_ADD_CASE + default: break; + } +} + +void V3DfgCache::invalidateByValue(DfgVertex* vtxp) { + switch (vtxp->type()) { +#define VERTEX_CACHE_ADD_CASE(t) \ + case t::dfgType(): \ + V3DfgCacheInternal::invalidateByValue(m_cache##t, reinterpret_cast(vtxp)); \ + break; + FOREACH_CACHED_VERTEX_TYPE(VERTEX_CACHE_ADD_CASE) +#undef VERTEX_CACHE_ADD_CASE + default: break; + } +} + +} // namespace V3DfgCacheInternal diff --git a/src/V3DfgCache.h b/src/V3DfgCache.h new file mode 100644 index 000000000..a85af80e0 --- /dev/null +++ b/src/V3DfgCache.h @@ -0,0 +1,402 @@ +// -*- mode: C++; c-file-style: "cc-mode" -*- +//************************************************************************* +// DESCRIPTION: Verilator: Dfg vertex cache to find existing vertices +// +// Code available from: https://verilator.org +// +//************************************************************************* +// +// Copyright 2003-2024 by Wilson Snyder. This program is free software; you +// can redistribute it and/or modify it under the terms of either the GNU +// Lesser General Public License Version 3 or the Perl Artistic License +// Version 2.0. +// SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0 +// +//************************************************************************* +// +// A cache for DfgGraph, to find existing vertices with identical inputs. +// +// Beware that if you use this data-structure, you must invalidate the +// cache any time you change the inputs of an existing vertex, otherwise +// you will have a very bad day. +// +//************************************************************************* + +#ifndef VERILATOR_V3DFGCACHE_H_ +#define VERILATOR_V3DFGCACHE_H_ + +#include "V3Dfg.h" + +#include + +// Template specializatoins must be defiend before they are used, +// so this is implemented in a namespace + +namespace V3DfgCacheInternal { + +// Hash constants by value, everything else by identity +inline V3Hash vertexHash(const DfgVertex* vtxp) { + if (const DfgConst* const constp = vtxp->cast()) return constp->num().toHash(); + return V3Hash{reinterpret_cast(vtxp)}; +} + +// Constants are equal by value, everything else is equal by identity +inline bool vertexEqual(const DfgVertex* ap, const DfgVertex* bp) { + if (ap == bp) return true; + if (ap->type() != bp->type()) return false; + if (const DfgConst* const aConstp = ap->cast()) { + const DfgConst* const bConstp = bp->as(); + return aConstp->num().isCaseEq(bConstp->num()); + } + return false; +} + +class KeySel final { + const DfgVertex* const m_fromp; + const uint32_t m_lsb; + const uint32_t m_width; + +public: + KeySel(DfgVertex* fromp, uint32_t lsb, uint32_t width) + : m_fromp{fromp} + , m_lsb{lsb} + , m_width{width} {} + + struct Hash final { + size_t operator()(const KeySel& key) const { + V3Hash hash{vertexHash(key.m_fromp)}; + hash += key.m_lsb; + hash += key.m_width; + return hash.value(); + } + }; + + struct Equal final { + bool operator()(const KeySel& a, const KeySel& b) const { + return a.m_lsb == b.m_lsb && a.m_width == b.m_width + && vertexEqual(a.m_fromp, b.m_fromp); + } + }; +}; + +class KeyUnary final { + const DfgVertex* const m_source0p; + +public: + KeyUnary(DfgVertex* source0p) + : m_source0p{source0p} {} + + struct Hash final { + size_t operator()(const KeyUnary& key) const { // + return vertexHash(key.m_source0p).value(); + } + }; + + struct Equal final { + bool operator()(const KeyUnary& a, const KeyUnary& b) const { + return vertexEqual(a.m_source0p, b.m_source0p); + } + }; +}; + +class KeyBinary final { + const DfgVertex* const m_source0p; + const DfgVertex* const m_source1p; + +public: + KeyBinary(DfgVertex* source0p, DfgVertex* source1p) + : m_source0p{source0p} + , m_source1p{source1p} {} + + struct Hash final { + size_t operator()(const KeyBinary& key) const { + V3Hash hash{vertexHash(key.m_source0p)}; + hash += vertexHash(key.m_source1p); + return hash.value(); + } + }; + + struct Equal final { + bool operator()(const KeyBinary& a, const KeyBinary& b) const { + return vertexEqual(a.m_source0p, b.m_source0p) + && vertexEqual(a.m_source1p, b.m_source1p); + } + }; +}; + +class KeyTernary final { + const DfgVertex* const m_source0p; + const DfgVertex* const m_source1p; + const DfgVertex* const m_source2p; + +public: + KeyTernary(DfgVertex* source0p, DfgVertex* source1p, DfgVertex* source2p) + : m_source0p{source0p} + , m_source1p{source1p} + , m_source2p{source2p} {} + + struct Hash final { + size_t operator()(const KeyTernary& key) const { + V3Hash hash{vertexHash(key.m_source0p)}; + hash += vertexHash(key.m_source1p); + hash += vertexHash(key.m_source2p); + return hash.value(); + } + }; + + struct Equal final { + bool operator()(const KeyTernary& a, const KeyTernary& b) const { + return vertexEqual(a.m_source0p, b.m_source0p) + && vertexEqual(a.m_source1p, b.m_source1p) + && vertexEqual(a.m_source2p, b.m_source2p); + } + }; +}; + +template +using Cache = std::unordered_map; + +using CacheSel = Cache; +using CacheUnary = Cache; +using CacheBinary = Cache; +using CacheTernary = Cache; + +// These return a reference to the mapped entry, inserting a nullptr if not yet exists +inline DfgSel*& getEntry(CacheSel& cache, AstNodeDType* dtypep, DfgVertex* src0p, uint32_t lsb) { + UASSERT_OBJ(VN_IS(dtypep, BasicDType), dtypep, "non-packed has no 'width()'"); + return cache + .emplace(std::piecewise_construct, // + std::forward_as_tuple(src0p, lsb, dtypep->width()), // + std::forward_as_tuple(nullptr)) + .first->second; +} + +inline DfgVertexUnary*& getEntry(CacheUnary& cache, AstNodeDType*, DfgVertex* src0p) { + return cache + .emplace(std::piecewise_construct, // + std::forward_as_tuple(src0p), // + std::forward_as_tuple(nullptr)) + .first->second; +} + +inline DfgVertexBinary*& getEntry(CacheBinary& cache, AstNodeDType*, DfgVertex* src0p, + DfgVertex* src1p) { + return cache + .emplace(std::piecewise_construct, // + std::forward_as_tuple(src0p, src1p), // + std::forward_as_tuple(nullptr)) + .first->second; +} + +inline DfgVertexTernary*& getEntry(CacheTernary& cache, AstNodeDType*, DfgVertex* src0p, + DfgVertex* src1p, DfgVertex* src2p) { + return cache + .emplace(std::piecewise_construct, // + std::forward_as_tuple(src0p, src1p, src2p), // + std::forward_as_tuple(nullptr)) + .first->second; +} + +// These return a reference to the mapped entry, inserting a nullptr if not yet exists +inline CacheSel::iterator find(CacheSel& cache, AstNodeDType* dtypep, DfgVertex* src0p, + uint32_t lsb) { + UASSERT_OBJ(VN_IS(dtypep, BasicDType), dtypep, "non-packed has no 'width()'"); + const uint32_t width = dtypep->width(); + return cache.find({src0p, lsb, width}); +} + +inline CacheUnary::iterator find(CacheUnary& cache, AstNodeDType*, DfgVertex* src0p) { + return cache.find({src0p}); +} + +inline CacheBinary::iterator find(CacheBinary& cache, AstNodeDType*, DfgVertex* src0p, + DfgVertex* src1p) { + return cache.find({src0p, src1p}); +} + +inline CacheTernary::iterator find(CacheTernary& cache, AstNodeDType*, DfgVertex* src0p, + DfgVertex* src1p, DfgVertex* src2p) { + return cache.find({src0p, src1p, src2p}); +} + +// These set the operands of a new vertex +inline void setOperands(DfgSel* vtxp, DfgVertex* fromp, uint32_t lsb) { + vtxp->fromp(fromp); + vtxp->lsb(lsb); +} + +inline void setOperands(DfgVertexUnary* vtxp, DfgVertex* src0p) { // + vtxp->relinkSource<0>(src0p); +} + +inline void setOperands(DfgVertexBinary* vtxp, DfgVertex* src0p, DfgVertex* src1p) { + vtxp->relinkSource<0>(src0p); + vtxp->relinkSource<1>(src1p); +} + +inline void setOperands(DfgVertexTernary* vtxp, DfgVertex* src0p, DfgVertex* src1p, + DfgVertex* src2p) { + vtxp->relinkSource<0>(src0p); + vtxp->relinkSource<1>(src1p); + vtxp->relinkSource<2>(src2p); +} + +// Get or create (and insert) vertex with given operands +template +inline Vertex* getOrCreate(DfgGraph& dfg, FileLine* flp, AstNodeDType* dtypep, Cache& cache, + Operands... operands) { + typename Cache::mapped_type& entrypr = getEntry(cache, dtypep, operands...); + if (!entrypr) { + Vertex* const newp = new Vertex{dfg, flp, dtypep}; + setOperands(newp, operands...); + entrypr = newp; + } + return reinterpret_cast(entrypr); +} + +// These add an existing vertex to the table, if an equivalent does not yet exist +inline void cache(CacheSel& cache, DfgSel* vtxp) { + DfgSel*& entrypr = getEntry(cache, vtxp->dtypep(), vtxp->fromp(), vtxp->lsb()); + if (!entrypr) entrypr = vtxp; +} + +inline void cache(CacheUnary& cache, DfgVertexUnary* vtxp) { + DfgVertexUnary*& entrypr = getEntry(cache, vtxp->dtypep(), vtxp->source<0>()); + if (!entrypr) entrypr = vtxp; +} + +inline void cache(CacheBinary& cache, DfgVertexBinary* vtxp) { + DfgVertexBinary*& entrypr + = getEntry(cache, vtxp->dtypep(), vtxp->source<0>(), vtxp->source<1>()); + if (!entrypr) entrypr = vtxp; +} + +inline void cache(CacheTernary& cache, DfgVertexTernary* vtxp) { + DfgVertexTernary*& entrypr + = getEntry(cache, vtxp->dtypep(), vtxp->source<0>(), vtxp->source<1>(), vtxp->source<2>()); + if (!entrypr) entrypr = vtxp; +} + +// These remove an existing vertex from the cache, if it is the cached vertex +inline void invalidateByValue(CacheSel& cache, DfgSel* vtxp) { + const auto it = find(cache, vtxp->dtypep(), vtxp->fromp(), vtxp->lsb()); + if (it != cache.end() && it->second == vtxp) cache.erase(it); +} + +inline void invalidateByValue(CacheUnary& cache, DfgVertexUnary* vtxp) { + const auto it = find(cache, vtxp->dtypep(), vtxp->source<0>()); + if (it != cache.end() && it->second == vtxp) cache.erase(it); +} + +inline void invalidateByValue(CacheBinary& cache, DfgVertexBinary* vtxp) { + const auto it = find(cache, vtxp->dtypep(), vtxp->source<0>(), vtxp->source<1>()); + if (it != cache.end() && it->second == vtxp) cache.erase(it); +} + +inline void invalidateByValue(CacheTernary& cache, DfgVertexTernary* vtxp) { + const auto it + = find(cache, vtxp->dtypep(), vtxp->source<0>(), vtxp->source<1>(), vtxp->source<2>()); + if (it != cache.end() && it->second == vtxp) cache.erase(it); +} + +// clang-format off + +// This type defines the cache type corresponding to a vertex type +template struct CacheTypeImpl : public CacheTypeImpl {}; +template <> struct CacheTypeImpl { using type = CacheSel; }; +template <> struct CacheTypeImpl { using type = CacheUnary; }; +template <> struct CacheTypeImpl { using type = CacheBinary; }; +template <> struct CacheTypeImpl { using type = CacheTernary; }; +template using CacheType = typename CacheTypeImpl::type; + +// Just add new lines in here for new vertex types you need to be able to cache + +#define FOREACH_CACHED_VERTEX_TYPE(macro) \ + macro(DfgSel) \ + macro(DfgNot) \ + macro(DfgNegate) \ + macro(DfgConcat) \ + macro(DfgNeq) \ + macro(DfgShiftL) \ + macro(DfgShiftR) \ + macro(DfgShiftRS) \ + macro(DfgAdd) \ + macro(DfgSub) \ + macro(DfgMul) \ + macro(DfgMulS) \ + macro(DfgEq) \ + macro(DfgAnd) \ + macro(DfgOr) \ + macro(DfgXor) \ + macro(DfgRedAnd) \ + macro(DfgRedOr) \ + macro(DfgRedXor) \ + macro(DfgCond) + +// clang-format on + +class V3DfgCache final { + DfgGraph& m_dfg; // The DfgGraph we are caching the vertices of + + // The per type caches +#define VERTEX_CACHE_DECLARE_LUT(t) CacheType m_cache##t; + FOREACH_CACHED_VERTEX_TYPE(VERTEX_CACHE_DECLARE_LUT) +#undef VERTEX_CACHE_DECLARE_LUT + + // Specializations return one of the above m_cache members + template + inline CacheType& cacheForType(); + +public: + V3DfgCache(DfgGraph& dfg) + : m_dfg{dfg} {} + + // Find a vertex of type 'Vertex', with the given operands, or create a new one and add it. + template + inline Vertex* getOrCreate(FileLine* flp, AstNodeDType* dtypep, Operands... operands); + + // Add an existing vertex of the table. If an equivalent already exists, then nothing happens. + void cache(DfgVertex* vtxp); + + // Remove an exiting vertex, it is the cached vertex. + void invalidateByValue(DfgVertex* vtxp); +}; + +// clang-format off +// The per-type specializations of 'V3DfgCache::cacheForType' +#define VERTEX_CACHE_DEFINE_LUT_SPECIALIZATION(t) \ + template <> inline CacheType& V3DfgCache::cacheForType() { return m_cache ## t; } +FOREACH_CACHED_VERTEX_TYPE(VERTEX_CACHE_DEFINE_LUT_SPECIALIZATION) +#undef VERTEX_CACHE_DEFINE_LUT_SPECIALIZATION +// clang-format on + +// Find a vertex of type 'Vertex', with the given operands, or create a new one and add it +template +Vertex* V3DfgCache::getOrCreate(FileLine* flp, AstNodeDType* dtypep, Operands... operands) { + static_assert(std::is_final::value, "Must invoke on final vertex type"); + constexpr bool isSel = std::is_same::value; + constexpr bool isUnary = !isSel && std::is_base_of::value; + constexpr bool isBinary = std::is_base_of::value; + constexpr bool isTernary = std::is_base_of::value; + static_assert(isSel || isUnary || isBinary || isTernary, + "'get' called with unknown vertex type"); + + static_assert(!isSel || sizeof...(Operands) == 2, // + "Wrong number of operands to DfgSel"); + static_assert(!isUnary || sizeof...(Operands) == 1, + "Wrong number of operands to unary vertex"); + static_assert(!isBinary || sizeof...(Operands) == 2, + "Wrong number of operands to binary vertex"); + static_assert(!isTernary || sizeof...(Operands) == 3, + "Wrong number of operands to ternary vertex"); + + return V3DfgCacheInternal::getOrCreate, Operands...>( + m_dfg, flp, dtypep, cacheForType(), operands...); +} + +} // namespace V3DfgCacheInternal + +// Export only the public interface class +using V3DfgCacheInternal::V3DfgCache; + +#endif // VERILATOR_V3DFGCACHE_H_ diff --git a/src/V3DfgPeephole.cpp b/src/V3DfgPeephole.cpp index e6fed4218..7b9ae8fbd 100644 --- a/src/V3DfgPeephole.cpp +++ b/src/V3DfgPeephole.cpp @@ -26,6 +26,7 @@ #include "V3DfgPeephole.h" #include "V3Dfg.h" +#include "V3DfgCache.h" #include "V3DfgPasses.h" #include "V3Stats.h" @@ -135,7 +136,6 @@ class V3DfgPeephole final : public DfgVisitor { // STATE DfgGraph& m_dfg; // The DfgGraph being visited V3DfgPeepholeContext& m_ctx; // The config structure - // bool m_changed = false; // Changed a vertex AstNodeDType* const m_bitDType = DfgVertex::dtypeForWidth(1); // Common, so grab it up front // Head of work list. Note that we want all next pointers in the list to be non-zero (including // that of the last element). This allows as to do two important things: detect if an element @@ -144,6 +144,9 @@ class V3DfgPeephole final : public DfgVisitor { // can easily check for the end of the list. DfgVertex* m_workListp = reinterpret_cast(this); + // Vertex lookup-table to avoid creating redundant vertices + V3DfgCache m_cache{m_dfg}; + #define APPLYING(id) if (checkApplying(VDfgPeepholePattern::id)) // METHODS @@ -151,7 +154,6 @@ class V3DfgPeephole final : public DfgVisitor { if (!m_ctx.m_enabled[id]) return false; UINFO(9, "Applying DFG patten " << id.ascii() << endl); ++m_ctx.m_count[id]; - // m_changed = true; return true; } @@ -180,27 +182,33 @@ class V3DfgPeephole final : public DfgVisitor { // the work list), but it will be deleted when the work list is processed. if (vtxp->getUser()) return; // Otherwise we can delete it now. + // Remove from cache + m_cache.invalidateByValue(vtxp); + // Unlink source edges + vtxp->forEachSourceEdge([](DfgEdge& edge, size_t) { edge.unlinkSource(); }); + // Should not have sinks + UASSERT_OBJ(!vtxp->hasSinks(), vtxp, "Should not delete used vertex"); + // VL_DO_DANGLING(vtxp->unlinkDelete(m_dfg), vtxp); } void replace(DfgVertex* vtxp, DfgVertex* replacementp) { + UASSERT_OBJ(vtxp != replacementp, vtxp, + "Should not try to replace with a vertex with itself"); + UASSERT_OBJ(vtxp->width() == replacementp->width(), vtxp, + "Replacement vertex has different width"); // Add sinks of replaced vertex to the work list addSinksToWorkList(vtxp); // Add replacement to the work list addToWorkList(replacementp); // Replace vertex with the replacement + vtxp->forEachSink([&](DfgVertex& sink) { m_cache.invalidateByValue(&sink); }); vtxp->replaceWith(replacementp); + replacementp->forEachSink([&](DfgVertex& sink) { m_cache.cache(&sink); }); // Vertex is now unused, so delete it deleteVertex(vtxp); } - void modified(DfgVertex* vtxp) { - // Add sinks of modified vertex to the work list - addSinksToWorkList(vtxp); - // Add the modified vertex itself to the work list - addToWorkList(vtxp); - } - // Shorthand static AstNodeDType* dtypeForWidth(uint32_t width) { return DfgVertex::dtypeForWidth(width); } @@ -211,23 +219,35 @@ class V3DfgPeephole final : public DfgVisitor { DfgConst* makeZero(FileLine* flp, uint32_t width) { return new DfgConst{m_dfg, flp, width}; } // Create a new vertex of the given type - template - Vertex* make(FileLine* flp, Args&&... args) { - static_assert(std::is_final::value, "Must invoke on final class"); - static_assert(!std::is_same::value, "Use 'makeZero' instead"); - static_assert(!std::is_base_of::value, "Can't create variables"); - // Create the new vertex - Vertex* const vtxp = new Vertex{m_dfg, flp, std::forward(args)...}; + template + Vertex* make(FileLine* flp, AstNodeDType* dtypep, Operands... operands) { + // Find or create an equivalent vertex + Vertex* const vtxp = m_cache.getOrCreate(flp, dtypep, operands...); + // Add to work list. - vtxp->template setUser(m_workListp); - m_workListp = vtxp; + DfgVertex*& workListNextp = vtxp->template user(); + if (!workListNextp) { + workListNextp = m_workListp; + m_workListp = vtxp; + } + // Return new node return vtxp; } + // Same as above, but 'flp' and 'dtypep' are taken from the given example vertex + template + Vertex* make(DfgVertex* examplep, Operands... operands) { + return make(examplep->fileline(), examplep->dtypep(), operands...); + } + + // Note: If any of the following transformers return true, then the vertex was replaced and the + // caller must not do any further changes, so the caller must check the return value, otherwise + // there will be hard to debug issues. + // Constant fold unary vertex, return true if folded template - bool foldUnary(Vertex* vtxp) { + VL_ATTR_WARN_UNUSED_RESULT bool foldUnary(Vertex* vtxp) { static_assert(std::is_base_of::value, "Must invoke on unary"); static_assert(std::is_final::value, "Must invoke on final class"); if (DfgConst* const srcp = vtxp->srcp()->template cast()) { @@ -243,7 +263,7 @@ class V3DfgPeephole final : public DfgVisitor { // Constant fold binary vertex, return true if folded template - bool foldBinary(Vertex* vtxp) { + VL_ATTR_WARN_UNUSED_RESULT bool foldBinary(Vertex* vtxp) { static_assert(std::is_base_of::value, "Must invoke on binary"); static_assert(std::is_final::value, "Must invoke on final class"); if (DfgConst* const lhsp = vtxp->lhsp()->template cast()) { @@ -259,29 +279,10 @@ class V3DfgPeephole final : public DfgVisitor { return false; } - // Rotate the expression tree rooted at 'vtxp' to the right ('vtxp->lhsp()' becomes root, - // producing a right-leaning tree). Warning: only valid for associative operations. - template - void rotateRight(Vertex* vtxp) { - static_assert(std::is_base_of::value, "Must invoke on binary"); - static_assert(std::is_final::value, "Must invoke on final class"); - DfgVertexBinary* const ap = vtxp; - DfgVertexBinary* const bp = vtxp->lhsp()->template as(); - UASSERT_OBJ(!bp->hasMultipleSinks(), vtxp, "Can't rotate a non-tree"); - ap->replaceWith(bp); - ap->lhsp(bp->rhsp()); - bp->rhsp(ap); - // Concatenation dtypes need to be fixed up, other associative nodes preserve types - if VL_CONSTEXPR_CXX17 (std::is_same::value) { - ap->dtypep(dtypeForWidth(ap->lhsp()->width() + ap->rhsp()->width())); - bp->dtypep(dtypeForWidth(bp->lhsp()->width() + bp->rhsp()->width())); - } - } - // Transformations that apply to all associative binary vertices. // Returns true if vtxp was replaced. template - bool associativeBinary(Vertex* vtxp) { + VL_ATTR_WARN_UNUSED_RESULT bool associativeBinary(Vertex* vtxp) { static_assert(std::is_base_of::value, "Must invoke on binary"); static_assert(std::is_final::value, "Must invoke on final class"); @@ -313,18 +314,9 @@ class V3DfgPeephole final : public DfgVisitor { foldOp(constp->num(), lConstp->num(), rlConstp->num()); // Replace vertex - if (!rVtxp->hasMultipleSinks()) { - rVtxp->lhsp(constp); - rVtxp->dtypep(vtxp->dtypep()); - replace(vtxp, rVtxp); - return true; - } else { - Vertex* const resp = make(flp, vtxp->dtypep()); - resp->lhsp(constp); - resp->rhsp(rVtxp->rhsp()); - replace(vtxp, resp); - return true; - } + Vertex* const resp = make(vtxp, constp, rVtxp->rhsp()); + replace(vtxp, resp); + return true; } } } @@ -342,79 +334,93 @@ class V3DfgPeephole final : public DfgVisitor { foldOp(constp->num(), lrConstp->num(), rConstp->num()); // Replace vertex - if (!lVtxp->hasMultipleSinks()) { - lVtxp->rhsp(constp); - lVtxp->dtypep(vtxp->dtypep()); - replace(vtxp, lVtxp); - return true; - } else { - Vertex* const resp = make(flp, vtxp->dtypep()); - resp->lhsp(lVtxp->lhsp()); - resp->rhsp(constp); - replace(vtxp, resp); - return true; - } + Vertex* const resp = make(vtxp, lVtxp->lhsp(), constp); + replace(vtxp, resp); + return true; } } } } // Make associative trees right leaning to reduce pattern variations, and for better CSE - while (vtxp->lhsp()->template is() && !vtxp->lhsp()->hasMultipleSinks()) { + bool changed = false; + while (true) { + Vertex* const lhsp = vtxp->lhsp()->template cast(); + if (!lhsp || lhsp->hasMultipleSinks()) break; + APPLYING(RIGHT_LEANING_ASSOC) { - rotateRight(vtxp); - modified(vtxp); - continue; + // Rotate the expression tree rooted at 'vtxp' to the right, producing a + // right-leaning tree + DfgVertex* const ap = lhsp->lhsp(); + DfgVertex* const bp = lhsp->rhsp(); + DfgVertex* const cp = vtxp->rhsp(); + + AstNodeDType* const rootDtyptp = vtxp->dtypep(); + AstNodeDType* childDtyptp = vtxp->dtypep(); + // Concatenation dtypes need to be fixed up, other associative nodes preserve + // types + if VL_CONSTEXPR_CXX17 (std::is_same::value) { + childDtyptp = dtypeForWidth(bp->width() + cp->width()); + } + + Vertex* const childp = make(vtxp->fileline(), childDtyptp, bp, cp); + Vertex* const rootp = make(lhsp->fileline(), rootDtyptp, ap, childp); + replace(vtxp, rootp); + changed = true; + vtxp = rootp; } - break; } - return false; + return changed; } // Transformations that apply to all commutative binary vertices - void commutativeBinary(DfgVertexBinary* vtxp) { - DfgVertex* const lhsp = vtxp->source<0>(); - DfgVertex* const rhsp = vtxp->source<1>(); + template + VL_ATTR_WARN_UNUSED_RESULT bool commutativeBinary(Vertex* vtxp) { + static_assert(std::is_base_of::value, "Must invoke on binary"); + static_assert(std::is_final::value, "Must invoke on final class"); + + DfgVertex* const lhsp = vtxp->lhsp(); + DfgVertex* const rhsp = vtxp->rhsp(); // Ensure Const is on left-hand side to simplify other patterns - if (lhsp->is()) return; + if (lhsp->is()) return false; if (rhsp->is()) { APPLYING(SWAP_CONST_IN_COMMUTATIVE_BINARY) { - vtxp->lhsp(rhsp); - vtxp->rhsp(lhsp); - modified(vtxp); - return; + Vertex* const replacementp = make(vtxp, rhsp, lhsp); + replace(vtxp, replacementp); + return true; } } // Ensure Not is on the left-hand side to simplify other patterns - if (lhsp->is()) return; + if (lhsp->is()) return false; if (rhsp->is()) { APPLYING(SWAP_NOT_IN_COMMUTATIVE_BINARY) { - vtxp->lhsp(rhsp); - vtxp->rhsp(lhsp); - modified(vtxp); - return; + Vertex* const replacementp = make(vtxp, rhsp, lhsp); + replace(vtxp, replacementp); + return true; } } - // If both sides are variable references, order the side in some defined way. This allows - // CSE to later merge 'a op b' with 'b op a'. + // If both sides are variable references, order the side in some defined way. This + // allows CSE to later merge 'a op b' with 'b op a'. if (lhsp->is() && rhsp->is()) { AstVar* const lVarp = lhsp->as()->varp(); AstVar* const rVarp = rhsp->as()->varp(); if (lVarp->name() > rVarp->name()) { APPLYING(SWAP_VAR_IN_COMMUTATIVE_BINARY) { - vtxp->lhsp(rhsp); - vtxp->rhsp(lhsp); - modified(vtxp); - return; + Vertex* const replacementp = make(vtxp, rhsp, lhsp); + replace(vtxp, replacementp); + return true; } } } + + return false; } // Bitwise operation with one side Const, and the other side a Concat template - bool tryPushBitwiseOpThroughConcat(Vertex* vtxp, DfgConst* constp, DfgConcat* concatp) { + VL_ATTR_WARN_UNUSED_RESULT bool tryPushBitwiseOpThroughConcat(Vertex* vtxp, DfgConst* constp, + DfgConcat* concatp) { UASSERT_OBJ(constp->dtypep() == concatp->dtypep(), vtxp, "Mismatched widths"); FileLine* const flp = vtxp->fileline(); @@ -432,24 +438,17 @@ class V3DfgPeephole final : public DfgVisitor { const uint32_t rWidth = rDtypep->width(); // The new Lhs vertex - Vertex* const newLhsp = make(flp, lDtypep); DfgConst* const newLhsConstp = makeZero(constp->fileline(), lWidth); newLhsConstp->num().opSel(constp->num(), width - 1, rWidth); - newLhsp->lhsp(newLhsConstp); - newLhsp->rhsp(concatp->lhsp()); + Vertex* const newLhsp = make(flp, lDtypep, newLhsConstp, concatp->lhsp()); // The new Rhs vertex - Vertex* const newRhsp = make(flp, rDtypep); DfgConst* const newRhsConstp = makeZero(constp->fileline(), rWidth); newRhsConstp->num().opSel(constp->num(), rWidth - 1, 0); - newRhsp->lhsp(newRhsConstp); - newRhsp->rhsp(concatp->rhsp()); + Vertex* const newRhsp = make(flp, rDtypep, newRhsConstp, concatp->rhsp()); // The replacement Concat vertex - DfgConcat* const newConcat - = make(concatp->fileline(), concatp->dtypep()); - newConcat->lhsp(newLhsp); - newConcat->rhsp(newRhsp); + DfgConcat* const newConcat = make(concatp, newLhsp, newRhsp); // Replace this vertex replace(vtxp, newConcat); @@ -460,13 +459,14 @@ class V3DfgPeephole final : public DfgVisitor { } template - bool tryPushCompareOpThroughConcat(Vertex* vtxp, DfgConst* constp, DfgConcat* concatp) { + VL_ATTR_WARN_UNUSED_RESULT bool tryPushCompareOpThroughConcat(Vertex* vtxp, DfgConst* constp, + DfgConcat* concatp) { UASSERT_OBJ(constp->dtypep() == concatp->dtypep(), vtxp, "Mismatched widths"); FileLine* const flp = vtxp->fileline(); - // If at least one of the sides of the Concat is constant, then push the Vertex past the - // Concat + // If at least one of the sides of the Concat is constant, then push the Vertex past + // the Concat if (concatp->lhsp()->is() || concatp->rhsp()->is()) { APPLYING(PUSH_COMPARE_OP_THROUGH_CONCAT) { const uint32_t width = concatp->width(); @@ -474,29 +474,25 @@ class V3DfgPeephole final : public DfgVisitor { const uint32_t rWidth = concatp->rhsp()->width(); // The new Lhs vertex - Vertex* const newLhsp = make(flp, m_bitDType); DfgConst* const newLhsConstp = makeZero(constp->fileline(), lWidth); newLhsConstp->num().opSel(constp->num(), width - 1, rWidth); - newLhsp->lhsp(newLhsConstp); - newLhsp->rhsp(concatp->lhsp()); + Vertex* const newLhsp + = make(flp, m_bitDType, newLhsConstp, concatp->lhsp()); // The new Rhs vertex - Vertex* const newRhsp = make(flp, m_bitDType); DfgConst* const newRhsConstp = makeZero(constp->fileline(), rWidth); newRhsConstp->num().opSel(constp->num(), rWidth - 1, 0); - newRhsp->lhsp(newRhsConstp); - newRhsp->rhsp(concatp->rhsp()); + Vertex* const newRhsp + = make(flp, m_bitDType, newRhsConstp, concatp->rhsp()); // The replacement Vertex DfgVertexBinary* const replacementp = std::is_same::value - ? make(concatp->fileline(), m_bitDType) + ? make(concatp->fileline(), m_bitDType, newLhsp, newRhsp) : nullptr; UASSERT_OBJ(replacementp, vtxp, "Unhandled vertex type in 'tryPushCompareOpThroughConcat': " << vtxp->typeName()); - replacementp->relinkSource<0>(newLhsp); - replacementp->relinkSource<1>(newRhsp); // Replace this vertex replace(vtxp, replacementp); @@ -507,7 +503,7 @@ class V3DfgPeephole final : public DfgVisitor { } template - bool tryPushBitwiseOpThroughReductions(Bitwise* vtxp) { + VL_ATTR_WARN_UNUSED_RESULT bool tryPushBitwiseOpThroughReductions(Bitwise* vtxp) { using Reduction = BitwiseToReduction; if (Reduction* const lRedp = vtxp->lhsp()->template cast()) { @@ -518,11 +514,8 @@ class V3DfgPeephole final : public DfgVisitor { && !lSrcp->hasMultipleSinks() && !rSrcp->hasMultipleSinks()) { APPLYING(PUSH_BITWISE_THROUGH_REDUCTION) { FileLine* const flp = vtxp->fileline(); - Bitwise* const bwp = make(flp, lSrcp->dtypep()); - bwp->lhsp(lSrcp); - bwp->rhsp(rSrcp); - Reduction* const redp = make(flp, m_bitDType); - redp->srcp(bwp); + Bitwise* const bwp = make(flp, lSrcp->dtypep(), lSrcp, rSrcp); + Reduction* const redp = make(flp, m_bitDType, bwp); replace(vtxp, redp); return true; } @@ -534,10 +527,10 @@ class V3DfgPeephole final : public DfgVisitor { } template - void optimizeReduction(Reduction* vtxp) { + VL_ATTR_WARN_UNUSED_RESULT bool optimizeReduction(Reduction* vtxp) { using Bitwise = ReductionToBitwise; - if (foldUnary(vtxp)) return; + if (foldUnary(vtxp)) return true; DfgVertex* const srcp = vtxp->srcp(); FileLine* const flp = vtxp->fileline(); @@ -546,7 +539,7 @@ class V3DfgPeephole final : public DfgVisitor { if (srcp->dtypep() == m_bitDType) { APPLYING(REMOVE_WIDTH_ONE_REDUCTION) { replace(vtxp, srcp); - return; + return true; } } @@ -554,22 +547,18 @@ class V3DfgPeephole final : public DfgVisitor { if (condp->thenp()->is() || condp->elsep()->is()) { APPLYING(PUSH_REDUCTION_THROUGH_COND_WITH_CONST_BRANCH) { // The new 'then' vertex - Reduction* const newThenp = make(flp, m_bitDType); - newThenp->srcp(condp->thenp()); + Reduction* const newThenp = make(flp, m_bitDType, condp->thenp()); // The new 'else' vertex - Reduction* const newElsep = make(flp, m_bitDType); - newElsep->srcp(condp->elsep()); + Reduction* const newElsep = make(flp, m_bitDType, condp->elsep()); // The replacement Cond vertex - DfgCond* const newCondp = make(condp->fileline(), m_bitDType); - newCondp->condp(condp->condp()); - newCondp->thenp(newThenp); - newCondp->elsep(newElsep); + DfgCond* const newCondp = make(condp->fileline(), m_bitDType, + condp->condp(), newThenp, newElsep); // Replace this vertex replace(vtxp, newCondp); - return; + return true; } } } @@ -578,31 +567,36 @@ class V3DfgPeephole final : public DfgVisitor { if (concatp->lhsp()->is() || concatp->rhsp()->is()) { APPLYING(PUSH_REDUCTION_THROUGH_CONCAT) { // Reduce the parts of the concatenation - Reduction* const lRedp = make(concatp->fileline(), m_bitDType); - lRedp->srcp(concatp->lhsp()); - Reduction* const rRedp = make(concatp->fileline(), m_bitDType); - rRedp->srcp(concatp->rhsp()); + Reduction* const lRedp + = make(concatp->fileline(), m_bitDType, concatp->lhsp()); + Reduction* const rRedp + = make(concatp->fileline(), m_bitDType, concatp->rhsp()); // Bitwise reduce the results - Bitwise* const replacementp = make(flp, m_bitDType); - replacementp->lhsp(lRedp); - replacementp->rhsp(rRedp); + Bitwise* const replacementp = make(flp, m_bitDType, lRedp, rRedp); replace(vtxp, replacementp); - return; + return true; } } } + + return false; } - void optimizeShiftRHS(DfgVertexBinary* vtxp) { - if (const DfgConcat* const concatp = vtxp->rhsp()->cast()) { + template + VL_ATTR_WARN_UNUSED_RESULT bool optimizeShiftRHS(Shift* vtxp) { + static_assert(std::is_base_of::value, "Must invoke on binary"); + static_assert(std::is_final::value, "Must invoke on final class"); + if (const DfgConcat* const concatp = vtxp->rhsp()->template cast()) { if (concatp->lhsp()->isZero()) { // Drop redundant zero extension APPLYING(REMOVE_REDUNDANT_ZEXT_ON_RHS_OF_SHIFT) { - vtxp->rhsp(concatp->rhsp()); - modified(vtxp); + Shift* const replacementp = make(vtxp, vtxp->lhsp(), concatp->rhsp()); + replace(vtxp, replacementp); + return true; } } } + return false; } // VISIT methods @@ -626,14 +620,14 @@ class V3DfgPeephole final : public DfgVisitor { if (foldUnary(vtxp)) return; - // Convert all Extend into Concat with zeros. This simplifies other patterns as they only - // need to handle Concat, which is more generic, and don't need special cases for + // Convert all Extend into Concat with zeros. This simplifies other patterns as they + // only need to handle Concat, which is more generic, and don't need special cases for // Extend. APPLYING(REPLACE_EXTEND) { - FileLine* const flp = vtxp->fileline(); - DfgConcat* const replacementp = make(flp, vtxp->dtypep()); - replacementp->lhsp(makeZero(flp, vtxp->width() - vtxp->srcp()->width())); - replacementp->rhsp(vtxp->srcp()); + DfgConcat* const replacementp = make( + vtxp, // + makeZero(vtxp->fileline(), vtxp->width() - vtxp->srcp()->width()), // + vtxp->srcp()); replace(vtxp, replacementp); return; } @@ -668,18 +662,14 @@ class V3DfgPeephole final : public DfgVisitor { if (condp->thenp()->is() || condp->elsep()->is()) { APPLYING(PUSH_NOT_THROUGH_COND) { // The new 'then' vertex - DfgNot* const newThenp = make(vtxp->fileline(), vtxp->dtypep()); - newThenp->srcp(condp->thenp()); + DfgNot* const newThenp = make(vtxp, condp->thenp()); // The new 'else' vertex - DfgNot* const newElsep = make(vtxp->fileline(), vtxp->dtypep()); - newElsep->srcp(condp->elsep()); + DfgNot* const newElsep = make(vtxp, condp->elsep()); // The replacement Cond vertex - DfgCond* const newCondp = make(condp->fileline(), vtxp->dtypep()); - newCondp->condp(condp->condp()); - newCondp->thenp(newThenp); - newCondp->elsep(newElsep); + DfgCond* const newCondp = make(condp->fileline(), vtxp->dtypep(), + condp->condp(), newThenp, newElsep); // Replace this vertex replace(vtxp, newCondp); @@ -701,9 +691,8 @@ class V3DfgPeephole final : public DfgVisitor { // Not of Eq if (DfgEq* const eqp = vtxp->srcp()->cast()) { APPLYING(REPLACE_NOT_EQ) { - DfgNeq* const replacementp = make(eqp->fileline(), vtxp->dtypep()); - replacementp->lhsp(eqp->lhsp()); - replacementp->rhsp(eqp->rhsp()); + DfgNeq* const replacementp + = make(eqp->fileline(), vtxp->dtypep(), eqp->lhsp(), eqp->rhsp()); replace(vtxp, replacementp); return; } @@ -712,9 +701,8 @@ class V3DfgPeephole final : public DfgVisitor { // Not of Neq if (DfgNeq* const neqp = vtxp->srcp()->cast()) { APPLYING(REPLACE_NOT_NEQ) { - DfgEq* const replacementp = make(neqp->fileline(), vtxp->dtypep()); - replacementp->lhsp(neqp->lhsp()); - replacementp->rhsp(neqp->rhsp()); + DfgEq* const replacementp = make(neqp->fileline(), vtxp->dtypep(), + neqp->lhsp(), neqp->rhsp()); replace(vtxp, replacementp); return; } @@ -730,11 +718,17 @@ class V3DfgPeephole final : public DfgVisitor { if (foldUnary(vtxp)) return; } - void visit(DfgRedOr* vtxp) override { optimizeReduction(vtxp); } + void visit(DfgRedOr* vtxp) override { + if (optimizeReduction(vtxp)) return; + } - void visit(DfgRedAnd* vtxp) override { optimizeReduction(vtxp); } + void visit(DfgRedAnd* vtxp) override { + if (optimizeReduction(vtxp)) return; + } - void visit(DfgRedXor* vtxp) override { optimizeReduction(vtxp); } + void visit(DfgRedXor* vtxp) override { + if (optimizeReduction(vtxp)) return; + } void visit(DfgSel* vtxp) override { DfgVertex* const fromp = vtxp->fromp(); @@ -771,15 +765,14 @@ class V3DfgPeephole final : public DfgVisitor { if (msb < rhsp->width()) { // If the select is entirely from rhs, then replace with sel from rhs APPLYING(REMOVE_SEL_FROM_RHS_OF_CONCAT) { // - vtxp->fromp(rhsp); - modified(vtxp); + DfgSel* const replacementp = make(vtxp, rhsp, vtxp->lsb()); + replace(vtxp, replacementp); } } else if (lsb >= rhsp->width()) { // If the select is entirely from the lhs, then replace with sel from lhs APPLYING(REMOVE_SEL_FROM_LHS_OF_CONCAT) { - vtxp->fromp(lhsp); - vtxp->lsb(lsb - rhsp->width()); - modified(vtxp); + DfgSel* const replacementp = make(vtxp, lhsp, lsb - rhsp->width()); + replace(vtxp, replacementp); } } else if (lsb == 0 || msb == concatp->width() - 1 // || lhsp->is() || rhsp->is() // @@ -792,20 +785,14 @@ class V3DfgPeephole final : public DfgVisitor { const uint32_t lSelWidth = width - rSelWidth; // The new Lhs vertex - DfgSel* const newLhsp = make(flp, dtypeForWidth(lSelWidth)); - newLhsp->fromp(lhsp); - newLhsp->lsb(0); + DfgSel* const newLhsp = make(flp, dtypeForWidth(lSelWidth), lhsp, 0U); // The new Rhs vertex - DfgSel* const newRhsp = make(flp, dtypeForWidth(rSelWidth)); - newRhsp->fromp(rhsp); - newRhsp->lsb(lsb); + DfgSel* const newRhsp = make(flp, dtypeForWidth(rSelWidth), rhsp, lsb); // The replacement Concat vertex DfgConcat* const newConcat - = make(concatp->fileline(), vtxp->dtypep()); - newConcat->lhsp(newLhsp); - newConcat->rhsp(newRhsp); + = make(concatp->fileline(), vtxp->dtypep(), newLhsp, newRhsp); // Replace this vertex replace(vtxp, newConcat); @@ -822,9 +809,8 @@ class V3DfgPeephole final : public DfgVisitor { const uint32_t newLsb = lsb % srcWidth; if (newLsb + width <= srcWidth) { APPLYING(PUSH_SEL_THROUGH_REPLICATE) { - vtxp->fromp(repp->srcp()); - vtxp->lsb(newLsb); - modified(vtxp); + DfgSel* const replacementp = make(vtxp, repp->srcp(), newLsb); + replace(vtxp, replacementp); } } } @@ -837,12 +823,11 @@ class V3DfgPeephole final : public DfgVisitor { UASSERT_OBJ(notp->srcp()->dtypep() == notp->dtypep(), notp, "Mismatched widths"); APPLYING(PUSH_SEL_THROUGH_NOT) { // Make Sel select from source of Not - vtxp->fromp(notp->srcp()); + DfgSel* const newSelp = make(vtxp, notp->srcp(), vtxp->lsb()); // Add Not after Sel - DfgNot* const replacementp = make(notp->fileline(), vtxp->dtypep()); - vtxp->replaceWith(replacementp); - replacementp->srcp(vtxp); - modified(vtxp); + DfgNot* const replacementp + = make(notp->fileline(), vtxp->dtypep(), newSelp); + replace(vtxp, replacementp); } } } @@ -850,11 +835,9 @@ class V3DfgPeephole final : public DfgVisitor { // Sel from Sel if (DfgSel* const selp = fromp->cast()) { APPLYING(REPLACE_SEL_FROM_SEL) { - // Make this Sel select from the source of the source Sel - vtxp->fromp(selp->fromp()); - // Adjust LSB - vtxp->lsb(lsb + selp->lsb()); - modified(vtxp); + // Make this Sel select from the source of the source Sel with adjusted LSB + DfgSel* const replacementp = make(vtxp, selp->fromp(), lsb + selp->lsb()); + replace(vtxp, replacementp); } } @@ -864,20 +847,14 @@ class V3DfgPeephole final : public DfgVisitor { if (condp->thenp()->is() || condp->elsep()->is()) { APPLYING(PUSH_SEL_THROUGH_COND) { // The new 'then' vertex - DfgSel* const newThenp = make(flp, vtxp->dtypep()); - newThenp->fromp(condp->thenp()); - newThenp->lsb(lsb); + DfgSel* const newThenp = make(vtxp, condp->thenp(), lsb); // The new 'else' vertex - DfgSel* const newElsep = make(flp, vtxp->dtypep()); - newElsep->fromp(condp->elsep()); - newElsep->lsb(lsb); + DfgSel* const newElsep = make(vtxp, condp->elsep(), lsb); // The replacement Cond vertex - DfgCond* const newCondp = make(condp->fileline(), vtxp->dtypep()); - newCondp->condp(condp->condp()); - newCondp->thenp(newThenp); - newCondp->elsep(newElsep); + DfgCond* const newCondp = make(condp->fileline(), vtxp->dtypep(), + condp->condp(), newThenp, newElsep); // Replace this vertex replace(vtxp, newCondp); @@ -892,13 +869,10 @@ class V3DfgPeephole final : public DfgVisitor { if (lsb == 0) { UASSERT_OBJ(shiftLp->lhsp()->width() >= width, vtxp, "input of shift narrow"); APPLYING(PUSH_SEL_THROUGH_SHIFTL) { - vtxp->fromp(shiftLp->lhsp()); - DfgShiftL* const newShiftLp - = make(shiftLp->fileline(), vtxp->dtypep()); - vtxp->replaceWith(newShiftLp); - newShiftLp->lhsp(vtxp); - newShiftLp->rhsp(shiftLp->rhsp()); - modified(vtxp); + DfgSel* const newSelp = make(vtxp, shiftLp->lhsp(), vtxp->lsb()); + DfgShiftL* const replacementp = make( + shiftLp->fileline(), vtxp->dtypep(), newSelp, shiftLp->rhsp()); + replace(vtxp, replacementp); } } } @@ -914,7 +888,7 @@ class V3DfgPeephole final : public DfgVisitor { if (associativeBinary(vtxp)) return; - commutativeBinary(vtxp); + if (commutativeBinary(vtxp)) return; DfgVertex* const lhsp = vtxp->lhsp(); DfgVertex* const rhsp = vtxp->rhsp(); @@ -925,25 +899,17 @@ class V3DfgPeephole final : public DfgVisitor { if (DfgNot* const lhsNotp = lhsp->cast()) { if (DfgNot* const rhsNotp = rhsp->cast()) { APPLYING(REPLACE_AND_OF_NOT_AND_NOT) { - DfgOr* const orp = make(flp, vtxp->dtypep()); - orp->lhsp(lhsNotp->srcp()); - orp->rhsp(rhsNotp->srcp()); - DfgNot* const notp = make(flp, vtxp->dtypep()); - notp->srcp(orp); + DfgOr* const orp = make(vtxp, lhsNotp->srcp(), rhsNotp->srcp()); + DfgNot* const notp = make(vtxp, orp); replace(vtxp, notp); return; } } if (DfgNeq* const rhsNeqp = rhsp->cast()) { APPLYING(REPLACE_AND_OF_NOT_AND_NEQ) { - DfgOr* const orp = make(flp, vtxp->dtypep()); - orp->lhsp(lhsNotp->srcp()); - DfgEq* const newRhsp = make(rhsp->fileline(), rhsp->dtypep()); - newRhsp->lhsp(rhsNeqp->lhsp()); - newRhsp->rhsp(rhsNeqp->rhsp()); - orp->rhsp(newRhsp); - DfgNot* const notp = make(flp, vtxp->dtypep()); - notp->srcp(orp); + DfgEq* const newRhsp = make(rhsp, rhsNeqp->lhsp(), rhsNeqp->rhsp()); + DfgOr* const orp = make(vtxp, lhsNotp->srcp(), newRhsp); + DfgNot* const notp = make(vtxp, orp); replace(vtxp, notp); return; } @@ -991,7 +957,7 @@ class V3DfgPeephole final : public DfgVisitor { if (associativeBinary(vtxp)) return; - commutativeBinary(vtxp); + if (commutativeBinary(vtxp)) return; DfgVertex* const lhsp = vtxp->lhsp(); DfgVertex* const rhsp = vtxp->rhsp(); @@ -1002,25 +968,17 @@ class V3DfgPeephole final : public DfgVisitor { if (DfgNot* const lhsNotp = lhsp->cast()) { if (DfgNot* const rhsNotp = rhsp->cast()) { APPLYING(REPLACE_OR_OF_NOT_AND_NOT) { - DfgAnd* const andp = make(flp, vtxp->dtypep()); - andp->lhsp(lhsNotp->srcp()); - andp->rhsp(rhsNotp->srcp()); - DfgNot* const notp = make(flp, vtxp->dtypep()); - notp->srcp(andp); + DfgAnd* const andp = make(vtxp, lhsNotp->srcp(), rhsNotp->srcp()); + DfgNot* const notp = make(vtxp, andp); replace(vtxp, notp); return; } } if (DfgNeq* const rhsNeqp = rhsp->cast()) { APPLYING(REPLACE_OR_OF_NOT_AND_NEQ) { - DfgAnd* const andp = make(flp, vtxp->dtypep()); - andp->lhsp(lhsNotp->srcp()); - DfgEq* const newRhsp = make(rhsp->fileline(), rhsp->dtypep()); - newRhsp->lhsp(rhsNeqp->lhsp()); - newRhsp->rhsp(rhsNeqp->rhsp()); - andp->rhsp(newRhsp); - DfgNot* const notp = make(flp, vtxp->dtypep()); - notp->srcp(andp); + DfgEq* const newRhsp = make(rhsp, rhsNeqp->lhsp(), rhsNeqp->rhsp()); + DfgAnd* const andp = make(vtxp, lhsNotp->srcp(), newRhsp); + DfgNot* const notp = make(vtxp, andp); replace(vtxp, notp); return; } @@ -1033,18 +991,16 @@ class V3DfgPeephole final : public DfgVisitor { if (lhsConcatp->lhsp()->dtypep() == rhsConcatp->lhsp()->dtypep()) { if (lhsConcatp->lhsp()->isZero() && rhsConcatp->rhsp()->isZero()) { APPLYING(REPLACE_OR_OF_CONCAT_ZERO_LHS_AND_CONCAT_RHS_ZERO) { - DfgConcat* const replacementp = make(flp, vtxp->dtypep()); - replacementp->lhsp(rhsConcatp->lhsp()); - replacementp->rhsp(lhsConcatp->rhsp()); + DfgConcat* const replacementp + = make(vtxp, rhsConcatp->lhsp(), lhsConcatp->rhsp()); replace(vtxp, replacementp); return; } } if (lhsConcatp->rhsp()->isZero() && rhsConcatp->lhsp()->isZero()) { APPLYING(REPLACE_OR_OF_CONCAT_LHS_ZERO_AND_CONCAT_ZERO_RHS) { - DfgConcat* const replacementp = make(flp, vtxp->dtypep()); - replacementp->lhsp(lhsConcatp->lhsp()); - replacementp->rhsp(rhsConcatp->rhsp()); + DfgConcat* const replacementp + = make(vtxp, lhsConcatp->lhsp(), rhsConcatp->rhsp()); replace(vtxp, replacementp); return; } @@ -1094,11 +1050,10 @@ class V3DfgPeephole final : public DfgVisitor { if (associativeBinary(vtxp)) return; - commutativeBinary(vtxp); + if (commutativeBinary(vtxp)) return; DfgVertex* const lhsp = vtxp->lhsp(); DfgVertex* const rhsp = vtxp->rhsp(); - FileLine* const flp = vtxp->fileline(); if (DfgConst* const lConstp = lhsp->cast()) { if (lConstp->isZero()) { @@ -1109,14 +1064,13 @@ class V3DfgPeephole final : public DfgVisitor { } if (lConstp->isOnes()) { APPLYING(REPLACE_XOR_WITH_ONES) { - DfgNot* const replacementp = make(flp, vtxp->dtypep()); - replacementp->srcp(rhsp); + DfgNot* const replacementp = make(vtxp, rhsp); replace(vtxp, replacementp); return; } } if (DfgConcat* const rConcatp = rhsp->cast()) { - tryPushBitwiseOpThroughConcat(vtxp, lConstp, rConcatp); + if (tryPushBitwiseOpThroughConcat(vtxp, lConstp, rConcatp)) return; return; } } @@ -1134,7 +1088,7 @@ class V3DfgPeephole final : public DfgVisitor { if (associativeBinary(vtxp)) return; - commutativeBinary(vtxp); + if (commutativeBinary(vtxp)) return; } void visit(DfgArraySel* vtxp) override { @@ -1168,9 +1122,8 @@ class V3DfgPeephole final : public DfgVisitor { if (vtxp->dtypep() == rSelp->fromp()->dtypep() && rSelp->lsb() == lConstp->width()) { APPLYING(REPLACE_CONCAT_ZERO_AND_SEL_TOP_WITH_SHIFTR) { - DfgShiftR* const replacementp = make(flp, vtxp->dtypep()); - replacementp->lhsp(rSelp->fromp()); - replacementp->rhsp(makeI32(flp, lConstp->width())); + DfgShiftR* const replacementp = make( + vtxp, rSelp->fromp(), makeI32(flp, lConstp->width())); replace(vtxp, replacementp); return; } @@ -1183,9 +1136,8 @@ class V3DfgPeephole final : public DfgVisitor { if (DfgSel* const lSelp = lhsp->cast()) { if (vtxp->dtypep() == lSelp->fromp()->dtypep() && lSelp->lsb() == 0) { APPLYING(REPLACE_CONCAT_SEL_BOTTOM_AND_ZERO_WITH_SHIFTL) { - DfgShiftL* const replacementp = make(flp, vtxp->dtypep()); - replacementp->lhsp(lSelp->fromp()); - replacementp->rhsp(makeI32(flp, rConstp->width())); + DfgShiftL* const replacementp = make( + vtxp, lSelp->fromp(), makeI32(flp, rConstp->width())); replace(vtxp, replacementp); return; } @@ -1197,12 +1149,10 @@ class V3DfgPeephole final : public DfgVisitor { if (DfgNot* const rNot = rhsp->cast()) { if (!lNot->hasMultipleSinks() && !rNot->hasMultipleSinks()) { APPLYING(PUSH_CONCAT_THROUGH_NOTS) { - vtxp->lhsp(lNot->srcp()); - vtxp->rhsp(rNot->srcp()); - DfgNot* const replacementp = make(flp, vtxp->dtypep()); - vtxp->replaceWith(replacementp); - replacementp->srcp(vtxp); - modified(vtxp); + DfgConcat* const newCatp + = make(vtxp, lNot->srcp(), rNot->srcp()); + DfgNot* const replacementp = make(vtxp, newCatp); + replace(vtxp, replacementp); return; } } @@ -1215,10 +1165,8 @@ class V3DfgPeephole final : public DfgVisitor { if (lSelp->lsb() == rSelp->lsb() + rSelp->width()) { // Two consecutive Sels, make a single Sel. const uint32_t width = lSelp->width() + rSelp->width(); - DfgSel* const joinedSelp = make(flp, dtypeForWidth(width)); - joinedSelp->fromp(rSelp->fromp()); - joinedSelp->lsb(rSelp->lsb()); - return joinedSelp; + return make(flp, dtypeForWidth(width), rSelp->fromp(), + rSelp->lsb()); } } return nullptr; @@ -1240,9 +1188,7 @@ class V3DfgPeephole final : public DfgVisitor { if (DfgSel* const jointSelp = joinSels(lSelp, rlSelp, flp)) { APPLYING(REPLACE_NESTED_CONCAT_OF_ADJOINING_SELS_ON_LHS) { DfgConcat* const replacementp - = make(flp, vtxp->dtypep()); - replacementp->lhsp(jointSelp); - replacementp->rhsp(rConcatp->rhsp()); + = make(vtxp, jointSelp, rConcatp->rhsp()); replace(vtxp, replacementp); return; } @@ -1256,9 +1202,7 @@ class V3DfgPeephole final : public DfgVisitor { if (DfgSel* const jointSelp = joinSels(lrlSelp, rSelp, flp)) { APPLYING(REPLACE_NESTED_CONCAT_OF_ADJOINING_SELS_ON_RHS) { DfgConcat* const replacementp - = make(flp, vtxp->dtypep()); - replacementp->lhsp(lConcatp->lhsp()); - replacementp->rhsp(jointSelp); + = make(vtxp, lConcatp->lhsp(), jointSelp); replace(vtxp, replacementp); return; } @@ -1280,7 +1224,7 @@ class V3DfgPeephole final : public DfgVisitor { void visit(DfgEq* vtxp) override { if (foldBinary(vtxp)) return; - commutativeBinary(vtxp); + if (commutativeBinary(vtxp)) return; DfgVertex* const lhsp = vtxp->lhsp(); DfgVertex* const rhsp = vtxp->rhsp(); @@ -1354,7 +1298,7 @@ class V3DfgPeephole final : public DfgVisitor { if (associativeBinary(vtxp)) return; - commutativeBinary(vtxp); + if (commutativeBinary(vtxp)) return; } void visit(DfgMulS* vtxp) override { @@ -1363,7 +1307,7 @@ class V3DfgPeephole final : public DfgVisitor { if (associativeBinary(vtxp)) return; - commutativeBinary(vtxp); + if (commutativeBinary(vtxp)) return; } void visit(DfgNeq* vtxp) override { @@ -1399,17 +1343,17 @@ class V3DfgPeephole final : public DfgVisitor { void visit(DfgShiftL* vtxp) override { if (foldBinary(vtxp)) return; - optimizeShiftRHS(vtxp); + if (optimizeShiftRHS(vtxp)) return; } void visit(DfgShiftR* vtxp) override { if (foldBinary(vtxp)) return; - optimizeShiftRHS(vtxp); + if (optimizeShiftRHS(vtxp)) return; } void visit(DfgShiftRS* vtxp) override { if (foldBinary(vtxp)) return; - optimizeShiftRHS(vtxp); + if (optimizeShiftRHS(vtxp)) return; } void visit(DfgSub* vtxp) override { @@ -1430,8 +1374,7 @@ class V3DfgPeephole final : public DfgVisitor { } if (vtxp->dtypep() == m_bitDType && rConstp->hasValue(1)) { APPLYING(REPLACE_SUB_WITH_NOT) { - DfgNot* const replacementp = make(vtxp->fileline(), m_bitDType); - replacementp->srcp(lhsp); + DfgNot* const replacementp = make(vtxp->fileline(), m_bitDType, lhsp); replace(vtxp, replacementp); return; } @@ -1471,10 +1414,9 @@ class V3DfgPeephole final : public DfgVisitor { if (DfgNot* const condNotp = condp->cast()) { if (!condp->hasMultipleSinks() || condNotp->hasMultipleSinks()) { APPLYING(SWAP_COND_WITH_NOT_CONDITION) { - vtxp->condp(condNotp->srcp()); - vtxp->thenp(elsep); - vtxp->elsep(thenp); - modified(vtxp); + DfgCond* const replacementp + = make(vtxp, condNotp->srcp(), elsep, thenp); + replace(vtxp, replacementp); return; } } @@ -1483,13 +1425,9 @@ class V3DfgPeephole final : public DfgVisitor { if (DfgNeq* const condNeqp = condp->cast()) { if (!condp->hasMultipleSinks()) { APPLYING(SWAP_COND_WITH_NEQ_CONDITION) { - DfgEq* const newCondp = make(condp->fileline(), condp->dtypep()); - newCondp->lhsp(condNeqp->lhsp()); - newCondp->rhsp(condNeqp->rhsp()); - vtxp->condp(newCondp); - vtxp->thenp(elsep); - vtxp->elsep(thenp); - modified(vtxp); + DfgEq* const newCondp = make(condp, condNeqp->lhsp(), condNeqp->rhsp()); + DfgCond* const replacementp = make(vtxp, newCondp, elsep, thenp); + replace(vtxp, replacementp); return; } } @@ -1500,13 +1438,11 @@ class V3DfgPeephole final : public DfgVisitor { if (!thenNotp->srcp()->is() && !elseNotp->srcp()->is() && !thenNotp->hasMultipleSinks() && !elseNotp->hasMultipleSinks()) { APPLYING(PULL_NOTS_THROUGH_COND) { + DfgCond* const newCondp = make( + vtxp, vtxp->condp(), thenNotp->srcp(), elseNotp->srcp()); DfgNot* const replacementp - = make(thenp->fileline(), vtxp->dtypep()); - vtxp->thenp(thenNotp->srcp()); - vtxp->elsep(elseNotp->srcp()); - vtxp->replaceWith(replacementp); - replacementp->srcp(vtxp); - modified(vtxp); + = make(thenp->fileline(), vtxp->dtypep(), newCondp); + replace(vtxp, replacementp); return; } } @@ -1520,13 +1456,11 @@ class V3DfgPeephole final : public DfgVisitor { if (constp->hasValue(1)) { if (thenAddp->rhsp() == elsep) { APPLYING(REPLACE_COND_INC) { - DfgConcat* const extp = make(flp, vtxp->dtypep()); - extp->rhsp(condp); - extp->lhsp(makeZero(flp, vtxp->width() - 1)); + DfgConcat* const extp = make( + vtxp, makeZero(flp, vtxp->width() - 1), condp); FileLine* const thenFlp = thenAddp->fileline(); - DfgAdd* const addp = make(thenFlp, vtxp->dtypep()); - addp->lhsp(thenAddp->rhsp()); - addp->rhsp(extp); + DfgAdd* const addp = make(thenFlp, vtxp->dtypep(), + thenAddp->rhsp(), extp); replace(vtxp, addp); return; } @@ -1540,13 +1474,11 @@ class V3DfgPeephole final : public DfgVisitor { if (constp->hasValue(1)) { if (thenSubp->lhsp() == elsep) { APPLYING(REPLACE_COND_DEC) { - DfgConcat* const extp = make(flp, vtxp->dtypep()); - extp->rhsp(condp); - extp->lhsp(makeZero(flp, vtxp->width() - 1)); + DfgConcat* const extp = make( + vtxp, makeZero(flp, vtxp->width() - 1), condp); FileLine* const thenFlp = thenSubp->fileline(); - DfgSub* const subp = make(thenFlp, vtxp->dtypep()); - subp->lhsp(thenSubp->lhsp()); - subp->rhsp(extp); + DfgSub* const subp = make(thenFlp, vtxp->dtypep(), + thenSubp->lhsp(), extp); replace(vtxp, subp); return; } @@ -1557,43 +1489,32 @@ class V3DfgPeephole final : public DfgVisitor { } if (vtxp->dtypep() == m_bitDType) { - AstNodeDType* const dtypep = vtxp->dtypep(); if (thenp->isZero()) { // a ? 0 : b becomes ~a & b APPLYING(REPLACE_COND_WITH_THEN_BRANCH_ZERO) { - DfgAnd* const repalcementp = make(flp, dtypep); - DfgNot* const notp = make(flp, dtypep); - notp->srcp(condp); - repalcementp->lhsp(notp); - repalcementp->rhsp(elsep); + DfgNot* const notp = make(vtxp, condp); + DfgAnd* const repalcementp = make(vtxp, notp, elsep); replace(vtxp, repalcementp); return; } } if (thenp->isOnes()) { // a ? 1 : b becomes a | b APPLYING(REPLACE_COND_WITH_THEN_BRANCH_ONES) { - DfgOr* const repalcementp = make(flp, dtypep); - repalcementp->lhsp(condp); - repalcementp->rhsp(elsep); + DfgOr* const repalcementp = make(vtxp, condp, elsep); replace(vtxp, repalcementp); return; } } if (elsep->isZero()) { // a ? b : 0 becomes a & b APPLYING(REPLACE_COND_WITH_ELSE_BRANCH_ZERO) { - DfgAnd* const repalcementp = make(flp, dtypep); - repalcementp->lhsp(condp); - repalcementp->rhsp(thenp); + DfgAnd* const repalcementp = make(vtxp, condp, thenp); replace(vtxp, repalcementp); return; } } if (elsep->isOnes()) { // a ? b : 1 becomes ~a | b APPLYING(REPLACE_COND_WITH_ELSE_BRANCH_ONES) { - DfgOr* const repalcementp = make(flp, dtypep); - DfgNot* const notp = make(flp, dtypep); - notp->srcp(condp); - repalcementp->lhsp(notp); - repalcementp->rhsp(thenp); + DfgNot* const notp = make(vtxp, condp); + DfgOr* const repalcementp = make(vtxp, notp, thenp); replace(vtxp, repalcementp); return; } @@ -1610,12 +1531,14 @@ class V3DfgPeephole final : public DfgVisitor { // DfgVertex::user is the next pointer of the work list elements const auto userDataInUse = m_dfg.userDataInUse(); - // Add all vertices to the work list. This also allocates all DfgVertex::user. + // Add all vertices to the work list, and to the vertex cache. + // This also allocates all DfgVertex::user. for (DfgVertex *vtxp = m_dfg.opVerticesBeginp(), *nextp; vtxp; vtxp = nextp) { nextp = vtxp->verticesNext(); if (VL_LIKELY(nextp)) VL_PREFETCH_RW(nextp); vtxp->setUser(m_workListp); m_workListp = vtxp; + m_cache.cache(vtxp); } // Process the work list diff --git a/src/astgen b/src/astgen index 1fa0ca3ca..942dc6747 100755 --- a/src/astgen +++ b/src/astgen @@ -1192,9 +1192,11 @@ def write_dfg_macros(filename): if node.isLeaf: emitBlock('''\ static constexpr VDfgType dfgType() {{ return VDfgType::at{t}; }}; + using Super = Dfg{s}; void accept(DfgVisitor& v) override {{ v.visit(this); }} ''', - t=node.name) + t=node.name, + s=node.superClass.name) for n in range(1, node.arity + 1): name, _, _ = node.getOp(n) diff --git a/test_regress/t/t_dfg_peephole.v b/test_regress/t/t_dfg_peephole.v index 7f0ea086e..b6745ef93 100644 --- a/test_regress/t/t_dfg_peephole.v +++ b/test_regress/t/t_dfg_peephole.v @@ -118,7 +118,7 @@ module t ( `signal(PUSH_NOT_THROUGH_COND, ~(rand_a[0] ? rand_a[4:0] : 5'hb)); `signal(REMOVE_NOT_NOT, ~~rand_a); `signal(REPLACE_NOT_NEQ, ~(rand_a != rand_b)); - `signal(REPLACE_NOT_EQ, ~(rand_a == rand_b)); + `signal(REPLACE_NOT_EQ, ~(srand_a == srand_b)); `signal(REPLACE_NOT_OF_CONST, ~4'd0); `signal(REPLACE_AND_OF_NOT_AND_NOT, ~rand_a[1] & ~rand_b[1]); `signal(REPLACE_AND_OF_NOT_AND_NEQ, ~rand_a[2] & (rand_b != 64'd2));