// -*- mode: C++; c-file-style: "cc-mode" -*- //************************************************************************* // DESCRIPTION: Verilator: Peephole optimizations over DfgGraph // // Code available from: https://verilator.org // //************************************************************************* // // This program is free software; you can redistribute it and/or modify it // under the terms of either the GNU Lesser General Public License Version 3 // or the Perl Artistic License Version 2.0. // SPDX-FileCopyrightText: 2003-2026 Wilson Snyder // SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0 // //************************************************************************* // // A pattern-matching based optimizer for DfgGraph. This is in some aspects // similar to V3Const, but more powerful in that it does not care about // ordering combinational statement. This is also less broadly applicable // than V3Const, as it does not apply to procedural statements with // sequential execution semantics. // // Each pattern can look at a certain number of source vertices to see // if a simplified form can be introduced. Some patterns also look at the // immediate sinks of some vertices. Ideally the algorithm should run to // fixed point (until nothing else changes). To do this efficiently, two // lists of vertices are maintained: // - the 'work' list contains vertices to be considered on the current // iteration // - the 'iter' list contains vertices whose whole neighborhood could be // considered on the next iteration // The 'work' list ensures simple cascading pattern applications are // handled in a single pass. The 'iter' list ensures the algorithm runs // to fixed point if no pattern looks deeper across the graph than the // neighborhood considered on the next iteration. // //************************************************************************* #include "V3PchAstNoMT.h" // VL_MT_DISABLED_CODE_UNIT #include "V3Dfg.h" #include "V3DfgCache.h" #include "V3DfgPasses.h" #include "V3DfgPeepholePatterns.h" #include "V3Stats.h" #include #include #include VL_DEFINE_DEBUG_FUNCTIONS; V3DfgPeepholeContext::V3DfgPeepholeContext(V3DfgContext& ctx, const std::string& label) : V3DfgSubContext{ctx, label, "Peephole"} { const auto checkEnabled = [this](VDfgPeepholePattern id) { std::string str{id.ascii()}; std::transform(str.begin(), str.end(), str.begin(), [](unsigned char c) { // return c == '_' ? '-' : std::tolower(c); }); m_enabled[id] = v3Global.opt.fDfgPeepholeEnabled(str); }; #define OPTIMIZATION_CHECK_ENABLED(id, name) checkEnabled(VDfgPeepholePattern::id); FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION(OPTIMIZATION_CHECK_ENABLED) #undef OPTIMIZATION_CHECK_ENABLED } V3DfgPeepholeContext::~V3DfgPeepholeContext() { const auto emitStat = [this](VDfgPeepholePattern id) { std::string str{id.ascii()}; std::transform(str.begin(), str.end(), str.begin(), [](unsigned char c) { // return c == '_' ? ' ' : std::tolower(c); }); addStat(str, m_count[id]); }; #define OPTIMIZATION_EMIT_STATS(id, name) emitStat(VDfgPeepholePattern::id); FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION(OPTIMIZATION_EMIT_STATS) #undef OPTIMIZATION_EMIT_STATS } // clang-format off template struct ReductionToBitwiseImpl {}; template <> struct ReductionToBitwiseImpl { using type = DfgAnd; }; template <> struct ReductionToBitwiseImpl { using type = DfgOr; }; template <> struct ReductionToBitwiseImpl { using type = DfgXor; }; template using ReductionToBitwise = typename ReductionToBitwiseImpl::type; template struct BitwiseToReductionImpl {}; template <> struct BitwiseToReductionImpl { using type = DfgRedAnd; }; template <> struct BitwiseToReductionImpl { using type = DfgRedOr; }; template <> struct BitwiseToReductionImpl { using type = DfgRedXor; }; template using BitwiseToReduction = typename BitwiseToReductionImpl::type; namespace { template void foldOp(V3Number& out, const V3Number& src); template <> void foldOp (V3Number& out, const V3Number& src) { out.opAssign(src); } template <> void foldOp (V3Number& out, const V3Number& src) { out.opExtendS(src, src.width()); } template <> void foldOp (V3Number& out, const V3Number& src) { out.opLogNot(src); } template <> void foldOp (V3Number& out, const V3Number& src) { out.opNegate(src); } template <> void foldOp (V3Number& out, const V3Number& src) { out.opNot(src); } template <> void foldOp (V3Number& out, const V3Number& src) { out.opRedAnd(src); } template <> void foldOp (V3Number& out, const V3Number& src) { out.opRedOr(src); } template <> void foldOp (V3Number& out, const V3Number& src) { out.opRedXor(src); } template void foldOp(V3Number& out, const V3Number& lhs, const V3Number& rhs); template <> void foldOp (V3Number& out, const V3Number& lhs, const V3Number& rhs) { out.opAdd(lhs, rhs); } template <> void foldOp (V3Number& out, const V3Number& lhs, const V3Number& rhs) { out.opAnd(lhs, rhs); } template <> void foldOp (V3Number& out, const V3Number& lhs, const V3Number& rhs) { out.opConcat(lhs, rhs); } template <> void foldOp (V3Number& out, const V3Number& lhs, const V3Number& rhs) { out.opDiv(lhs, rhs); } template <> void foldOp (V3Number& out, const V3Number& lhs, const V3Number& rhs) { out.opDivS(lhs, rhs); } template <> void foldOp (V3Number& out, const V3Number& lhs, const V3Number& rhs) { out.opEq(lhs, rhs); } template <> void foldOp (V3Number& out, const V3Number& lhs, const V3Number& rhs) { out.opGt(lhs, rhs); } template <> void foldOp (V3Number& out, const V3Number& lhs, const V3Number& rhs) { out.opGtS(lhs, rhs); } template <> void foldOp (V3Number& out, const V3Number& lhs, const V3Number& rhs) { out.opGte(lhs, rhs); } template <> void foldOp (V3Number& out, const V3Number& lhs, const V3Number& rhs) { out.opGteS(lhs, rhs); } template <> void foldOp (V3Number& out, const V3Number& lhs, const V3Number& rhs) { out.opLogAnd(lhs, rhs); } template <> void foldOp (V3Number& out, const V3Number& lhs, const V3Number& rhs) { out.opLogEq(lhs, rhs); } template <> void foldOp (V3Number& out, const V3Number& lhs, const V3Number& rhs) { out.opLogIf(lhs, rhs); } template <> void foldOp (V3Number& out, const V3Number& lhs, const V3Number& rhs) { out.opLogOr(lhs, rhs); } template <> void foldOp (V3Number& out, const V3Number& lhs, const V3Number& rhs) { out.opLt(lhs, rhs); } template <> void foldOp (V3Number& out, const V3Number& lhs, const V3Number& rhs) { out.opLtS(lhs, rhs); } template <> void foldOp (V3Number& out, const V3Number& lhs, const V3Number& rhs) { out.opLte(lhs, rhs); } template <> void foldOp (V3Number& out, const V3Number& lhs, const V3Number& rhs) { out.opLteS(lhs, rhs); } template <> void foldOp (V3Number& out, const V3Number& lhs, const V3Number& rhs) { out.opModDiv(lhs, rhs); } template <> void foldOp (V3Number& out, const V3Number& lhs, const V3Number& rhs) { out.opModDivS(lhs, rhs); } template <> void foldOp (V3Number& out, const V3Number& lhs, const V3Number& rhs) { out.opMul(lhs, rhs); } template <> void foldOp (V3Number& out, const V3Number& lhs, const V3Number& rhs) { out.opMulS(lhs, rhs); } template <> void foldOp (V3Number& out, const V3Number& lhs, const V3Number& rhs) { out.opNeq(lhs, rhs); } template <> void foldOp (V3Number& out, const V3Number& lhs, const V3Number& rhs) { out.opOr(lhs, rhs); } template <> void foldOp (V3Number& out, const V3Number& lhs, const V3Number& rhs) { out.opPow(lhs, rhs); } template <> void foldOp (V3Number& out, const V3Number& lhs, const V3Number& rhs) { out.opPowSS(lhs, rhs); } template <> void foldOp (V3Number& out, const V3Number& lhs, const V3Number& rhs) { out.opPowSU(lhs, rhs); } template <> void foldOp (V3Number& out, const V3Number& lhs, const V3Number& rhs) { out.opPowUS(lhs, rhs); } template <> void foldOp (V3Number& out, const V3Number& lhs, const V3Number& rhs) { out.opRepl(lhs, rhs); } template <> void foldOp (V3Number& out, const V3Number& lhs, const V3Number& rhs) { out.opShiftL(lhs, rhs); } template <> void foldOp (V3Number& out, const V3Number& lhs, const V3Number& rhs) { out.opShiftR(lhs, rhs); } template <> void foldOp (V3Number& out, const V3Number& lhs, const V3Number& rhs) { out.opShiftRS(lhs, rhs, lhs.width()); } template <> void foldOp (V3Number& out, const V3Number& lhs, const V3Number& rhs) { out.opSub(lhs, rhs); } template <> void foldOp (V3Number& out, const V3Number& lhs, const V3Number& rhs) { out.opXor(lhs, rhs); } } // clang-format on class V3DfgPeephole final : public DfgVisitor { // TYPES struct VertexInfo final { size_t m_workListIndex = 0; // Position of this vertx m_workList (0 means not in list) size_t m_iterListIndex = 0; // Position of this vertx m_iterList (0 means not in list) size_t m_generation = 0; // Generation number of this vertex - for uniqueness check size_t m_id = 0; // Unique vertex ID (0 means unassigned) - for sorting }; // STATE DfgGraph& m_dfg; // The DfgGraph being visited V3DfgPeepholeContext& m_ctx; // The config structure const DfgDataType& m_bitDType = DfgDataType::packed(1); // Common, so grab it up front std::vector m_workList; // List of vertices processed in current interation std::vector m_iterList; // Vertices to start from on next iteration DfgUserMap m_vInfo = m_dfg.makeUserMap(); // Map to VertexInfo V3DfgCache m_cache{m_dfg}; // Vertex cache to avoid creating redundant vertices DfgVertex* m_vtxp = nullptr; // Currently considered vertex size_t m_currentGeneration = 0; // Current generation number size_t m_lastId = 0; // Last unique vertex ID assigned // STATIC STATE static V3DebugBisect s_debugBisect; // Debug aid #define APPLYING(id) if (checkApplying(VDfgPeepholePattern::id)) // METHODS bool checkApplying(VDfgPeepholePattern id) { if (VL_UNLIKELY(!m_ctx.m_enabled[id] || s_debugBisect.isStopped())) return false; UINFO(9, "Applying DFG pattern " << id.ascii()); ++m_ctx.m_count[id]; return true; } void incrementGeneration() { ++m_currentGeneration; // TODO: could sweep on overflow } void addToWorkList(DfgVertex* vtxp) { VertexInfo& vInfo = m_vInfo[*vtxp]; // If already on work list, ignore if (vInfo.m_workListIndex) return; // Add to work list vInfo.m_workListIndex = m_workList.size(); m_workList.push_back(vtxp); } void removeFromWorkList(DfgVertex* vtxp) { VertexInfo& vInfo = m_vInfo[*vtxp]; // m_workList[0] is always nullptr, fine to assign same here m_workList[vInfo.m_workListIndex] = nullptr; vInfo.m_workListIndex = 0; } void addToIterList(DfgVertex* vtxp) { VertexInfo& vInfo = m_vInfo[*vtxp]; // If already on iter list, ignore if (vInfo.m_iterListIndex) return; // Add to iter list vInfo.m_iterListIndex = m_iterList.size(); m_iterList.push_back(vtxp); } void removeFromIterList(DfgVertex* vtxp) { VertexInfo& vInfo = m_vInfo[*vtxp]; // m_iterList[0] is always nullptr, fine to assign same here m_iterList[vInfo.m_iterListIndex] = nullptr; vInfo.m_iterListIndex = 0; } void deleteVertex(DfgVertex* vtxp) { UASSERT_OBJ(!m_vInfo[vtxp].m_workListIndex, vtxp, "Deleted Vertex is in work list"); UASSERT_OBJ(!vtxp->hasSinks(), vtxp, "Should not delete used vertex"); // Invalidate cache entry m_cache.invalidate(vtxp); // It might be in the iter list, remove it removeFromIterList(vtxp); // Gather source vertices - they might be duplicates, make unique using generation number incrementGeneration(); std::vector srcps; srcps.reserve(vtxp->nInputs()); vtxp->foreachSource([&](DfgVertex& src) { // If it's a variable, add to work list to see if it became redundant if (src.is()) { addToWorkList(&src); return false; } // Gather unique sources VertexInfo& vInfo = m_vInfo[src]; if (vInfo.m_generation != m_currentGeneration) srcps.push_back(&src); vInfo.m_generation = m_currentGeneration; return false; }); // This pass only removes variables that are either not driven in this graph, // or are not observable outside the graph. If there is also no external write // to the variable and no references in other graph then delete the Ast var too. const DfgVertexVar* const varp = vtxp->cast(); if (varp && !varp->isVolatile() && !varp->hasDfgRefs()) { AstNode* const nodep = varp->nodep(); VL_DO_DANGLING(vtxp->unlinkDelete(m_dfg), vtxp); VL_DO_DANGLING(nodep->unlinkFrBack()->deleteTree(), nodep); } else { VL_DO_DANGLING(vtxp->unlinkDelete(m_dfg), vtxp); } // Partition sources into used/unused after removing the vertex const auto mid = std::stable_partition(srcps.begin(), srcps.end(), [](DfgVertex* srcp) { // return srcp->hasSinks(); }); // Add used sources to the iter list - their sinks have changed for (auto it = srcps.begin(); it != mid; ++it) addToIterList(*it); // Recursively delete unused sources for (auto it = mid; it != srcps.end(); ++it) { // Remove from work list removeFromWorkList(*it); // Delete vertex deleteVertex(*it); } } // Replace 'm_vtxp' with the given vertex void replace(DfgVertex* resp) { // Should not be in the work list UASSERT_OBJ(!m_vInfo[m_vtxp].m_workListIndex, m_vtxp, "Replaced Vertex is in work list"); // Debug bisect check const auto debugCallback = [&]() -> void { UINFO(0, "Problematic DfgPeephole replacement: " << m_vtxp << " -> " << resp); m_dfg.sourceCone({m_vtxp, resp}); const auto cone = m_dfg.sourceCone({m_vtxp, resp}); m_dfg.dumpDotFilePrefixed("peephole-broken", [&](const DfgVertex& v) { // return cone->count(&v); }); }; if (VL_UNLIKELY(s_debugBisect.stop(debugCallback))) return; // Add sources of the original vertex to the iter list - their sinks are changing m_vtxp->foreachSource([&](DfgVertex& src) { addToIterList(&src); return false; }); // Remove sinks of the original vertex from the cache - their inputs are changing m_vtxp->foreachSink([&](DfgVertex& dst) { m_cache.invalidate(&dst); return false; }); // Replace vertex with the replacement m_vtxp->replaceWith(resp); // Re-cache all sinks of the replacement resp->foreachSink([&](DfgVertex& dst) { m_cache.cache(&dst); return false; }); // Original vertex is now unused, so delete it deleteVertex(m_vtxp); // Add new vertex to iter list to consider neighborhood addToIterList(resp); // Add new vertex and sinks to work list as likely to match addToWorkList(resp); resp->foreachSink([&](DfgVertex& dst) { addToWorkList(&dst); return false; }); } // Create a 32-bit DfgConst vertex DfgConst* makeI32(FileLine* flp, uint32_t val) { return new DfgConst{m_dfg, flp, 32, val}; } // Create a DfgConst vertex with the given width and value zero DfgConst* makeZero(FileLine* flp, uint32_t width) { return new DfgConst{m_dfg, flp, width, 0}; } // Create a DfgConst vertex with the given width and value all ones DfgConst* makeOnes(FileLine* flp, uint32_t width) { DfgConst* const resp = makeZero(flp, width); resp->num().setAllBits1(); return resp; } // Create a new vertex of the given type template Vertex* make(FileLine* flp, const DfgDataType& dtype, Operands... operands) { // Find or create an equivalent vertex Vertex* const vtxp = m_cache.getOrCreate(flp, dtype, operands...); // Sanity check UASSERT_OBJ(vtxp->dtype() == dtype, vtxp, "Vertex dtype mismatch"); if (VL_UNLIKELY(v3Global.opt.debugCheck())) vtxp->typeCheck(m_dfg); // Assign vertex ID VertexInfo& vInfo = m_vInfo[vtxp]; if (!vInfo.m_id) vInfo.m_id = ++m_lastId; // Add to work list addToWorkList(vtxp); // Return new node return vtxp; } // Same as above, but 'flp' and 'dtypep' are taken from the given example vertex template Vertex* make(const DfgVertex* examplep, Operands... operands) { return make(examplep->fileline(), examplep->dtype(), operands...); } // Check two vertex are the same, or the same constant value static bool isSame(const DfgVertex* ap, const DfgVertex* bp) { if (ap == bp) return true; if (ap->dtype() != bp->dtype()) return false; const DfgConst* const aConstp = ap->cast(); if (!aConstp) return false; const DfgConst* const bConstp = bp->cast(); if (!bConstp) return false; return aConstp->num().isCaseEq(bConstp->num()); } static bool isZero(const DfgVertex* vtxp) { if (const DfgConst* const constp = vtxp->cast()) return constp->isZero(); return false; } static bool isOnes(const DfgVertex* vtxp) { if (const DfgConst* const constp = vtxp->cast()) return constp->isOnes(); return false; } static bool isEqOne(const DfgVertex* vtxp) { if (const DfgConst* const constp = vtxp->cast()) return constp->num().isEqOne(); return false; } static bool areAdjacent(uint32_t& lsb, const DfgSel* lSelp, const DfgSel* rSelp) { if (!isSame(lSelp->srcp(), rSelp->srcp())) return false; if (lSelp->lsb() + lSelp->width() == rSelp->lsb()) { lsb = lSelp->lsb(); return true; } if (lSelp->lsb() == rSelp->lsb() + rSelp->width()) { lsb = rSelp->lsb(); return true; } return false; } // Note: If any of the following transformers return true, then the vertex was replaced and the // caller must not do any further changes, so the caller must check the return value, otherwise // there will be hard to debug issues. // Constant fold unary vertex, return true if folded template VL_ATTR_WARN_UNUSED_RESULT bool foldUnary(Vertex* const vtxp) { static_assert(std::is_base_of::value, "Must invoke on unary"); static_assert(std::is_final::value, "Must invoke on final class"); if (DfgConst* const srcp = vtxp->srcp()->template cast()) { APPLYING(FOLD_UNARY) { DfgConst* const resultp = makeZero(vtxp->fileline(), vtxp->width()); foldOp(resultp->num(), srcp->num()); replace(resultp); return true; } } return false; } // Constant fold binary vertex, return true if folded template VL_ATTR_WARN_UNUSED_RESULT bool foldBinary(Vertex* const vtxp) { static_assert(std::is_base_of::value, "Must invoke on binary"); static_assert(std::is_final::value, "Must invoke on final class"); if (DfgConst* const lhsp = vtxp->inputp(0)->template cast()) { if (DfgConst* const rhsp = vtxp->inputp(1)->template cast()) { APPLYING(FOLD_BINARY) { DfgConst* const resultp = makeZero(vtxp->fileline(), vtxp->width()); foldOp(resultp->num(), lhsp->num(), rhsp->num()); replace(resultp); return true; } } } return false; } // Transformations that apply to all associative binary vertices. // Returns true if vtxp was replaced. template VL_ATTR_WARN_UNUSED_RESULT bool associativeBinary(Vertex* const vtxp) { static_assert(std::is_base_of::value, "Must invoke on binary"); static_assert(std::is_final::value, "Must invoke on final class"); DfgVertex* const lhsp = vtxp->lhsp(); DfgVertex* const rhsp = vtxp->rhsp(); FileLine* const flp = vtxp->fileline(); DfgConst* const lConstp = lhsp->cast(); DfgConst* const rConstp = rhsp->cast(); if (lConstp && rConstp) { APPLYING(FOLD_ASSOC_BINARY) { DfgConst* const resultp = makeZero(flp, vtxp->width()); foldOp(resultp->num(), lConstp->num(), rConstp->num()); replace(resultp); return true; } } if (lConstp) { if (Vertex* const rVtxp = rhsp->cast()) { if (DfgConst* const rlConstp = rVtxp->lhsp()->template cast()) { APPLYING(FOLD_ASSOC_BINARY_LHS_OF_RHS) { // Fold constants const uint32_t width = std::is_same::value ? lConstp->width() + rlConstp->width() : vtxp->width(); DfgConst* const constp = makeZero(flp, width); foldOp(constp->num(), lConstp->num(), rlConstp->num()); // Replace vertex replace(make(vtxp, constp, rVtxp->rhsp())); return true; } } } } if (rConstp) { if (Vertex* const lVtxp = lhsp->cast()) { if (DfgConst* const lrConstp = lVtxp->rhsp()->template cast()) { APPLYING(FOLD_ASSOC_BINARY_RHS_OF_LHS) { // Fold constants const uint32_t width = std::is_same::value ? lrConstp->width() + rConstp->width() : vtxp->width(); DfgConst* const constp = makeZero(flp, width); foldOp(constp->num(), lrConstp->num(), rConstp->num()); // Replace vertex replace(make(vtxp, lVtxp->lhsp(), constp)); return true; } } } } // Make associative trees right leaning to reduce pattern variations, and for better CSE if (Vertex* const alhsp = vtxp->lhsp()->template cast()) { if (!alhsp->hasMultipleSinks()) { DfgVertex* const ap = alhsp->lhsp(); DfgVertex* const bp = alhsp->rhsp(); DfgVertex* const cp = vtxp->rhsp(); // Only do this if the rhs is not th same as the operands of the LHS, otherwise // SWAP_SIDE_IN_COMMUTATIVE_BINARY can get in a loop with this pattern. if (ap != cp && bp != cp) { APPLYING(RIGHT_LEANING_ASSOC) { // Rotate the expression tree rooted at 'vtxp' to the right, // producing a right-leaning tree // Concatenation dtypes need adjusting, other assoc vertices preserve types const DfgDataType& childDType = std::is_same::value ? DfgDataType::packed(bp->width() + cp->width()) : vtxp->dtype(); Vertex* const bcp = make(vtxp->fileline(), childDType, bp, cp); replace(make(alhsp->fileline(), vtxp->dtype(), ap, bcp)); return true; } } } } // Attempt to reuse associative binary expressions if hey already exist, e.g.: // '(a OP (b OP c))' -> '(a OP b) OP c', iff '(a OP b)' already exists, or // '(a OP c) OP b' iff '(a OP c)' already exists and the vertex is commutative. // Only do this is 'b OP c' has a single use and can subsequently be removed, // otherwise there is no improvement. if (!rhsp->hasMultipleSinks()) { if (Vertex* rVtxp = rhsp->template cast()) { DfgVertex* const rlVtxp = rVtxp->lhsp(); DfgVertex* const rrVtxp = rVtxp->rhsp(); const DfgDataType& dtype = std::is_same::value ? DfgDataType::packed(lhsp->width() + rlVtxp->width()) : vtxp->dtype(); if (Vertex* const existingp = m_cache.get(dtype, lhsp, rlVtxp)) { UASSERT_OBJ(existingp->hasSinks(), vtxp, "Existing vertex should be used"); if (existingp != rhsp) { APPLYING(REUSE_ASSOC_BINARY_LHS_WITH_LHS_OF_RHS) { replace(make(vtxp, existingp, rrVtxp)); return true; } } } // Concat is not commutative if VL_CONSTEXPR_CXX17 (!std::is_same::value) { if (Vertex* const existingp = m_cache.get(dtype, lhsp, rrVtxp)) { UASSERT_OBJ(existingp->hasSinks(), vtxp, "Existing vertex should be used"); if (existingp != rhsp) { APPLYING(REUSE_ASSOC_BINARY_LHS_WITH_RHS_OF_RHS) { replace(make(vtxp, existingp, rlVtxp)); return true; } } } } } } return false; } // Transformations that apply to all commutative binary vertices template VL_ATTR_WARN_UNUSED_RESULT bool commutativeBinary(Vertex* const vtxp) { static_assert(std::is_base_of::value, "Must invoke on binary"); static_assert(std::is_final::value, "Must invoke on final class"); DfgVertex* const lhsp = vtxp->lhsp(); DfgVertex* const rhsp = vtxp->rhsp(); // If constant LHS, push through cond if used once. (Enables branch combining) if (DfgConst* const lConstp = lhsp->cast()) { if (DfgCond* const rCondp = rhsp->cast()) { if (!rCondp->hasMultipleSinks()) { APPLYING(PUSH_COMMUTATIVE_BINARY_THROUGH_COND) { DfgVertex* const tp = make(vtxp, lConstp, rCondp->thenp()); DfgVertex* const ep = make(vtxp, lConstp, rCondp->elsep()); replace(make(vtxp, rCondp->condp(), tp, ep)); return true; } } } return false; } // Ensure Const is on left-hand side to simplify other patterns { const bool lIsConst = lhsp->is(); const bool rIsConst = rhsp->is(); if (lIsConst != rIsConst) { if (rIsConst) { APPLYING(SWAP_CONST_IN_COMMUTATIVE_BINARY) { replace(make(vtxp, rhsp, lhsp)); return true; } } return false; } } // Ensure Not is on the left-hand side to simplify other patterns { const bool lIsNot = lhsp->is(); const bool rIsNot = rhsp->is(); if (lIsNot != rIsNot) { if (rIsNot) { APPLYING(SWAP_NOT_IN_COMMUTATIVE_BINARY) { replace(make(vtxp, rhsp, lhsp)); return true; } } return false; } } // Ensure same vertex is on the right-hand side to simplify other patterns { const bool lIsSame = lhsp->is(); const bool rIsSame = rhsp->is(); if (lIsSame != rIsSame) { if (lIsSame) { APPLYING(SWAP_SAME_IN_COMMUTATIVE_BINARY) { replace(make(vtxp, rhsp, lhsp)); return true; } } return false; } } // Otherwise put sides in order based on unique iD, this makes // 'a op b' and 'b op a' end up the same for better combining. { const VertexInfo& lInfo = m_vInfo[lhsp]; const VertexInfo& rInfo = m_vInfo[rhsp]; if (lInfo.m_id > rInfo.m_id) { APPLYING(SWAP_SIDE_IN_COMMUTATIVE_BINARY) { replace(make(vtxp, rhsp, lhsp)); return true; } } } return false; } // Transformations that apply to all distributive and associative binary // vertices 'Other' is the type that is distributive over 'Vertex', // that is: a Other (b Vertex c) == (a Other b) Vertex (a Other c) template VL_ATTR_WARN_UNUSED_RESULT bool distributiveAndAssociativeBinary(Vertex* vtxp) { DfgVertex* const lhsp = vtxp->lhsp(); DfgVertex* const rhsp = vtxp->rhsp(); if (!lhsp->hasMultipleSinks() && !rhsp->hasMultipleSinks()) { // Convert '(a Other b) Vertex (a Other c)' and associative // variations to 'a Other (b Vertex c)' if (Other* const lp = lhsp->cast()) { if (Other* const rp = rhsp->cast()) { DfgVertex* const llp = lp->lhsp(); DfgVertex* const lrp = lp->rhsp(); DfgVertex* const rlp = rp->lhsp(); DfgVertex* const rrp = rp->rhsp(); DfgVertex* ap = nullptr; DfgVertex* bp = nullptr; DfgVertex* cp = nullptr; if (llp == rlp) { ap = llp; bp = lrp; cp = rrp; } else if (llp == rrp) { ap = llp; bp = lrp; cp = rlp; } else if (lrp == rlp) { ap = lrp; bp = llp; cp = rrp; } else if (lrp == rrp) { ap = lrp; bp = llp; cp = rlp; } if (ap) { APPLYING(REPLACE_DISTRIBUTIVE_BINARY) { replace(make(vtxp, ap, make(lhsp, bp, cp))); return true; } } } } } return false; } // Bitwise operation with one side Const, and the other side a Concat template VL_ATTR_WARN_UNUSED_RESULT bool tryPushBitwiseOpThroughConcat(Vertex* const vtxp, DfgConst* constp, DfgConcat* concatp) { FileLine* const flp = vtxp->fileline(); // If at least one of the sides of the Concat constant, or width 1 (i.e.: can be // further simplified), then push the Vertex past the Concat if (concatp->lhsp()->is() || concatp->rhsp()->is() // || concatp->lhsp()->dtype() == m_bitDType || concatp->rhsp()->dtype() == m_bitDType) { APPLYING(PUSH_BITWISE_OP_THROUGH_CONCAT) { const uint32_t width = concatp->width(); const DfgDataType& lDtype = concatp->lhsp()->dtype(); const DfgDataType& rDtype = concatp->rhsp()->dtype(); const uint32_t lWidth = lDtype.size(); const uint32_t rWidth = rDtype.size(); // The new Lhs vertex DfgConst* const newLhsConstp = makeZero(constp->fileline(), lWidth); newLhsConstp->num().opSel(constp->num(), width - 1, rWidth); Vertex* const newLhsp = make(flp, lDtype, newLhsConstp, concatp->lhsp()); // The new Rhs vertex DfgConst* const newRhsConstp = makeZero(constp->fileline(), rWidth); newRhsConstp->num().opSel(constp->num(), rWidth - 1, 0); Vertex* const newRhsp = make(flp, rDtype, newRhsConstp, concatp->rhsp()); // Replace this vertex replace(make(concatp, newLhsp, newRhsp)); return true; } } return false; } template VL_ATTR_WARN_UNUSED_RESULT bool tryPushCompareOpThroughConcat(Vertex* const vtxp, DfgConst* constp, DfgConcat* concatp) { FileLine* const flp = vtxp->fileline(); // If at least one of the sides of the Concat is constant, then push the Vertex past // the Concat if (concatp->lhsp()->is() || concatp->rhsp()->is()) { APPLYING(PUSH_COMPARE_OP_THROUGH_CONCAT) { const uint32_t width = concatp->width(); const uint32_t lWidth = concatp->lhsp()->width(); const uint32_t rWidth = concatp->rhsp()->width(); // The new Lhs vertex DfgConst* const newLhsConstp = makeZero(constp->fileline(), lWidth); newLhsConstp->num().opSel(constp->num(), width - 1, rWidth); Vertex* const newLhsp = make(flp, m_bitDType, newLhsConstp, concatp->lhsp()); // The new Rhs vertex DfgConst* const newRhsConstp = makeZero(constp->fileline(), rWidth); newRhsConstp->num().opSel(constp->num(), rWidth - 1, 0); Vertex* const newRhsp = make(flp, m_bitDType, newRhsConstp, concatp->rhsp()); // The replacement Vertex DfgVertexBinary* const resp = std::is_same::value ? make(concatp->fileline(), m_bitDType, newLhsp, newRhsp) : nullptr; UASSERT_OBJ(resp, vtxp, "Unhandled vertex type in 'tryPushCompareOpThroughConcat': " << vtxp->typeName()); // Replace this vertex replace(resp); return true; } } return false; } template VL_ATTR_WARN_UNUSED_RESULT bool tryPushBitwiseOpThroughReductions(Bitwise* const vtxp) { using Reduction = BitwiseToReduction; if (Reduction* const lRedp = vtxp->lhsp()->template cast()) { if (Reduction* const rRedp = vtxp->rhsp()->template cast()) { DfgVertex* const lSrcp = lRedp->srcp(); DfgVertex* const rSrcp = rRedp->srcp(); if (lSrcp->dtype() == rSrcp->dtype() && lSrcp->width() <= VL_QUADSIZE && !lSrcp->is() && !rSrcp->is() && !lSrcp->hasMultipleSinks() && !rSrcp->hasMultipleSinks()) { APPLYING(PUSH_BITWISE_THROUGH_REDUCTION) { FileLine* const flp = vtxp->fileline(); Bitwise* const bwp = make(flp, lSrcp->dtype(), lSrcp, rSrcp); replace(make(flp, m_bitDType, bwp)); return true; } } } } return false; } template VL_ATTR_WARN_UNUSED_RESULT bool tryReplaceBitwiseWithReduction(Bitwise* vtxp) { UASSERT_OBJ(vtxp->width() == 1, vtxp, "Width must be 1"); using Reduction = BitwiseToReduction; DfgVertex* const lhsp = vtxp->lhsp(); DfgVertex* const rhsp = vtxp->rhsp(); if (DfgSel* const lSelp = lhsp->template cast()) { DfgSel* rSelp = rhsp->template cast(); DfgVertex* extrap = nullptr; if (!rSelp) { if (Bitwise* const rBitwisep = rhsp->template cast()) { rSelp = rBitwisep->lhsp()->template cast(); extrap = rBitwisep->rhsp(); } } if (rSelp) { uint32_t lsb = 0; if (areAdjacent(lsb, lSelp, rSelp)) { APPLYING(REPLACE_BITWISE_OF_SELS_WITH_REDUCTION) { const DfgDataType& dtype = DfgDataType::packed(lSelp->width() + rSelp->width()); DfgSel* const newSelp = make(lSelp->fileline(), dtype, lSelp->srcp(), lsb); DfgVertex* resp = make(vtxp, newSelp); if (extrap) resp = make(vtxp, resp, extrap); replace(resp); return true; } } } } if (Reduction* const lRedp = lhsp->template cast()) { Reduction* rRedp = rhsp->template cast(); DfgVertex* extrap = nullptr; if (!rRedp) { if (Bitwise* const rBitwisep = rhsp->template cast()) { rRedp = rBitwisep->lhsp()->template cast(); extrap = rBitwisep->rhsp(); } } if (rRedp) { if (DfgSel* const lSelp = lRedp->srcp()->template cast()) { if (DfgSel* const rSelp = rRedp->srcp()->template cast()) { uint32_t lsb = 0; if (areAdjacent(lsb, lSelp, rSelp)) { APPLYING(REPLACE_BITWISE_OF_REDUCTION_OF_SELS_WITH_REDUCTION) { const DfgDataType& dtype = DfgDataType::packed(lSelp->width() + rSelp->width()); DfgSel* const newSelp = make(lSelp->fileline(), dtype, lSelp->srcp(), lsb); DfgVertex* resp = make(vtxp, newSelp); if (extrap) resp = make(vtxp, resp, extrap); replace(resp); return true; } } } } } } return false; } template VL_ATTR_WARN_UNUSED_RESULT bool optimizeReduction(Reduction* const vtxp) { using Bitwise = ReductionToBitwise; if (foldUnary(vtxp)) return true; DfgVertex* const srcp = vtxp->srcp(); FileLine* const flp = vtxp->fileline(); // Reduction of 1-bit value if (srcp->dtype() == m_bitDType) { APPLYING(REMOVE_WIDTH_ONE_REDUCTION) { replace(srcp); return true; } } if (DfgCond* const condp = srcp->cast()) { if (condp->thenp()->is() || condp->elsep()->is()) { APPLYING(PUSH_REDUCTION_THROUGH_COND_WITH_CONST_BRANCH) { // The new 'then' vertex Reduction* const newThenp = make(flp, m_bitDType, condp->thenp()); // The new 'else' vertex Reduction* const newElsep = make(flp, m_bitDType, condp->elsep()); // The replacement Cond vertex DfgCond* const newCondp = make(condp->fileline(), m_bitDType, condp->condp(), newThenp, newElsep); // Replace this vertex replace(newCondp); return true; } } } if (DfgConcat* const concatp = srcp->cast()) { if (concatp->lhsp()->is() || concatp->rhsp()->is() || concatp->lhsp()->dtype() == m_bitDType || concatp->rhsp()->dtype() == m_bitDType) { APPLYING(PUSH_REDUCTION_THROUGH_CONCAT) { // Reduce the parts of the concatenation Reduction* const lRedp = make(concatp->fileline(), m_bitDType, concatp->lhsp()); Reduction* const rRedp = make(concatp->fileline(), m_bitDType, concatp->rhsp()); // Bitwise reduce the results replace(make(flp, m_bitDType, lRedp, rRedp)); return true; } } } if (Bitwise* const bitwisep = vtxp->srcp()->template cast()) { if (!bitwisep->hasMultipleSinks()) { if (bitwisep->lhsp()->template is() || bitwisep->rhsp()->template is()) { APPLYING(PUSH_REDUCTION_THROUGH_BITWISE_OF_CONCAT) { Reduction* const newLhsp = make(flp, m_bitDType, bitwisep->lhsp()); Reduction* const newRhsp = make(flp, m_bitDType, bitwisep->rhsp()); replace(make(flp, m_bitDType, newLhsp, newRhsp)); return true; } } if (DfgSel* const lSelp = bitwisep->lhsp()->template cast()) { if (DfgSel* const rSelp = bitwisep->rhsp()->template cast()) { uint32_t lsb = 0; if (areAdjacent(lsb, lSelp, rSelp)) { APPLYING(PUSH_REDUCTION_THROUGH_BITWISE_OF_SELS) { const DfgDataType& dtype = DfgDataType::packed(lSelp->width() + rSelp->width()); DfgSel* const newSelp = make(lSelp->fileline(), dtype, lSelp->srcp(), lsb); replace(make(vtxp, newSelp)); return true; } } } } } } return false; } template VL_ATTR_WARN_UNUSED_RESULT bool optimizeShiftRHS(Shift* const vtxp) { static_assert(std::is_base_of::value, "Must invoke on binary"); static_assert(std::is_final::value, "Must invoke on final class"); if (const DfgConcat* const concatp = vtxp->rhsp()->template cast()) { if (isZero(concatp->lhsp())) { // Drop redundant zero extension APPLYING(REMOVE_REDUNDANT_ZEXT_ON_RHS_OF_SHIFT) { replace(make(vtxp, vtxp->lhsp(), concatp->rhsp())); return true; } } } return false; } // Given an operand of an Add, return the term that could be used for conveting to CountOnes // Result is a tulpe of (Vertex, Lsb, Width) std::tuple addToCountOnesTerm(DfgVertex* vtxp) { if (DfgConcat* const oCatp = vtxp->cast()) { if (isZero(oCatp->lhsp())) { if (DfgCountOnes* const countOnesp = oCatp->rhsp()->cast()) { // Zero extended count ones if (DfgSel* const selp = countOnesp->srcp()->cast()) { return {selp->fromp(), selp->lsb(), selp->width()}; } } else if (DfgSel* const selp = oCatp->rhsp()->cast()) { // Zero extended single bit select if (selp->dtype() == m_bitDType) { // return {selp->fromp(), selp->lsb(), selp->width()}; } } } return {nullptr, 0, 0}; } if (DfgCountOnes* const countOnesp = vtxp->cast()) { // Simple count ones if (DfgSel* const selp = countOnesp->srcp()->cast()) { return {selp->fromp(), selp->lsb(), selp->width()}; } return {nullptr, 0, 0}; } if (DfgSel* const oSelp = vtxp->cast()) { if (oSelp->lsb() == 0) { // Truncated count ones if (DfgCountOnes* const countOnesp = oSelp->fromp()->cast()) { // Zero extended count ones if (DfgSel* const selp = countOnesp->srcp()->cast()) { return {selp->fromp(), selp->lsb(), selp->width()}; } } } // Single bit select if (oSelp->dtype() == m_bitDType) { // return {oSelp->fromp(), oSelp->lsb(), 1}; } return {nullptr, 0, 0}; } // Altered form of extended MSB if (DfgShiftR* const shiftrp = vtxp->cast()) { if (DfgConst* const rConstp = shiftrp->rhsp()->cast()) { if (rConstp->toU32() == shiftrp->width() - 1) { return {shiftrp->lhsp(), shiftrp->width() - 1, 1}; } } return {nullptr, 0, 0}; } // Not applicable return {nullptr, 0, 0}; } // VISIT methods void visit(DfgVertex*) override {} //========================================================================= // DfgVertexUnary //========================================================================= void visit(DfgExtend* const vtxp) override { if (foldUnary(vtxp)) return; // Convert all Extend into Concat with zeros. This simplifies other patterns as they // only need to handle Concat, which is more generic, and don't need special cases for // Extend. APPLYING(REPLACE_EXTEND) { DfgVertex* const zerop = makeZero(vtxp->fileline(), vtxp->width() - vtxp->srcp()->width()); replace(make(vtxp, zerop, vtxp->srcp())); return; } } void visit(DfgExtendS* const vtxp) override { if (foldUnary(vtxp)) return; } void visit(DfgLogNot* const vtxp) override { if (foldUnary(vtxp)) return; } void visit(DfgNegate* const vtxp) override { if (foldUnary(vtxp)) return; } void visit(DfgNot* const vtxp) override { if (foldUnary(vtxp)) return; // Not of Cond if (DfgCond* const condp = vtxp->srcp()->cast()) { // If at least one of the branches are a constant, push the Not past the Cond if (condp->thenp()->is() || condp->elsep()->is()) { APPLYING(PUSH_NOT_THROUGH_COND) { // The new 'then' vertex DfgNot* const newThenp = make(vtxp, condp->thenp()); // The new 'else' vertex DfgNot* const newElsep = make(vtxp, condp->elsep()); // The replacement Cond vertex DfgCond* const newCondp = make(condp->fileline(), vtxp->dtype(), condp->condp(), newThenp, newElsep); // Replace this vertex replace(newCondp); return; } } } // Not of Not if (DfgNot* const notp = vtxp->srcp()->cast()) { APPLYING(REMOVE_NOT_NOT) { replace(notp->srcp()); return; } } if (!vtxp->srcp()->hasMultipleSinks()) { // Not of Eq if (DfgEq* const eqp = vtxp->srcp()->cast()) { APPLYING(REPLACE_NOT_EQ) { replace( make(eqp->fileline(), vtxp->dtype(), eqp->lhsp(), eqp->rhsp())); return; } } // Not of Neq if (DfgNeq* const neqp = vtxp->srcp()->cast()) { APPLYING(REPLACE_NOT_NEQ) { replace( make(neqp->fileline(), vtxp->dtype(), neqp->lhsp(), neqp->rhsp())); return; } } } } void visit(DfgRedOr* const vtxp) override { if (optimizeReduction(vtxp)) return; } void visit(DfgRedAnd* const vtxp) override { if (optimizeReduction(vtxp)) return; } void visit(DfgRedXor* const vtxp) override { if (optimizeReduction(vtxp)) return; } void visit(DfgSel* const vtxp) override { DfgVertex* const fromp = vtxp->fromp(); FileLine* const flp = vtxp->fileline(); const uint32_t lsb = vtxp->lsb(); const uint32_t width = vtxp->width(); const uint32_t msb = lsb + width - 1; if (DfgConst* const constp = fromp->cast()) { APPLYING(FOLD_SEL) { DfgConst* const resp = makeZero(flp, width); resp->num().opSel(constp->num(), msb, lsb); replace(resp); return; } } // Full width select, replace with the source. if (fromp->width() == width) { UASSERT_OBJ(lsb == 0, fromp, "Out of range select should have been fixed up earlier"); APPLYING(REMOVE_FULL_WIDTH_SEL) { replace(fromp); return; } } // Sel from Concat if (DfgConcat* const concatp = fromp->cast()) { DfgVertex* const lhsp = concatp->lhsp(); DfgVertex* const rhsp = concatp->rhsp(); if (msb < rhsp->width()) { // If the select is entirely from rhs, then replace with sel from rhs APPLYING(REMOVE_SEL_FROM_RHS_OF_CONCAT) { // replace(make(vtxp, rhsp, vtxp->lsb())); return; } } else if (lsb >= rhsp->width()) { // If the select is entirely from the lhs, then replace with sel from lhs APPLYING(REMOVE_SEL_FROM_LHS_OF_CONCAT) { replace(make(vtxp, lhsp, lsb - rhsp->width())); return; } } } if (DfgReplicate* const repp = fromp->cast()) { // If the Sel is wholly into the source of the Replicate, push the Sel through the // Replicate and apply it directly to the source of the Replicate. const uint32_t srcWidth = repp->srcp()->width(); if (width <= srcWidth) { const uint32_t newLsb = lsb % srcWidth; if (newLsb + width <= srcWidth) { APPLYING(PUSH_SEL_THROUGH_REPLICATE) { replace(make(vtxp, repp->srcp(), newLsb)); return; } } } } // Sel from Not if (DfgNot* const notp = fromp->cast()) { // Replace "Sel from Not" with "Not of Sel" if (!notp->hasMultipleSinks()) { APPLYING(PUSH_SEL_THROUGH_NOT) { // Make Sel select from source of Not DfgSel* const newSelp = make(vtxp, notp->srcp(), vtxp->lsb()); // Add Not after Sel replace(make(notp->fileline(), vtxp->dtype(), newSelp)); return; } } } // Sel from Sel if (DfgSel* const selp = fromp->cast()) { APPLYING(REPLACE_SEL_FROM_SEL) { // Select from the source of the source Sel with adjusted LSB replace(make(vtxp, selp->fromp(), lsb + selp->lsb())); return; } } // Sel from Cond if (DfgCond* const condp = fromp->cast()) { if (!condp->hasMultipleSinks()) { APPLYING(PUSH_SEL_THROUGH_COND) { // The new 'then' vertex DfgSel* const newThenp = make(vtxp, condp->thenp(), lsb); // The new 'else' vertex DfgSel* const newElsep = make(vtxp, condp->elsep(), lsb); // The replacement Cond vertex DfgCond* const newCondp = make(condp->fileline(), vtxp->dtype(), condp->condp(), newThenp, newElsep); // Replace this vertex replace(newCondp); return; } } } // Sel from ShiftL if (DfgShiftL* const shiftLp = fromp->cast()) { // If selecting bottom bits of left shift, push the Sel before the shift if (lsb == 0) { UASSERT_OBJ(shiftLp->lhsp()->width() >= width, vtxp, "input of shift narrow"); APPLYING(PUSH_SEL_THROUGH_SHIFTL) { DfgSel* const newSelp = make(vtxp, shiftLp->lhsp(), vtxp->lsb()); replace(make(vtxp, newSelp, shiftLp->rhsp())); return; } } } // Sel from a partial variable or narrowed vertex { DfgSplicePacked* splicep = fromp->cast(); if (DfgVarPacked* const varp = fromp->cast()) { // Must be a splice, otherwise it would have been inlined if (varp->tmpForp() && varp->srcp()) splicep = varp->srcp()->as(); } if (splicep) { DfgVertex* driverp = nullptr; uint32_t driverLsb = 0; splicep->foreachDriver([&](DfgVertex& src, const uint32_t dLsb) { const uint32_t dMsb = dLsb + src.width() - 1; // If it does not cover the whole searched bit range, move on if (lsb < dLsb || dMsb < msb) return false; // Save the driver driverp = &src; driverLsb = dLsb; return true; }); if (driverp) { APPLYING(PUSH_SEL_THROUGH_SPLICE) { replace(make(vtxp, driverp, lsb - driverLsb)); return; } } } } } void visit(DfgMux* const vtxp) override { DfgVertex* const fromp = vtxp->fromp(); DfgVertex* const lsbp = vtxp->lsbp(); FileLine* const flp = vtxp->fileline(); if (DfgConst* const lsbConstp = lsbp->cast()) { APPLYING(REPLACE_MUX_WITH_SEL) { replace(make(vtxp, fromp, lsbConstp->num().toUInt())); return; } } if (isZero(fromp)) { APPLYING(FOLD_MUX_FROM_ZERO) { replace(makeZero(flp, vtxp->width())); return; } } if (isOnes(fromp)) { APPLYING(FOLD_MUX_FROM_ONES) { replace(makeOnes(flp, vtxp->width())); return; } } } //========================================================================= // DfgVertexBinary - bitwise //========================================================================= void visit(DfgAnd* const vtxp) override { DfgVertex* const lhsp = vtxp->lhsp(); DfgVertex* const rhsp = vtxp->rhsp(); if (isSame(lhsp, rhsp)) { APPLYING(REMOVE_AND_WITH_SELF) { replace(lhsp); return; } } if (associativeBinary(vtxp)) return; if (commutativeBinary(vtxp)) return; FileLine* const flp = vtxp->fileline(); // Bubble pushing (De Morgan) if (!lhsp->hasMultipleSinks() || !rhsp->hasMultipleSinks()) { if (DfgNot* const lhsNotp = lhsp->cast()) { if (DfgNot* const rhsNotp = rhsp->cast()) { APPLYING(REPLACE_AND_OF_NOT_AND_NOT) { DfgOr* const orp = make(vtxp, lhsNotp->srcp(), rhsNotp->srcp()); replace(make(vtxp, orp)); return; } } if (DfgNeq* const rhsNeqp = rhsp->cast()) { APPLYING(REPLACE_AND_OF_NOT_AND_NEQ) { DfgEq* const newRhsp = make(rhsp, rhsNeqp->lhsp(), rhsNeqp->rhsp()); DfgOr* const orp = make(vtxp, lhsNotp->srcp(), newRhsp); replace(make(vtxp, orp)); return; } } } } if (DfgConst* const lConstp = lhsp->cast()) { if (lConstp->isZero()) { APPLYING(REPLACE_AND_WITH_ZERO) { replace(lConstp); return; } } if (lConstp->isOnes()) { APPLYING(REMOVE_AND_WITH_ONES) { replace(rhsp); return; } } if (DfgConcat* const rhsConcatp = rhsp->cast()) { if (tryPushBitwiseOpThroughConcat(vtxp, lConstp, rhsConcatp)) return; } } if (distributiveAndAssociativeBinary(vtxp)) return; if (tryPushBitwiseOpThroughReductions(vtxp)) return; if (DfgNot* const lhsNotp = lhsp->cast()) { // ~A & A is all zeroes if (lhsNotp->srcp() == rhsp) { APPLYING(REPLACE_CONTRADICTORY_AND) { replace(makeZero(flp, vtxp->width())); return; } } // ~A & (A & _) or ~A & (_ & A) is all zeroes if (DfgAnd* const rhsAndp = rhsp->cast()) { if (lhsNotp->srcp() == rhsAndp->lhsp() || lhsNotp->srcp() == rhsAndp->rhsp()) { APPLYING(REPLACE_CONTRADICTORY_AND_3) { replace(makeZero(flp, vtxp->width())); return; } } } } if (vtxp->dtype() == m_bitDType) { if (tryReplaceBitwiseWithReduction(vtxp)) return; } } void visit(DfgOr* const vtxp) override { DfgVertex* const lhsp = vtxp->lhsp(); DfgVertex* const rhsp = vtxp->rhsp(); if (isSame(lhsp, rhsp)) { APPLYING(REMOVE_OR_WITH_SELF) { replace(lhsp); return; } } if (associativeBinary(vtxp)) return; if (commutativeBinary(vtxp)) return; FileLine* const flp = vtxp->fileline(); // Bubble pushing (De Morgan) if (!lhsp->hasMultipleSinks() || !rhsp->hasMultipleSinks()) { if (DfgNot* const lhsNotp = lhsp->cast()) { if (DfgNot* const rhsNotp = rhsp->cast()) { APPLYING(REPLACE_OR_OF_NOT_AND_NOT) { DfgAnd* const andp = make(vtxp, lhsNotp->srcp(), rhsNotp->srcp()); replace(make(vtxp, andp)); return; } } if (DfgNeq* const rhsNeqp = rhsp->cast()) { APPLYING(REPLACE_OR_OF_NOT_AND_NEQ) { DfgEq* const newRhsp = make(rhsp, rhsNeqp->lhsp(), rhsNeqp->rhsp()); DfgAnd* const andp = make(vtxp, lhsNotp->srcp(), newRhsp); replace(make(vtxp, andp)); return; } } } } if (DfgConcat* const lhsConcatp = lhsp->cast()) { if (DfgConcat* const rhsConcatp = rhsp->cast()) { if (lhsConcatp->lhsp()->dtype() == rhsConcatp->lhsp()->dtype()) { if (isZero(lhsConcatp->lhsp()) && isZero(rhsConcatp->rhsp())) { APPLYING(REPLACE_OR_OF_CONCAT_ZERO_LHS_AND_CONCAT_RHS_ZERO) { replace(make(vtxp, rhsConcatp->lhsp(), lhsConcatp->rhsp())); return; } } if (isZero(lhsConcatp->rhsp()) && isZero(rhsConcatp->lhsp())) { APPLYING(REPLACE_OR_OF_CONCAT_LHS_ZERO_AND_CONCAT_ZERO_RHS) { replace(make(vtxp, lhsConcatp->lhsp(), rhsConcatp->rhsp())); return; } } } } } if (DfgConst* const lConstp = lhsp->cast()) { if (lConstp->isZero()) { APPLYING(REMOVE_OR_WITH_ZERO) { replace(rhsp); return; } } if (lConstp->isOnes()) { APPLYING(REPLACE_OR_WITH_ONES) { replace(lhsp); return; } } if (DfgConcat* const rhsConcatp = rhsp->cast()) { if (tryPushBitwiseOpThroughConcat(vtxp, lConstp, rhsConcatp)) return; } } if (distributiveAndAssociativeBinary(vtxp)) return; if (tryPushBitwiseOpThroughReductions(vtxp)) return; if (DfgNot* const lhsNotp = lhsp->cast()) { // ~A | A is all ones if (lhsNotp->srcp() == rhsp) { APPLYING(REPLACE_TAUTOLOGICAL_OR) { DfgConst* const resp = makeZero(flp, vtxp->width()); resp->num().setAllBits1(); replace(resp); return; } } // ~A | (A | _) or ~A | (_ | A) is all ones if (DfgOr* const rhsOrp = rhsp->cast()) { if (lhsNotp->srcp() == rhsOrp->lhsp() || lhsNotp->srcp() == rhsOrp->rhsp()) { APPLYING(REPLACE_TAUTOLOGICAL_OR_3) { DfgConst* const resp = makeZero(flp, vtxp->width()); resp->num().setAllBits1(); replace(resp); return; } } } } if (vtxp->dtype() == m_bitDType) { if (tryReplaceBitwiseWithReduction(vtxp)) return; } } void visit(DfgXor* const vtxp) override { DfgVertex* const lhsp = vtxp->lhsp(); DfgVertex* const rhsp = vtxp->rhsp(); if (isSame(lhsp, rhsp)) { APPLYING(REPLACE_XOR_WITH_SELF) { replace(makeZero(vtxp->fileline(), vtxp->width())); return; } } if (associativeBinary(vtxp)) return; if (commutativeBinary(vtxp)) return; if (DfgConst* const lConstp = lhsp->cast()) { if (lConstp->isZero()) { APPLYING(REMOVE_XOR_WITH_ZERO) { replace(rhsp); return; } } if (lConstp->isOnes()) { APPLYING(REPLACE_XOR_WITH_ONES) { replace(make(vtxp, rhsp)); return; } } if (DfgConcat* const rConcatp = rhsp->cast()) { if (tryPushBitwiseOpThroughConcat(vtxp, lConstp, rConcatp)) return; } } if (tryPushBitwiseOpThroughReductions(vtxp)) return; if (vtxp->dtype() == m_bitDType) { if (tryReplaceBitwiseWithReduction(vtxp)) return; } } //========================================================================= // DfgVertexBinary - other //========================================================================= void visit(DfgAdd* const vtxp) override { if (associativeBinary(vtxp)) return; if (commutativeBinary(vtxp)) return; DfgVertex* const lhsp = vtxp->lhsp(); DfgVertex* const rhsp = vtxp->rhsp(); FileLine* const flp = vtxp->fileline(); if (isZero(lhsp)) { APPLYING(REMOVE_ADD_ZERO) { replace(rhsp); return; } } const std::tuple lTerm = addToCountOnesTerm(lhsp); if (DfgVertex* const lVtxp = std::get<0>(lTerm)) { std::tuple rTerm = addToCountOnesTerm(rhsp); DfgVertex* extrap = nullptr; if (!std::get<0>(rTerm)) { if (DfgAdd* const rAddp = rhsp->cast()) { rTerm = addToCountOnesTerm(rAddp->lhsp()); extrap = rAddp->rhsp(); } } if (DfgVertex* const rVtxp = std::get<0>(rTerm)) { if (isSame(lVtxp, rVtxp)) { const uint32_t lLsb = std::get<1>(lTerm); const uint32_t rLsb = std::get<1>(rTerm); const uint32_t lWidth = std::get<2>(lTerm); const uint32_t rWidth = std::get<2>(rTerm); bool adjoined = true; uint32_t lsb = 0; if (lLsb + lWidth == rLsb) { lsb = lLsb; } else if (lLsb == rLsb + rWidth) { lsb = rLsb; } else { adjoined = false; } if (adjoined) { APPLYING(REPLACE_ADD_WITH_COUNT_ONES) { DfgSel* const selp = make(vtxp->fileline(), DfgDataType::packed(lWidth + rWidth), lVtxp, lsb); DfgVertex* resp = make(flp, DfgDataType::packed(32), selp); if (vtxp->width() > 32U) { resp = make(vtxp, makeZero(flp, vtxp->width() - 32U), resp); } else if (vtxp->width() < 32U) { resp = make(vtxp, resp, 0U); } if (extrap) resp = make(vtxp, resp, extrap); replace(resp); return; } } } } } } void visit(DfgArraySel* const vtxp) override { DfgConst* const idxp = vtxp->bitp()->cast(); if (!idxp) return; DfgVarArray* const varp = vtxp->fromp()->cast(); if (!varp) return; if (varp->varp()->isForced()) return; if (varp->varp()->isSigUserRWPublic()) return; DfgVertex* const srcp = varp->srcp(); if (!srcp) return; if (DfgSpliceArray* const splicep = srcp->cast()) { DfgVertex* const driverp = splicep->driverAt(idxp->toSizeT()); if (!driverp) return; DfgUnitArray* const uap = driverp->cast(); if (!uap) return; if (uap->srcp()->is()) return; // If driven by a variable that had a Driver in DFG, it is partial if (DfgVertexVar* const dvarp = uap->srcp()->cast()) { if (dvarp->srcp()) return; } APPLYING(INLINE_ARRAYSEL_SPLICE) { replace(uap->srcp()); return; } } if (DfgUnitArray* const uap = srcp->cast()) { UASSERT_OBJ(idxp->toSizeT() == 0, vtxp, "Array index out of range"); if (uap->srcp()->is()) return; // If driven by a variable that had a Driver in DFG, it is partial if (DfgVertexVar* const dvarp = uap->srcp()->cast()) { if (dvarp->srcp()) return; } APPLYING(INLINE_ARRAYSEL_UNIT) { replace(uap->srcp()); return; } } } void visit(DfgConcat* const vtxp) override { if (associativeBinary(vtxp)) return; DfgVertex* const lhsp = vtxp->lhsp(); DfgVertex* const rhsp = vtxp->rhsp(); FileLine* const flp = vtxp->fileline(); if (isZero(lhsp)) { DfgConst* const lConstp = lhsp->as(); if (DfgSel* const rSelp = rhsp->cast()) { if (vtxp->dtype() == rSelp->fromp()->dtype() && rSelp->lsb() == lConstp->width()) { APPLYING(REPLACE_CONCAT_ZERO_AND_SEL_TOP_WITH_SHIFTR) { replace( make(vtxp, rSelp->fromp(), makeI32(flp, lConstp->width()))); return; } } } } if (isZero(rhsp)) { DfgConst* const rConstp = rhsp->as(); if (DfgSel* const lSelp = lhsp->cast()) { if (vtxp->dtype() == lSelp->fromp()->dtype() && lSelp->lsb() == 0) { APPLYING(REPLACE_CONCAT_SEL_BOTTOM_AND_ZERO_WITH_SHIFTL) { replace( make(vtxp, lSelp->fromp(), makeI32(flp, rConstp->width()))); return; } } } } if (DfgNot* const lNot = lhsp->cast()) { if (DfgNot* const rNot = rhsp->cast()) { if (!lNot->hasMultipleSinks() && !rNot->hasMultipleSinks()) { APPLYING(PUSH_CONCAT_THROUGH_NOTS) { DfgConcat* const newCatp = make(vtxp, lNot->srcp(), rNot->srcp()); replace(make(vtxp, newCatp)); return; } } } } if (DfgSel* const lSelp = lhsp->cast()) { if (DfgSel* const rSelp = rhsp->cast()) { if (isSame(lSelp->fromp(), rSelp->fromp())) { if (lSelp->lsb() == rSelp->lsb() + rSelp->width()) { APPLYING(REMOVE_CONCAT_OF_ADJOINING_SELS) { replace(make(vtxp, rSelp->fromp(), rSelp->lsb())); return; } } } } if (DfgConcat* const rConcatp = rhsp->cast()) { if (DfgSel* const rlSelp = rConcatp->lhsp()->cast()) { if (isSame(lSelp->fromp(), rlSelp->fromp())) { if (lSelp->lsb() == rlSelp->lsb() + rlSelp->width()) { APPLYING(REPLACE_NESTED_CONCAT_OF_ADJOINING_SELS_ON_LHS) { const uint32_t width = lSelp->width() + rlSelp->width(); const DfgDataType& dtype = DfgDataType::packed(width); DfgSel* const selp = make(flp, dtype, rlSelp->fromp(), rlSelp->lsb()); replace(make(vtxp, selp, rConcatp->rhsp())); return; } } } } } } if (DfgSel* const rSelp = rhsp->cast()) { if (DfgConcat* const lConcatp = lhsp->cast()) { if (DfgSel* const lrSelp = lConcatp->rhsp()->cast()) { if (isSame(lrSelp->fromp(), rSelp->fromp())) { if (lrSelp->lsb() == rSelp->lsb() + rSelp->width()) { APPLYING(REPLACE_NESTED_CONCAT_OF_ADJOINING_SELS_ON_RHS) { const uint32_t width = lrSelp->width() + rSelp->width(); const DfgDataType& dtype = DfgDataType::packed(width); DfgSel* const selp = make(flp, dtype, rSelp->fromp(), rSelp->lsb()); replace(make(vtxp, lConcatp->lhsp(), selp)); return; } } } } } } if (DfgConst* const lConstp = lhsp->cast()) { if (DfgCond* const rCondp = rhsp->cast()) { DfgVertex* const rtVtxp = rCondp->thenp(); DfgVertex* const reVtxp = rCondp->elsep(); DfgConst* const rtConstp = rtVtxp->cast(); DfgConst* const reConstp = reVtxp->cast(); if (!rCondp->hasMultipleSinks() && (rtConstp || reConstp)) { APPLYING(PUSH_CONCAT_THROUGH_COND_LHS) { DfgVertex* const thenp = [&]() -> DfgVertex* { FileLine* const rtFlp = rtVtxp->fileline(); if (rtConstp) { DfgConst* const constp = makeZero(rtFlp, vtxp->width()); constp->num().opConcat(lConstp->num(), rtConstp->num()); return constp; } return make(rtFlp, vtxp->dtype(), lConstp, rtVtxp); }(); DfgVertex* const elsep = [&]() -> DfgVertex* { FileLine* const reFlp = reVtxp->fileline(); if (reConstp) { DfgConst* const constp = makeZero(reFlp, vtxp->width()); constp->num().opConcat(lConstp->num(), reConstp->num()); return constp; } return make(reFlp, vtxp->dtype(), lConstp, reVtxp); }(); replace(make(vtxp, rCondp->condp(), thenp, elsep)); return; } } } } if (DfgConst* const rConstp = rhsp->cast()) { if (DfgCond* const lCondp = lhsp->cast()) { DfgVertex* const ltVtxp = lCondp->thenp(); DfgVertex* const leVtxp = lCondp->elsep(); DfgConst* const ltConstp = ltVtxp->cast(); DfgConst* const leConstp = leVtxp->cast(); if (!lCondp->hasMultipleSinks() && (ltConstp || leConstp)) { APPLYING(PUSH_CONCAT_THROUGH_COND_RHS) { DfgVertex* const thenp = [&]() -> DfgVertex* { FileLine* const ltFlp = ltVtxp->fileline(); if (ltConstp) { DfgConst* const constp = makeZero(ltFlp, vtxp->width()); constp->num().opConcat(ltConstp->num(), rConstp->num()); return constp; } return make(ltFlp, vtxp->dtype(), ltVtxp, rConstp); }(); DfgVertex* const elsep = [&]() -> DfgVertex* { FileLine* const leFlp = leVtxp->fileline(); if (leConstp) { DfgConst* const constp = makeZero(leFlp, vtxp->width()); constp->num().opConcat(leConstp->num(), rConstp->num()); return constp; } return make(leFlp, vtxp->dtype(), leVtxp, rConstp); }(); replace(make(vtxp, lCondp->condp(), thenp, elsep)); return; } } } } // Attempt to narrow a concatenation that produces unused bits on the edges { const uint32_t vMsb = vtxp->width() - 1; // MSB of the concatenation const uint32_t lLsb = vtxp->rhsp()->width(); // LSB of the LHS const uint32_t rMsb = lLsb - 1; // MSB of the RHS // Check each sink, and record the range of bits used by them uint32_t lsb = vMsb; // LSB used by a sink uint32_t msb = 0; // MSB used by a sink bool hasCrossSink = false; // True if some sinks use bits from both sides vtxp->foreachSink([&](DfgVertex& sink) { // Record bits used by DfgSel sinks if (const DfgSel* const selp = sink.cast()) { const uint32_t selLsb = selp->lsb(); const uint32_t selMsb = selLsb + selp->width() - 1; lsb = std::min(lsb, selLsb); msb = std::max(msb, selMsb); hasCrossSink |= selMsb >= lLsb && rMsb >= selLsb; return false; } // Ignore non-observable variable sinks. These will be eliminated. if (const DfgVarPacked* const varp = sink.cast()) { if (!varp->hasSinks() && !varp->isObserved()) return false; } // Otherwise the whole value is used lsb = 0; msb = vMsb; return true; }); if (hasCrossSink && (vMsb > msb || lsb > 0)) { APPLYING(NARROW_CONCAT) { // Narrowed RHS DfgVertex* nRhsp = rhsp; if (lsb != 0) nRhsp = make(flp, DfgDataType::packed(rMsb - lsb + 1), rhsp, lsb); // Narrowed LHS DfgVertex* nLhsp = lhsp; if (msb != vMsb) nLhsp = make(flp, DfgDataType::packed(msb - lLsb + 1), lhsp, 0U); // Narrowed concatenation DfgVertex* const catp = make(flp, DfgDataType::packed(msb - lsb + 1), nLhsp, nRhsp); // Need to insert via a partial splice to avoid infinite matching, // this splice will be eliminated on later visits to its sinks. DfgVertex::ScopeCache scopeCache; DfgSplicePacked* const sp = new DfgSplicePacked{m_dfg, flp, vtxp->dtype()}; sp->addDriver(catp, lsb, flp); m_vInfo[sp].m_id = ++m_lastId; replace(sp); return; } } } } void visit(DfgDiv* const vtxp) override { if (foldBinary(vtxp)) return; } void visit(DfgDivS* const vtxp) override { if (foldBinary(vtxp)) return; } void visit(DfgEq* const vtxp) override { if (foldBinary(vtxp)) return; if (commutativeBinary(vtxp)) return; DfgVertex* const lhsp = vtxp->lhsp(); DfgVertex* const rhsp = vtxp->rhsp(); FileLine* const flp = vtxp->fileline(); if (isSame(lhsp, rhsp)) { APPLYING(FOLD_SELF_EQ) { replace(makeOnes(flp, 1)); return; } } if (DfgConst* const lhsConstp = lhsp->cast()) { if (DfgConcat* const rhsConcatp = rhsp->cast()) { if (tryPushCompareOpThroughConcat(vtxp, lhsConstp, rhsConcatp)) return; } } } void visit(DfgGt* const vtxp) override { if (foldBinary(vtxp)) return; DfgVertex* const lhsp = vtxp->lhsp(); DfgVertex* const rhsp = vtxp->rhsp(); FileLine* const flp = vtxp->fileline(); if (isSame(lhsp, rhsp)) { APPLYING(FOLD_SELF_GT) { replace(makeZero(flp, 1)); return; } } } void visit(DfgGtS* const vtxp) override { if (foldBinary(vtxp)) return; DfgVertex* const lhsp = vtxp->lhsp(); DfgVertex* const rhsp = vtxp->rhsp(); FileLine* const flp = vtxp->fileline(); if (isSame(lhsp, rhsp)) { APPLYING(FOLD_SELF_GTS) { replace(makeZero(flp, 1)); return; } } } void visit(DfgGte* const vtxp) override { if (foldBinary(vtxp)) return; DfgVertex* const lhsp = vtxp->lhsp(); DfgVertex* const rhsp = vtxp->rhsp(); FileLine* const flp = vtxp->fileline(); if (isSame(lhsp, rhsp)) { APPLYING(FOLD_SELF_GTE) { replace(makeOnes(flp, 1)); return; } } } void visit(DfgGteS* const vtxp) override { if (foldBinary(vtxp)) return; DfgVertex* const lhsp = vtxp->lhsp(); DfgVertex* const rhsp = vtxp->rhsp(); FileLine* const flp = vtxp->fileline(); if (isSame(lhsp, rhsp)) { APPLYING(FOLD_SELF_GTES) { replace(makeOnes(flp, 1)); return; } } } void visit(DfgLogAnd* const vtxp) override { if (foldBinary(vtxp)) return; DfgVertex* const lhsp = vtxp->lhsp(); DfgVertex* const rhsp = vtxp->rhsp(); if (lhsp->width() == 1 && rhsp->width() == 1) { APPLYING(REPLACE_LOGAND_WITH_AND) { replace(make(vtxp, lhsp, rhsp)); return; } } } void visit(DfgLogEq* const vtxp) override { if (foldBinary(vtxp)) return; } void visit(DfgLogIf* const vtxp) override { if (foldBinary(vtxp)) return; } void visit(DfgLogOr* const vtxp) override { if (foldBinary(vtxp)) return; DfgVertex* const lhsp = vtxp->lhsp(); DfgVertex* const rhsp = vtxp->rhsp(); if (lhsp->width() == 1 && rhsp->width() == 1) { APPLYING(REPLACE_LOGOR_WITH_OR) { replace(make(vtxp, lhsp, rhsp)); return; } } } void visit(DfgLt* const vtxp) override { if (foldBinary(vtxp)) return; DfgVertex* const lhsp = vtxp->lhsp(); DfgVertex* const rhsp = vtxp->rhsp(); FileLine* const flp = vtxp->fileline(); if (isSame(lhsp, rhsp)) { APPLYING(FOLD_SELF_LT) { replace(makeZero(flp, 1)); return; } } } void visit(DfgLtS* const vtxp) override { if (foldBinary(vtxp)) return; DfgVertex* const lhsp = vtxp->lhsp(); DfgVertex* const rhsp = vtxp->rhsp(); FileLine* const flp = vtxp->fileline(); if (isSame(lhsp, rhsp)) { APPLYING(FOLD_SELF_LTS) { replace(makeZero(flp, 1)); return; } } } void visit(DfgLte* const vtxp) override { if (foldBinary(vtxp)) return; DfgVertex* const lhsp = vtxp->lhsp(); DfgVertex* const rhsp = vtxp->rhsp(); FileLine* const flp = vtxp->fileline(); if (isSame(lhsp, rhsp)) { APPLYING(FOLD_SELF_LTE) { replace(makeOnes(flp, 1)); return; } } } void visit(DfgLteS* const vtxp) override { if (foldBinary(vtxp)) return; DfgVertex* const lhsp = vtxp->lhsp(); DfgVertex* const rhsp = vtxp->rhsp(); FileLine* const flp = vtxp->fileline(); if (isSame(lhsp, rhsp)) { APPLYING(FOLD_SELF_LTES) { replace(makeOnes(flp, 1)); return; } } } void visit(DfgModDiv* const vtxp) override { if (foldBinary(vtxp)) return; } void visit(DfgModDivS* const vtxp) override { if (foldBinary(vtxp)) return; } void visit(DfgMul* const vtxp) override { if (associativeBinary(vtxp)) return; if (commutativeBinary(vtxp)) return; } void visit(DfgMulS* const vtxp) override { if (associativeBinary(vtxp)) return; if (commutativeBinary(vtxp)) return; } void visit(DfgNeq* const vtxp) override { if (foldBinary(vtxp)) return; DfgVertex* const lhsp = vtxp->lhsp(); DfgVertex* const rhsp = vtxp->rhsp(); FileLine* const flp = vtxp->fileline(); if (isSame(lhsp, rhsp)) { APPLYING(FOLD_SELF_NEQ) { replace(makeZero(flp, 1)); return; } } } void visit(DfgPow* const vtxp) override { if (foldBinary(vtxp)) return; } void visit(DfgPowSS* const vtxp) override { if (foldBinary(vtxp)) return; } void visit(DfgPowSU* const vtxp) override { if (foldBinary(vtxp)) return; } void visit(DfgPowUS* const vtxp) override { if (foldBinary(vtxp)) return; } void visit(DfgReplicate* const vtxp) override { if (vtxp->dtype() == vtxp->srcp()->dtype()) { APPLYING(REMOVE_REPLICATE_ONCE) { replace(vtxp->srcp()); return; } } if (foldBinary(vtxp)) return; } void visit(DfgShiftL* const vtxp) override { if (foldBinary(vtxp)) return; if (optimizeShiftRHS(vtxp)) return; DfgVertex* const lhsp = vtxp->lhsp(); DfgVertex* const rhsp = vtxp->rhsp(); if (DfgShiftL* const lShiftLp = lhsp->cast()) { if (!lShiftLp->hasMultipleSinks() && rhsp->dtype() == lShiftLp->rhsp()->dtype()) { APPLYING(REPLACE_SHIFTL_SHIFTL) { DfgAdd* const addp = make(rhsp, rhsp, lShiftLp->rhsp()); replace(make(vtxp, lShiftLp->lhsp(), addp)); return; } } } if (DfgConst* const rConstp = rhsp->cast()) { const uint32_t shiftAmount = rConstp->toU32(); if (shiftAmount == 0) { APPLYING(REMOVE_SHIFTL_ZERO) { replace(lhsp); return; } } if (shiftAmount >= vtxp->width()) { APPLYING(REPLACE_SHIFTL_OVER) { replace(makeZero(vtxp->fileline(), vtxp->width())); return; } } if (DfgConcat* const lConcatp = lhsp->cast()) { if (!lConcatp->hasMultipleSinks()) { APPLYING(REPLACE_SHIFTL_CAT) { DfgVertex* const lRhsp = lConcatp->rhsp(); DfgVertex* const lLhsp = lConcatp->lhsp(); // Compute widths of 3 possible result parts const uint32_t rW = shiftAmount; const uint32_t mW = std::min(lRhsp->width(), vtxp->width() - shiftAmount); const uint32_t lW = vtxp->width() - mW - rW; // Construct the result FileLine* const flp = lConcatp->fileline(); DfgVertex* const rp = makeZero(flp, shiftAmount); DfgVertex* const mp = make(flp, DfgDataType::packed(mW), lRhsp, 0U); DfgVertex* np = make(flp, DfgDataType::packed(mW + rW), mp, rp); if (!lW) { replace(np); return; } DfgVertex* const lp = make(flp, DfgDataType::packed(lW), lLhsp, 0U); np = make(vtxp, lp, np); replace(np); return; } } } if (DfgSel* const lSelp = lhsp->cast()) { if (!lSelp->hasMultipleSinks()) { APPLYING(REPLACE_SHIFTL_SEL) { const uint32_t nSelWidth = lSelp->width() - shiftAmount; DfgVertex* const nSelp = make(lSelp->fileline(), DfgDataType::packed(nSelWidth), lSelp->fromp(), lSelp->lsb()); replace( make(vtxp, nSelp, makeZero(vtxp->fileline(), shiftAmount))); return; } } } if (DfgCond* const lCondp = lhsp->cast()) { if (!lCondp->hasMultipleSinks()) { APPLYING(PUSH_SHIFTL_THROUGH_COND) { DfgShiftL* const tp = make(vtxp, lCondp->thenp(), rConstp); DfgShiftL* const ep = make(vtxp, lCondp->elsep(), rConstp); replace(make(vtxp, lCondp->condp(), tp, ep)); return; } } } } } void visit(DfgShiftR* const vtxp) override { if (foldBinary(vtxp)) return; if (optimizeShiftRHS(vtxp)) return; DfgVertex* const lhsp = vtxp->lhsp(); DfgVertex* const rhsp = vtxp->rhsp(); if (DfgShiftR* const lShiftRp = lhsp->cast()) { if (!lShiftRp->hasMultipleSinks() && rhsp->dtype() == lShiftRp->rhsp()->dtype()) { APPLYING(REPLACE_SHIFTR_SHIFTR) { DfgAdd* const addp = make(rhsp, rhsp, lShiftRp->rhsp()); replace(make(vtxp, lShiftRp->lhsp(), addp)); return; } } } if (DfgConst* const rConstp = rhsp->cast()) { const uint32_t shiftAmount = rConstp->toU32(); if (shiftAmount == 0) { APPLYING(REMOVE_SHIFTR_ZERO) { replace(lhsp); return; } } if (shiftAmount >= vtxp->width()) { APPLYING(REPLACE_SHIFTR_OVER) { replace(makeZero(vtxp->fileline(), vtxp->width())); return; } } if (DfgConcat* const lConcatp = lhsp->cast()) { if (!lConcatp->hasMultipleSinks()) { APPLYING(REPLACE_SHIFTR_CAT) { DfgVertex* const lRhsp = lConcatp->rhsp(); DfgVertex* const lLhsp = lConcatp->lhsp(); // Compute widths of 3 possible result parts const uint32_t lW = shiftAmount; const uint32_t mW = std::min(lLhsp->width(), vtxp->width() - shiftAmount); const uint32_t rW = vtxp->width() - mW - lW; // Construct the result FileLine* const flp = lConcatp->fileline(); DfgVertex* const lp = makeZero(flp, shiftAmount); DfgVertex* const mp = make(flp, DfgDataType::packed(mW), lLhsp, lLhsp->width() - mW); DfgVertex* np = make(flp, DfgDataType::packed(lW + mW), lp, mp); if (!rW) { replace(np); return; } DfgVertex* const rp = make(flp, DfgDataType::packed(rW), lRhsp, lRhsp->width() - rW); np = make(vtxp, np, rp); replace(np); return; } } } if (DfgSel* const lSelp = lhsp->cast()) { if (!lSelp->hasMultipleSinks()) { APPLYING(REPLACE_SHIFTR_SEL) { const uint32_t nSelWidth = lSelp->width() - shiftAmount; DfgVertex* const nSelp = make(lSelp->fileline(), DfgDataType::packed(nSelWidth), lSelp->fromp(), lSelp->lsb() + shiftAmount); replace( make(vtxp, makeZero(vtxp->fileline(), shiftAmount), nSelp)); return; } } } if (DfgCond* const lCondp = lhsp->cast()) { if (!lCondp->hasMultipleSinks()) { APPLYING(PUSH_SHIFTR_THROUGH_COND) { DfgShiftR* const tp = make(vtxp, lCondp->thenp(), rConstp); DfgShiftR* const ep = make(vtxp, lCondp->elsep(), rConstp); replace(make(vtxp, lCondp->condp(), tp, ep)); return; } } } } } void visit(DfgShiftRS* const vtxp) override { if (foldBinary(vtxp)) return; if (optimizeShiftRHS(vtxp)) return; } void visit(DfgSub* const vtxp) override { if (foldBinary(vtxp)) return; DfgVertex* const lhsp = vtxp->lhsp(); DfgVertex* const rhsp = vtxp->rhsp(); if (DfgConst* const rConstp = rhsp->cast()) { if (rConstp->isZero()) { APPLYING(REMOVE_SUB_ZERO) { replace(lhsp); return; } } if (vtxp->dtype() == m_bitDType && rConstp->hasValue(1)) { APPLYING(REPLACE_SUB_WITH_NOT) { replace(make(vtxp->fileline(), m_bitDType, lhsp)); return; } } } } //========================================================================= // DfgVertexTernary //========================================================================= void visit(DfgCond* const vtxp) override { DfgVertex* const condp = vtxp->condp(); DfgVertex* const thenp = vtxp->thenp(); DfgVertex* const elsep = vtxp->elsep(); FileLine* const flp = vtxp->fileline(); if (condp->dtype() != m_bitDType) return; if (isOnes(condp)) { APPLYING(REMOVE_COND_WITH_TRUE_CONDITION) { replace(thenp); return; } } if (isZero(condp)) { APPLYING(REMOVE_COND_WITH_FALSE_CONDITION) { replace(elsep); return; } } if (isSame(thenp, elsep)) { APPLYING(REMOVE_COND_WITH_BRANCHES_SAME) { replace(elsep); return; } } if (DfgNot* const condNotp = condp->cast()) { if (!condp->hasMultipleSinks() || condNotp->hasMultipleSinks()) { APPLYING(SWAP_COND_WITH_NOT_CONDITION) { replace(make(vtxp, condNotp->srcp(), elsep, thenp)); return; } } } if (DfgNeq* const condNeqp = condp->cast()) { if (!condp->hasMultipleSinks()) { APPLYING(SWAP_COND_WITH_NEQ_CONDITION) { DfgEq* const newCondp = make(condp, condNeqp->lhsp(), condNeqp->rhsp()); replace(make(vtxp, newCondp, elsep, thenp)); return; } } } if (DfgNot* const thenNotp = thenp->cast()) { if (DfgNot* const elseNotp = elsep->cast()) { if (!thenNotp->srcp()->is() && !elseNotp->srcp()->is() && !thenNotp->hasMultipleSinks() && !elseNotp->hasMultipleSinks()) { APPLYING(PULL_NOTS_THROUGH_COND) { DfgCond* const newCondp = make( vtxp, vtxp->condp(), thenNotp->srcp(), elseNotp->srcp()); replace(make(thenp->fileline(), vtxp->dtype(), newCondp)); return; } } } } if (DfgOr* const condOrp = condp->cast()) { if (DfgCond* const thenCondp = thenp->cast()) { if (!thenCondp->hasMultipleSinks()) { if (condOrp->lhsp() == thenCondp->condp()) { // '(a | b) ? (a ? x : y) : z' -> 'a ? x : b ? y : z' APPLYING(REPLACE_COND_OR_THEN_COND_LHS) { DfgCond* const ep = make(thenCondp, condOrp->rhsp(), thenCondp->elsep(), elsep); replace(make(vtxp, condOrp->lhsp(), thenCondp->thenp(), ep)); return; } } if (condOrp->rhsp() == thenCondp->condp()) { // '(a | b) ? (a ? x : y) : z' -> 'a ? x : b ? y : z' APPLYING(REPLACE_COND_OR_THEN_COND_RHS) { DfgCond* const ep = make(thenCondp, condOrp->lhsp(), thenCondp->elsep(), elsep); replace(make(vtxp, condOrp->rhsp(), thenCondp->thenp(), ep)); return; } } } } } if (vtxp->width() > 1) { // 'cond ? a + 1 : a' -> 'a + cond' if (DfgAdd* const thenAddp = thenp->cast()) { if (DfgConst* const constp = thenAddp->lhsp()->cast()) { if (constp->hasValue(1)) { if (thenAddp->rhsp() == elsep) { APPLYING(REPLACE_COND_INC) { DfgConcat* const extp = make( vtxp, makeZero(flp, vtxp->width() - 1), condp); FileLine* const thenFlp = thenAddp->fileline(); replace( make(thenFlp, vtxp->dtype(), thenAddp->rhsp(), extp)); return; } } } } } // 'cond ? a - 1 : a' -> 'a - cond' if (DfgSub* const thenSubp = thenp->cast()) { if (DfgConst* const constp = thenSubp->rhsp()->cast()) { if (constp->hasValue(1)) { if (thenSubp->lhsp() == elsep) { APPLYING(REPLACE_COND_DEC) { DfgConcat* const extp = make( vtxp, makeZero(flp, vtxp->width() - 1), condp); FileLine* const thenFlp = thenSubp->fileline(); replace( make(thenFlp, vtxp->dtype(), thenSubp->lhsp(), extp)); return; } } } } } } if (vtxp->dtype() == m_bitDType) { if (isZero(thenp)) { // a ? 0 : b becomes ~a & b APPLYING(REPLACE_COND_WITH_THEN_BRANCH_ZERO) { replace(make(vtxp, make(vtxp, condp), elsep)); return; } } if (thenp == condp) { // a ? a : b becomes a | b APPLYING(REPLACE_COND_WITH_THEN_BRANCH_COND) { replace(make(vtxp, condp, elsep)); return; } } if (elsep == condp) { // a ? b : a becomes a & b APPLYING(REPLACE_COND_WITH_ELSE_BRANCH_COND) { replace(make(vtxp, condp, thenp)); return; } } if (isOnes(thenp)) { // a ? 1 : b becomes a | b APPLYING(REPLACE_COND_WITH_THEN_BRANCH_ONES) { replace(make(vtxp, condp, elsep)); return; } } if (isZero(elsep)) { // a ? b : 0 becomes a & b APPLYING(REPLACE_COND_WITH_ELSE_BRANCH_ZERO) { replace(make(vtxp, condp, thenp)); return; } } if (isOnes(elsep)) { // a ? b : 1 becomes ~a | b APPLYING(REPLACE_COND_WITH_ELSE_BRANCH_ONES) { replace(make(vtxp, make(vtxp, condp), thenp)); return; } } } if (DfgConcat* const tConcatp = thenp->cast()) { if (DfgConcat* const eConcatp = elsep->cast()) { DfgVertex* const tRhsp = tConcatp->rhsp(); DfgVertex* const tLhsp = tConcatp->lhsp(); DfgVertex* const eRhsp = eConcatp->rhsp(); DfgVertex* const eLhsp = eConcatp->lhsp(); if (isSame(tRhsp, eRhsp)) { APPLYING(REPLACE_COND_SAME_CAT_RHS) { DfgCond* const newCondp = make(flp, tLhsp->dtype(), condp, tLhsp, eLhsp); replace(make(vtxp, newCondp, tRhsp)); return; } } if (isSame(tLhsp, eLhsp)) { APPLYING(REPLACE_COND_SAME_CAT_LHS) { DfgCond* const newCondp = make(flp, tRhsp->dtype(), condp, tRhsp, eRhsp); replace(make(vtxp, tLhsp, newCondp)); return; } } } } if (isZero(elsep) && isEqOne(thenp)) { APPLYING(REPLACE_COND_CONST_ONE_ZERO) { DfgVertex* resp = condp; if (const uint32_t extend = vtxp->width() - 1) { DfgConst* const zerop = makeZero(flp, extend); resp = make(vtxp, zerop, resp); } replace(resp); return; } } if (isZero(thenp) && isEqOne(elsep)) { APPLYING(REPLACE_COND_CONST_ZERO_ONE) { DfgVertex* resp = make(condp, condp); if (const uint32_t extend = vtxp->width() - 1) { DfgConst* const zerop = makeZero(vtxp->fileline(), extend); resp = make(vtxp, zerop, resp); } replace(resp); return; } } if (DfgCond* const tCondp = thenp->cast()) { if (isSame(condp, tCondp->condp())) { APPLYING(REPLACE_COND_SAME_COND_THEN) { replace(make(vtxp, condp, tCondp->thenp(), elsep)); return; } } } if (DfgCond* const eCondp = elsep->cast()) { if (isSame(condp, eCondp->condp())) { APPLYING(REPLACE_COND_SAME_COND_ELSE) { replace(make(vtxp, condp, thenp, eCondp->elsep())); return; } } } } void visit(DfgVertexVar* const vtxp) override { if (vtxp->hasSinks()) return; if (vtxp->isObserved()) return; if (vtxp->defaultp()) return; // If undriven, or driven from another var, it is completely redundant. if (!vtxp->srcp() || vtxp->srcp()->is()) { APPLYING(REMOVE_VAR) { deleteVertex(vtxp); return; } } // Otherwise remove if there is only one sink that is not a removable variable bool foundOne = false; const bool keep = vtxp->srcp()->foreachSink([&](DfgVertex& sink) { // Ignore non-observable variable sinks. These can be eliminated. if (const DfgVertexVar* const varp = sink.cast()) { if (!varp->hasSinks() && !varp->isObserved()) return false; } if (foundOne) return true; foundOne = true; return false; }); if (!keep) { APPLYING(REMOVE_VAR) { deleteVertex(vtxp); return; } } } V3DfgPeephole(DfgGraph& dfg, V3DfgPeepholeContext& ctx) : m_dfg{dfg} , m_ctx{ctx} { // Assign vertex IDs m_dfg.forEachVertex([&](DfgVertex& vtx) { m_vInfo[vtx].m_id = ++m_lastId; }); // Initialize the work list and iter list. They can't get bigger than // m_dfg.size(), but new vertices are created in the loop, so over alloacte m_workList.reserve(m_dfg.size() * 2); m_iterList.reserve(m_dfg.size() * 2); // Need a nullptr at index 0 so VertexInfo::m_*ListIndex == 0 can check membership m_workList.push_back(nullptr); m_iterList.push_back(nullptr); // Add all variable vertices to the work list. Do this first so they are processed // last. This order has a better chance of preserving original variables in case // they are needed to hold intermediate results. for (DfgVertexVar& vtx : m_dfg.varVertices()) addToWorkList(&vtx); // Add all operation vertices to the work list for (DfgVertex& vtx : m_dfg.opVertices()) addToWorkList(&vtx); // Process iteratively while (true) { // Process the work list - keep the placeholder at index 0 while (m_workList.size() > 1) { VL_RESTORER(m_vtxp); // Pop up head of work list m_vtxp = m_workList.back(); m_workList.pop_back(); if (!m_vtxp) continue; // If removed from worklist (deleted), move on m_vInfo[m_vtxp].m_workListIndex = 0; // No longer on work list // Variables are special, just visit them, the visit might delete them if (DfgVertexVar* const varp = m_vtxp->cast()) { visit(varp); continue; } // Unsued vertices should have been removed immediately UASSERT_OBJ(m_vtxp->hasSinks(), m_vtxp, "Operation vertex should have sinks"); // Check if an equivalent vertex exists, if so replace this vertex with it if (DfgVertex* const sampep = m_cache.cache(m_vtxp)) { APPLYING(REPLACE_WITH_EQUIVALENT) { replace(sampep); continue; } } // Visit vertex, might get deleted in the process iterate(m_vtxp); } // If nothing was added to the iter list, we can stop if (m_iterList.size() == 1) break; // Expand the iter list to visit the whole neighborhood of vertices // within a fixed number of hops from the enqueued vertices. This // enables patterns that look deeper across the graph to be reconsidered. { size_t begin = 0; size_t end = m_iterList.size(); for (size_t hops = 1; hops <= 4; ++hops) { for (size_t i = begin; i < end; ++i) { DfgVertex* const vtxp = m_iterList[i]; if (!vtxp) continue; vtxp->foreachSink([&](DfgVertex& dst) { addToIterList(&dst); return false; }); vtxp->foreachSource([&](DfgVertex& src) { addToIterList(&src); return false; }); } begin = end; end = m_iterList.size(); } } // Move vertices in the iter list to the work list for the next iteration for (DfgVertex* const vtxp : m_iterList) { if (!vtxp) continue; removeFromIterList(vtxp); addToWorkList(vtxp); } // Reset the iter list m_iterList.resize(1); } } #undef APPLYING public: static void apply(DfgGraph& dfg, V3DfgPeepholeContext& ctx) { V3DfgPeephole{dfg, ctx}; } }; V3DebugBisect V3DfgPeephole::s_debugBisect{"DfgPeephole"}; void V3DfgPasses::peephole(DfgGraph& dfg, V3DfgPeepholeContext& ctx) { if (!v3Global.opt.fDfgPeephole()) return; V3DfgPeephole::apply(dfg, ctx); }