diff --git a/src/V3Dfg.h b/src/V3Dfg.h index ecc4d421a..d1378e34a 100644 --- a/src/V3Dfg.h +++ b/src/V3Dfg.h @@ -826,6 +826,16 @@ bool DfgVertex::isCheaperThanLoad() const { if (!lCatp->isZero()) return false; return catp->rhsp()->isCheaperThanLoad(); } + // Reduction of a cheap vertex + if (const DfgRedOr* const redOrp = cast()) { + return redOrp->srcp()->isCheaperThanLoad(); + } + if (const DfgRedAnd* const redAndp = cast()) { + return redAndp->srcp()->isCheaperThanLoad(); + } + if (const DfgRedXor* const redXorp = cast()) { + return redXorp->srcp()->isCheaperThanLoad(); + } // Otherwise probably not return false; } diff --git a/src/V3DfgPeephole.cpp b/src/V3DfgPeephole.cpp index 8d6a8549a..7c09632a6 100644 --- a/src/V3DfgPeephole.cpp +++ b/src/V3DfgPeephole.cpp @@ -353,6 +353,7 @@ class V3DfgPeephole final : public DfgVisitor { // Check two vertex are the same, or the same constant value static bool isSame(const DfgVertex* ap, const DfgVertex* bp) { if (ap == bp) return true; + if (ap->dtype() != bp->dtype()) return false; const DfgConst* const aConstp = ap->cast(); if (!aConstp) return false; const DfgConst* const bConstp = bp->cast(); @@ -370,6 +371,24 @@ class V3DfgPeephole final : public DfgVisitor { return false; } + static bool isEqOne(const DfgVertex* vtxp) { + if (const DfgConst* const constp = vtxp->cast()) return constp->num().isEqOne(); + return false; + } + + static bool areAdjacent(uint32_t& lsb, const DfgSel* lSelp, const DfgSel* rSelp) { + if (!isSame(lSelp->srcp(), rSelp->srcp())) return false; + if (lSelp->lsb() + lSelp->width() == rSelp->lsb()) { + lsb = lSelp->lsb(); + return true; + } + if (lSelp->lsb() == rSelp->lsb() + rSelp->width()) { + lsb = rSelp->lsb(); + return true; + } + return false; + } + // Note: If any of the following transformers return true, then the vertex was replaced and the // caller must not do any further changes, so the caller must check the return value, otherwise // there will be hard to debug issues. @@ -472,22 +491,26 @@ class V3DfgPeephole final : public DfgVisitor { // Make associative trees right leaning to reduce pattern variations, and for better CSE if (Vertex* const alhsp = vtxp->lhsp()->template cast()) { if (!alhsp->hasMultipleSinks()) { - APPLYING(RIGHT_LEANING_ASSOC) { - // Rotate the expression tree rooted at 'vtxp' to the right, producing a - // right-leaning tree - DfgVertex* const ap = alhsp->lhsp(); - DfgVertex* const bp = alhsp->rhsp(); - DfgVertex* const cp = vtxp->rhsp(); + DfgVertex* const ap = alhsp->lhsp(); + DfgVertex* const bp = alhsp->rhsp(); + DfgVertex* const cp = vtxp->rhsp(); + // Only do this if the rhs is not th same as the operands of the LHS, otherwise + // SWAP_SIDE_IN_COMMUTATIVE_BINARY can get in a loop with this pattern. + if (ap != cp && bp != cp) { + APPLYING(RIGHT_LEANING_ASSOC) { + // Rotate the expression tree rooted at 'vtxp' to the right, + // producing a right-leaning tree - // Concatenation dtypes need adjusting, other assoc vertices preserve types - const DfgDataType& childDType - = std::is_same::value - ? DfgDataType::packed(bp->width() + cp->width()) - : vtxp->dtype(); + // Concatenation dtypes need adjusting, other assoc vertices preserve types + const DfgDataType& childDType + = std::is_same::value + ? DfgDataType::packed(bp->width() + cp->width()) + : vtxp->dtype(); - Vertex* const bcp = make(vtxp->fileline(), childDType, bp, cp); - replace(make(alhsp->fileline(), vtxp->dtype(), ap, bcp)); - return true; + Vertex* const bcp = make(vtxp->fileline(), childDType, bp, cp); + replace(make(alhsp->fileline(), vtxp->dtype(), ap, bcp)); + return true; + } } } } @@ -543,6 +566,21 @@ class V3DfgPeephole final : public DfgVisitor { DfgVertex* const lhsp = vtxp->lhsp(); DfgVertex* const rhsp = vtxp->rhsp(); + // If constant LHS, push through cond if used once. (Enables branch combining) + if (DfgConst* const lConstp = lhsp->cast()) { + if (DfgCond* const rCondp = rhsp->cast()) { + if (!rCondp->hasMultipleSinks()) { + APPLYING(PUSH_COMMUTATIVE_BINARY_THROUGH_COND) { + DfgVertex* const tp = make(vtxp, lConstp, rCondp->thenp()); + DfgVertex* const ep = make(vtxp, lConstp, rCondp->elsep()); + replace(make(vtxp, rCondp->condp(), tp, ep)); + return true; + } + } + } + return false; + } + // Ensure Const is on left-hand side to simplify other patterns { const bool lIsConst = lhsp->is(); @@ -738,7 +776,8 @@ class V3DfgPeephole final : public DfgVisitor { if (Reduction* const rRedp = vtxp->rhsp()->template cast()) { DfgVertex* const lSrcp = lRedp->srcp(); DfgVertex* const rSrcp = rRedp->srcp(); - if (lSrcp->dtype() == rSrcp->dtype() && lSrcp->width() <= 64 + if (lSrcp->dtype() == rSrcp->dtype() && lSrcp->width() <= VL_QUADSIZE + && !lSrcp->is() && !rSrcp->is() && !lSrcp->hasMultipleSinks() && !rSrcp->hasMultipleSinks()) { APPLYING(PUSH_BITWISE_THROUGH_REDUCTION) { FileLine* const flp = vtxp->fileline(); @@ -753,6 +792,73 @@ class V3DfgPeephole final : public DfgVisitor { return false; } + template + VL_ATTR_WARN_UNUSED_RESULT bool tryReplaceBitwiseWithReduction(Bitwise* vtxp) { + UASSERT_OBJ(vtxp->width() == 1, vtxp, "Width must be 1"); + using Reduction = BitwiseToReduction; + + DfgVertex* const lhsp = vtxp->lhsp(); + DfgVertex* const rhsp = vtxp->rhsp(); + + if (DfgSel* const lSelp = lhsp->template cast()) { + DfgSel* rSelp = rhsp->template cast(); + DfgVertex* extrap = nullptr; + if (!rSelp) { + if (Bitwise* const rBitwisep = rhsp->template cast()) { + rSelp = rBitwisep->lhsp()->template cast(); + extrap = rBitwisep->rhsp(); + } + } + if (rSelp) { + uint32_t lsb = 0; + if (areAdjacent(lsb, lSelp, rSelp)) { + APPLYING(REPLACE_BITWISE_OF_SELS_WITH_REDUCTION) { + const DfgDataType& dtype + = DfgDataType::packed(lSelp->width() + rSelp->width()); + DfgSel* const newSelp + = make(lSelp->fileline(), dtype, lSelp->srcp(), lsb); + DfgVertex* resp = make(vtxp, newSelp); + if (extrap) resp = make(vtxp, resp, extrap); + replace(resp); + return true; + } + } + } + } + + if (Reduction* const lRedp = lhsp->template cast()) { + Reduction* rRedp = rhsp->template cast(); + DfgVertex* extrap = nullptr; + if (!rRedp) { + if (Bitwise* const rBitwisep = rhsp->template cast()) { + rRedp = rBitwisep->lhsp()->template cast(); + extrap = rBitwisep->rhsp(); + } + } + if (rRedp) { + if (DfgSel* const lSelp = lRedp->srcp()->template cast()) { + if (DfgSel* const rSelp = rRedp->srcp()->template cast()) { + uint32_t lsb = 0; + if (areAdjacent(lsb, lSelp, rSelp)) { + APPLYING(REPLACE_BITWISE_OF_REDUCTION_OF_SELS_WITH_REDUCTION) { + const DfgDataType& dtype + = DfgDataType::packed(lSelp->width() + rSelp->width()); + DfgSel* const newSelp + = make(lSelp->fileline(), dtype, lSelp->srcp(), lsb); + DfgVertex* resp = make(vtxp, newSelp); + if (extrap) resp = make(vtxp, resp, extrap); + replace(resp); + return true; + } + } + } + } + } + } + + return false; + } + template VL_ATTR_WARN_UNUSED_RESULT bool optimizeReduction(Reduction* const vtxp) { using Bitwise = ReductionToBitwise; @@ -791,7 +897,9 @@ class V3DfgPeephole final : public DfgVisitor { } if (DfgConcat* const concatp = srcp->cast()) { - if (concatp->lhsp()->is() || concatp->rhsp()->is()) { + if (concatp->lhsp()->is() || concatp->rhsp()->is() + || concatp->lhsp()->dtype() == m_bitDType + || concatp->rhsp()->dtype() == m_bitDType) { APPLYING(PUSH_REDUCTION_THROUGH_CONCAT) { // Reduce the parts of the concatenation Reduction* const lRedp @@ -806,6 +914,38 @@ class V3DfgPeephole final : public DfgVisitor { } } + if (Bitwise* const bitwisep = vtxp->srcp()->template cast()) { + if (!bitwisep->hasMultipleSinks()) { + if (bitwisep->lhsp()->template is() + || bitwisep->rhsp()->template is()) { + APPLYING(PUSH_REDUCTION_THROUGH_BITWISE_OF_CONCAT) { + Reduction* const newLhsp + = make(flp, m_bitDType, bitwisep->lhsp()); + Reduction* const newRhsp + = make(flp, m_bitDType, bitwisep->rhsp()); + replace(make(flp, m_bitDType, newLhsp, newRhsp)); + return true; + } + } + + if (DfgSel* const lSelp = bitwisep->lhsp()->template cast()) { + if (DfgSel* const rSelp = bitwisep->rhsp()->template cast()) { + uint32_t lsb = 0; + if (areAdjacent(lsb, lSelp, rSelp)) { + APPLYING(PUSH_REDUCTION_THROUGH_BITWISE_OF_SELS) { + const DfgDataType& dtype + = DfgDataType::packed(lSelp->width() + rSelp->width()); + DfgSel* const newSelp + = make(lSelp->fileline(), dtype, lSelp->srcp(), lsb); + replace(make(vtxp, newSelp)); + return true; + } + } + } + } + } + } + return false; } @@ -1011,9 +1151,7 @@ class V3DfgPeephole final : public DfgVisitor { // Sel from Cond if (DfgCond* const condp = fromp->cast()) { - // If at least one of the branches are a constant, push the select past the cond - if (!condp->hasMultipleSinks() - && (condp->thenp()->is() || condp->elsep()->is())) { + if (!condp->hasMultipleSinks()) { APPLYING(PUSH_SEL_THROUGH_COND) { // The new 'then' vertex DfgSel* const newThenp = make(vtxp, condp->thenp(), lsb); @@ -1096,7 +1234,7 @@ class V3DfgPeephole final : public DfgVisitor { FileLine* const flp = vtxp->fileline(); // Bubble pushing (De Morgan) - if (!lhsp->hasMultipleSinks() && !rhsp->hasMultipleSinks()) { + if (!lhsp->hasMultipleSinks() || !rhsp->hasMultipleSinks()) { if (DfgNot* const lhsNotp = lhsp->cast()) { if (DfgNot* const rhsNotp = rhsp->cast()) { APPLYING(REPLACE_AND_OF_NOT_AND_NOT) { @@ -1116,15 +1254,15 @@ class V3DfgPeephole final : public DfgVisitor { } } - if (DfgConst* const lhsConstp = lhsp->cast()) { - if (lhsConstp->isZero()) { + if (DfgConst* const lConstp = lhsp->cast()) { + if (lConstp->isZero()) { APPLYING(REPLACE_AND_WITH_ZERO) { - replace(lhsConstp); + replace(lConstp); return; } } - if (lhsConstp->isOnes()) { + if (lConstp->isOnes()) { APPLYING(REMOVE_AND_WITH_ONES) { replace(rhsp); return; @@ -1132,7 +1270,7 @@ class V3DfgPeephole final : public DfgVisitor { } if (DfgConcat* const rhsConcatp = rhsp->cast()) { - if (tryPushBitwiseOpThroughConcat(vtxp, lhsConstp, rhsConcatp)) return; + if (tryPushBitwiseOpThroughConcat(vtxp, lConstp, rhsConcatp)) return; } } @@ -1159,6 +1297,10 @@ class V3DfgPeephole final : public DfgVisitor { } } } + + if (vtxp->dtype() == m_bitDType) { + if (tryReplaceBitwiseWithReduction(vtxp)) return; + } } void visit(DfgOr* const vtxp) override { @@ -1179,7 +1321,7 @@ class V3DfgPeephole final : public DfgVisitor { FileLine* const flp = vtxp->fileline(); // Bubble pushing (De Morgan) - if (!lhsp->hasMultipleSinks() && !rhsp->hasMultipleSinks()) { + if (!lhsp->hasMultipleSinks() || !rhsp->hasMultipleSinks()) { if (DfgNot* const lhsNotp = lhsp->cast()) { if (DfgNot* const rhsNotp = rhsp->cast()) { APPLYING(REPLACE_OR_OF_NOT_AND_NOT) { @@ -1218,15 +1360,15 @@ class V3DfgPeephole final : public DfgVisitor { } } - if (DfgConst* const lhsConstp = lhsp->cast()) { - if (lhsConstp->isZero()) { + if (DfgConst* const lConstp = lhsp->cast()) { + if (lConstp->isZero()) { APPLYING(REMOVE_OR_WITH_ZERO) { replace(rhsp); return; } } - if (lhsConstp->isOnes()) { + if (lConstp->isOnes()) { APPLYING(REPLACE_OR_WITH_ONES) { replace(lhsp); return; @@ -1234,7 +1376,7 @@ class V3DfgPeephole final : public DfgVisitor { } if (DfgConcat* const rhsConcatp = rhsp->cast()) { - if (tryPushBitwiseOpThroughConcat(vtxp, lhsConstp, rhsConcatp)) return; + if (tryPushBitwiseOpThroughConcat(vtxp, lConstp, rhsConcatp)) return; } } @@ -1265,6 +1407,10 @@ class V3DfgPeephole final : public DfgVisitor { } } } + + if (vtxp->dtype() == m_bitDType) { + if (tryReplaceBitwiseWithReduction(vtxp)) return; + } } void visit(DfgXor* const vtxp) override { @@ -1297,11 +1443,14 @@ class V3DfgPeephole final : public DfgVisitor { } if (DfgConcat* const rConcatp = rhsp->cast()) { if (tryPushBitwiseOpThroughConcat(vtxp, lConstp, rConcatp)) return; - return; } } if (tryPushBitwiseOpThroughReductions(vtxp)) return; + + if (vtxp->dtype() == m_bitDType) { + if (tryReplaceBitwiseWithReduction(vtxp)) return; + } } //========================================================================= @@ -1452,14 +1601,30 @@ class V3DfgPeephole final : public DfgVisitor { if (DfgConst* const lConstp = lhsp->cast()) { if (DfgCond* const rCondp = rhsp->cast()) { - if (!rCondp->hasMultipleSinks()) { - DfgVertex* const rtVtxp = rCondp->thenp(); - DfgVertex* const reVtxp = rCondp->elsep(); + DfgVertex* const rtVtxp = rCondp->thenp(); + DfgVertex* const reVtxp = rCondp->elsep(); + DfgConst* const rtConstp = rtVtxp->cast(); + DfgConst* const reConstp = reVtxp->cast(); + if (!rCondp->hasMultipleSinks() && (rtConstp || reConstp)) { APPLYING(PUSH_CONCAT_THROUGH_COND_LHS) { - DfgConcat* const thenp - = make(rtVtxp->fileline(), vtxp->dtype(), lConstp, rtVtxp); - DfgConcat* const elsep - = make(reVtxp->fileline(), vtxp->dtype(), lConstp, reVtxp); + DfgVertex* const thenp = [&]() -> DfgVertex* { + FileLine* const rtFlp = rtVtxp->fileline(); + if (rtConstp) { + DfgConst* const constp = makeZero(rtFlp, vtxp->width()); + constp->num().opConcat(lConstp->num(), rtConstp->num()); + return constp; + } + return make(rtFlp, vtxp->dtype(), lConstp, rtVtxp); + }(); + DfgVertex* const elsep = [&]() -> DfgVertex* { + FileLine* const reFlp = reVtxp->fileline(); + if (reConstp) { + DfgConst* const constp = makeZero(reFlp, vtxp->width()); + constp->num().opConcat(lConstp->num(), reConstp->num()); + return constp; + } + return make(reFlp, vtxp->dtype(), lConstp, reVtxp); + }(); replace(make(vtxp, rCondp->condp(), thenp, elsep)); return; } @@ -1469,14 +1634,31 @@ class V3DfgPeephole final : public DfgVisitor { if (DfgConst* const rConstp = rhsp->cast()) { if (DfgCond* const lCondp = lhsp->cast()) { - if (!lCondp->hasMultipleSinks()) { - DfgVertex* const ltVtxp = lCondp->thenp(); - DfgVertex* const leVtxp = lCondp->elsep(); + DfgVertex* const ltVtxp = lCondp->thenp(); + DfgVertex* const leVtxp = lCondp->elsep(); + DfgConst* const ltConstp = ltVtxp->cast(); + DfgConst* const leConstp = leVtxp->cast(); + if (!lCondp->hasMultipleSinks() && (ltConstp || leConstp)) { APPLYING(PUSH_CONCAT_THROUGH_COND_RHS) { - DfgConcat* const thenp - = make(ltVtxp->fileline(), vtxp->dtype(), ltVtxp, rConstp); - DfgConcat* const elsep - = make(leVtxp->fileline(), vtxp->dtype(), leVtxp, rConstp); + DfgVertex* const thenp = [&]() -> DfgVertex* { + FileLine* const ltFlp = ltVtxp->fileline(); + if (ltConstp) { + DfgConst* const constp = makeZero(ltFlp, vtxp->width()); + constp->num().opConcat(ltConstp->num(), rConstp->num()); + return constp; + } + return make(ltFlp, vtxp->dtype(), ltVtxp, rConstp); + }(); + DfgVertex* const elsep = [&]() -> DfgVertex* { + FileLine* const leFlp = leVtxp->fileline(); + if (leConstp) { + DfgConst* const constp = makeZero(leFlp, vtxp->width()); + constp->num().opConcat(leConstp->num(), rConstp->num()); + return constp; + } + return make(leFlp, vtxp->dtype(), leVtxp, rConstp); + }(); + replace(make(vtxp, lCondp->condp(), thenp, elsep)); return; } @@ -1688,31 +1870,82 @@ class V3DfgPeephole final : public DfgVisitor { DfgVertex* const lhsp = vtxp->lhsp(); DfgVertex* const rhsp = vtxp->rhsp(); + if (DfgShiftL* const lShiftLp = lhsp->cast()) { + if (!lShiftLp->hasMultipleSinks() && rhsp->dtype() == lShiftLp->rhsp()->dtype()) { + APPLYING(REPLACE_SHIFTL_SHIFTL) { + DfgAdd* const addp = make(rhsp, rhsp, lShiftLp->rhsp()); + replace(make(vtxp, lShiftLp->lhsp(), addp)); + return; + } + } + } + if (DfgConst* const rConstp = rhsp->cast()) { + const uint32_t shiftAmount = rConstp->toU32(); + + if (shiftAmount == 0) { + APPLYING(REMOVE_SHIFTL_ZERO) { + replace(lhsp); + return; + } + } + + if (shiftAmount >= vtxp->width()) { + APPLYING(REPLACE_SHIFTL_OVER) { + replace(makeZero(vtxp->fileline(), vtxp->width())); + return; + } + } + if (DfgConcat* const lConcatp = lhsp->cast()) { - if (!lConcatp->hasMultipleSinks() - && lConcatp->lhsp()->width() == rConstp->toU32()) { + if (!lConcatp->hasMultipleSinks()) { APPLYING(REPLACE_SHIFTL_CAT) { - DfgVertex* const zerop - = makeZero(lConcatp->fileline(), lConcatp->lhsp()->width()); - replace(make(vtxp, lConcatp->rhsp(), zerop)); + DfgVertex* const lRhsp = lConcatp->rhsp(); + DfgVertex* const lLhsp = lConcatp->lhsp(); + // Compute widths of 3 possible result parts + const uint32_t rW = shiftAmount; + const uint32_t mW = std::min(lRhsp->width(), vtxp->width() - shiftAmount); + const uint32_t lW = vtxp->width() - mW - rW; + // Construct the result + FileLine* const flp = lConcatp->fileline(); + DfgVertex* const rp = makeZero(flp, shiftAmount); + DfgVertex* const mp + = make(flp, DfgDataType::packed(mW), lRhsp, 0U); + DfgVertex* np = make(flp, DfgDataType::packed(mW + rW), mp, rp); + if (!lW) { + replace(np); + return; + } + DfgVertex* const lp + = make(flp, DfgDataType::packed(lW), lLhsp, 0U); + np = make(vtxp, lp, np); + replace(np); return; } } } - if (DfgShiftR* const lShiftRp = lhsp->cast()) { - if (!lShiftRp->hasMultipleSinks() && isSame(rConstp, lShiftRp->rhsp())) { - if (DfgConcat* const llCatp = lShiftRp->lhsp()->cast()) { - const uint32_t shiftAmount = rConstp->toU32(); - if (!llCatp->hasMultipleSinks() - && llCatp->rhsp()->width() == shiftAmount) { - APPLYING(REPLACE_SHIFTRL_CAT) { - DfgConst* const zerop = makeZero(llCatp->fileline(), shiftAmount); - replace(make(vtxp, llCatp->lhsp(), zerop)); - return; - } - } + if (DfgSel* const lSelp = lhsp->cast()) { + if (!lSelp->hasMultipleSinks()) { + APPLYING(REPLACE_SHIFTL_SEL) { + const uint32_t nSelWidth = lSelp->width() - shiftAmount; + DfgVertex* const nSelp + = make(lSelp->fileline(), DfgDataType::packed(nSelWidth), + lSelp->fromp(), lSelp->lsb()); + replace( + make(vtxp, nSelp, makeZero(vtxp->fileline(), shiftAmount))); + return; + } + } + } + + if (DfgCond* const lCondp = lhsp->cast()) { + if (!lCondp->hasMultipleSinks()) { + APPLYING(PUSH_SHIFTL_THROUGH_COND) { + DfgShiftL* const tp = make(vtxp, lCondp->thenp(), rConstp); + DfgShiftL* const ep = make(vtxp, lCondp->elsep(), rConstp); + replace(make(vtxp, lCondp->condp(), tp, ep)); + return; } } } @@ -1722,6 +1955,90 @@ class V3DfgPeephole final : public DfgVisitor { void visit(DfgShiftR* const vtxp) override { if (foldBinary(vtxp)) return; if (optimizeShiftRHS(vtxp)) return; + + DfgVertex* const lhsp = vtxp->lhsp(); + DfgVertex* const rhsp = vtxp->rhsp(); + + if (DfgShiftR* const lShiftRp = lhsp->cast()) { + if (!lShiftRp->hasMultipleSinks() && rhsp->dtype() == lShiftRp->rhsp()->dtype()) { + APPLYING(REPLACE_SHIFTR_SHIFTR) { + DfgAdd* const addp = make(rhsp, rhsp, lShiftRp->rhsp()); + replace(make(vtxp, lShiftRp->lhsp(), addp)); + return; + } + } + } + + if (DfgConst* const rConstp = rhsp->cast()) { + const uint32_t shiftAmount = rConstp->toU32(); + + if (shiftAmount == 0) { + APPLYING(REMOVE_SHIFTR_ZERO) { + replace(lhsp); + return; + } + } + + if (shiftAmount >= vtxp->width()) { + APPLYING(REPLACE_SHIFTR_OVER) { + replace(makeZero(vtxp->fileline(), vtxp->width())); + return; + } + } + + if (DfgConcat* const lConcatp = lhsp->cast()) { + if (!lConcatp->hasMultipleSinks()) { + APPLYING(REPLACE_SHIFTR_CAT) { + DfgVertex* const lRhsp = lConcatp->rhsp(); + DfgVertex* const lLhsp = lConcatp->lhsp(); + // Compute widths of 3 possible result parts + const uint32_t lW = shiftAmount; + const uint32_t mW = std::min(lLhsp->width(), vtxp->width() - shiftAmount); + const uint32_t rW = vtxp->width() - mW - lW; + // Construct the result + FileLine* const flp = lConcatp->fileline(); + DfgVertex* const lp = makeZero(flp, shiftAmount); + DfgVertex* const mp = make(flp, DfgDataType::packed(mW), lLhsp, + lLhsp->width() - mW); + DfgVertex* np = make(flp, DfgDataType::packed(lW + mW), lp, mp); + if (!rW) { + replace(np); + return; + } + DfgVertex* const rp = make(flp, DfgDataType::packed(rW), lRhsp, + lRhsp->width() - rW); + np = make(vtxp, np, rp); + replace(np); + return; + } + } + } + + if (DfgSel* const lSelp = lhsp->cast()) { + if (!lSelp->hasMultipleSinks()) { + APPLYING(REPLACE_SHIFTR_SEL) { + const uint32_t nSelWidth = lSelp->width() - shiftAmount; + DfgVertex* const nSelp + = make(lSelp->fileline(), DfgDataType::packed(nSelWidth), + lSelp->fromp(), lSelp->lsb() + shiftAmount); + replace( + make(vtxp, makeZero(vtxp->fileline(), shiftAmount), nSelp)); + return; + } + } + } + + if (DfgCond* const lCondp = lhsp->cast()) { + if (!lCondp->hasMultipleSinks()) { + APPLYING(PUSH_SHIFTR_THROUGH_COND) { + DfgShiftR* const tp = make(vtxp, lCondp->thenp(), rConstp); + DfgShiftR* const ep = make(vtxp, lCondp->elsep(), rConstp); + replace(make(vtxp, lCondp->condp(), tp, ep)); + return; + } + } + } + } } void visit(DfgShiftRS* const vtxp) override { @@ -1892,6 +2209,12 @@ class V3DfgPeephole final : public DfgVisitor { return; } } + if (elsep == condp) { // a ? b : a becomes a & b + APPLYING(REPLACE_COND_WITH_ELSE_BRANCH_COND) { + replace(make(vtxp, condp, thenp)); + return; + } + } if (isOnes(thenp)) { // a ? 1 : b becomes a | b APPLYING(REPLACE_COND_WITH_THEN_BRANCH_ONES) { replace(make(vtxp, condp, elsep)); @@ -1911,6 +2234,75 @@ class V3DfgPeephole final : public DfgVisitor { } } } + + if (DfgConcat* const tConcatp = thenp->cast()) { + if (DfgConcat* const eConcatp = elsep->cast()) { + DfgVertex* const tRhsp = tConcatp->rhsp(); + DfgVertex* const tLhsp = tConcatp->lhsp(); + DfgVertex* const eRhsp = eConcatp->rhsp(); + DfgVertex* const eLhsp = eConcatp->lhsp(); + + if (isSame(tRhsp, eRhsp)) { + APPLYING(REPLACE_COND_SAME_CAT_RHS) { + DfgCond* const newCondp + = make(flp, tLhsp->dtype(), condp, tLhsp, eLhsp); + replace(make(vtxp, newCondp, tRhsp)); + return; + } + } + + if (isSame(tLhsp, eLhsp)) { + APPLYING(REPLACE_COND_SAME_CAT_LHS) { + DfgCond* const newCondp + = make(flp, tRhsp->dtype(), condp, tRhsp, eRhsp); + replace(make(vtxp, tLhsp, newCondp)); + return; + } + } + } + } + + if (isZero(elsep) && isEqOne(thenp)) { + APPLYING(REPLACE_COND_CONST_ONE_ZERO) { + DfgVertex* resp = condp; + if (const uint32_t extend = vtxp->width() - 1) { + DfgConst* const zerop = makeZero(flp, extend); + resp = make(vtxp, zerop, resp); + } + replace(resp); + return; + } + } + + if (isZero(thenp) && isEqOne(elsep)) { + APPLYING(REPLACE_COND_CONST_ZERO_ONE) { + DfgVertex* resp = make(condp, condp); + if (const uint32_t extend = vtxp->width() - 1) { + DfgConst* const zerop = makeZero(vtxp->fileline(), extend); + resp = make(vtxp, zerop, resp); + } + replace(resp); + return; + } + } + + if (DfgCond* const tCondp = thenp->cast()) { + if (isSame(condp, tCondp->condp())) { + APPLYING(REPLACE_COND_SAME_COND_THEN) { + replace(make(vtxp, condp, tCondp->thenp(), elsep)); + return; + } + } + } + + if (DfgCond* const eCondp = elsep->cast()) { + if (isSame(condp, eCondp->condp())) { + APPLYING(REPLACE_COND_SAME_COND_ELSE) { + replace(make(vtxp, condp, thenp, eCondp->elsep())); + return; + } + } + } } void visit(DfgVertexVar* const vtxp) override { diff --git a/src/V3DfgPeepholePatterns.h b/src/V3DfgPeepholePatterns.h index 6c5db5b9c..9eed0d456 100644 --- a/src/V3DfgPeepholePatterns.h +++ b/src/V3DfgPeepholePatterns.h @@ -39,11 +39,14 @@ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, PULL_NOTS_THROUGH_COND) \ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, PUSH_BITWISE_OP_THROUGH_CONCAT) \ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, PUSH_BITWISE_THROUGH_REDUCTION) \ + _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, PUSH_COMMUTATIVE_BINARY_THROUGH_COND) \ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, PUSH_COMPARE_OP_THROUGH_CONCAT) \ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, PUSH_CONCAT_THROUGH_COND_LHS) \ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, PUSH_CONCAT_THROUGH_COND_RHS) \ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, PUSH_CONCAT_THROUGH_NOTS) \ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, PUSH_NOT_THROUGH_COND) \ + _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, PUSH_REDUCTION_THROUGH_BITWISE_OF_CONCAT) \ + _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, PUSH_REDUCTION_THROUGH_BITWISE_OF_SELS) \ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, PUSH_REDUCTION_THROUGH_CONCAT) \ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, PUSH_REDUCTION_THROUGH_COND_WITH_CONST_BRANCH) \ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, PUSH_SEL_THROUGH_COND) \ @@ -51,6 +54,8 @@ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, PUSH_SEL_THROUGH_REPLICATE) \ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, PUSH_SEL_THROUGH_SHIFTL) \ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, PUSH_SEL_THROUGH_SPLICE) \ + _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, PUSH_SHIFTL_THROUGH_COND) \ + _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, PUSH_SHIFTR_THROUGH_COND) \ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REMOVE_AND_WITH_ONES) \ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REMOVE_AND_WITH_SELF) \ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REMOVE_CONCAT_OF_ADJOINING_SELS) \ @@ -65,6 +70,8 @@ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REMOVE_REPLICATE_ONCE) \ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REMOVE_SEL_FROM_LHS_OF_CONCAT) \ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REMOVE_SEL_FROM_RHS_OF_CONCAT) \ + _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REMOVE_SHIFTL_ZERO) \ + _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REMOVE_SHIFTR_ZERO) \ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REMOVE_SUB_ZERO) \ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REMOVE_VAR) \ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REMOVE_WIDTH_ONE_REDUCTION) \ @@ -72,12 +79,21 @@ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REPLACE_AND_OF_NOT_AND_NEQ) \ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REPLACE_AND_OF_NOT_AND_NOT) \ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REPLACE_AND_WITH_ZERO) \ + _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REPLACE_BITWISE_OF_REDUCTION_OF_SELS_WITH_REDUCTION) \ + _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REPLACE_BITWISE_OF_SELS_WITH_REDUCTION) \ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REPLACE_CONCAT_SEL_BOTTOM_AND_ZERO_WITH_SHIFTL) \ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REPLACE_CONCAT_ZERO_AND_SEL_TOP_WITH_SHIFTR) \ + _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REPLACE_COND_CONST_ONE_ZERO) \ + _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REPLACE_COND_CONST_ZERO_ONE) \ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REPLACE_COND_DEC) \ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REPLACE_COND_INC) \ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REPLACE_COND_OR_THEN_COND_LHS) \ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REPLACE_COND_OR_THEN_COND_RHS) \ + _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REPLACE_COND_SAME_CAT_LHS) \ + _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REPLACE_COND_SAME_CAT_RHS) \ + _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REPLACE_COND_SAME_COND_ELSE) \ + _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REPLACE_COND_SAME_COND_THEN) \ + _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REPLACE_COND_WITH_ELSE_BRANCH_COND) \ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REPLACE_COND_WITH_ELSE_BRANCH_ONES) \ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REPLACE_COND_WITH_ELSE_BRANCH_ZERO) \ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REPLACE_COND_WITH_THEN_BRANCH_COND) \ @@ -100,7 +116,13 @@ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REPLACE_OR_WITH_ONES) \ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REPLACE_SEL_FROM_SEL) \ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REPLACE_SHIFTL_CAT) \ - _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REPLACE_SHIFTRL_CAT) \ + _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REPLACE_SHIFTL_OVER) \ + _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REPLACE_SHIFTL_SEL) \ + _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REPLACE_SHIFTL_SHIFTL) \ + _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REPLACE_SHIFTR_CAT) \ + _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REPLACE_SHIFTR_OVER) \ + _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REPLACE_SHIFTR_SEL) \ + _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REPLACE_SHIFTR_SHIFTR) \ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REPLACE_SUB_WITH_NOT) \ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REPLACE_TAUTOLOGICAL_OR) \ _FOR_EACH_DFG_PEEPHOLE_OPTIMIZATION_APPLY(macro, REPLACE_TAUTOLOGICAL_OR_3) \ diff --git a/test_regress/t/t_dfg_peephole.v b/test_regress/t/t_dfg_peephole.v index b1033e51d..d703f0886 100644 --- a/test_regress/t/t_dfg_peephole.v +++ b/test_regress/t/t_dfg_peephole.v @@ -192,6 +192,7 @@ module t ( `signal(REPLACE_COND_WITH_THEN_BRANCH_COND, rand_a[0] ? rand_a[0] : rand_a[1]); `signal(REPLACE_COND_WITH_THEN_BRANCH_ZERO, rand_a[0] ? 1'd0 : rand_a[1]); `signal(REPLACE_COND_WITH_THEN_BRANCH_ONES, rand_a[0] ? 1'd1 : rand_a[1]); + `signal(REPLACE_COND_WITH_ELSE_BRANCH_COND, rand_a[0] ? rand_a[1] : rand_a[0]); `signal(REPLACE_COND_WITH_ELSE_BRANCH_ZERO, rand_a[0] ? rand_a[1] : 1'd0); `signal(REPLACE_COND_WITH_ELSE_BRANCH_ONES, rand_a[0] ? rand_a[1] : 1'd1); `signal(INLINE_ARRAYSEL_SPLICE, array[0]); @@ -246,6 +247,50 @@ module t ( `signal(REUSE_ASSOC_LHS_WITH_RHS_OF_RHS_XOR_COMMON, rand_a[23:4] ^ rand_a[39:20]); `signal(REUSE_ASSOC_LHS_WITH_RHS_OF_RHS_XOR, rand_a[23:4] ^ (~rand_b[24:5] ^ rand_a[39:20])); + `signal(REPLACE_COND_CONST_ONE_ZERO, rand_a[0] ? 8'b1 : 8'b0); + `signal(REPLACE_COND_CONST_ZERO_ONE, rand_a[0] ? 8'b0 : 8'b1); + `signal(REPLACE_COND_CAT_LHS_CONST_ONE_ZERO, rand_a[0] ? {8'b1, rand_b[0]} : {8'b0, rand_b[1]}); + `signal(REPLACE_COND_CAT_LHS_CONST_ZERO_ONE, rand_a[0] ? {8'b0, rand_b[0]} : {8'b1, rand_b[1]}); + `signal(REPLACE_COND_SAME_CAT_LHS, rand_a[0] ? {8'd0, rand_b[0]} : {8'd0, rand_b[1]}); + `signal(REPLACE_COND_SAME_CAT_RHS, rand_a[0] ? {rand_b[0], 8'd0} : {rand_b[1], 8'd0}); + `signal(REPLACE_COND_SAM_COND_THEN, rand_a[0] ? (rand_a[0] ? rand_b[1:0] : rand_b[3:2]) : rand_b[5:4]); + `signal(REPLACE_COND_SAM_COND_ELSE, rand_a[0] ? rand_b[1:0] : (rand_a[0] ? rand_b[3:2] : rand_b[5:4])); + `signal(REMOVE_SHIFTL_ZERO, rand_a << 0); + `signal(REPLACE_SHIFTL_OVER, rand_a << 64); + `signal(REPLACE_SHIFTL_SEL, rand_a[27:0] << 4); + `signal(REMOVE_SHIFTR_ZERO, rand_a >> 0); + `signal(REPLACE_SHIFTR_OVER, rand_a >> 64); + `signal(REPLACE_SHIFTR_SEL, rand_a[28:1] >> 4); + `signal(PUSH_BITWISE_OP_THROUGH_COND_AND, 4'd2 & (rand_a[0] ? 4'd7 : 4'd4)); + `signal(PUSH_BITWISE_OP_THROUGH_COND_OR, 4'd2 | (rand_a[0] ? 4'd7 : 4'd4)); + `signal(PUSH_BITWISE_OP_THROUGH_COND_XOR, 4'd2 ^ (rand_a[0] ? 4'd7 : 4'd4)); + `signal(SIMPLIFY_COND_THEN, rand_a[0] ? {1'b0, ~rand_a[0]} : 2'b0); + `signal(SIMPLIFY_COND_ELSE, rand_a[0] ? 2'b0 : {1'b0, ~rand_a[0]}); + `signal(PUSH_CONCAT_THROUGH_COND_LHS_A, {5'd0, rand_a[0] ? {rand_b[4], 1'b0} : 2'b0}); + `signal(PUSH_CONCAT_THROUGH_COND_LHS_B, {5'd0, rand_a[0] ? 2'b0 : {rand_b[4], 1'b0}}); + `signal(PUSH_CONCAT_THROUGH_COND_RHS_A, {rand_a[0] ? {rand_b[5], 1'b0} : 2'b0, 5'd0}); + `signal(PUSH_CONCAT_THROUGH_COND_RHS_B, {rand_a[0] ? 2'b0 : {rand_b[5], 1'b0}, 5'd0}); + `signal(PUSH_REDUCTION_THROUGH_BITWISE_OF_CONCAT_AND, &({rand_a[0], rand_a[2]} & rand_b[1:0])); + `signal(PUSH_REDUCTION_THROUGH_BITWISE_OF_CONCAT_OR, |({rand_a[0], rand_a[2]} | rand_b[1:0])); + `signal(PUSH_REDUCTION_THROUGH_BITWISE_OF_CONCAT_XOR, ^({rand_a[0], rand_a[2]} ^ rand_b[1:0])); + `signal(PUSH_REDUCTION_THROUGH_BITWISE_OF_SELS_AND_A, &(rand_a[55+:2] & rand_a[53+:2])); + `signal(PUSH_REDUCTION_THROUGH_BITWISE_OF_SELS_OR_A, |(rand_a[55+:2] | rand_a[53+:2])); + `signal(PUSH_REDUCTION_THROUGH_BITWISE_OF_SELS_XOR_A, ^(rand_a[55+:2] ^ rand_a[53+:2])); + `signal(PUSH_REDUCTION_THROUGH_BITWISE_OF_SELS_AND_B, &(rand_a[57+:2] & rand_a[59+:2])); + `signal(PUSH_REDUCTION_THROUGH_BITWISE_OF_SELS_OR_B, |(rand_a[57+:2] | rand_a[59+:2])); + `signal(PUSH_REDUCTION_THROUGH_BITWISE_OF_SELS_XOR_B, ^(rand_a[57+:2] ^ rand_a[59+:2])); + `signal(PUSH_SHIFTL_THROUGH_COND, (rand_a[0] ? rand_a >> 1 : rand_b >> 1) << 1); + `signal(PUSH_SHIFTR_THROUGH_COND, (rand_a[0] ? rand_a << 1 : rand_b << 1) >> 1); + `signal(REPLACE_BITWISE_OF_REDUCTION_OF_SELS_WITH_REDUCTION_AND_A, (&rand_a[10 +:2]) & (&rand_a[12 +: 2])); + `signal(REPLACE_BITWISE_OF_REDUCTION_OF_SELS_WITH_REDUCTION_OR_A, (|rand_a[10 +:2]) | (|rand_a[12 +: 2])); + `signal(REPLACE_BITWISE_OF_REDUCTION_OF_SELS_WITH_REDUCTION_XOR_A, (^rand_a[10 +:2]) ^ (^rand_a[12 +: 2])); + `signal(REPLACE_BITWISE_OF_REDUCTION_OF_SELS_WITH_REDUCTION_AND_B, (&rand_a[12 +:2]) & (&rand_a[10 +: 2])); + `signal(REPLACE_BITWISE_OF_REDUCTION_OF_SELS_WITH_REDUCTION_OR_B, (|rand_a[12 +:2]) | (|rand_a[10 +: 2])); + `signal(REPLACE_BITWISE_OF_REDUCTION_OF_SELS_WITH_REDUCTION_XOR_B, (^rand_a[12 +:2]) ^ (^rand_a[10 +: 2])); + `signal(REPLACE_SHIFTL_SHIFTL, rand_a << 2 << 3); + `signal(REPLACE_SHIFTR_SHIFTR, rand_a >> 2 >> 3); + `signal(PUSH_COMMUTATIVE_BINARY_THROUGH_COND, 58'h1 + (rand_a[0] ? rand_b[1 +: 58] : ~rand_b[1 +: 58])); + // Operators that should work wiht mismatched widths `signal(MISMATCHED_ShiftL,const_a << 4'd2); `signal(MISMATCHED_ShiftR,const_a >> 4'd2); @@ -273,9 +318,6 @@ module t ( end `signal(PUSH_SEL_THROUGH_SPLICE, sel_from_partial_tmp[1:0]); - `signal(PUSH_CONCAT_THROUGH_COND_LHS, {5'd0, rand_a[0] ? {rand_b[4], 1'b0} : {1'b0, rand_b[6]}}); - `signal(PUSH_CONCAT_THROUGH_COND_RHS, {rand_a[0] ? {rand_b[5], 1'b0} : {1'b0, rand_b[7]}, 5'd0}); - `signal(REPLACE_SHIFTL_CAT, {31'd0, rand_a[42 +: 7]} << 31); `signal(REPLACE_SHIFTRL_CAT, {rand_a[13 +: 7], rand_b[8 +: 27]} >> 27 << 27);