diff --git a/docs/guide/exe_verilator.rst b/docs/guide/exe_verilator.rst index c428f17f3..24e22b852 100644 --- a/docs/guide/exe_verilator.rst +++ b/docs/guide/exe_verilator.rst @@ -660,8 +660,7 @@ Summary: .. option:: -fno-bit-scan-loops - Rarely needed. Disable converting leading-one and count-ones loops into - bit-reduction operations. + Rarely needed. Disable converting bit counting loops into built-in operations. .. option:: -fno-case diff --git a/include/verilated_funcs.h b/include/verilated_funcs.h index 682899f4c..88da40387 100644 --- a/include/verilated_funcs.h +++ b/include/verilated_funcs.h @@ -925,7 +925,6 @@ static inline IData VL_MOSTSETBITP1_Q(QData lhs) VL_PURE { #endif } static inline IData VL_MOSTSETBITP1_W(int words, WDataInP const lwp) VL_PURE { - // MSB set bit plus one; similar to FLS. 0=value is zero for (int i = words - 1; i >= 0; --i) { // Shorter worst case if predict not taken if (VL_UNLIKELY(lwp[i])) return i * VL_EDATASIZE + VL_MOSTSETBITP1_I(lwp[i]); diff --git a/src/V3AstNodeExpr.h b/src/V3AstNodeExpr.h index 392aee936..c60515758 100644 --- a/src/V3AstNodeExpr.h +++ b/src/V3AstNodeExpr.h @@ -5745,11 +5745,10 @@ public: void numberOperate(V3Number& out, const V3Number& lhs) override { out.opMostSetBitP1(lhs); } string emitVerilog() override { return "%f$mostsetbitp1(%l)"; } string emitC() override { return "VL_MOSTSETBITP1_%lq(%lW, %P, %li)"; } - bool cleanOut() const override { return false; } + bool cleanOut() const override { return true; } bool cleanLhs() const override { return true; } bool sizeMattersLhs() const override { return false; } int instrCount() const override { return widthInstrs() * 16; } - bool isSystemFunc() const override { return true; } }; class AstNToI final : public AstNodeUniop { // String to any-size integral diff --git a/src/V3Options.h b/src/V3Options.h index 88b1359ba..3aebaea91 100644 --- a/src/V3Options.h +++ b/src/V3Options.h @@ -392,7 +392,7 @@ private: // MEMBERS (optimizations) bool m_fAcycSimp; // main switch: -fno-acyc-simp: acyclic pre-optimizations bool m_fAssemble; // main switch: -fno-assemble: assign assemble - bool m_fBitScanLoops; // main switch: -fno-bit-scan-loops: lower priority-encoder loops to CLZ + bool m_fBitScanLoops; // main switch: -fno-bit-scan-loops: convert bit scan loops to builtins bool m_fCaseDecoder; // main switch: -fno-case-decoder: case decoder conversion bool m_fCaseTable; // main switch: -fno-case-table: case table conversion bool m_fCaseTree; // main switch: -fno-case-tree: case tree conversion diff --git a/src/V3Unroll.cpp b/src/V3Unroll.cpp index 2707af6ce..43bf5e376 100644 --- a/src/V3Unroll.cpp +++ b/src/V3Unroll.cpp @@ -423,11 +423,11 @@ class UnrollAllVisitor final : VNVisitor { UnrolllBindings m_bindings; // Variable bindings // METHODS - // Peel value-preserving width casts/truncations (Extend/ExtendS, or a low-bits Sel - // with constant lsb 0) to reach the underlying VarRef; nullptr for anything else - // (e.g. an arithmetic offset like idx+1 or idx*2). - static AstVarRef* unwrapToVarRef(AstNodeExpr* nodep) { - while (nodep) { + // Peel value-preserving width casts (Extend/ExtendS, or a low-bits Sel with lsb 0) to the + // underlying VarRef. A Sel kept narrower than 'minWidth' is a lossy narrowing (idx[1:0]) + // and is rejected. + static AstVarRef* unwrapToVarRef(AstNodeExpr* nodep, int minWidth) { + while (true) { if (AstVarRef* const refp = VN_CAST(nodep, VarRef)) return refp; if (AstExtend* const ep = VN_CAST(nodep, Extend)) { nodep = ep->lhsp(); @@ -435,45 +435,54 @@ class UnrollAllVisitor final : VNVisitor { nodep = ep->lhsp(); } else if (AstSel* const sp = VN_CAST(nodep, Sel)) { const AstConst* const lsbp = VN_CAST(sp->lsbp(), Const); - if (!lsbp || lsbp->toUInt() != 0) return nullptr; + if (!lsbp || lsbp->toUInt() != 0 || sp->width() < minWidth) return nullptr; nodep = sp->fromp(); } else { return nullptr; } } - return nullptr; } - // True if 'nodep' is exactly 'var + 1' (commutative), where the var operand is the - // given variable reached through value-preserving casts only. + // True if 'nodep' is exactly '1 + var' for 'vscp' (V3Const puts the constant on the LHS). + // Passing the add's width as minWidth rejects a lossy increment like 32'(i[1:0]) + 1. bool isVarPlus1(AstNode* nodep, const AstVarScope* vscp) { AstAdd* const addp = VN_CAST(nodep, Add); - if (!addp) return false; - const auto isOne = [](AstNodeExpr* p) -> bool { - const AstConst* const cp = VN_CAST(p, Const); - return cp && !cp->num().isFourState() && cp->toUInt() == 1; - }; - const AstVarRef* const l = unwrapToVarRef(addp->lhsp()); - const AstVarRef* const r = unwrapToVarRef(addp->rhsp()); - if (isOne(addp->rhsp()) && l && l->varScopep() == vscp) return true; - if (isOne(addp->lhsp()) && r && r->varScopep() == vscp) return true; - return false; + if (!addp || !addp->lhsp()->isOne()) return false; + const AstVarRef* const r = unwrapToVarRef(addp->rhsp(), addp->width()); + return r && r->varScopep() == vscp; } - // Match a strict ascending loop bound 'idx < W', written either way and signed or - // unsigned: (W > idx) [Gt/GtS] or (idx < W) [Lt/LtS]. Sets wp and idxRefp. + // Resize the 32-bit reduction to the accumulator width; truncating the low bits matches + // the original counted loop's wrap-around. + static AstNodeExpr* resizeToWidth(AstNodeExpr* exprp, const AstVarRef* targetRefp) { + const int width = targetRefp->width(); + if (width == 32) return exprp; + FileLine* const flp = exprp->fileline(); + if (width < 32) return new AstSel{flp, exprp, 0, width}; + AstExtend* const extp = new AstExtend{flp, exprp}; + extp->dtypeFrom(targetRefp); + return extp; + } + // Match a strict ascending loop bound 'idx < W'. V3Const canonicalizes this to the + // 'W > idx' form (Gt unsigned, GtS signed), so only that form is matched. static bool ascendingBound(AstNodeExpr* condp, AstConst*& wp, AstVarRef*& idxRefp) { - if (VN_IS(condp, Gt) || VN_IS(condp, GtS)) { - AstNodeBiop* const bp = VN_AS(condp, NodeBiop); - wp = VN_CAST(bp->lhsp(), Const); - idxRefp = VN_CAST(bp->rhsp(), VarRef); - } else if (VN_IS(condp, Lt) || VN_IS(condp, LtS)) { - AstNodeBiop* const bp = VN_AS(condp, NodeBiop); - idxRefp = VN_CAST(bp->lhsp(), VarRef); - wp = VN_CAST(bp->rhsp(), Const); - } else { - return false; - } + if (!VN_IS(condp, Gt) && !VN_IS(condp, GtS)) return false; + AstNodeBiop* const bp = VN_AS(condp, NodeBiop); + wp = VN_CAST(bp->lhsp(), Const); + idxRefp = VN_CAST(bp->rhsp(), VarRef); return wp && idxRefp && !wp->num().isFourState(); } + // Recognize the redundant in-range guard Verilator auto-inserts for a select into a + // non-power-of-two vector. V3Const canonicalizes 'idx <= C' to '(C >= idx)' (Gte/GteS, + // const on the LHS), so only that form occurs; with C >= W-1 it is always true for idx + // in 0..W-1. + static bool isInRangeGuard(AstNodeExpr* condp, const AstVarScope* idxVscp, uint32_t width, + int addrBits) { + if (!VN_IS(condp, Gte) && !VN_IS(condp, GteS)) return false; + AstNodeBiop* const bp = VN_AS(condp, NodeBiop); + const AstConst* const cp = VN_CAST(bp->lhsp(), Const); + if (!cp || cp->num().isFourState() || cp->toUInt() < width - 1) return false; + const AstVarRef* const r = unwrapToVarRef(bp->rhsp(), addrBits); + return r && r->varScopep() == idxVscp; + } // Recognize a single-bit scan loop over all W bits of 'vec' (idx 0..W-1, target // pre-zeroed) and lower it to a bit-reduction primitive. Two idioms are matched: // target = 0; idx = 0; @@ -482,27 +491,24 @@ class UnrollAllVisitor final : VNVisitor { // = idx + 1 => target = $mostsetbitp1(vec) (leading-one / bit-width) // = target + 1 => target = $countones(vec) (population count) bool tryLowerBitScanLoop(AstLoop* loopp) { - // Body must be exactly: LoopTest, If, Assign(increment) AstLoopTest* const testp = VN_CAST(loopp->stmtsp(), LoopTest); if (!testp) return false; AstIf* const ifp = VN_CAST(testp->nextp(), If); if (!ifp) return false; AstAssign* const incp = VN_CAST(ifp->nextp(), Assign); if (!incp || incp->nextp()) return false; - // Loop test: a strict ascending bound 'idx < W' (W > idx), signed or unsigned AstConst* wp = nullptr; AstVarRef* idxRefp = nullptr; if (!ascendingBound(testp->condp(), wp, idxRefp)) return false; AstVarScope* const idxVscp = idxRefp->varScopep(); const uint32_t width = wp->toUInt(); - // Index must start at 0 (so it takes exactly the values 0..W-1) + // Bits needed to address all W bits of 'vec' (clog2(W)); a narrower index is lossy. + const int addrBits = width <= 1 ? 1 : V3Number::log2b(width - 1) + 1; const AstConst* const idxInitp = m_bindings.get(idxVscp); - if (!idxInitp || idxInitp->num().isFourState() || idxInitp->toUInt() != 0) return false; - // Increment must be exactly: idx = idx + 1 + if (!idxInitp || !idxInitp->isZero()) return false; AstVarRef* const incLhsp = VN_CAST(incp->lhsp(), VarRef); if (!incLhsp || incLhsp->varScopep() != idxVscp) return false; if (!isVarPlus1(incp->rhsp(), idxVscp)) return false; - // If: no else, and 'then' is exactly one assignment 'target = ' if (ifp->elsesp()) return false; AstAssign* const thenp = VN_CAST(ifp->thensp(), Assign); if (!thenp || thenp->nextp()) return false; @@ -513,47 +519,46 @@ class UnrollAllVisitor final : VNVisitor { const bool isLeadingOne = isVarPlus1(thenp->rhsp(), idxVscp); const bool isCountOnes = !isLeadingOne && isVarPlus1(thenp->rhsp(), targetVscp); if (!isLeadingOne && !isCountOnes) return false; - // $countones yields a 32-bit result; only fold when the accumulator fits in 32 bits. - if (isCountOnes && targetRefp->width() > 32) return false; - // If-cond must select exactly bit 'idx' of some vector 'vec' (vec != idx/target) - AstNodeExpr* vecExprp = nullptr; - ifp->condp()->foreach([&](AstSel* selp) { - if (vecExprp || selp->width() != 1) return; - const AstVarRef* const fromp = VN_CAST(selp->fromp(), VarRef); - if (!fromp) return; - const AstVarScope* const fromVscp = fromp->varScopep(); - if (fromVscp == idxVscp || fromVscp == targetVscp) return; - const AstVarRef* const idxInSel = unwrapToVarRef(selp->lsbp()); - if (idxInSel && idxInSel->varScopep() == idxVscp) vecExprp = selp->fromp(); - }); - if (!vecExprp) return false; - // The loop must scan exactly all bits of 'vec' + // If-cond is the 1-bit select 'vec[idx]', possibly wrapped in the redundant in-range + // guard Verilator auto-inserts (as 'guard && sel') for a non-power-of-two vector: + // '(idx <= W-1) && vec[idx]' (default / --x-assign 0; a LogAnd), or + // '(idx <= W-1) ? vec[idx] : ' (--x-assign unique; a Cond). + // The guard is always true for idx in 0..W-1, so peel it to reach the select. Any + // other compound condition (e.g. 'vec[idx] && en') leaves a non-select, rejected below. + AstNodeExpr* condp = ifp->condp(); + if (AstLogAnd* const andp = VN_CAST(condp, LogAnd)) { + if (isInRangeGuard(andp->lhsp(), idxVscp, width, addrBits)) condp = andp->rhsp(); + } else if (AstCond* const ternp = VN_CAST(condp, Cond)) { + if (isInRangeGuard(ternp->condp(), idxVscp, width, addrBits)) condp = ternp->thenp(); + } + AstSel* const selp = VN_CAST(condp, Sel); + if (!selp || selp->width() != 1) return false; + const AstVarRef* const fromp = VN_CAST(selp->fromp(), VarRef); + if (!fromp) return false; + const AstVarScope* const fromVscp = fromp->varScopep(); + if (fromVscp == idxVscp || fromVscp == targetVscp) return false; + AstNodeExpr* const vecExprp = selp->fromp(); + // Must scan all W bits of 'vec', indexed by exactly 'idx' (address kept >= clog2(W), + // so a lossy narrowing like vec[idx[2:0]] is rejected). if (static_cast(width) != vecExprp->width()) return false; + const AstVarRef* const idxInSel = unwrapToVarRef(selp->lsbp(), addrBits); + if (!idxInSel || idxInSel->varScopep() != idxVscp) return false; // 'target' must be const-0 immediately before the loop (collected in m_bindings), // so that an all-zero 'vec' yields 0, matching $mostsetbitp1's definition. const AstConst* const targetInitp = m_bindings.get(targetVscp); - if (!targetInitp || targetInitp->num().isFourState() || targetInitp->toUInt() != 0) { - return false; - } - // Lower the loop to: target = (vec); idx = W; - // The 'idx = W' store preserves the counted loop's exit value, so the rewrite is - // sound even if idx is read after the loop (dead-code elimination drops it if not). + if (!targetInitp || !targetInitp->isZero()) return false; + // Rewrite to 'target = (vec); idx = W'. The 'idx = W' store preserves the + // loop's exit value, so this is sound even if idx is read afterwards (else DCE drops it). FileLine* const flp = loopp->fileline(); AstNodeExpr* reducep; if (isLeadingOne) { - AstMostSetBitP1* const msbp = new AstMostSetBitP1{flp, vecExprp->cloneTree(false)}; - msbp->dtypeFrom(targetRefp); - reducep = msbp; + reducep = new AstMostSetBitP1{flp, vecExprp->cloneTree(false)}; } else { - // $countones must yield a 32-bit result (DFG invariant); narrow afterwards to - // the accumulator width to match the loop's wrap-around behaviour. AstCountOnes* const conep = new AstCountOnes{flp, vecExprp->cloneTree(false)}; conep->dtypeSetInteger2State(); - reducep - = (targetRefp->width() < 32) - ? static_cast(new AstSel{flp, conep, 0, targetRefp->width()}) - : static_cast(conep); + reducep = conep; } + reducep = resizeToWidth(reducep, targetRefp); AstAssign* const newp = new AstAssign{flp, targetRefp->cloneTree(false), reducep}; newp->addNext(new AstAssign{flp, incLhsp->cloneTree(false), wp->cloneTree(false)}); loopp->replaceWith(newp); @@ -596,7 +601,7 @@ class UnrollAllVisitor final : VNVisitor { m_bindings.set(lhsp->varScopep(), valp); } - // Recognize a leading-one / priority-encoder loop and lower it to $mostsetbitp1 + // Recognize a bit counting loop and lower it to a builtin if (v3Global.opt.fBitScanLoops() && tryLowerBitScanLoop(nodep)) return; // Attempt to unroll this loop diff --git a/test_regress/t/t_bit_scan_loops.py b/test_regress/t/t_bit_scan_loops.py index f6ef9fd12..8d0b83c51 100755 --- a/test_regress/t/t_bit_scan_loops.py +++ b/test_regress/t/t_bit_scan_loops.py @@ -4,22 +4,21 @@ # This program is free software; you can redistribute it and/or modify it # under the terms of either the GNU Lesser General Public License Version 3 # or the Perl Artistic License Version 2.0. -# SPDX-FileCopyrightText: 2024 Wilson Snyder +# SPDX-FileCopyrightText: 2026 Wilson Snyder # SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0 import vltest_bootstrap -test.scenarios('simulator') +test.scenarios('vlt') -test.compile(verilator_flags2=['--stats']) +# --unroll-count 0 so the loops are recognized without relying on unrolling. +test.compile(verilator_flags2=['--stats', '--unroll-count', '0']) -# Exactly 4 loops must lower to $mostsetbitp1: the 3 canonical I/Q/W leading-one -# loops plus the unsigned-index one. The 6 negative loops (incl. W != width) must -# be left alone -- a wrong lowering would push the count above 4. +# The leading-one positives lower to $mostsetbitp1, the count-ones positive to +# $countones; the negatives are left as loops (a wrong lowering would raise a count). test.file_grep(test.stats, r'Optimizations, Loop unrolling, Lowered priority-encoder to mostsetbitp1\s+(\d+)', - 4) -# And the one count-ones loop must lower to $countones. + 8) test.file_grep(test.stats, r'Optimizations, Loop unrolling, Lowered count-set-bits to countones\s+(\d+)', 1) diff --git a/test_regress/t/t_bit_scan_loops.v b/test_regress/t/t_bit_scan_loops.v index 2fc56f7fd..fa2c1df00 100644 --- a/test_regress/t/t_bit_scan_loops.v +++ b/test_regress/t/t_bit_scan_loops.v @@ -1,78 +1,168 @@ // DESCRIPTION: Verilator: Verilog Test module // -// Exercises the priority-encoder / leading-one ("bit-width", find-last-set) loop -// idiom that V3Unroll lowers to $mostsetbitp1 (CLZ). -// -// Positive cases (MUST lower): canonical for(b=0;b64b) runtime paths. -// Negative cases (MUST NOT lower): non-canonical scans whose result differs -// from $mostsetbitp1(vec). These both (a) self-check their own values and -// (b) are counted by the .py via --stats so a wrong lowering is caught. +// Exercises the bit-counting loop idioms that V3Unroll lowers to builtins: +// leading-one for (b=0;b $mostsetbitp1(vec) +// count-ones for (b=0;b $countones(vec) +// Positives must lower (counted via --stats by the .py); negatives compute a +// different value than the builtin and so must be left as loops. // // This file ONLY is placed under the Creative Commons Public Domain. -// SPDX-FileCopyrightText: 2024 Wilson Snyder +// SPDX-FileCopyrightText: 2026 Wilson Snyder // SPDX-License-Identifier: CC0-1.0 +// verilog_format: off +`define stop $stop +`define checkh(gotv,expv) do if ((gotv) !== (expv)) begin $write("%%Error: %s:%0d: got=%0x exp=%0x (%s !== %s)\n", `__FILE__,`__LINE__, (gotv), (expv), `"gotv`", `"expv`"); `stop; end while(0); +// verilog_format: on + module t ( input clk ); - // ---- positive: should lower (I / Q / W paths) ---- - logic [31:0] p32; logic [5:0] n32; - logic [47:0] p48; logic [6:0] n48; - logic [79:0] p80; logic [6:0] n80; - logic [31:0] pc; logic [5:0] nc; // population count (count-ones) -> $countones - logic [31:0] pu; logic [5:0] nu; // unsigned loop var (still $mostsetbitp1) - always_comb begin n32 = 0; for (int b = 0; b < 32; b++) if (p32[b]) n32 = 6'(b + 1); end - always_comb begin n48 = 0; for (int b = 0; b < 48; b++) if (p48[b]) n48 = 7'(b + 1); end - always_comb begin n80 = 0; for (int b = 0; b < 80; b++) if (p80[b]) n80 = 7'(b + 1); end - always_comb begin nc = 0; for (int b = 0; b < 32; b++) if (pc[b]) nc = nc + 1; end - always_comb begin nu = 0; for (int unsigned b = 0; b < 32; b++) if (pu[b]) nu = 6'(b + 1); end + // ---- positives: must lower ---- + logic [31:0] p32; + logic [5:0] n32; // I path, narrow target (select resize) + logic [47:0] p48; + logic [6:0] n48; // Q path + logic [79:0] p80; + logic [6:0] n80; // W path + logic [31:0] pu; + logic [5:0] nu; // unsigned loop index + logic [31:0] p32e; + logic [31:0] n32e; // 32-bit target (no resize) + logic [31:0] p32w; + logic [39:0] n40; // >32-bit target (extend resize) + logic [31:0] pc; + logic [5:0] nc; // count-ones -> $countones + logic [31:0] kvec; // const (set in initial) -> exercises $mostsetbitp1 fold + logic [5:0] kn; + initial kvec = 32'h0000_0100; + logic [31:0] kvec0; // const 0 -> $mostsetbitp1(0)=0 (covers the zero path) + logic [5:0] kn0; + initial kvec0 = 32'h0; + always_comb begin + n32 = 0; + for (int b = 0; b < 32; b++) if (p32[b]) n32 = 6'(b + 1); + end + always_comb begin + n48 = 0; + for (int b = 0; b < 48; b++) if (p48[b]) n48 = 7'(b + 1); + end + always_comb begin + n80 = 0; + for (int b = 0; b < 80; b++) if (p80[b]) n80 = 7'(b + 1); + end + always_comb begin + nu = 0; + for (int unsigned b = 0; b < 32; b++) if (pu[b]) nu = 6'(b + 1); + end + always_comb begin + n32e = 0; + for (int b = 0; b < 32; b++) if (p32e[b]) n32e = 32'(b + 1); + end + always_comb begin + n40 = 0; + for (int b = 0; b < 32; b++) if (p32w[b]) n40 = 40'(b + 1); + end + always_comb begin + nc = 0; + for (int b = 0; b < 32; b++) if (pc[b]) nc = nc + 1; + end + always_comb begin + kn = 0; + for (int b = 0; b < 32; b++) if (kvec[b]) kn = 6'(b + 1); + end + always_comb begin + kn0 = 0; + for (int b = 0; b < 32; b++) if (kvec0[b]) kn0 = 6'(b + 1); + end - // ---- negative: must NOT lower (each scans a subset / computes a different value) ---- - logic [31:0] vn; - logic [5:0] e_step2; // step 2 (even bits only) - logic [6:0] e_start1; // starts at 1 (skips bit 0) - logic [6:0] e_mul; // target = idx*2 + 1 - logic [5:0] e_off; // selects vec[idx+1] - logic [5:0] e_noP1; // target = idx (no +1) - logic [31:0] vw; // separate vec with a set bit above the scan bound - logic [5:0] e_narrow; // bound W=16 != width(vec)=32 (scans only a prefix) - always_comb begin e_step2 = 0; for (int b = 0; b < 32; b += 2) if (vn[b]) e_step2 = 6'(b + 1); end - always_comb begin e_start1 = 0; for (int b = 1; b < 32; b++) if (vn[b]) e_start1 = 7'(b + 1); end - always_comb begin e_mul = 0; for (int b = 0; b < 32; b++) if (vn[b]) e_mul = 7'(2 * b + 1); end - always_comb begin e_off = 0; for (int b = 0; b < 31; b++) if (vn[b + 1]) e_off = 6'(b + 1); end - always_comb begin e_noP1 = 0; for (int b = 0; b < 32; b++) if (vn[b]) e_noP1 = 6'(b); end - always_comb begin e_narrow = 0; for (int b = 0; b < 16; b++) if (vw[b]) e_narrow = 6'(b + 1); end + // ---- negatives: must NOT lower (each yields a different value than the builtin) ---- + logic [31:0] vn; // shared input, bits {2,4,5,7} + logic [31:0] vw; // has a set bit above the scan bound + logic [31:0] vt; // for the truncated-index case + logic en1; // runtime gate for the compound-condition case + logic [5:0] e_step2; + logic [6:0] e_start1; + logic [6:0] e_mul; + logic [5:0] e_off; + logic [5:0] e_noP1; + logic [5:0] e_narrow; + logic [5:0] e_comp; + logic [5:0] e_trunc; + always_comb begin + e_step2 = 0; + for (int b = 0; b < 32; b += 2) if (vn[b]) e_step2 = 6'(b + 1); + end + always_comb begin + e_start1 = 0; + for (int b = 1; b < 32; b++) if (vn[b]) e_start1 = 7'(b + 1); + end + always_comb begin + e_mul = 0; + for (int b = 0; b < 32; b++) if (vn[b]) e_mul = 7'(2 * b + 1); + end + always_comb begin + e_off = 0; + for (int b = 0; b < 31; b++) if (vn[b+1]) e_off = 6'(b + 1); + end + always_comb begin + e_noP1 = 0; + for (int b = 0; b < 32; b++) if (vn[b]) e_noP1 = 6'(b); + end + always_comb begin + e_narrow = 0; + for (int b = 0; b < 16; b++) if (vw[b]) e_narrow = 6'(b + 1); + end + always_comb begin + e_comp = 0; + for (int b = 0; b < 32; b++) if (vn[b] && en1) e_comp = 6'(b + 1); + end + // verilator lint_off WIDTHEXPAND + always_comb begin + e_trunc = 0; + for (int b = 0; b < 32; b++) if (vt[b[2:0]]) e_trunc = 6'(b + 1); + end + // verilator lint_on WIDTHEXPAND - integer cyc = 0; - integer fails = 0; + int cyc = 0; always @(posedge clk) begin cyc <= cyc + 1; if (cyc == 0) begin - p32 <= 32'h8000_0000; // MSB -> 32 - p48 <= 48'h0; p48[47] <= 1'b1; // MSB -> 48 (Q path) - p80 <= 80'h0; p80[79] <= 1'b1; // MSB -> 80 (W path) - pc <= 32'hf0f0_f0f0; // 16 set bits - pu <= 32'h0001_0000; // bit 16 -> 17 - vn <= 32'h0000_00b4; // bits {2,4,5,7} set - vw <= 32'h0010_0008; // bits {3,20}: prefix-scan(16)->4, full->21 + p32 <= 32'h8000_0000; + p48 <= 48'h0; + p48[47] <= 1'b1; + p80 <= 80'h0; + p80[79] <= 1'b1; + pu <= 32'h0001_0000; // bit 16 + p32e <= 32'h8000_0000; + p32w <= 32'h8000_0000; + pc <= 32'hf0f0_f0f0; // 16 ones + vn <= 32'h0000_00b4; // bits {2,4,5,7} + vw <= 32'h0010_0008; // bits {3,20} + vt <= 32'h0000_0080; // bit 7 + en1 <= 1'b0; // gate off -> compound loop yields 0 end else if (cyc == 1) begin - if (n32 !== 6'd32) fails = fails + 1; - if (n48 !== 7'd48) fails = fails + 1; - if (n80 !== 7'd80) fails = fails + 1; - if (nc !== 6'd16) fails = fails + 1; // popcount(0xF0F0F0F0) - if (nu !== 6'd17) fails = fails + 1; // unsigned-loop leading-one, bit 16 -> 17 - // negatives, hand-computed for vn = 0xB4 (set bits 2,4,5,7): - if (e_step2 !== 6'd5) fails = fails + 1; // highest even set bit (4) + 1 - if (e_start1 !== 7'd8) fails = fails + 1; // highest set bit in [1,32) (7) + 1 - if (e_mul !== 7'd15) fails = fails + 1; // 2*7 + 1 - if (e_off !== 6'd7) fails = fails + 1; // idx where vec[idx+1]; highest = 6 -> 7 - if (e_noP1 !== 6'd7) fails = fails + 1; // highest set bit (7), no +1 - if (e_narrow !== 6'd4) fails = fails + 1; // W != width: only low 16 bits scanned (bit 3) - if (fails == 0) $write("*-* All Finished *-*\n"); - else $write("FAILED: %0d mismatches\n", fails); + `checkh(n32, 6'd32); + `checkh(n48, 7'd48); + `checkh(n80, 7'd80); + `checkh(nu, 6'd17); // unsigned-index leading-one, bit 16 -> 17 + `checkh(n32e, 32'd32); + `checkh(n40, 40'd32); + `checkh(nc, 6'd16); // popcount(0xF0F0F0F0) + `checkh(kn, 6'd9); // mostsetbitp1(0x100), constant-folded + `checkh(kn0, 6'd0); // mostsetbitp1(0)=0, constant-folded (zero path) + // negatives, hand-computed for vn = 0xB4 (bits 2,4,5,7): + `checkh(e_step2, 6'd5); // highest even set bit (4) + 1 + `checkh(e_start1, 7'd8); // highest set bit in [1,32) (7) + 1 + `checkh(e_mul, 7'd15); // 2*7 + 1 + `checkh(e_off, 6'd7); // idx where vec[idx+1]; highest 6 -> 7 + `checkh(e_noP1, 6'd7); // highest set bit (7), no +1 + `checkh(e_narrow, 6'd4); // W=16 != width(vec): only low bits scanned (bit 3) + `checkh(e_comp, 6'd0); // && en1 (=0); a wrong lowering would give 8 + `checkh(e_trunc, 6'd32); // vt[b[2:0]] last hits b=31; a wrong lowering would give 8 + $write("*-* All Finished *-*\n"); $finish; end end diff --git a/test_regress/t/t_bit_scan_loops_off.py b/test_regress/t/t_bit_scan_loops_off.py index f11dffe28..cdf34ea55 100755 --- a/test_regress/t/t_bit_scan_loops_off.py +++ b/test_regress/t/t_bit_scan_loops_off.py @@ -4,21 +4,20 @@ # This program is free software; you can redistribute it and/or modify it # under the terms of either the GNU Lesser General Public License Version 3 # or the Perl Artistic License Version 2.0. -# SPDX-FileCopyrightText: 2024 Wilson Snyder +# SPDX-FileCopyrightText: 2026 Wilson Snyder # SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0 import vltest_bootstrap -test.scenarios('simulator') +test.scenarios('vlt') # Reuse the same design; only the optimization switch differs. test.top_filename = "t/t_bit_scan_loops.v" -test.compile(verilator_flags2=['--stats', '-fno-bit-scan-loops']) +test.compile(verilator_flags2=['--stats', '--unroll-count', '0', '-fno-bit-scan-loops']) -# With the optimization disabled, no loop may be lowered (count 0 / line absent). -test.file_grep_not(test.stats, - r'Lowered priority-encoder to mostsetbitp1\s+[1-9]') +# With the optimization disabled, nothing lowers. +test.file_grep(test.stats, r'Lowered priority-encoder to mostsetbitp1\s+([0-9])', 0) test.execute() diff --git a/test_regress/t/t_bit_scan_loops_xassign0.py b/test_regress/t/t_bit_scan_loops_xassign0.py new file mode 100755 index 000000000..083a30cba --- /dev/null +++ b/test_regress/t/t_bit_scan_loops_xassign0.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +# DESCRIPTION: Verilator: Verilog Test driver/expect definition +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of either the GNU Lesser General Public License Version 3 +# or the Perl Artistic License Version 2.0. +# SPDX-FileCopyrightText: 2026 Wilson Snyder +# SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0 + +import vltest_bootstrap + +test.scenarios('vlt') + +# Reuse the same design. '--x-assign 0' makes the auto-inserted out-of-range guard on a +# non-power-of-two bit-select a plain '(idx <= W-1) && vec[idx]' (AstLogAnd), rather than +# the ternary '(idx <= W-1) ? vec[idx] : ' (AstCond) produced under the driver's default +# '--x-assign unique'. This exercises the matcher's other guard-peel branch. +test.top_filename = "t/t_bit_scan_loops.v" + +test.compile(verilator_flags2=['--stats', '--unroll-count', '0', '--x-assign', '0']) + +# Same lowering counts as the default run -- only the guard shape differs, not the result. +test.file_grep(test.stats, + r'Optimizations, Loop unrolling, Lowered priority-encoder to mostsetbitp1\s+(\d+)', + 8) +test.file_grep(test.stats, + r'Optimizations, Loop unrolling, Lowered count-set-bits to countones\s+(\d+)', + 1) + +test.execute() + +test.passes()