diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt index 76b9a9cfa..9dd1fd5cd 100644 --- a/kernel/CMakeLists.txt +++ b/kernel/CMakeLists.txt @@ -25,6 +25,8 @@ yosys_core(kernel celledges.h celltypes.h compute_graph.h + compressor_tree.cc + compressor_tree.h consteval.h constids.inc cost.cc @@ -80,7 +82,6 @@ yosys_core(kernel topo_scc.h utils.h version.cc - wallace_tree.h yosys.cc yosys_common.h yosys_config.h diff --git a/kernel/compressor_tree.cc b/kernel/compressor_tree.cc new file mode 100644 index 000000000..de2260d26 --- /dev/null +++ b/kernel/compressor_tree.cc @@ -0,0 +1,328 @@ +#include "compressor_tree.h" + +YOSYS_NAMESPACE_BEGIN + +namespace CompressorTree +{ + +std::pair emit_compressor_32(Module *module, SigSpec a, SigSpec b, SigSpec c, int width) +{ + SigSpec sum = module->addWire(NEW_ID, width); + SigSpec cout = module->addWire(NEW_ID, width); + module->addFa(NEW_ID, a, b, c, cout, sum); + + SigSpec carry; + carry.append(State::S0); + carry.append(cout.extract(0, width - 1)); + return {sum, carry}; +} + +std::pair emit_compressor_42(Module *module, SigSpec a, SigSpec b, SigSpec c, SigSpec d, int width) +{ + // First FA: a + b + c -> s0 + SigSpec s0 = module->addWire(NEW_ID, width); + SigSpec cout_h_full = module->addWire(NEW_ID, width); + module->addFa(NEW_ID, a, b, c, cout_h_full, s0); + + // cin[0] = 0, cin[i] = cout_h_full[i-1] + SigSpec cin; + cin.append(State::S0); + if (width > 1) + cin.append(cout_h_full.extract(0, width - 1)); + + // Second FA: s0 + d + cin -> sum + SigSpec sum = module->addWire(NEW_ID, width); + SigSpec carry_full = module->addWire(NEW_ID, width); + module->addFa(NEW_ID, s0, d, cin, carry_full, sum); + + SigSpec carry; + carry.append(State::S0); + if (width > 1) + carry.append(carry_full.extract(0, width - 1)); + + return {sum, carry}; +} + +std::vector generate_partial_products(Module *module, SigSpec a, SigSpec b, bool a_signed, bool b_signed, int width) { + int width_a = GetSize(a); + int width_b = GetSize(b); + std::vector products; + products.reserve(width_a + 3); + + for (int i = 0; i < width_a; i++) { + SigBit ai = a[i]; + + // b_shifted = (0_i ## b) + SigSpec b_shifted = SigSpec(State::S0, i); + b_shifted.append(b); + b_shifted.extend_u0(width, false); + + // row = b_shifted & replicate(a[i], width) + SigSpec ai_rep = SigSpec(ai, width); + SigSpec row = module->addWire(NEW_ID, width); + module->addAnd(NEW_ID, b_shifted, ai_rep, row); + + // Apply Modified Baugh-Wooley inversions for this row + bool row_is_bottom = (i == width_a - 1); + bool any_inversion = (row_is_bottom && b_signed) || a_signed; + + if (any_inversion) { + std::vector mask(width, RTLIL::State::S0); + + for (int j = 0; j < width_b; j++) { + int col = i + j; + if (col < 0 || col >= width) + continue; + bool col_is_right = (j == width_b - 1); + // Flip masks + bool invert = (row_is_bottom && b_signed) ^ (col_is_right && a_signed); + if (invert) + mask[col] = RTLIL::State::S1; + } + + // Skip the xor entirely if the mask is all zeroes + bool nonzero = false; + for (auto s : mask) + if (s == RTLIL::State::S1) { + nonzero = true; + break; + } + if (nonzero) { + SigSpec inverted = module->addWire(NEW_ID, width); + module->addXor(NEW_ID, row, SigSpec(RTLIL::Const(mask)), inverted); + row = inverted; + } + } + + products.push_back({row, 0}); + } + + // Correction constants + auto push_one_at = [&](int col) { + if (col < 0 || col >= width) + return; + std::vector v(width, RTLIL::State::S0); + v[col] = RTLIL::State::S1; + products.push_back({SigSpec(RTLIL::Const(v)), 0}); + }; + + if (b_signed) + push_one_at(width_a - 1); + if (a_signed) + push_one_at(width_b - 1); + if (a_signed || b_signed) + push_one_at(width_a + width_b - 1); + + return products; +} + +std::pair reduce_scheduled(Module *module, std::vector operands, int width, Strategy strategy, int *out_compressor_count, int *out_final_depth) { + int levels = 0; + int fa_count = 0; + int c42_count = 0; + int final_depth = 0; + + for (auto &op : operands) + op.sig.extend_u0(width); + + // Only compress operands ready at current level + for (int level = 0; operands.size() > 2; level++) { + // Partition operands into ready and waiting + std::vector ready; + std::vector waiting; + ready.reserve(operands.size()); + for (auto &op : operands) { + if (op.depth <= level) + ready.push_back(op); + else + waiting.push_back(op); + } + + if (ready.size() < 3) { + levels++; + continue; + } + + // Apply compressors to ready operands + std::vector compressed; + compressed.reserve(ready.size()); + size_t i = 0; + + // PREFER_42 attempts 4:2 grouping greedily (falls back to 3:2 for the residual) + // FA_ONLY skips + // DADDA = PREFER_42 (TODO: inspect column heights?) + bool try_42 = (strategy == Strategy::PREFER_42 || strategy == Strategy::DADDA); + + while (i < ready.size()) { + size_t remaining = ready.size() - i; + + if (try_42 && remaining >= 4) { + DepthSig a = ready[i + 0]; + DepthSig b = ready[i + 1]; + DepthSig c = ready[i + 2]; + DepthSig d = ready[i + 3]; + + auto [sum, carry] = emit_compressor_42(module, a.sig, b.sig, c.sig, d.sig, width); + int dmax = std::max({a.depth, b.depth, c.depth, d.depth}); + + compressed.push_back({sum, dmax + 2}); + compressed.push_back({carry, dmax + 2}); + + fa_count += 2; + c42_count += 1; + i += 4; + } else if (remaining >= 3) { + DepthSig a = ready[i + 0]; + DepthSig b = ready[i + 1]; + DepthSig c = ready[i + 2]; + + auto [sum, carry] = emit_compressor_32(module, a.sig, b.sig, c.sig, width); + int dmax = std::max({a.depth, b.depth, c.depth}); + + compressed.push_back({sum, dmax + 1}); + compressed.push_back({carry, dmax + 1}); + + fa_count += 1; + i += 3; + } else { + // Uncompressed operands pass through to next level + for (; i < ready.size(); i++) + compressed.push_back(ready[i]); + break; + } + } + + // Merge compressed with waiting operands + for (auto &op : waiting) + compressed.push_back(op); + + operands = std::move(compressed); + levels++; + } + + if(out_compressor_count) + *out_compressor_count = fa_count; + if (operands.size() == 0) { + if (out_final_depth) + *out_final_depth = 0; + return {SigSpec(State::S0, width), SigSpec(State::S0, width)}; + } + if (operands.size() == 1) { + if (out_final_depth) + *out_final_depth = operands[0].depth; + return {operands[0].sig, SigSpec(State::S0, width)}; + } + + final_depth = std::max(operands[0].depth, operands[1].depth); + if (out_final_depth) + *out_final_depth = final_depth; + log_assert(operands.size() == 2); + log(" CompressorTree::reduce_scheduled: %d levels, %d $fa (%d as 4:2), final depth %d\n", levels, fa_count, c42_count, final_depth); + return {operands[0].sig, operands[1].sig}; +} + +void emit_kogge_stone(Module *module, SigSpec a, SigSpec b, SigSpec y) +{ + int width = GetSize(y); + log_assert(GetSize(a) == width); + log_assert(GetSize(b) == width); + + if (width == 0) + return; + + if (width == 1) { + module->addXorGate(NEW_ID, a[0], b[0], y[0]); + return; + } + + // Bit level gen and prop + std::vector g_pre(width), p_pre(width); + for (int i = 0; i < width; i++) { + SigBit gi = module->addWire(NEW_ID); + SigBit pi = module->addWire(NEW_ID); + module->addAndGate(NEW_ID, a[i], b[i], gi); + module->addXorGate(NEW_ID, a[i], b[i], pi); + g_pre[i] = gi; + p_pre[i] = pi; + } + + // Propagate (g, p) through ceil(log2 W) levels + std::vector g = g_pre; + std::vector p = p_pre; + int num_levels = 0; + + while ((1 << num_levels) < width) + num_levels++; + + for (int k = 1; k <= num_levels; k++) { + int s = 1 << (k - 1); + std::vector g_next(width), p_next(width); + for (int i = 0; i < width; i++) { + if (i < s) { + // Nothing to do + g_next[i] = g[i]; + p_next[i] = p[i]; + } else { + // g_i^k = g_i | (p_i & g_(i-s)) + SigBit and_pg = module->addWire(NEW_ID); + module->addAndGate(NEW_ID, p[i], g[i - s], and_pg); + SigBit gnew = module->addWire(NEW_ID); + module->addOrGate(NEW_ID, g[i], and_pg, gnew); + g_next[i] = gnew; + + // p_i^k = p_i & p_(i-s) + if (k < num_levels) { + SigBit pnew = module->addWire(NEW_ID); + module->addAndGate(NEW_ID, p[i], p[i - s], pnew); + p_next[i] = pnew; + } else { + // Skip last level + p_next[i] = State::Sx; + } + } + } + + g = std::move(g_next); + p = std::move(p_next); + } + + // Sum layer, g[i] is COUT of bit i + // With CIN 0: + // sum[0] = p_pre[0] + // sum[i] = p_pre[i] ^ g[i-1] ... + module->connect(y[0], p_pre[0]); + for (int i = 1; i < width; i++) + module->addXorGate(NEW_ID, p_pre[i], g[i - 1], y[i]); +} + +Cell *emit_final_adder(Module *module, SigSpec a, SigSpec b, SigSpec y, FinalAdder choice) { + switch (choice) { + case FinalAdder::DEFAULT: + case FinalAdder::RIPPLE: { + return module->addAdd(NEW_ID, a, b, y, false); + } + case FinalAdder::PARALLEL_PREFIX: { + emit_kogge_stone(module, a, b, y); + return nullptr; + } + } + log_assert(false && "CompressorTree::emit_final_adder: invalid choice"); + return nullptr; +} + +FinalAdder pick_final_adder(int width, int final_depth, FinalMode mode) { + switch (mode) { + case FinalMode::RIPPLE: return FinalAdder::RIPPLE; + case FinalMode::PREFIX: return FinalAdder::PARALLEL_PREFIX; + case FinalMode::AUTO: + default: { + bool wide = width >= RIPPLE_PREFIX_WIDTH_THRESHOLD; + bool deep = final_depth >= RIPPLE_PREFIX_DEPTH_THRESHOLD; + return (wide && deep) ? FinalAdder::PARALLEL_PREFIX : FinalAdder::DEFAULT; + } + } +} + +} // namespace CompressorTree + +YOSYS_NAMESPACE_END diff --git a/kernel/compressor_tree.h b/kernel/compressor_tree.h new file mode 100644 index 000000000..5ac9d1c3e --- /dev/null +++ b/kernel/compressor_tree.h @@ -0,0 +1,117 @@ +/** + * Generalized compressor-tree utilities for multi-operand addition + * + * Terminology: + * - compressor: $fa viewed as reducing N inputs to M outputs (sum + shifted carry) (N:M compressor) + * - level: A stage of parallel compression operations + * - depth: Maximum number of N:M compressor levels from any input to a signal + * + * Supported compressors: + * - 3:2 compressor + * - 4:2 compressor + * + * References: + * - "Some schemes for parallel multipliers" (https://www.acsel-lab.com/arithmetic/arith6/papers/ARITH6_Dadda.pdf) + * - "Basilisk: Achieving Competitive Performance with Open EDA Tools" (https://arxiv.org/pdf/2405.03523) + * - "Binary Adder Architectures for Cell-Based VLSI and their Synthesis" (https://iis-people.ee.ethz.ch/~zimmi/publications/adder_arch.pdf) + * - "A Suggestion for a Fast Multiplier" (https://www.ece.ucdavis.edu/~vojin/CLASSES/EEC280/Web-page/papers/Arithmetic/Wallace_mult.pdf) + */ + +#ifndef COMPRESSOR_TREE_H +#define COMPRESSOR_TREE_H + +#include "kernel/sigtools.h" +#include "kernel/yosys.h" + +YOSYS_NAMESPACE_BEGIN + +namespace CompressorTree +{ + +// Width and depth thresholds below which a ripple is preferred over parallel-prefix +// NOTE: Based on "Binary Adder Architectures for Cell-Based VLSI and their Synthesis" (Tables 4.7, 4.9) - the threshold +// should be the point where Kogge-Stone isn't strictly less efficient than RCA +constexpr int RIPPLE_PREFIX_WIDTH_THRESHOLD = 16; +constexpr int RIPPLE_PREFIX_DEPTH_THRESHOLD = 5; + +enum class Strategy { + FA_ONLY, // 3:2 compressors + PREFER_42, // Prefer 4:2 grouping when >=4 operands ready + DADDA, // Defer compression until column counts exceed +}; + +struct DepthSig { + SigSpec sig; + int depth; +}; + +enum class FinalAdder { + DEFAULT, // emit $add and let downstream techmap pick + RIPPLE, // emit $add with explicit narrow hint + PARALLEL_PREFIX, // emit $add with PARALLEL_PREFIX +}; + +enum class FinalMode { + AUTO, + RIPPLE, + PREFIX, +}; + +std::pair emit_compressor_32(Module *module, SigSpec a, SigSpec b, SigSpec c, int width); +std::pair emit_compressor_42(Module *module, SigSpec a, SigSpec b, SigSpec c, SigSpec d, int width); + +/** + * generate_partial_products() - Generate partial products for FMA concat + * @module:The Yosys module to which the compressors will be added + * @a: Signal A + * @b: Signal B + * @a_signed: Whether signal A is signed + * @b_signed: Whether signal B is signed + * @width: Target width + * + * Return: Partial-product matrix as a set of depth-0 vectors + */ +std::vector generate_partial_products(Module *module, SigSpec a, SigSpec b, bool a_signed, bool b_signed, int width); + +/** + * reduce_scheduled() - Reduce multiple operands to two using a compressor tree + * @module: The Yosys module to which the compressors will be added + * @operands: Vector of operands to be reduced + * @sigs: Vector of input signals (operands) to be reduced + * @width: Target bit-width to which all operands will be zero-extended + * @strategy: Compression strategy to use + * @out_compressor_count: Optional pointer to return the number of $fa cells emitted + * @out_final_depth: Optional pointer to return the final depth of the scheduled tree + * + * Return: The final two reduced operands, that are to be fed into an adder + */ +std::pair reduce_scheduled(Module *module, std::vector operands, int width, Strategy strategy, int *out_compressor_count = nullptr, int *out_final_depth = nullptr); + +/** + * emit_kogge_stone() - Emit a Kogge-Stone parallel-prefix adder + * @module: The Yosys module to which the gates will be added + * @a: Signal A + * @b: Signal B + * @y: Signal Y = (A + B) mod 2^W + */ +void emit_kogge_stone(Module *module, SigSpec a, SigSpec b, SigSpec y); + +/** + * emit_final_adder() - Emit the final carry-propagate addition between the two reduced vectors + * @module:The Yosys module to which the compressors will be added + * @a: Signal A + * @b: Signal B + * @y: Signal Y + * @choice: Adder type to instantiate + * + * Return: Cell* of the emitted instance + */ +Cell *emit_final_adder(Module *module, SigSpec a, SigSpec b, SigSpec y, FinalAdder choice); + +FinalAdder pick_final_adder(int width, int final_depth, FinalMode mode); + +} // namespace CompressorTree + +YOSYS_NAMESPACE_END + +#endif // COMPRESSOR_TREE_H \ No newline at end of file diff --git a/kernel/wallace_tree.h b/kernel/wallace_tree.h deleted file mode 100644 index eb3513803..000000000 --- a/kernel/wallace_tree.h +++ /dev/null @@ -1,112 +0,0 @@ -/** - * Wallace tree utilities for multi-operand addition using carry-save adders - * - * Terminology: - * - compressor: $fa viewed as reducing 3 inputs to 2 outputs (sum + shifted carry) (3:2 compressor) - * - level: A stage of parallel compression operations - * - depth: Maximum number of 3:2 compressor levels from any input to a signal - * - * References: - * - "Binary Adder Architectures for Cell-Based VLSI and their Synthesis" (https://iis-people.ee.ethz.ch/~zimmi/publications/adder_arch.pdf) - * - "A Suggestion for a Fast Multiplier" (https://www.ece.ucdavis.edu/~vojin/CLASSES/EEC280/Web-page/papers/Arithmetic/Wallace_mult.pdf) - */ - -#ifndef WALLACE_TREE_H -#define WALLACE_TREE_H - -#include "kernel/sigtools.h" -#include "kernel/yosys.h" - -YOSYS_NAMESPACE_BEGIN - -inline std::pair emit_fa(Module *module, SigSpec a, SigSpec b, SigSpec c, int width) -{ - SigSpec sum = module->addWire(NEW_ID, width); - SigSpec cout = module->addWire(NEW_ID, width); - - module->addFa(NEW_ID, a, b, c, cout, sum); - - SigSpec carry; - carry.append(State::S0); - carry.append(cout.extract(0, width - 1)); - return {sum, carry}; -} - -/** - * wallace_reduce_scheduled() - Reduce multiple operands to two using a Wallace tree - * @module: The Yosys module to which the compressors will be added - * @sigs: Vector of input signals (operands) to be reduced - * @width: Target bit-width to which all operands will be zero-extended - * @compressor_count: Optional pointer to return the number of $fa cells emitted - * - * Return: The final two reduced operands, that are to be fed into an adder - */ -inline std::pair wallace_reduce_scheduled(Module *module, std::vector &sigs, int width, int *compressor_count = nullptr) -{ - struct DepthSig { - SigSpec sig; - int depth; - }; - - for (auto &s : sigs) - s.extend_u0(width); - - std::vector operands; - operands.reserve(sigs.size()); - for (auto &s : sigs) - operands.push_back({s, 0}); - - // Number of $fa's emitted - if (compressor_count) - *compressor_count = 0; - - // Only compress operands ready at current level - for (int level = 0; operands.size() > 2; level++) { - // Partition operands into ready and waiting - std::vector ready, waiting; - for (auto &op : operands) { - if (op.depth <= level) - ready.push_back(op); - else - waiting.push_back(op); - } - - if (ready.size() < 3) - continue; - - // Apply compressors to ready operands - std::vector compressed; - size_t i = 0; - while (i + 2 < ready.size()) { - auto [sum, carry] = emit_fa(module, ready[i].sig, ready[i + 1].sig, ready[i + 2].sig, width); - int new_depth = std::max({ready[i].depth, ready[i + 1].depth, ready[i + 2].depth}) + 1; - compressed.push_back({sum, new_depth}); - compressed.push_back({carry, new_depth}); - if (compressor_count) - (*compressor_count)++; - i += 3; - } - // Uncompressed operands pass through to next level - for (; i < ready.size(); i++) - compressed.push_back(ready[i]); - // Merge compressed with waiting operands - for (auto &op : waiting) - compressed.push_back(op); - - operands = std::move(compressed); - } - - if (operands.size() == 0) - return {SigSpec(State::S0, width), SigSpec(State::S0, width)}; - else if (operands.size() == 1) - return {operands[0].sig, SigSpec(State::S0, width)}; - else { - log_assert(operands.size() == 2); - log(" Wallace tree depth: %d levels of $fa + 1 final $add\n", std::max(operands[0].depth, operands[1].depth)); - return {operands[0].sig, operands[1].sig}; - } -} - -YOSYS_NAMESPACE_END - -#endif diff --git a/passes/techmap/arith_tree.cc b/passes/techmap/arith_tree.cc index 259e0c5bb..39217f817 100644 --- a/passes/techmap/arith_tree.cc +++ b/passes/techmap/arith_tree.cc @@ -1,5 +1,5 @@ /** - * Replaces chains of $add/$sub and $macc cells with carry-save adder trees + * Replaces chains of $add/$sub/$alu and $macc cells with carry-save compression trees * * Terminology: * - parent: Cells that consume another cell's output @@ -7,9 +7,9 @@ * - chain: Connected path of chainable cells */ +#include "kernel/compressor_tree.h" #include "kernel/macc.h" #include "kernel/sigtools.h" -#include "kernel/wallace_tree.h" #include "kernel/yosys.h" #include @@ -17,49 +17,57 @@ USING_YOSYS_NAMESPACE PRIVATE_NAMESPACE_BEGIN -struct Operand { - SigSpec sig; - bool is_signed; - bool negate; +struct ArithTreeOptions { + CompressorTree::Strategy strategy = CompressorTree::Strategy::PREFER_42; + CompressorTree::FinalMode final_mode = CompressorTree::FinalMode::RIPPLE; + bool fma_fusion = true; }; -struct Traversal { +struct ArithTreeWorker { + const ArithTreeOptions &opt; + Module *module; SigMap sigmap; + dict> bit_consumers; dict fanout; - Traversal(Module *module) : sigmap(module) - { - for (auto cell : module->cells()) - for (auto &conn : cell->connections()) - if (cell->input(conn.first)) - for (auto bit : sigmap(conn.second)) - bit_consumers[bit].insert(cell); - for (auto &pair : bit_consumers) - fanout[pair.first] = pair.second.size(); + pool addsub; + pool alu; + pool macc; + + struct Operand { + SigSpec sig; + bool is_signed; + bool negate; + // With FMA, when both factors are set, the operand represents a product to + // be expanded into partial products at extraction time, is_signed then + // applies to factor_a, and factor_b carries its own signedness + SigSpec factor_b; // empty for regular operands + bool factor_b_signed = false; + }; + + ArithTreeWorker(const ArithTreeOptions &opt, Module *module) : opt(opt), module(module), sigmap(module) + { + // Build traversal data + for (auto cell : module->cells()) { + for (auto &[name, sig] : cell->connections()) { + if (cell->input(name)) { + for (auto bit : sigmap(sig)) { + bit_consumers[bit].insert(cell); + } + } + } + } + + for (auto &[sig, consumers] : bit_consumers) + fanout[sig] = consumers.size(); for (auto wire : module->wires()) if (wire->port_output) for (auto bit : sigmap(SigSpec(wire))) fanout[bit]++; - } -}; -struct Cells { - pool addsub; - pool alu; - pool macc; - - static bool is_addsub(Cell *cell) { return cell->type == ID($add) || cell->type == ID($sub); } - - static bool is_alu(Cell *cell) { return cell->type == ID($alu); } - - static bool is_macc(Cell *cell) { return cell->type == ID($macc) || cell->type == ID($macc_v2); } - - bool empty() { return addsub.empty() && alu.empty() && macc.empty(); } - - Cells(Module *module) - { + // Collect cell data for (auto cell : module->cells()) { if (is_addsub(cell)) addsub.insert(cell); @@ -69,59 +77,55 @@ struct Cells { macc.insert(cell); } } -}; -struct AluInfo { - Cells &cells; - Traversal &traversal; - bool is_subtract(Cell *cell) - { - SigSpec bi = traversal.sigmap(cell->getPort(ID::BI)); - SigSpec ci = traversal.sigmap(cell->getPort(ID::CI)); + bool is_addsub(Cell *cell) { + return cell->type == ID($add) || cell->type == ID($sub); + } + + bool is_alu(Cell *cell) { + return cell->type == ID($alu); + } + + bool is_macc(Cell *cell) { + return cell->type == ID($macc) || cell->type == ID($macc_v2); + } + + bool is_sub(Cell *cell) { + SigSpec bi = sigmap(cell->getPort(ID::BI)); + SigSpec ci = sigmap(cell->getPort(ID::CI)); return GetSize(bi) == 1 && bi[0] == State::S1 && GetSize(ci) == 1 && ci[0] == State::S1; } bool is_add(Cell *cell) { - SigSpec bi = traversal.sigmap(cell->getPort(ID::BI)); - SigSpec ci = traversal.sigmap(cell->getPort(ID::CI)); + SigSpec bi = sigmap(cell->getPort(ID::BI)); + SigSpec ci = sigmap(cell->getPort(ID::CI)); return GetSize(bi) == 1 && bi[0] == State::S0 && GetSize(ci) == 1 && ci[0] == State::S0; } bool is_chainable(Cell *cell) { - if (!(is_add(cell) || is_subtract(cell))) + if (!(is_add(cell) || is_sub(cell))) return false; - - for (auto bit : traversal.sigmap(cell->getPort(ID::X))) - if (traversal.fanout.count(bit) && traversal.fanout[bit] > 0) + for (auto bit : sigmap(cell->getPort(ID::X))) + if (fanout.count(bit) && fanout[bit] > 0) return false; - for (auto bit : traversal.sigmap(cell->getPort(ID::CO))) - if (traversal.fanout.count(bit) && traversal.fanout[bit] > 0) + for (auto bit : sigmap(cell->getPort(ID::CO))) + if (fanout.count(bit) && fanout[bit] > 0) return false; - return true; } -}; - -struct Rewriter { - Module *module; - Cells &cells; - Traversal traversal; - AluInfo alu_info; - - Rewriter(Module *module, Cells &cells) : module(module), cells(cells), traversal(module), alu_info{cells, traversal} {} Cell *sole_chainable_consumer(SigSpec sig, const pool &candidates) { Cell *consumer = nullptr; for (auto bit : sig) { - if (!traversal.fanout.count(bit) || traversal.fanout[bit] != 1) + if (!fanout.count(bit) || fanout[bit] != 1) return nullptr; - if (!traversal.bit_consumers.count(bit) || traversal.bit_consumers[bit].size() != 1) + if (!bit_consumers.count(bit) || bit_consumers[bit].size() != 1) return nullptr; - Cell *c = *traversal.bit_consumers[bit].begin(); + Cell *c = *bit_consumers[bit].begin(); if (!candidates.count(c)) return nullptr; @@ -137,7 +141,7 @@ struct Rewriter { { dict parent_of; for (auto cell : candidates) { - Cell *consumer = sole_chainable_consumer(traversal.sigmap(cell->getPort(ID::Y)), candidates); + Cell *consumer = sole_chainable_consumer(sigmap(cell->getPort(ID::Y)), candidates); if (consumer && consumer != cell) parent_of[cell] = consumer; } @@ -177,12 +181,12 @@ struct Rewriter { { pool bits; for (auto cell : chain) - for (auto bit : traversal.sigmap(cell->getPort(ID::Y))) + for (auto bit : sigmap(cell->getPort(ID::Y))) bits.insert(bit); return bits; } - static bool overlaps(SigSpec sig, const pool &bits) + bool overlaps(SigSpec sig, const pool &bits) { for (auto bit : sig) if (bits.count(bit)) @@ -195,17 +199,16 @@ struct Rewriter { bool parent_subtracts; if (parent->type == ID($sub)) parent_subtracts = true; - else if (cells.is_alu(parent)) - parent_subtracts = alu_info.is_subtract(parent); + else if (is_alu(parent)) + parent_subtracts = is_sub(parent); else return false; if (!parent_subtracts) return false; - // Check if any bit of child's Y connects to parent's B - SigSpec child_y = traversal.sigmap(child->getPort(ID::Y)); - SigSpec parent_b = traversal.sigmap(parent->getPort(ID::B)); + SigSpec child_y = sigmap(child->getPort(ID::Y)); + SigSpec parent_b = sigmap(parent->getPort(ID::B)); for (auto bit : child_y) for (auto pbit : parent_b) if (bit == pbit) @@ -244,21 +247,20 @@ struct Rewriter { for (auto cell : chain) { bool cell_neg = negated.count(cell) ? negated[cell] : false; - SigSpec a = traversal.sigmap(cell->getPort(ID::A)); - SigSpec b = traversal.sigmap(cell->getPort(ID::B)); + SigSpec a = sigmap(cell->getPort(ID::A)); + SigSpec b = sigmap(cell->getPort(ID::B)); bool a_signed = cell->getParam(ID::A_SIGNED).as_bool(); bool b_signed = cell->getParam(ID::B_SIGNED).as_bool(); - bool b_sub = (cell->type == ID($sub)) || (cells.is_alu(cell) && alu_info.is_subtract(cell)); + bool b_sub = (cell->type == ID($sub)) || (is_alu(cell) && is_sub(cell)); - // Only add operands not produced by other chain cells if (!overlaps(a, chain_bits)) { - operands.push_back({a, a_signed, cell_neg}); + operands.push_back({a, a_signed, cell_neg, SigSpec(), false}); if (cell_neg) neg_compensation++; } if (!overlaps(b, chain_bits)) { bool neg = cell_neg ^ b_sub; - operands.push_back({b, b_signed, neg}); + operands.push_back({b, b_signed, neg, SigSpec(), false}); if (neg) neg_compensation++; } @@ -272,63 +274,83 @@ struct Rewriter { neg_compensation = 0; for (auto &term : macc.terms) { - // Bail on multiplication - if (GetSize(term.in_b) != 0) - return false; - operands.push_back({term.in_a, term.is_signed, term.do_subtract}); + if (GetSize(term.in_b) != 0) { + if (!opt.fma_fusion) + return false; + + // Preserve term as a multiplicative operand which is expanded into partial products + Operand op; + op.sig = term.in_a; + op.is_signed = term.is_signed; + op.negate = term.do_subtract; + op.factor_b = term.in_b; + op.factor_b_signed = term.is_signed; + operands.push_back(op); + continue; + } + operands.push_back({term.in_a, term.is_signed, term.do_subtract, SigSpec(), false}); if (term.do_subtract) neg_compensation++; } return true; } - SigSpec extend_operand(SigSpec sig, bool is_signed, int width) + std::vector build_operand_pool(std::vector &operands, int width, int &neg_compensation) { - if (GetSize(sig) < width) { - SigBit pad; - if (is_signed && GetSize(sig) > 0) - pad = sig[GetSize(sig) - 1]; - else - pad = State::S0; - sig.append(SigSpec(pad, width - GetSize(sig))); - } - if (GetSize(sig) > width) - sig = sig.extract(0, width); - return sig; - } - - void replace_with_carry_save_tree(std::vector &operands, SigSpec result_y, int neg_compensation, const char *desc) - { - int width = GetSize(result_y); - std::vector extended; - extended.reserve(operands.size() + 1); + // Expand operands into a flat list of signals for reduction + std::vector pool; + pool.reserve(operands.size() * 2); for (auto &op : operands) { - SigSpec s = extend_operand(op.sig, op.is_signed, width); - if (op.negate) - s = module->Not(NEW_ID, s); - extended.push_back(s); + if (GetSize(op.factor_b) == 0) { + // Additive operand + op.sig.extend_u0(width, op.is_signed); + if (op.negate) + op.sig = module->Not(NEW_ID, op.sig); + pool.push_back({op.sig, 0}); + } else { + // Multiplicative operand + auto pps = CompressorTree::generate_partial_products(module, op.sig, op.factor_b, op.is_signed, op.factor_b_signed, width); + + if (!op.negate) { + for (auto &pp : pps) + pool.push_back(pp); + continue; + } + + auto [pa, pb] = CompressorTree::reduce_scheduled(module, pps, width, opt.strategy); + SigSpec p = module->addWire(NEW_ID, width); + module->addAdd(NEW_ID, pa, pb, p, false); + SigSpec np = module->addWire(NEW_ID, width); + module->addNot(NEW_ID, p, np); + pool.push_back({np, 0}); + neg_compensation++; + } } - // Add correction for negated operands (-x = ~x + 1 so 1 per negation) if (neg_compensation > 0) - extended.push_back(SigSpec(neg_compensation, width)); + pool.push_back({SigSpec(neg_compensation, width), 0}); - int compressor_count; - auto [a, b] = wallace_reduce_scheduled(module, extended, width, &compressor_count); - log(" %s -> %d $fa + 1 $add (%d operands, module %s)\n", desc, compressor_count, (int)operands.size(), module); + return pool; + } - // Emit final add - module->addAdd(NEW_ID, a, b, result_y, false); + void emit_tree(std::vector &operands, SigSpec result_y, int neg_compensation) + { + int width = GetSize(result_y); + auto pool = build_operand_pool(operands, width, neg_compensation); + int final_depth = 0; + auto [a, b] = CompressorTree::reduce_scheduled(module, std::move(pool), width, opt.strategy, nullptr, &final_depth); + auto final_choice = CompressorTree::pick_final_adder(width, final_depth, opt.final_mode); + CompressorTree::emit_final_adder(module, a, b, result_y, final_choice); } void process_chains() { pool candidates; - for (auto cell : cells.addsub) + for (auto cell : addsub) candidates.insert(cell); - for (auto cell : cells.alu) - if (alu_info.is_chainable(cell)) + for (auto cell : alu) + if (is_chainable(cell)) candidates.insert(cell); if (candidates.empty()) @@ -354,7 +376,7 @@ struct Rewriter { for (auto c : chain) to_remove.insert(c); - replace_with_carry_save_tree(operands, root->getPort(ID::Y), neg_compensation, "Replaced add/sub chain"); + emit_tree(operands, root->getPort(ID::Y), neg_compensation); } for (auto cell : to_remove) @@ -363,48 +385,66 @@ struct Rewriter { void process_maccs() { - for (auto cell : cells.macc) { + pool to_remove; + for (auto cell : macc) { std::vector operands; int neg_compensation; if (!extract_macc_operands(cell, operands, neg_compensation)) continue; - if (operands.size() < 3) + if (operands.size() < 1) continue; - - replace_with_carry_save_tree(operands, cell->getPort(ID::Y), neg_compensation, "Replaced $macc"); - module->remove(cell); + int mul_terms = 0; + for (auto &op : operands) + if (GetSize(op.factor_b) > 0) + mul_terms++; + bool has_mul = (mul_terms > 0); + if (mul_terms == 1 && operands.size() == 1) + continue; + if (!has_mul && operands.size() < 3) + continue; + emit_tree(operands, cell->getPort(ID::Y), neg_compensation); + to_remove.insert(cell); } + for (auto cell : to_remove) + module->remove(cell); +} + + void run() + { + if (addsub.empty() && alu.empty() && macc.empty()) + return; + + process_chains(); + process_maccs(); } }; -void run(Module *module) -{ - Cells cells(module); - - if (cells.empty()) - return; - - Rewriter rewriter{module, cells}; - rewriter.process_chains(); - rewriter.process_maccs(); -} - struct ArithTreePass : public Pass { - ArithTreePass() : Pass("arith_tree", "convert add/sub/macc chains to carry-save adder trees") {} + ArithTreePass() : Pass("arith_tree", "convert add/sub/macc/alu chains to carry-save adder trees") {} void help() override { // |---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---| log("\n"); - log(" arith_tree [selection]\n"); + log(" arith_tree [options] [selection]\n"); log("\n"); log("This pass replaces chains of $add/$sub cells, $alu cells (with constant\n"); - log("BI/CI), and $macc/$macc_v2 cells (without multiplications) with carry-save\n"); - log("adder trees using $fa cells and a single final $add.\n"); + log("BI/CI), and $macc/$macc_v2 cells with carry-save adder trees \n"); + log("using $fa cells and a single final adder.\n"); log("\n"); - log("The tree uses Wallace-tree scheduling: at each level, ready operands are\n"); - log("grouped into triplets and compressed via full adders, giving\n"); - log("O(log_{1.5} N) depth for N input operands.\n"); + log(" -strategy \n"); + log(" Compressor strategy. 'fa' uses only 3:2 full-adder groupings\n"); + log(" '42' (the default) prefers 4:2 compressor groupings, with\n"); + log(" fallback to 3:2 compressors for residuals\n"); + log("\n"); + log(" -final \n"); + log(" Selects the architecture used for the final two-vector add.\n"); + log("\n"); + log(" -no-fma\n"); + log(" Disable fused multiply-add expansion in $macc cells\n"); + log("\n"); + log("The default behaviour delivers 4:2 compression, FMA fusion, and a\n"); + log("final standard adder\n"); log("\n"); } @@ -412,15 +452,39 @@ struct ArithTreePass : public Pass { { log_header(design, "Executing ARITH_TREE pass.\n"); + ArithTreeOptions opt; + size_t argidx; - for (argidx = 1; argidx < args.size(); argidx++) + for (argidx = 1; argidx < args.size(); argidx++) { + const std::string &arg = args[argidx]; + if (arg == "-strategy" && argidx + 1 < args.size()) { + const std::string &v = args[++argidx]; + if (v == "fa") { opt.strategy = CompressorTree::Strategy::FA_ONLY; } + else if (v == "42") { opt.strategy = CompressorTree::Strategy::PREFER_42; } + else { log_cmd_error("arith_tree: unknown -strategy '%s'\n", v.c_str()); } + continue; + } + if (arg == "-final" && argidx + 1 < args.size()) { + const std::string &v = args[++argidx]; + if (v == "auto") { opt.final_mode = CompressorTree::FinalMode::AUTO; } + else if (v == "ripple") { opt.final_mode = CompressorTree::FinalMode::RIPPLE; } + else if (v == "prefix") { opt.final_mode = CompressorTree::FinalMode::PREFIX; } + else { log_cmd_error("arith_tree: unknown -final '%s'\n", v.c_str()); } + continue; + } + if (arg == "-no-fma") { + opt.fma_fusion = false; + continue; + } break; + } extra_args(args, argidx, design); - for (auto module : design->selected_modules()) { - run(module); + for (auto mod : design->selected_modules()) { + ArithTreeWorker worker(opt, mod); + worker.run(); } } } ArithTreePass; -PRIVATE_NAMESPACE_END +PRIVATE_NAMESPACE_END \ No newline at end of file diff --git a/passes/techmap/booth.cc b/passes/techmap/booth.cc index 630c83f8c..972edc431 100644 --- a/passes/techmap/booth.cc +++ b/passes/techmap/booth.cc @@ -58,7 +58,7 @@ synth -top my_design -booth #include "kernel/sigtools.h" #include "kernel/yosys.h" #include "kernel/macc.h" -#include "kernel/wallace_tree.h" +#include "kernel/compressor_tree.h" USING_YOSYS_NAMESPACE PRIVATE_NAMESPACE_BEGIN @@ -386,7 +386,11 @@ struct BoothPassWorker { // Later on yosys will clean up unused constants // DebugDumpAlignPP(aligned_pp); - auto [wtree_a, wtree_b] = wallace_reduce_scheduled(module, aligned_pp, z_sz); + std::vector operands; + operands.reserve(aligned_pp.size()); + for (auto &s : aligned_pp) + operands.push_back({s, 0}); + auto [wtree_a, wtree_b] = CompressorTree::reduce_scheduled(module, std::move(operands), z_sz, CompressorTree::Strategy::FA_ONLY); // Debug code: Dump out the csa trees // DumpCSATrees(debug_csa_trees); diff --git a/passes/tests/CMakeLists.txt b/passes/tests/CMakeLists.txt index 007339ac5..5cedacf81 100644 --- a/passes/tests/CMakeLists.txt +++ b/passes/tests/CMakeLists.txt @@ -7,6 +7,9 @@ yosys_test_pass(cell yosys_test_pass(abcloop test_abcloop.cc ) +yosys_test_pass(kogge_stone + test_kogge_stone.cc +) yosys_pass(raise_error raise_error.cc diff --git a/passes/tests/test_kogge_stone.cc b/passes/tests/test_kogge_stone.cc new file mode 100644 index 000000000..95b97bb14 --- /dev/null +++ b/passes/tests/test_kogge_stone.cc @@ -0,0 +1,101 @@ +#include "kernel/compressor_tree.h" +#include "kernel/yosys.h" + +USING_YOSYS_NAMESPACE +PRIVATE_NAMESPACE_BEGIN + +static void build_lcu_adder(Module *module, SigSpec a, SigSpec b, SigSpec y) +{ + int width = GetSize(y); + + SigSpec p = module->Xor(NEW_ID, a, b); + SigSpec g = module->And(NEW_ID, a, b); + + SigSpec co = module->addWire(NEW_ID, width); + Cell *lcu = module->addCell(NEW_ID, ID($lcu)); + lcu->setParam(ID::WIDTH, width); + lcu->setPort(ID::P, p); + lcu->setPort(ID::G, g); + lcu->setPort(ID::CI, State::S0); + lcu->setPort(ID::CO, co); + + SigSpec carry_in; + carry_in.append(State::S0); + carry_in.append(co.extract(0, width - 1)); + module->addXor(NEW_ID, p, carry_in, y); +} + +static Module *make_module(Design *design, IdString name, int width) +{ + Module *module = design->addModule(name); + + Wire *a = module->addWire(ID(a), width); + a->port_input = true; + Wire *b = module->addWire(ID(b), width); + b->port_input = true; + Wire *y = module->addWire(ID(y), width); + y->port_output = true; + module->fixup_ports(); + + return module; +} + +struct TestKoggeStonePass : public Pass { + TestKoggeStonePass() : Pass("test_kogge_stone", "build adders for Kogge-Stone equivalence testing") {} + void help() override + { + // |---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---| + log("\n"); + log(" test_kogge_stone [options]\n"); + log("\n"); + log("Build two modules implementing an unsigned 'y = a + b' adder of a given width,\n"); + log("and compare various internal Kogge-Stone adders.\n"); + log("\n"); + log(" -width N\n"); + log(" width of the operands and result (default = 16)\n"); + log("\n"); + log(" -gold name\n"); + log(" name of the $lcu-based reference module (default = gold)\n"); + log("\n"); + log(" -gate name\n"); + log(" name of the emit_kogge_stone() module (default = gate)\n"); + log("\n"); + } + void execute(std::vector args, Design *design) override + { + int width = 16; + IdString gold_name = ID(gold); + IdString gate_name = ID(gate); + + size_t argidx; + for (argidx = 1; argidx < args.size(); argidx++) { + if (args[argidx] == "-width" && argidx + 1 < args.size()) { + width = atoi(args[++argidx].c_str()); + continue; + } + if (args[argidx] == "-gold" && argidx + 1 < args.size()) { + gold_name = RTLIL::escape_id(args[++argidx]); + continue; + } + if (args[argidx] == "-gate" && argidx + 1 < args.size()) { + gate_name = RTLIL::escape_id(args[++argidx]); + continue; + } + break; + } + extra_args(args, argidx, design, false); + + if (width < 1) + log_cmd_error("Width must be at least 1.\n"); + + log_header(design, "Executing TEST_KOGGE_STONE pass (width=%d).\n", width); + + Module *gold = make_module(design, gold_name, width); + build_lcu_adder(gold, gold->wire(ID(a)), gold->wire(ID(b)), gold->wire(ID(y))); + + Module *gate = make_module(design, gate_name, width); + CompressorTree::emit_kogge_stone(gate, gate->wire(ID(a)), gate->wire(ID(b)), gate->wire(ID(y))); + } +} TestKoggeStonePass; + +PRIVATE_NAMESPACE_END diff --git a/tests/arith_tree/arith_tree_42.ys b/tests/arith_tree/arith_tree_42.ys new file mode 100644 index 000000000..f0657cde3 --- /dev/null +++ b/tests/arith_tree/arith_tree_42.ys @@ -0,0 +1,122 @@ +read_verilog <