Add muladd rule.

passes/opt/CMakeLists.txt

@@ -83,6 +83,7 @@ pmgen_command(peepopt
 	peepopt_shiftpow2.pmg
 	peepopt_muldiv.pmg
 	peepopt_muldiv_c.pmg
+	peepopt_muladd.pmg
 	peepopt_formal_clockgateff.pmg
 	PREFIX
 		peepopt

passes/opt/peepopt.cc

@@ -29,6 +29,80 @@ bool did_something;
 // scratchpad configurations for pmgen
 int shiftadd_max_ratio;
 
+pool<SigBit> muladd_keep_bits, muladd_mul_bits;
+int muladd_min_product_width;
+int muladd_max_chain_depth;
+
+struct MuladdLevel {
+	Cell *adder;
+	IdString port;
+	SigSpec product;
+};
+
+IdString muladd_other_port(IdString name)
+{
+	return name == ID::A ? ID::B : ID::A;
+}
+
+IdString muladd_width_param(IdString name)
+{
+	return name == ID::A ? ID::A_WIDTH : ID::B_WIDTH;
+}
+
+bool muladd_is_product(Cell *cell)
+{
+	if (cell == nullptr || cell->type != ID($mul))
+		return false;
+	int operand_width = GetSize(cell->getPort(ID::A)) + GetSize(cell->getPort(ID::B));
+	return operand_width >= muladd_min_product_width;
+}
+
+bool muladd_holds_product(const SigSpec &sig)
+{
+	for (auto bit : sig)
+		if (muladd_mul_bits.count(bit))
+			return true;
+	return false;
+}
+
+bool muladd_signal_kept(const SigSpec &sig)
+{
+	for (auto bit : sig)
+		if (muladd_keep_bits.count(bit))
+			return true;
+	return false;
+}
+
+bool muladd_signals_overlap(const SigSpec &lhs, const SigSpec &rhs)
+{
+	pool<SigBit> lhs_bits(lhs.begin(), lhs.end());
+	for (auto bit : rhs)
+		if (lhs_bits.count(bit))
+			return true;
+	return false;
+}
+
+// reassociating is only exact when both hold
+bool muladd_levels_compatible(Cell *upper, Cell *lower)
+{
+	if (upper->getParam(ID::Y_WIDTH).as_int() > lower->getParam(ID::Y_WIDTH).as_int())
+		return false;
+	// the parameter is a bool of any width
+	return lower->getParam(ID::A_SIGNED).as_bool() == upper->getParam(ID::A_SIGNED).as_bool();
+}
+
+void muladd_rotate(Cell *outer, IdString outer_port, const vector<MuladdLevel> &levels, const SigSpec &addend)
+{
+	outer->setPort(outer_port, levels.front().product);
+	outer->setParam(muladd_width_param(outer_port), GetSize(levels.front().product));
+
+	for (int i = 0; i < GetSize(levels); i++) {
+		SigSpec moved = i + 1 < GetSize(levels) ? levels[i + 1].product : addend;
+		levels[i].adder->setPort(levels[i].port, moved);
+		levels[i].adder->setParam(muladd_width_param(levels[i].port), GetSize(moved));
+	}
+}
+
 // Helper function, removes LSB 0s
 SigSpec remove_bottom_padding(SigSpec sig)
 {
@@ -39,6 +113,20 @@ SigSpec remove_bottom_padding(SigSpec sig)
 
 #include "passes/opt/peepopt_pm.h"
 
+void collect_muladd_bits(peepopt_pm &pm)
+{
+	muladd_keep_bits.clear();
+	muladd_mul_bits.clear();
+	for (auto wire : pm.module->wires())
+		if (wire->get_bool_attribute(ID::keep))
+			for (auto bit : pm.sigmap(wire))
+				muladd_keep_bits.insert(bit);
+	for (auto cell : pm.module->cells())
+		if (cell->type == ID($mul))
+			for (auto bit : pm.sigmap(cell->getPort(ID::Y)))
+				muladd_mul_bits.insert(bit);
+}
+
 struct PeepoptPass : public Pass {
 	PeepoptPass() : Pass("peepopt", "collection of peephole optimizers") { }
 	void help() override
@@ -74,6 +162,13 @@ struct PeepoptPass : public Pass {
 		log("                Scratchpad: 'peepopt.shiftpow2.max_data_multiple' (default: 2)\n");
 		log("                limits padding for out-of-range select values.\n");
 		log("\n");
+		log("   * muladd - Replace ((P+A*B)+C*D)+E with ((P+E)+A*B)+C*D, so that DSP\n");
+		log("                inference can give both multipliers a post-adder.\n");
+		log("                Scratchpad: 'peepopt.muladd.min_product_width' (default: 11)\n");
+		log("                is the smallest A_WIDTH+B_WIDTH that counts as a product.\n");
+		log("                Scratchpad: 'peepopt.muladd.max_chain_depth' (default: 64,\n");
+		log("                max 256) limits how far the operand is sunk.\n");
+		log("\n");
 		log("If -formalclk is specified it instead employs the following rules:\n");
 		log("\n");
 		log("   * clockgateff - Replace latch based clock gating patterns with a flip-flop\n");
@@ -103,6 +198,12 @@ struct PeepoptPass : public Pass {
 		// 2x implies there is a constant shift larger than the input-data which should be extremely rare
 		shiftadd_max_ratio = design->scratchpad_get_int("peepopt.shiftadd.max_data_multiple", 2);
 
+		// 11 is the A_WIDTH+B_WIDTH ice40_dsp asks for
+		muladd_min_product_width = design->scratchpad_get_int("peepopt.muladd.min_product_width", 11);
+		muladd_max_chain_depth = design->scratchpad_get_int("peepopt.muladd.max_chain_depth", 64);
+		// the walk recurses per level, so an unbounded setting overflows the stack
+		muladd_max_chain_depth = std::min(muladd_max_chain_depth, 256);
+
 		for (auto module : design->selected_modules())
 		{
 			did_something = true;
@@ -124,6 +225,10 @@ struct PeepoptPass : public Pass {
 					pm.run_shiftpow2();
 					pm.run_muldiv();
 					pm.run_muldiv_c();
+					if (!did_something) {
+						collect_muladd_bits(pm);
+						pm.run_muladd();
+					}
 				}
 			}
 		}

passes/opt/peepopt_muladd.pmg

@@ -0,0 +1,132 @@
+// Sink a non-product operand to the bottom of an $add chain, lifting every
+// product one adder up: ((P + M1) + M2) + C  ->  ((P + C) + M1) + M2
+
+pattern muladd
+udata <vector<MuladdLevel>> chain
+state <IdString> innerAB outerAB walkPort nextPort
+state <SigSpec> sigC
+state <Cell*> walkAdder
+
+match mul
+	select mul->type == $mul
+	// the product must feed the adder and nothing else
+	select nusers(port(mul, \Y)) == 2
+	select muladd_is_product(mul)
+endmatch
+
+match inner
+	select inner->type == $add
+	select nusers(port(inner, \Y)) == 2
+	choice <IdString> AB {\A, \B}
+	index <SigSpec> port(inner, AB) === port(mul, \Y)
+	set innerAB AB
+endmatch
+
+match outer
+	select outer->type == $add
+	choice <IdString> AB {\A, \B}
+	index <SigSpec> port(outer, AB) === port(inner, \Y)
+	filter outer != inner
+	filter muladd_levels_compatible(outer, inner)
+	set outerAB AB
+	set sigC port(outer, AB == \A ? \B : \A)
+endmatch
+
+code walkAdder walkPort
+{
+	if (muladd_holds_product(sigC))
+		reject;
+
+	if (!module->processes.empty())
+		reject;
+
+	chain.clear();
+	walkAdder = inner;
+	walkPort = innerAB;
+	subpattern(walk);
+	reject;
+}
+endcode
+
+subpattern walk
+arg walkAdder walkPort outer outerAB sigC
+
+code
+{
+	if (muladd_signal_kept(port(walkAdder, \Y)))
+		reject;
+
+	if (muladd_signals_overlap(sigC, port(walkAdder, \Y)))
+		reject;
+
+	for (auto &level : chain)
+		if (level.adder == walkAdder)
+			reject;
+
+	if (GetSize(chain) >= muladd_max_chain_depth)
+		reject;
+}
+endcode
+
+code
+{
+	chain.push_back({walkAdder, walkPort, port(walkAdder, walkPort)});
+}
+finally
+	chain.pop_back();
+endcode
+
+// the product that ends the chain
+match bottom
+	select bottom->type == $mul
+	select muladd_is_product(bottom)
+	index <SigSpec> port(bottom, \Y) === port(walkAdder, muladd_other_port(walkPort))
+	semioptional
+endmatch
+
+code
+{
+	if (bottom) {
+		did_something = true;
+		log("muladd pattern in %s: %s sinks past %d adder(s)\n", module, log_signal(sigC), GetSize(chain));
+
+		muladd_rotate(outer, muladd_other_port(outerAB), chain, sigC);
+
+		blacklist(outer);
+		for (auto &level : chain)
+			blacklist(level.adder);
+		accept;
+	}
+}
+endcode
+
+// else the next adder down
+match below
+	if !bottom
+	select below->type == $add
+	select nusers(port(below, \Y)) == 2
+	choice <IdString> AB {\A, \B}
+	index <SigSpec> port(below, \Y) === port(walkAdder, muladd_other_port(walkPort))
+	filter muladd_levels_compatible(walkAdder, below)
+	set nextPort AB
+endmatch
+
+match belowmul
+	if below
+	select belowmul->type == $mul
+	select nusers(port(belowmul, \Y)) == 2
+	select muladd_is_product(belowmul)
+	index <SigSpec> port(belowmul, \Y) === port(below, nextPort)
+endmatch
+
+code walkAdder walkPort
+{
+	if (below == nullptr)
+		reject;
+
+	walkAdder = below;
+	walkPort = nextPort;
+	subpattern(walk);
+	reject;
+}
+endcode

passes/opt/peepopt_muldiv_c.pmg

@@ -119,6 +119,7 @@ code
 	autoremove(div);
 
 	// Log, fixup, accept
+	did_something = true;
 	log("muldiv_const pattern in %s: mul=%s, div=%s\n", module, mul, div);
 	mul->fixup_parameters();
 	accept;

tests/arch/ice40/bug2030.ys

@@ -0,0 +1,49 @@
+read_verilog <<EOT
+module top(
+	input clk,
+	input signed [5:0] px, py, tx, a, b,
+	output reg signed [11:0] out
+);
+	reg signed [5:0] tx_r;
+
+	always @(posedge clk)
+		tx_r <= tx;
+
+	always @(posedge clk)
+		out <= px * a + py * b + tx_r;
+endmodule
+
+module top2(
+	input clk,
+	input signed [15:0] pretranslate_x, pretranslate_y, translate_x, a, b,
+	output reg [23:0] out
+);
+	reg signed [15:0] translate_x_r;
+
+	always @(posedge clk)
+		translate_x_r <= translate_x;
+
+	always @(posedge clk)
+		out <= pretranslate_x * a + pretranslate_y * b + translate_x_r;
+endmodule
+EOT
+proc
+design -save read
+
+hierarchy -top top
+equiv_opt -run :prove -multiclock -assert -map +/ice40/cells_sim.v synth_ice40 -dsp
+clk2fflogic
+miter -equiv -flatten -make_assert -make_outputs gold gate miter
+sat -set-init-zero -seq 4 -verify -prove-asserts -show-ports miter
+
+design -load postopt
+cd top
+select -assert-count 2 t:SB_MAC16
+select -assert-none t:SB_MAC16 %% t:* %D
+
+design -load read
+hierarchy -top top2
+synth_ice40 -dsp
+cd top2
+select -assert-count 2 t:SB_MAC16
+select -assert-none t:SB_MAC16 %% t:* %D

tests/peepopt/muladd.ys

@@ -0,0 +1,82 @@
+read_verilog <<EOT
+module top(input signed [7:0] a, b, c, d, input signed [15:0] e, output signed [15:0] y);
+	assign y = a*b + c*d + e;
+endmodule
+EOT
+
+prep -top top
+
+# both products share one adder
+select -assert-count 1 t:$mul %co2 t:$add %i
+equiv_opt -assert peepopt
+design -load postopt
+select -assert-count 2 t:$mul %co2 t:$add %i
+design -reset
+
+read_verilog <<EOT
+module top(input [7:0] a, b, c, d, f, g, input [15:0] e, output [15:0] y);
+	assign y = e + (a*b + c*d + f*g);
+endmodule
+
+EOT
+prep -top top
+equiv_opt -assert peepopt
+design -load postopt
+select -assert-count 3 t:$mul %co2 t:$add %i
+design -reset
+# Reject patterns
+
+read_verilog <<EOT
+module sumfanout(input signed [7:0] a, b, c, d, input signed [15:0] e, output signed [15:0] y, z);
+	wire signed [15:0] t = a*b + c*d; assign y = t + e, z = t;
+endmodule
+// the kept wire only aliases the partial sum, so the rule has to map it first
+module keptalias(input signed [7:0] a, b, c, d, input signed [15:0] e, output signed [15:0] y);
+	wire signed [15:0] t = a*b + c*d;
+	(* keep *) wire signed [15:0] probe; assign probe = t, y = t + e;
+endmodule
+// $add takes one signedness for both operands, so the operands cannot swap
+module signedness(input signed [7:0] x, a, b, input [7:0] c, output [15:0] y);
+	wire [15:0] t = x + a*b; assign y = t + c;
+endmodule
+// the partial sum is truncated and re-extended, so the sum really would change
+module truncated(input signed [7:0] a, b, c, d, input signed [15:0] e, output signed [15:0] y);
+	wire signed [9:0] t = a*b + c*d; assign y = t + e;
+endmodule
+// moving the second copy into the inner adder would feed it from its own output
+module sumtwice(input [7:0] x, a, b, output [7:0] y);
+	wire [7:0] t = x + a*b; assign y = t + t;
+endmodule
+// the chain has no top, so the product would travel around it forever
+module combcycle(input signed [7:0] a, b, input signed [15:0] e, output signed [15:0] o);
+	wire signed [15:0] t1, t2; assign t1 = t2 + a*b, t2 = t1 + e, o = t1;
+endmodule
+// swapping two products
+module allproducts(input signed [7:0] a, b, c, d, f, g, output signed [15:0] y);
+	assign y = a*b + c*d + f*g;
+endmodule
+EOT
+
+opt_clean
+peepopt
+cd sumfanout; select -assert-count 1 t:$mul %co2 t:$add %i; cd ..
+cd keptalias; select -assert-count 1 t:$mul %co2 t:$add %i; cd ..
+cd signedness; select -assert-count 1 t:$mul %co2 t:$add %i; cd ..
+cd truncated; select -assert-count 1 t:$mul %co2 t:$add %i; cd ..
+cd sumtwice; select -assert-count 1 t:$mul %co2 t:$add %i; cd ..
+cd combcycle; select -assert-count 1 t:$mul %co2 t:$add %i; cd ..
+cd allproducts; select -assert-count 2 t:$mul %co2 t:$add %i; cd ..
+design -reset
+
+read_verilog <<EOT
+module top(input [11:0] a, c, d, input [31:0] e, output [31:0] y);
+	wire [31:0] q = (a * 16'd5140) / (257 * 2);
+	assign y = q + c*d + e;
+endmodule
+EOT
+
+hierarchy -top top
+peepopt
+clean
+select -assert-count 2 t:$mul %co2 t:$add %i
+design -reset