Improve hierarchical DPI wrapper scheduling performance (#2583) (#5734)

docs/guide/exe_verilator.rst

@@ -2154,6 +2154,13 @@ The grammar of configuration commands is as follows:
    :option:`/*verilator&32;public_flat*/`, etc., metacomments. See
    also :ref:`VPI Example`.
 
+.. option:: profile_data -hier-dpi "<function_name>" -cost <cost_value>
+
+   Internal profiling data inserted during :vlopt:`--hierarchical`; specifies
+   execution cost of a hierarchical DPI wrappers for modules with
+   :option:`/*verilator&32;hier_block*/` metacomment. See
+   :ref:`Hierarchical Verilation`.
+
 .. option:: profile_data -mtask "<mtask_hash>" -cost <cost_value>
 
    Feeds profile-guided optimization data into the Verilator algorithms in

src/V3AstNodeOther.h

@@ -643,6 +643,7 @@ class AstCFunc final : public AstNode {
     bool m_dpiImportWrapper : 1;  // Wrapper for invoking DPI import prototype from generated code
     bool m_needProcess : 1;  // Needs access to VlProcess of the caller
     bool m_recursive : 1;  // Recursive or part of recursion
+    int m_cost;  // Function call cost
 public:
     AstCFunc(FileLine* fl, const string& name, AstScope* scopep, const string& rtnType = "")
         : ASTGEN_SUPER_CFunc(fl) {
@@ -671,6 +672,7 @@ public:
         m_dpiImportPrototype = false;
         m_dpiImportWrapper = false;
         m_recursive = false;
+        m_cost = v3Global.opt.instrCountDpi();  // As proxy for unknown general DPI cost
     }
     ASTGEN_MEMBERS_AstCFunc;
     string name() const override VL_MT_STABLE { return m_name; }
@@ -685,9 +687,7 @@ public:
     }
     //
     void name(const string& name) override { m_name = name; }
-    int instrCount() const override {
+    int instrCount() const override { return m_cost; }
-        return dpiImportPrototype() ? v3Global.opt.instrCountDpi() : 0;
-    }
     VBoolOrUnknown isConst() const { return m_isConst; }
     void isConst(bool flag) { m_isConst.setTrueOrFalse(flag); }
     void isConst(VBoolOrUnknown flag) { m_isConst = flag; }
@@ -746,6 +746,7 @@ public:
     bool isCoroutine() const { return m_rtnType == "VlCoroutine"; }
     void recursive(bool flag) { m_recursive = flag; }
     bool recursive() const { return m_recursive; }
+    void cost(int cost) { m_cost = cost; }
     // Special methods
     bool emptyBody() const {
         return argsp() == nullptr && initsp() == nullptr && stmtsp() == nullptr

src/V3Config.cpp

@@ -533,11 +533,13 @@ public:
 // Resolve modules and files in the design
 
 class V3ConfigResolver final {
+    enum ProfileDataMode : uint8_t { NONE = 0, MTASK = 1, HIER_DPI = 2 };
     V3ConfigModuleResolver m_modules;  // Access to module names (with wildcards)
     V3ConfigFileResolver m_files;  // Access to file names (with wildcards)
     V3ConfigScopeTraceResolver m_scopeTraces;  // Regexp to trace enables
     std::unordered_map<string, std::unordered_map<string, uint64_t>>
         m_profileData;  // Access to profile_data records
+    uint8_t m_mode = NONE;
     FileLine* m_profileFileLine = nullptr;
 
     V3ConfigResolver() = default;
@@ -552,10 +554,21 @@ public:
     V3ConfigFileResolver& files() { return m_files; }
     V3ConfigScopeTraceResolver& scopeTraces() { return m_scopeTraces; }
 
-    void addProfileData(FileLine* fl, const string& model, const string& key, uint64_t cost) {
+    void addProfileData(FileLine* fl, const string& hierDpi, uint64_t cost) {
+        // Empty key for hierarchical DPI wrapper costs.
+        addProfileData(fl, hierDpi, "", cost, HIER_DPI);
+    }
+    void addProfileData(FileLine* fl, const string& model, const string& key, uint64_t cost,
+                        ProfileDataMode mode = MTASK) {
         if (!m_profileFileLine) m_profileFileLine = fl;
         if (cost == 0) cost = 1;  // Cost 0 means delete (or no data)
         m_profileData[model][key] += cost;
+        m_mode |= mode;
+    }
+    bool containsMTaskProfileData() const { return m_mode & MTASK; }
+    uint64_t getProfileData(const string& hierDpi) const {
+        // Empty key for hierarchical DPI wrapper costs.
+        return getProfileData(hierDpi, "");
     }
     uint64_t getProfileData(const string& model, const string& key) const {
         const auto mit = m_profileData.find(model);
@@ -619,6 +632,10 @@ void V3Config::addModulePragma(const string& module, VPragmaType pragma) {
     V3ConfigResolver::s().modules().at(module).addModulePragma(pragma);
 }
 
+void V3Config::addProfileData(FileLine* fl, const string& hierDpi, uint64_t cost) {
+    V3ConfigResolver::s().addProfileData(fl, hierDpi, cost);
+}
+
 void V3Config::addProfileData(FileLine* fl, const string& model, const string& key,
                               uint64_t cost) {
     V3ConfigResolver::s().addProfileData(fl, model, key, cost);
@@ -724,6 +741,9 @@ void V3Config::applyVarAttr(AstNodeModule* modulep, AstNodeFTask* ftaskp, AstVar
     if (vp) vp->apply(varp);
 }
 
+uint64_t V3Config::getProfileData(const string& hierDpi) {
+    return V3ConfigResolver::s().getProfileData(hierDpi);
+}
 uint64_t V3Config::getProfileData(const string& model, const string& key) {
     return V3ConfigResolver::s().getProfileData(model, key);
 }
@@ -736,6 +756,10 @@ bool V3Config::getScopeTraceOn(const string& scope) {
 
 void V3Config::contentsPushText(const string& text) { return WildcardContents::pushText(text); }
 
+bool V3Config::containsMTaskProfileData() {
+    return V3ConfigResolver::s().containsMTaskProfileData();
+}
+
 bool V3Config::waive(FileLine* filelinep, V3ErrorCode code, const string& message) {
     V3ConfigFile* filep = V3ConfigResolver::s().files().resolve(filelinep->filename());
     if (!filep) return false;

src/V3Config.h

@@ -38,6 +38,7 @@ public:
                                const string& match);
     static void addInline(FileLine* fl, const string& module, const string& ftask, bool on);
     static void addModulePragma(const string& module, VPragmaType pragma);
+    static void addProfileData(FileLine* fl, const string& hierDpi, uint64_t cost);
     static void addProfileData(FileLine* fl, const string& model, const string& key,
                                uint64_t cost);
     static void addScopeTraceOn(bool on, const string& scope, int levels);
@@ -51,12 +52,15 @@ public:
     static void applyModule(AstNodeModule* modulep);
     static void applyVarAttr(AstNodeModule* modulep, AstNodeFTask* ftaskp, AstVar* varp);
 
+    static uint64_t getProfileData(const string& hierDpi);
     static uint64_t getProfileData(const string& model, const string& key);
     static FileLine* getProfileDataFileLine();
     static bool getScopeTraceOn(const string& scope);
 
     static void contentsPushText(const string& text);
 
+    static bool containsMTaskProfileData();
+
     static bool waive(FileLine* filelinep, V3ErrorCode code, const string& message);
 };
 

src/V3ExecGraph.cpp

@@ -528,9 +528,11 @@ void fillinCosts(V3Graph* execMTaskGraphp) {
 
     if (missingProfiles) {
         if (FileLine* const fl = V3Config::getProfileDataFileLine()) {
-            fl->v3warn(PROFOUTOFDATE, "Profile data for mtasks may be out of date. "
+            if (V3Config::containsMTaskProfileData()) {
-                                          << missingProfiles << " of " << totalEstimates
+                fl->v3warn(PROFOUTOFDATE, "Profile data for mtasks may be out of date. "
-                                          << " mtasks had no data");
+                                              << missingProfiles << " of " << totalEstimates
+                                              << " mtasks had no data");
+            }
         }
     }
 }

src/V3OrderParallel.cpp

@@ -1710,7 +1710,11 @@ class DpiImportCallVisitor final : public VNVisitor {
         if (nodep->dpiImportWrapper()) {
             if (nodep->dpiPure() ? !v3Global.opt.threadsDpiPure()
                                  : !v3Global.opt.threadsDpiUnpure()) {
-                m_hasDpiHazard = true;
+                // If hierarchical DPI wrapper cost is not found or is of a 0 cost,
+                // we have a normal DPI which induces DPI hazard by default.
+                m_hasDpiHazard = V3Config::getProfileData(nodep->cname()) == 0;
+                UINFO(9, "DPI wrapper '" << nodep->cname()
+                                         << "' has dpi hazard = " << m_hasDpiHazard << endl);
             }
         }
         iterateChildren(nodep);

src/V3ProtectLib.cpp

@@ -19,6 +19,7 @@
 #include "V3ProtectLib.h"
 
 #include "V3Hasher.h"
+#include "V3InstrCount.h"
 #include "V3String.h"
 #include "V3Task.h"
 
@@ -98,6 +99,32 @@ class ProtectVisitor final : public VNVisitor {
         txtp->addNodesp(new AstComment{fl, comment});
     }
 
+    void configSection(AstNodeModule* modp, AstTextBlock* txtp, FileLine* fl) {
+        txtp->addText(fl, "\n`ifdef VERILATOR\n");
+        txtp->addText(fl, "`verilator_config\n");
+
+        // The `eval` function is called inside both update functions. As those functions
+        // are created by text bashing, we need to find cost of `_eval` which is the first function
+        // with a real cost in AST.
+        uint32_t cost = 0;
+        modp->foreach([&cost](AstCFunc* cfuncp) {
+            if (cfuncp->name() == "_eval") cost = V3InstrCount::count(cfuncp, false);
+        });
+        txtp->addText(fl, "profile_data -hier-dpi \"" + m_libName
+                              + "_protectlib_combo_update\" -cost 64'd" + std::to_string(cost)
+                              + "\n");
+        txtp->addText(fl, "profile_data -hier-dpi \"" + m_libName
+                              + "_protectlib_seq_update\" -cost 64'd" + std::to_string(cost)
+                              + "\n");
+
+        // Mark remaining NDA protectlib wrapper DPIs as non-hazardous by deliberately forwarding
+        // them with non-zero cost.
+        txtp->addText(fl, "profile_data -hier-dpi \"" + m_libName
+                              + "_protectlib_combo_ignore\" -cost 64'd1\n");
+        txtp->addText(fl, "`verilog\n");
+        txtp->addText(fl, "`endif\n");
+    }
+
     void hashComment(AstTextBlock* txtp, FileLine* fl) {
         addComment(txtp, fl, "Checks to make sure the .sv wrapper and library agree");
     }
@@ -283,6 +310,9 @@ class ProtectVisitor final : public VNVisitor {
         txtp->addText(fl, "final " + m_libName + "_protectlib_final(handle__V);\n\n");
 
         txtp->addText(fl, "endmodule\n");
+
+        configSection(modp, txtp, fl);
+
         m_vfilep->tblockp(txtp);
     }
 

src/V3Task.cpp

@@ -27,6 +27,7 @@
 
 #include "V3Task.h"
 
+#include "V3Config.h"
 #include "V3Const.h"
 #include "V3EmitCBase.h"
 #include "V3Graph.h"
@@ -372,6 +373,7 @@ class TaskVisitor final : public VNVisitor {
     // STATE - across all visitors
     DpiCFuncs m_dpiNames;  // Map of all created DPI functions
     VDouble0 m_statInlines;  // Statistic tracking
+    VDouble0 m_statHierDpisWithCosts;  // Statistic tracking
 
     // METHODS
 
@@ -980,6 +982,11 @@ class TaskVisitor final : public VNVisitor {
         funcp->isMethod(false);
         funcp->protect(false);
         funcp->dpiPure(nodep->dpiPure());
+
+        const int cost = static_cast<int>(V3Config::getProfileData(funcp->name()));
+        m_statHierDpisWithCosts += (cost != 0);
+        funcp->cost(cost);
+
         // Add DPI Import to top, since it's a global function
         m_topScopep->scopep()->addBlocksp(funcp);
         makePortList(nodep, funcp);
@@ -1259,6 +1266,8 @@ class TaskVisitor final : public VNVisitor {
         if (nodep->name() == "new") cfuncp->isConstructor(true);
         if (cfuncp->dpiExportImpl()) cfuncp->cname(nodep->cname());
 
+        if (cfuncp->dpiImportWrapper()) cfuncp->cname(nodep->cname());
+
         if (!nodep->dpiImport() && !nodep->taskPublic()) {
             // Need symbol table
             cfuncp->argTypes(EmitCBase::symClassVar());
@@ -1613,7 +1622,11 @@ public:
         : m_statep{statep} {
         iterate(nodep);
     }
-    ~TaskVisitor() { V3Stats::addStat("Optimizations, Functions inlined", m_statInlines); }
+    ~TaskVisitor() {
+        V3Stats::addStat("Optimizations, Functions inlined", m_statInlines);
+        V3Stats::addStat("Optimizations, Hierarchical DPI wrappers with costs",
+                         m_statHierDpisWithCosts);
+    }
 };
 
 //######################################################################

src/verilog.l

@@ -141,6 +141,7 @@ vnum    {vnum1}|{vnum2}|{vnum3}|{vnum4}|{vnum5}
   -?"-cost"             { FL; return yVLT_D_COST; }
   -?"-file"             { FL; return yVLT_D_FILE; }
   -?"-function"         { FL; return yVLT_D_FUNCTION; }
+  -?"-hier-dpi"         { FL; return yVLT_D_HIER_DPI; }
   -?"-levels"           { FL; return yVLT_D_LEVELS; }
   -?"-lines"            { FL; return yVLT_D_LINES; }
   -?"-match"            { FL; return yVLT_D_MATCH; }

src/verilog.y

@@ -492,6 +492,7 @@ BISONPRE_VERSION(3.7,%define api.header.include {"V3ParseBison.h"})
 %token<fl>              yVLT_D_COST     "--cost"
 %token<fl>              yVLT_D_FILE     "--file"
 %token<fl>              yVLT_D_FUNCTION "--function"
+%token<fl>              yVLT_D_HIER_DPI "--hier-dpi"
 %token<fl>              yVLT_D_LEVELS   "--levels"
 %token<fl>              yVLT_D_LINES    "--lines"
 %token<fl>              yVLT_D_MATCH    "--match"
@@ -7655,6 +7656,8 @@ vltItem:
                         { V3Config::addCaseParallel(*$2, 0); }
         |       yVLT_PARALLEL_CASE vltDFile yVLT_D_LINES yaINTNUM
                         { V3Config::addCaseParallel(*$2, $4->toUInt()); }
+        |       yVLT_PROFILE_DATA vltDHierDpi vltDCost
+                        { V3Config::addProfileData($<fl>1, *$2, $3->toUQuad()); }
         |       yVLT_PROFILE_DATA vltDModel vltDMtask vltDCost
                         { V3Config::addProfileData($<fl>1, *$2, *$3, $4->toUQuad()); }
         ;
@@ -7699,6 +7702,10 @@ vltDFile<strp>:  // --file <arg>
                 yVLT_D_FILE str                         { $$ = $2; }
         ;
 
+vltDHierDpi<strp>:  // --hier-dpi <arg>
+                yVLT_D_HIER_DPI str                     { $$ = $2; }
+        ;
+
 vltDLevels<nump>:  // --levels <arg>
                 yVLT_D_LEVELS yaINTNUM                  { $$ = $2; }
         ;

test_regress/t/t_hier_block_perf.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+# DESCRIPTION: Verilator: Verilog Test driver/expect definition
+#
+# Copyright 2025 by Wilson Snyder. This program is free software; you
+# can redistribute it and/or modify it under the terms of either the GNU
+# Lesser General Public License Version 3 or the Perl Artistic License
+# Version 2.0.
+# SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0
+
+import vltest_bootstrap
+
+test.scenarios('vlt_all')
+test.init_benchmarksim()
+test.cycles = (int(test.benchmark) if test.benchmark else 1000000)
+test.sim_time = test.cycles * 10 + 1000
+THREADS = int(os.environ["SIM_THREADS"]) if "SIM_THREADS" in os.environ else 2
+
+test.compile(benchmarksim=1,
+             v_flags2=[
+                 "+define+SIM_CYCLES=" + str(test.cycles), "--prof-exec", "--hierarchical",
+                 "--stats"
+             ],
+             threads=(THREADS if test.vltmt else 1))
+
+test.file_grep(test.obj_dir + "/V" + test.name + "__hier.dir/V" + test.name + "__stats.txt",
+               r'Optimizations, Hierarchical DPI wrappers with costs\s+(\d+)', 3)
+
+test.execute(all_run_flags=[
+    "+verilator+prof+exec+start+2",
+    " +verilator+prof+exec+window+2",
+    " +verilator+prof+exec+file+" + test.obj_dir + "/profile_exec.dat",
+    " +verilator+prof+vlt+file+" + test.obj_dir + "/profile.vlt"])  # yapf:disable
+
+test.run(cmd=[
+    os.environ["VERILATOR_ROOT"] + "/bin/verilator_gantt", test.obj_dir +
+    "/profile_exec.dat", "--vcd " + test.obj_dir + "/profile_exec.vcd", "| tee " + test.obj_dir +
+    "/gantt.log"
+])
+
+test.passes()

test_regress/t/t_hier_block_perf.v

@@ -0,0 +1,314 @@
+// DESCRIPTION: Verilator: Verilog Test module
+//
+// This file ONLY is placed under the Creative Commons Public Domain, for
+// any use, without warranty, 2025 by Wilson Snyder.
+// SPDX-License-Identifier: CC0-1.0
+
+// based on t_gate_ormux
+
+`ifndef HIER_CORES
+   `define HIER_CORES 3
+`endif
+
+`ifndef MAIN_CORES
+   `define MAIN_CORES 1
+`endif
+
+module t (/*AUTOARG*/
+   // Inputs
+   clk
+   );
+   input clk;
+
+   generate
+      for (genvar i = 0; i < `MAIN_CORES; ++i) NonHierCore mainCore(clk);
+   endgenerate
+
+   generate
+      for (genvar i = 0; i < `HIER_CORES; ++i) Core hierCore(clk);
+   endgenerate
+endmodule
+
+module Core(input clk); /* verilator hier_block */
+      reg [63:0]   crc;
+      logic [31:0] rdata;
+      logic [31:0] rdata2;
+      wire [31:0]  wdata = crc[31:0];
+      wire [15:0]  sel = {11'h0, crc[36:32]};
+      wire         we = crc[48];
+
+      Test test (
+                 // Outputs
+                 .rdata                    (rdata[31:0]),
+                 .rdata2                   (rdata2[31:0]),
+                 // Inputs
+                 .clk                      (clk),
+                 .we                       (we),
+                 .sel                      (sel[15:0]),
+                 .wdata                    (wdata[31:0]));
+      wire [63:0] result = {rdata2, rdata};
+
+      Check check(.clk(clk), .crc(crc), .result(result), .rdata(rdata), .rdata2(rdata2));
+endmodule
+
+module NonHierCore(input clk);
+      reg [63:0]   crc;
+      logic [31:0] rdata;
+      logic [31:0] rdata2;
+      wire [31:0]  wdata = crc[31:0];
+      wire [15:0]  sel = {11'h0, crc[36:32]};
+      wire         we = crc[48];
+
+      Test test (
+                 // Outputs
+                 .rdata                    (rdata[31:0]),
+                 .rdata2                   (rdata2[31:0]),
+                 // Inputs
+                 .clk                      (clk),
+                 .we                       (we),
+                 .sel                      (sel[15:0]),
+                 .wdata                    (wdata[31:0]));
+      wire [63:0] result = {rdata2, rdata};
+
+      Check check(.clk(clk), .crc(crc), .result(result), .rdata(rdata), .rdata2(rdata2));
+endmodule
+
+module Check(
+      input clk,
+      output reg [63:0] crc,
+      input wire [63:0] result,
+      input logic [31:0] rdata,
+      input logic [31:0] rdata2
+   );
+   integer      cyc = 0;
+   reg [63:0]   sum;
+
+   always @ (posedge clk) begin
+`ifdef TEST_VERBOSE
+      $write("[%0t] cyc==%0d crc=%x result=%x\n", $time, cyc, crc, result);
+`endif
+      cyc <= cyc + 1;
+      crc <= {crc[62:0], crc[63] ^ crc[2] ^ crc[0]};
+      sum <= result ^ {sum[62:0], sum[63] ^ sum[2] ^ sum[0]};
+      if (rdata2 != rdata) $stop;
+      if (cyc==0) begin
+         // Setup
+         crc <= 64'h5aef0c8d_d70a4497;
+         sum <= '0;
+      end
+      else if (cyc<10) begin
+         sum <= '0;
+      end
+      else if (cyc == 99) begin
+         $write("[%0t] cyc==%0d crc=%x sum=%x\n", $time, cyc, crc, sum);
+         if (crc !== 64'hc77bb9b3784ea091) $stop;
+`define EXPECTED_SUM 64'h8977713eb467bc86
+         if (sum !== `EXPECTED_SUM) $stop;
+      end
+      else if (cyc == `SIM_CYCLES) begin
+         $write("[%0t] cyc==%0d crc=%x sum=%x\n", $time, cyc, crc, sum);
+         $write("*-* All Finished *-*\n");
+         $finish;
+      end
+   end
+endmodule
+
+module Test(/*AUTOARG*/
+   // Outputs
+   rdata, rdata2,
+   // Inputs
+   clk, we, sel, wdata
+   );
+   input clk;
+   input we;
+   input [15:0] sel;
+   input [31:0] wdata;
+   output logic [31:0] rdata;
+   output logic [31:0] rdata2;
+
+   logic               we_d1r;
+   logic [15:0]        sel_d1r;
+   logic [31:0]        wdata_d1r;
+   always_ff @ (posedge clk) begin
+      we_d1r <= we;
+      sel_d1r <= sel;
+      wdata_d1r <= wdata;
+   end
+
+   reg [31:0] csr0000;
+   reg [31:0] csr0001;
+   reg [31:0] csr0002;
+   reg [31:0] csr0003;
+   reg [31:0] csr0004;
+   reg [31:0] csr0005;
+   reg [31:0] csr0006;
+   reg [31:0] csr0007;
+   reg [31:0] csr0008;
+   reg [31:0] csr0009;
+   reg [31:0] csr000a;
+   reg [31:0] csr000b;
+   reg [31:0] csr000c;
+   reg [31:0] csr000d;
+   reg [31:0] csr000e;
+   reg [31:0] csr000f;
+   wire [31:0] csr0010 = 32'h33675230;
+   wire [31:0] csr0011 = 32'h00fa2144;
+   wire [31:0] csr0012 = 32'h6a5e8e10;
+   wire [31:0] csr0013 = 32'h000a5b5e;
+   wire [31:0] csr0014 = 32'h002fe51b;
+   wire [31:0] csr0015 = 32'h00027e00;
+   wire [31:0] csr0016 = 32'h0000e3c0;
+   wire [31:0] csr0017 = 32'h00efcf16;
+   wire [31:0] csr0018 = 32'h007a2600;
+   wire [31:0] csr0019 = 32'h0a4a9f10;
+   wire [31:0] csr001a = 32'h7d789de3;
+   wire [31:0] csr001b = 32'h40f655f9;
+   wire [31:0] csr001c = 32'hadad01f4;
+   wire [31:0] csr001d = 32'h02e7b33c;
+   wire [31:0] csr001e = 32'h12101533;
+   wire [31:0] csr001f = 32'h2cc1cce5;
+   initial begin
+      csr0000 = 32'he172d365;
+      csr0001 = 32'h35cc25e2;
+      csr0002 = 32'haf48436e;
+      csr0003 = 32'h135e55e4;
+      csr0004 = 32'h5fd6e48a;
+      csr0005 = 32'hb07d34ad;
+      csr0006 = 32'h2aa05deb;
+      csr0007 = 32'hfe97b680;
+      csr0008 = 32'h960f20bb;
+      csr0009 = 32'h251129f0;
+      csr000a = 32'hef3d2f93;
+      csr000b = 32'hef4bc127;
+      csr000c = 32'h3dfecb10;
+      csr000d = 32'h1b4690f5;
+      csr000e = 32'ha07822ab;
+      csr000f = 32'hf817cbf6;
+   end
+
+   always_ff @ (posedge clk) begin
+      if (we_d1r && sel_d1r == 16'h0000) csr0000 <= wdata_d1r;
+      if (we_d1r && sel_d1r == 16'h0001) csr0001 <= wdata_d1r;
+      if (we_d1r && sel_d1r == 16'h0002) csr0002 <= wdata_d1r;
+      if (we_d1r && sel_d1r == 16'h0003) csr0003 <= wdata_d1r;
+      if (we_d1r && sel_d1r == 16'h0004) csr0004 <= wdata_d1r;
+      if (we_d1r && sel_d1r == 16'h0005) csr0005 <= wdata_d1r;
+      if (we_d1r && sel_d1r == 16'h0006) csr0006 <= wdata_d1r;
+      if (we_d1r && sel_d1r == 16'h0007) csr0007 <= wdata_d1r;
+      if (we_d1r && sel_d1r == 16'h0008) csr0008 <= wdata_d1r;
+      if (we_d1r && sel_d1r == 16'h0009) csr0009 <= wdata_d1r;
+      if (we_d1r && sel_d1r == 16'h000a) csr000a <= wdata_d1r;
+      if (we_d1r && sel_d1r == 16'h000b) csr000b <= wdata_d1r;
+      if (we_d1r && sel_d1r == 16'h000c) csr000c <= wdata_d1r;
+      if (we_d1r && sel_d1r == 16'h000d) csr000d <= wdata_d1r;
+      if (we_d1r && sel_d1r == 16'h000e) csr000e <= wdata_d1r;
+      if (we_d1r && sel_d1r == 16'h000f) csr000f <= wdata_d1r;
+   end
+
+   wire dec0000 = sel_d1r[15:6] == 0 && !sel_d1r[5] && !sel_d1r[4] && !sel_d1r[3] && !sel_d1r[2] && !sel_d1r[1] && !sel_d1r[0];
+   wire dec0001 = sel_d1r[15:6] == 0 && !sel_d1r[5] && !sel_d1r[4] && !sel_d1r[3] && !sel_d1r[2] && !sel_d1r[1] &&  sel_d1r[0];
+   wire dec0002 = sel_d1r[15:6] == 0 && !sel_d1r[5] && !sel_d1r[4] && !sel_d1r[3] && !sel_d1r[2] &&  sel_d1r[1] && !sel_d1r[0];
+   wire dec0003 = sel_d1r[15:6] == 0 && !sel_d1r[5] && !sel_d1r[4] && !sel_d1r[3] && !sel_d1r[2] &&  sel_d1r[1] &&  sel_d1r[0];
+   wire dec0004 = sel_d1r[15:6] == 0 && !sel_d1r[5] && !sel_d1r[4] && !sel_d1r[3] &&  sel_d1r[2] && !sel_d1r[1] && !sel_d1r[0];
+   wire dec0005 = sel_d1r[15:6] == 0 && !sel_d1r[5] && !sel_d1r[4] && !sel_d1r[3] &&  sel_d1r[2] && !sel_d1r[1] &&  sel_d1r[0];
+   wire dec0006 = sel_d1r[15:6] == 0 && !sel_d1r[5] && !sel_d1r[4] && !sel_d1r[3] &&  sel_d1r[2] &&  sel_d1r[1] && !sel_d1r[0];
+   wire dec0007 = sel_d1r[15:6] == 0 && !sel_d1r[5] && !sel_d1r[4] && !sel_d1r[3] &&  sel_d1r[2] &&  sel_d1r[1] &&  sel_d1r[0];
+   wire dec0008 = sel_d1r[15:6] == 0 && !sel_d1r[5] && !sel_d1r[4] &&  sel_d1r[3] && !sel_d1r[2] && !sel_d1r[1] && !sel_d1r[0];
+   wire dec0009 = sel_d1r[15:6] == 0 && !sel_d1r[5] && !sel_d1r[4] &&  sel_d1r[3] && !sel_d1r[2] && !sel_d1r[1] &&  sel_d1r[0];
+   wire dec000a = sel_d1r[15:6] == 0 && !sel_d1r[5] && !sel_d1r[4] &&  sel_d1r[3] && !sel_d1r[2] &&  sel_d1r[1] && !sel_d1r[0];
+   wire dec000b = sel_d1r[15:6] == 0 && !sel_d1r[5] && !sel_d1r[4] &&  sel_d1r[3] && !sel_d1r[2] &&  sel_d1r[1] &&  sel_d1r[0];
+   wire dec000c = sel_d1r[15:6] == 0 && !sel_d1r[5] && !sel_d1r[4] &&  sel_d1r[3] &&  sel_d1r[2] && !sel_d1r[1] && !sel_d1r[0];
+   wire dec000d = sel_d1r[15:6] == 0 && !sel_d1r[5] && !sel_d1r[4] &&  sel_d1r[3] &&  sel_d1r[2] && !sel_d1r[1] &&  sel_d1r[0];
+   wire dec000e = sel_d1r[15:6] == 0 && !sel_d1r[5] && !sel_d1r[4] &&  sel_d1r[3] &&  sel_d1r[2] &&  sel_d1r[1] && !sel_d1r[0];
+   wire dec000f = sel_d1r[15:6] == 0 && !sel_d1r[5] && !sel_d1r[4] &&  sel_d1r[3] &&  sel_d1r[2] &&  sel_d1r[1] &&  sel_d1r[0];
+   wire dec0010 = sel_d1r[15:6] == 0 && !sel_d1r[5] &&  sel_d1r[4] && !sel_d1r[3] && !sel_d1r[2] && !sel_d1r[1] && !sel_d1r[0];
+   wire dec0011 = sel_d1r[15:6] == 0 && !sel_d1r[5] &&  sel_d1r[4] && !sel_d1r[3] && !sel_d1r[2] && !sel_d1r[1] &&  sel_d1r[0];
+   wire dec0012 = sel_d1r[15:6] == 0 && !sel_d1r[5] &&  sel_d1r[4] && !sel_d1r[3] && !sel_d1r[2] &&  sel_d1r[1] && !sel_d1r[0];
+   wire dec0013 = sel_d1r[15:6] == 0 && !sel_d1r[5] &&  sel_d1r[4] && !sel_d1r[3] && !sel_d1r[2] &&  sel_d1r[1] &&  sel_d1r[0];
+   wire dec0014 = sel_d1r[15:6] == 0 && !sel_d1r[5] &&  sel_d1r[4] && !sel_d1r[3] &&  sel_d1r[2] && !sel_d1r[1] && !sel_d1r[0];
+   wire dec0015 = sel_d1r[15:6] == 0 && !sel_d1r[5] &&  sel_d1r[4] && !sel_d1r[3] &&  sel_d1r[2] && !sel_d1r[1] &&  sel_d1r[0];
+   wire dec0016 = sel_d1r[15:6] == 0 && !sel_d1r[5] &&  sel_d1r[4] && !sel_d1r[3] &&  sel_d1r[2] &&  sel_d1r[1] && !sel_d1r[0];
+   wire dec0017 = sel_d1r[15:6] == 0 && !sel_d1r[5] &&  sel_d1r[4] && !sel_d1r[3] &&  sel_d1r[2] &&  sel_d1r[1] &&  sel_d1r[0];
+   wire dec0018 = sel_d1r[15:6] == 0 && !sel_d1r[5] &&  sel_d1r[4] &&  sel_d1r[3] && !sel_d1r[2] && !sel_d1r[1] && !sel_d1r[0];
+   wire dec0019 = sel_d1r[15:6] == 0 && !sel_d1r[5] &&  sel_d1r[4] &&  sel_d1r[3] && !sel_d1r[2] && !sel_d1r[1] &&  sel_d1r[0];
+   wire dec001a = sel_d1r[15:6] == 0 && !sel_d1r[5] &&  sel_d1r[4] &&  sel_d1r[3] && !sel_d1r[2] &&  sel_d1r[1] && !sel_d1r[0];
+   wire dec001b = sel_d1r[15:6] == 0 && !sel_d1r[5] &&  sel_d1r[4] &&  sel_d1r[3] && !sel_d1r[2] &&  sel_d1r[1] &&  sel_d1r[0];
+   wire dec001c = sel_d1r[15:6] == 0 && !sel_d1r[5] &&  sel_d1r[4] &&  sel_d1r[3] &&  sel_d1r[2] && !sel_d1r[1] && !sel_d1r[0];
+   wire dec001d = sel_d1r[15:6] == 0 && !sel_d1r[5] &&  sel_d1r[4] &&  sel_d1r[3] &&  sel_d1r[2] && !sel_d1r[1] &&  sel_d1r[0];
+   wire dec001e = sel_d1r[15:6] == 0 && !sel_d1r[5] &&  sel_d1r[4] &&  sel_d1r[3] &&  sel_d1r[2] &&  sel_d1r[1] && !sel_d1r[0];
+   wire dec001f = sel_d1r[15:6] == 0 && !sel_d1r[5] &&  sel_d1r[4] &&  sel_d1r[3] &&  sel_d1r[2] &&  sel_d1r[1] &&  sel_d1r[0];
+
+   assign rdata = (32'h0
+                   | {32{dec0000}} & csr0000
+                   | {32{dec0001}} & csr0001
+                   | {32{dec0002}} & csr0002
+                   | {32{dec0003}} & csr0003
+                   | {32{dec0004}} & csr0004
+                   | {32{dec0005}} & csr0005
+                   | {32{dec0006}} & csr0006
+                   | {32{dec0007}} & csr0007
+                   | {32{dec0008}} & csr0008
+                   | {32{dec0009}} & csr0009
+                   | {32{dec000a}} & csr000a
+                   | {32{dec000b}} & csr000b
+                   | {32{dec000c}} & csr000c
+                   | {32{dec000d}} & csr000d
+                   | {32{dec000e}} & csr000e
+                   | {32{dec000f}} & csr000f
+                   | {32{dec0010}} & csr0010
+                   | {32{dec0011}} & csr0011
+                   | {32{dec0012}} & csr0012
+                   | {32{dec0013}} & csr0013
+                   | {32{dec0014}} & csr0014
+                   | {32{dec0015}} & csr0015
+                   | {32{dec0016}} & csr0016
+                   | {32{dec0017}} & csr0017
+                   | {32{dec0018}} & csr0018
+                   | {32{dec0019}} & csr0019
+                   | {32{dec001a}} & csr001a
+                   | {32{dec001b}} & csr001b
+                   | {32{dec001c}} & csr001c
+                   | {32{dec001d}} & csr001d
+                   | {32{dec001e}} & csr001e
+                   | {32{dec001f}} & csr001f
+                  );
+
+   always_comb begin
+      case (sel_d1r)
+        16'h0000: rdata2 = csr0000;
+        16'h0001: rdata2 = csr0001;
+        16'h0002: rdata2 = csr0002;
+        16'h0003: rdata2 = csr0003;
+        16'h0004: rdata2 = csr0004;
+        16'h0005: rdata2 = csr0005;
+        16'h0006: rdata2 = csr0006;
+        16'h0007: rdata2 = csr0007;
+        16'h0008: rdata2 = csr0008;
+        16'h0009: rdata2 = csr0009;
+        16'h000a: rdata2 = csr000a;
+        16'h000b: rdata2 = csr000b;
+        16'h000c: rdata2 = csr000c;
+        16'h000d: rdata2 = csr000d;
+        16'h000e: rdata2 = csr000e;
+        16'h000f: rdata2 = csr000f;
+        16'h0010: rdata2 = csr0010;
+        16'h0011: rdata2 = csr0011;
+        16'h0012: rdata2 = csr0012;
+        16'h0013: rdata2 = csr0013;
+        16'h0014: rdata2 = csr0014;
+        16'h0015: rdata2 = csr0015;
+        16'h0016: rdata2 = csr0016;
+        16'h0017: rdata2 = csr0017;
+        16'h0018: rdata2 = csr0018;
+        16'h0019: rdata2 = csr0019;
+        16'h001a: rdata2 = csr001a;
+        16'h001b: rdata2 = csr001b;
+        16'h001c: rdata2 = csr001c;
+        16'h001d: rdata2 = csr001d;
+        16'h001e: rdata2 = csr001e;
+        16'h001f: rdata2 = csr001f;
+        default: rdata2 = 0;
+      endcase
+   end
+
+endmodule