From 642efbb563ca67b8f34f1a616f06e25b8a09119a Mon Sep 17 00:00:00 2001
From: Ross Thompson <ross1728@gmail.com>
Date: Thu, 26 Aug 2021 10:41:42 -0500
Subject: [PATCH 01/16] Converted the icache type from logic to state type.

---
 wally-pipelined/regression/wave.do       | 58 +++++++----------
 wally-pipelined/src/cache/ICacheCntrl.sv | 83 ++++++++++++------------
 wally-pipelined/src/cache/icache.sv      | 39 +++++++++--
 3 files changed, 97 insertions(+), 83 deletions(-)

diff --git a/wally-pipelined/regression/wave.do b/wally-pipelined/regression/wave.do
index a12e5cda8..48efbe285 100644
--- a/wally-pipelined/regression/wave.do
+++ b/wally-pipelined/regression/wave.do
@@ -314,8 +314,6 @@ add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM w
 add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group valid/dirty /testbench/dut/hart/lsu/dcache/ClearValid
 add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group valid/dirty /testbench/dut/hart/lsu/dcache/SetDirty
 add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group valid/dirty /testbench/dut/hart/lsu/dcache/ClearDirty
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM read} /testbench/dut/hart/lsu/dcache/SRAMAdr
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM read} /testbench/dut/hart/lsu/dcache/genblk1/cacheLRU/MemPAdrM
 add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM read} -group way0 {/testbench/dut/hart/lsu/dcache/MemWay[0]/ReadDataBlockWayMaskedM}
 add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM read} -group way0 {/testbench/dut/hart/lsu/dcache/MemWay[0]/WayHit}
 add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM read} -group way0 {/testbench/dut/hart/lsu/dcache/MemWay[0]/Valid}
@@ -340,31 +338,29 @@ add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM r
 add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM read} /testbench/dut/hart/lsu/dcache/ReadDataBlockWayMaskedM
 add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM read} /testbench/dut/hart/lsu/dcache/ReadDataWordM
 add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM read} /testbench/dut/hart/lsu/dcache/ReadDataWordMuxM
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM read} /testbench/dut/hart/lsu/dcache/genblk1/cacheLRU/BlockReplacementBits
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM read} /testbench/dut/hart/lsu/dcache/genblk1/cacheLRU/ReplacementBits
-add wave -noupdate -expand -group lsu -expand -group dcache -expand -group Victim /testbench/dut/hart/lsu/dcache/VictimTag
-add wave -noupdate -expand -group lsu -expand -group dcache -expand -group Victim /testbench/dut/hart/lsu/dcache/VictimWay
-add wave -noupdate -expand -group lsu -expand -group dcache -expand -group Victim /testbench/dut/hart/lsu/dcache/VictimDirtyWay
-add wave -noupdate -expand -group lsu -expand -group dcache -expand -group Victim /testbench/dut/hart/lsu/dcache/VictimDirty
-add wave -noupdate -expand -group lsu -expand -group dcache -expand -group {CPU side} /testbench/dut/hart/lsu/dcache/MemRWM
-add wave -noupdate -expand -group lsu -expand -group dcache -expand -group {CPU side} /testbench/dut/hart/lsu/dcache/MemAdrE
-add wave -noupdate -expand -group lsu -expand -group dcache -expand -group {CPU side} /testbench/dut/hart/lsu/dcache/MemPAdrM
-add wave -noupdate -expand -group lsu -expand -group dcache -expand -group {CPU side} /testbench/dut/hart/lsu/dcache/Funct3M
-add wave -noupdate -expand -group lsu -expand -group dcache -expand -group {CPU side} /testbench/dut/hart/lsu/dcache/Funct7M
-add wave -noupdate -expand -group lsu -expand -group dcache -expand -group {CPU side} /testbench/dut/hart/lsu/dcache/AtomicM
-add wave -noupdate -expand -group lsu -expand -group dcache -expand -group {CPU side} /testbench/dut/hart/lsu/dcache/CacheableM
-add wave -noupdate -expand -group lsu -expand -group dcache -expand -group {CPU side} /testbench/dut/hart/lsu/dcache/WriteDataM
-add wave -noupdate -expand -group lsu -expand -group dcache -expand -group {CPU side} /testbench/dut/hart/lsu/dcache/ReadDataM
-add wave -noupdate -expand -group lsu -expand -group dcache -expand -group {CPU side} /testbench/dut/hart/lsu/dcache/DCacheStall
-add wave -noupdate -expand -group lsu -expand -group dcache -expand -group status /testbench/dut/hart/lsu/dcache/WayHit
-add wave -noupdate -expand -group lsu -expand -group dcache -expand -group status -color {Medium Orchid} /testbench/dut/hart/lsu/dcache/CacheHit
-add wave -noupdate -expand -group lsu -expand -group dcache -expand -group status /testbench/dut/hart/lsu/dcache/FetchCount
-add wave -noupdate -expand -group lsu -expand -group dcache -expand -group {Memory Side} /testbench/dut/hart/lsu/dcache/AHBPAdr
-add wave -noupdate -expand -group lsu -expand -group dcache -expand -group {Memory Side} /testbench/dut/hart/lsu/dcache/AHBRead
-add wave -noupdate -expand -group lsu -expand -group dcache -expand -group {Memory Side} /testbench/dut/hart/lsu/dcache/AHBWrite
-add wave -noupdate -expand -group lsu -expand -group dcache -expand -group {Memory Side} /testbench/dut/hart/lsu/dcache/AHBAck
-add wave -noupdate -expand -group lsu -expand -group dcache -expand -group {Memory Side} /testbench/dut/hart/lsu/dcache/HRDATA
-add wave -noupdate -expand -group lsu -expand -group dcache -expand -group {Memory Side} /testbench/dut/hart/lsu/dcache/HWDATA
+add wave -noupdate -expand -group lsu -expand -group dcache -group Victim /testbench/dut/hart/lsu/dcache/VictimTag
+add wave -noupdate -expand -group lsu -expand -group dcache -group Victim /testbench/dut/hart/lsu/dcache/VictimWay
+add wave -noupdate -expand -group lsu -expand -group dcache -group Victim /testbench/dut/hart/lsu/dcache/VictimDirtyWay
+add wave -noupdate -expand -group lsu -expand -group dcache -group Victim /testbench/dut/hart/lsu/dcache/VictimDirty
+add wave -noupdate -expand -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/MemRWM
+add wave -noupdate -expand -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/MemAdrE
+add wave -noupdate -expand -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/MemPAdrM
+add wave -noupdate -expand -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/Funct3M
+add wave -noupdate -expand -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/Funct7M
+add wave -noupdate -expand -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/AtomicM
+add wave -noupdate -expand -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/CacheableM
+add wave -noupdate -expand -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/WriteDataM
+add wave -noupdate -expand -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/ReadDataM
+add wave -noupdate -expand -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/DCacheStall
+add wave -noupdate -expand -group lsu -expand -group dcache -group status /testbench/dut/hart/lsu/dcache/WayHit
+add wave -noupdate -expand -group lsu -expand -group dcache -group status -color {Medium Orchid} /testbench/dut/hart/lsu/dcache/CacheHit
+add wave -noupdate -expand -group lsu -expand -group dcache -group status /testbench/dut/hart/lsu/dcache/FetchCount
+add wave -noupdate -expand -group lsu -expand -group dcache -group {Memory Side} /testbench/dut/hart/lsu/dcache/AHBPAdr
+add wave -noupdate -expand -group lsu -expand -group dcache -group {Memory Side} /testbench/dut/hart/lsu/dcache/AHBRead
+add wave -noupdate -expand -group lsu -expand -group dcache -group {Memory Side} /testbench/dut/hart/lsu/dcache/AHBWrite
+add wave -noupdate -expand -group lsu -expand -group dcache -group {Memory Side} /testbench/dut/hart/lsu/dcache/AHBAck
+add wave -noupdate -expand -group lsu -expand -group dcache -group {Memory Side} /testbench/dut/hart/lsu/dcache/HRDATA
+add wave -noupdate -expand -group lsu -expand -group dcache -group {Memory Side} /testbench/dut/hart/lsu/dcache/HWDATA
 add wave -noupdate -expand -group lsu -group dtlb /testbench/dut/hart/lsu/dmmu/genblk1/tlb/tlbcontrol/EffectivePrivilegeMode
 add wave -noupdate -expand -group lsu -group dtlb /testbench/dut/hart/lsu/dmmu/genblk1/tlb/tlbcontrol/Translate
 add wave -noupdate -expand -group lsu -group dtlb /testbench/dut/hart/lsu/dmmu/genblk1/tlb/tlbcontrol/DisableTranslation
@@ -476,16 +472,10 @@ add wave -noupdate -group UART /testbench/dut/uncore/uart/uart/HSELUART
 add wave -noupdate -group UART /testbench/dut/uncore/uart/uart/HADDR
 add wave -noupdate -group UART /testbench/dut/uncore/uart/uart/HWRITE
 add wave -noupdate -group UART /testbench/dut/uncore/uart/uart/HWDATA
-add wave -noupdate /testbench/dut/hart/lsu/dcache/subwordread/HRDATA
-add wave -noupdate /testbench/dut/hart/lsu/dcache/subwordread/HSIZED
 add wave -noupdate /testbench/dut/hart/lsu/dcache/subwordread/offset0
 add wave -noupdate /testbench/dut/hart/lsu/dcache/subwordread/offset1
 add wave -noupdate /testbench/dut/hart/lsu/dcache/subwordread/offset2
 add wave -noupdate /testbench/dut/hart/lsu/dcache/subwordread/offset3
-add wave -noupdate /testbench/dut/hart/lsu/dcache/subwordread/genblk1/offset4
-add wave -noupdate /testbench/dut/hart/lsu/dcache/subwordread/genblk1/offset5
-add wave -noupdate /testbench/dut/hart/lsu/dcache/subwordread/genblk1/offset6
-add wave -noupdate /testbench/dut/hart/lsu/dcache/subwordread/genblk1/offset7
 add wave -noupdate /testbench/dut/hart/ExceptionM
 add wave -noupdate /testbench/dut/hart/PendingInterruptM
 add wave -noupdate /testbench/dut/hart/TrapM
@@ -506,4 +496,4 @@ configure wave -griddelta 40
 configure wave -timeline 0
 configure wave -timelineunits ns
 update
-WaveRestoreZoom {1280 ns} {3534 ns}
+WaveRestoreZoom {0 ns} {325837 ns}
diff --git a/wally-pipelined/src/cache/ICacheCntrl.sv b/wally-pipelined/src/cache/ICacheCntrl.sv
index 3881e9cb6..fc08f2d4f 100644
--- a/wally-pipelined/src/cache/ICacheCntrl.sv
+++ b/wally-pipelined/src/cache/ICacheCntrl.sv
@@ -71,51 +71,51 @@ module ICacheCntrl #(parameter BLOCKLEN = 256)
    );
 
   // FSM states
-  localparam STATE_READY = 'h0;
-  localparam STATE_HIT_SPILL = 'h1; // spill, block 0 hit
-  localparam STATE_HIT_SPILL_MISS_FETCH_WDV = 'h2; // block 1 miss, issue read to AHB and wait data.
-  localparam STATE_HIT_SPILL_MISS_FETCH_DONE = 'h3; // write data into SRAM/LUT
-  localparam STATE_HIT_SPILL_MERGE = 'h4;   // Read block 0 of CPU access, should be able to optimize into STATE_HIT_SPILL.
+  typedef enum {STATE_READY,
+		STATE_HIT_SPILL, // spill, block 0 hit
+		STATE_HIT_SPILL_MISS_FETCH_WDV, // block 1 miss, issue read to AHB and wait data.
+		STATE_HIT_SPILL_MISS_FETCH_DONE, // write data into SRAM/LUT
+		STATE_HIT_SPILL_MERGE,   // Read block 0 of CPU access, should be able to optimize into STATE_HIT_SPILL.
 
-  // a challenge is the spill signal gets us out of the ready state and moves us to
-  // 1 of the 2 spill branches.  However the original fsm design had us return to
-  // the ready state when the spill + hits/misses were fully resolved.  The problem
-  // is the spill signal is based on PCPF so when we return to READY to check if the
-  // cache has a hit it still expresses spill.  We can fix in 1 of two ways.
-  // 1. we can add 1 extra state at the end of each spill branch to returns the instruction
-  // to the CPU advancing the CPU and icache to the next instruction.
-  // 2. We can assert a signal which is delayed 1 cycle to suppress the spill when we get
-  // to the READY state.
-  // The first first option is more robust and increases the number of states by 2.  The
-  // second option is seams like it should work, but I worry there is a hidden interaction 
-  // between CPU stalling and that register.
-  // Picking option 1.
+		// a challenge is the spill signal gets us out of the ready state and moves us to
+		// 1 of the 2 spill branches.  However the original fsm design had us return to
+		// the ready state when the spill + hits/misses were fully resolved.  The problem
+		// is the spill signal is based on PCPF so when we return to READY to check if the
+		// cache has a hit it still expresses spill.  We can fix in 1 of two ways.
+		// 1. we can add 1 extra state at the end of each spill branch to returns the instruction
+		// to the CPU advancing the CPU and icache to the next instruction.
+		// 2. We can assert a signal which is delayed 1 cycle to suppress the spill when we get
+		// to the READY state.
+		// The first first option is more robust and increases the number of states by 2.  The
+		// second option is seams like it should work, but I worry there is a hidden interaction 
+		// between CPU stalling and that register.
+		// Picking option 1.
 
-  localparam STATE_HIT_SPILL_FINAL = 'h5; // this state replicates STATE_READY's replay of the
-  // spill access but does nto consider spill.  It also does not do another operation.
+		STATE_HIT_SPILL_FINAL, // this state replicates STATE_READY's replay of the
+		// spill access but does nto consider spill.  It also does not do another operation.
   
+		STATE_MISS_FETCH_WDV, // aligned miss, issue read to AHB and wait for data.
+		STATE_MISS_FETCH_DONE, // write data into SRAM/LUT
+		STATE_MISS_READ, // read block 1 from SRAM/LUT  
 
-  localparam STATE_MISS_FETCH_WDV = 'h6; // aligned miss, issue read to AHB and wait for data.
-  localparam STATE_MISS_FETCH_DONE = 'h7; // write data into SRAM/LUT
-  localparam STATE_MISS_READ = 'h8; // read block 1 from SRAM/LUT  
+		STATE_MISS_SPILL_FETCH_WDV, // spill, miss on block 0, issue read to AHB and wait
+		STATE_MISS_SPILL_FETCH_DONE, // write data into SRAM/LUT
+		STATE_MISS_SPILL_READ1, // read block 0 from SRAM/LUT
+		STATE_MISS_SPILL_2, // return to ready if hit or do second block update.
+		STATE_MISS_SPILL_2_START, // return to ready if hit or do second block update.  
+		STATE_MISS_SPILL_MISS_FETCH_WDV, // miss on block 1, issue read to AHB and wait
+		STATE_MISS_SPILL_MISS_FETCH_DONE, // write data to SRAM/LUT
+		STATE_MISS_SPILL_MERGE, // read block 0 of CPU access,
 
-  localparam STATE_MISS_SPILL_FETCH_WDV = 'h9; // spill, miss on block 0, issue read to AHB and wait
-  localparam STATE_MISS_SPILL_FETCH_DONE = 'ha; // write data into SRAM/LUT
-  localparam STATE_MISS_SPILL_READ1 = 'hb; // read block 0 from SRAM/LUT
-  localparam STATE_MISS_SPILL_2 = 'hc; // return to ready if hit or do second block update.
-  localparam STATE_MISS_SPILL_2_START = 'hd; // return to ready if hit or do second block update.  
-  localparam STATE_MISS_SPILL_MISS_FETCH_WDV = 'he; // miss on block 1, issue read to AHB and wait
-  localparam STATE_MISS_SPILL_MISS_FETCH_DONE = 'hf; // write data to SRAM/LUT
-  localparam STATE_MISS_SPILL_MERGE = 'h10; // read block 0 of CPU access,
+		STATE_MISS_SPILL_FINAL, // this state replicates STATE_READY's replay of the
+		// spill access but does nto consider spill.  It also does not do another operation.
 
-  localparam STATE_MISS_SPILL_FINAL = 'h11; // this state replicates STATE_READY's replay of the
-  // spill access but does nto consider spill.  It also does not do another operation.
+		STATE_INVALIDATE, // *** not sure if invalidate or evict? invalidate by cache block or address?
+		STATE_TLB_MISS,
+		STATE_TLB_MISS_DONE
+		} statetype;
   
 
-  localparam STATE_INVALIDATE = 'h12; // *** not sure if invalidate or evict? invalidate by cache block or address?
-  localparam STATE_TLB_MISS = 'h13;
-  localparam STATE_TLB_MISS_DONE = 'h14;
-
   
   localparam AHBByteLength = `XLEN / 8;
   localparam AHBOFFETWIDTH = $clog2(AHBByteLength);
@@ -129,7 +129,7 @@ module ICacheCntrl #(parameter BLOCKLEN = 256)
   localparam integer 	       PA_WIDTH = `PA_BITS - 2;
   
 
-  logic [4:0] 		       CurrState, NextState;
+  statetype CurrState, NextState;
   logic 		       hit, spill;
   logic 		       SavePC;
   logic [1:0] 		       PCMux;
@@ -187,10 +187,9 @@ module ICacheCntrl #(parameter BLOCKLEN = 256)
 
 
   // the FSM is always runing, do not stall.
-  flopr #(5) stateReg(.clk(clk),
-		      .reset(reset),
-		      .d(NextState),
-		      .q(CurrState));
+  always_ff @(posedge clk, posedge reset)
+    if (reset)    CurrState <= #1 STATE_READY;
+    else CurrState <= #1 NextState;
 
   assign spill = PCPF[4:1] == 4'b1111 ? 1'b1 : 1'b0;
   assign hit = ICacheMemReadValid; // note ICacheMemReadValid is hit.
diff --git a/wally-pipelined/src/cache/icache.sv b/wally-pipelined/src/cache/icache.sv
index 99c43df51..954828fd5 100644
--- a/wally-pipelined/src/cache/icache.sv
+++ b/wally-pipelined/src/cache/icache.sv
@@ -70,15 +70,15 @@ module icache
   
   
   ICacheMem #(.BLOCKLEN(BLOCKLEN), .NUMLINES(NUMLINES)) 
-  cachemem(
-           .*,
-           // Stall it if the pipeline is stalled, unless we're stalling it and we're ending our stall
+  cachemem(.clk,
+	   .reset,
            .flush(FlushMem),
+	   .PCTagF,
+	   .PCNextIndexF,
            .WriteEnable(ICacheMemWriteEnable),
            .WriteLine(ICacheMemWriteData),
-           .ReadLineF(ReadLineF),
-           .HitF(ICacheMemReadValid)
-	   );
+	   .ReadLineF,
+           .HitF(ICacheMemReadValid));
 
   always_comb begin
     case (PCTagF[4:1])
@@ -105,7 +105,32 @@ module icache
   end
 
 
-  ICacheCntrl #(.BLOCKLEN(BLOCKLEN)) controller(.*);
+  ICacheCntrl #(.BLOCKLEN(BLOCKLEN)) 
+  controller(.clk,
+	     .reset,
+	     .StallF,
+	     .StallD,
+	     .FlushD,
+	     .PCNextF,
+	     .PCPF,
+	     .ICacheMemReadData,
+	     .ICacheMemReadValid,
+	     .PCTagF,
+	     .PCNextIndexF, 
+	     .ICacheReadEn,
+	     .ICacheMemWriteEnable,
+	     .ICacheMemWriteData,
+	     .CompressedF,
+	     .FinalInstrRawF,
+	     .ICacheStallF,
+	     . EndFetchState,
+	     .ITLBMissF,
+	     .ITLBWriteF,
+	     .WalkerInstrPageFaultF,
+	     .InstrInF,
+	     .InstrAckF,
+	     .InstrPAdrF,
+	     .InstrReadF);
 
   // For now, assume no writes to executable memory
   assign FlushMem = 1'b0;

From c83f0a2e994df17c91087d21adbb2078d3a75f95 Mon Sep 17 00:00:00 2001
From: Ross Thompson <ross1728@gmail.com>
Date: Thu, 26 Aug 2021 10:49:54 -0500
Subject: [PATCH 02/16] Removed unused logic in icache.

---
 wally-pipelined/src/cache/ICacheCntrl.sv | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/wally-pipelined/src/cache/ICacheCntrl.sv b/wally-pipelined/src/cache/ICacheCntrl.sv
index fc08f2d4f..db5592a05 100644
--- a/wally-pipelined/src/cache/ICacheCntrl.sv
+++ b/wally-pipelined/src/cache/ICacheCntrl.sv
@@ -150,7 +150,6 @@ module ICacheCntrl #(parameter BLOCKLEN = 256)
   
   localparam [31:0]  	     NOP = 32'h13;
 
-  logic 			 reset_q;
   logic [1:0] 			 PCMux_q;
   
   
@@ -455,19 +454,5 @@ module ICacheCntrl #(parameter BLOCKLEN = 256)
 		      .d(PCPreFinalF[1]),
 		      .q(PCPreFinalF_q[1]));
   assign FinalInstrRawF = spill ? {ICacheMemReadData[15:0], SpillDataBlock0} : ICacheMemReadData;
-
-  // There is a frustrating issue on the first access.
-  // The cache will not contain any valid data but will contain x's on
-  // reset. This makes FinalInstrRawF invalid.  On the first cycle out of
-  // reset this register will pickup this x and it will propagate throughout
-  // the cpu causing simulation failure, most likely a trap for invalid instruction.
-  // Reset must be held 1 cycle longer to prevent this issue. additionally the
-  // reset should be to a NOP rather than 0.
-
-  // register reset
-  flop #(1) resetReg (.clk(clk),
-		      .d(reset),
-		      .q(reset_q));
-  
   
 endmodule

From b3849d8abbb9e1bd4fcc7af59dcd13b0f9fbf83f Mon Sep 17 00:00:00 2001
From: Ross Thompson <ross1728@gmail.com>
Date: Thu, 26 Aug 2021 10:58:19 -0500
Subject: [PATCH 03/16] Moved data path logic from icacheCntrl to icache.

---
 wally-pipelined/src/cache/ICacheCntrl.sv      | 34 +++----------------
 wally-pipelined/src/cache/icache.sv           | 26 ++++++++++++--
 .../testbench/testbench-imperas.sv            |  2 +-
 wally-pipelined/testbench/testbench-linux.sv  |  2 +-
 4 files changed, 30 insertions(+), 34 deletions(-)

diff --git a/wally-pipelined/src/cache/ICacheCntrl.sv b/wally-pipelined/src/cache/ICacheCntrl.sv
index db5592a05..ea3257aa8 100644
--- a/wally-pipelined/src/cache/ICacheCntrl.sv
+++ b/wally-pipelined/src/cache/ICacheCntrl.sv
@@ -48,12 +48,8 @@ module ICacheCntrl #(parameter BLOCKLEN = 256)
    output logic 	       ICacheMemWriteEnable,
    output logic [BLOCKLEN-1:0] ICacheMemWriteData,
 
-   // Outputs to rest of ifu
-   // High if the instruction in the fetch stage is compressed
-   output logic 	       CompressedF,
    // The instruction that was requested
    // If this instruction is compressed, upper 16 bits may be the next 16 bits or may be zeros
-   output logic [31:0] 	       FinalInstrRawF,
 
    // Outputs to pipeline control stuff
    output logic 	       ICacheStallF, EndFetchState,
@@ -67,7 +63,10 @@ module ICacheCntrl #(parameter BLOCKLEN = 256)
    input logic 		       InstrAckF,
    // The read we request from main memory
    output logic [`PA_BITS-1:0] InstrPAdrF,
-   output logic 	       InstrReadF
+   output logic 	       InstrReadF,
+
+   output logic 	       spill,
+   output logic 	       spillSave
    );
 
   // FSM states
@@ -130,12 +129,11 @@ module ICacheCntrl #(parameter BLOCKLEN = 256)
   
 
   statetype CurrState, NextState;
-  logic 		       hit, spill;
+  logic 		       hit;
   logic 		       SavePC;
   logic [1:0] 		       PCMux;
   logic 		       CntReset;
   logic 		       PreCntEn, CntEn;
-  logic 		       spillSave;
   logic 		       UnalignedSelect;
   logic 		       FetchCountFlag;
   localparam FetchCountThreshold = WORDSPERLINE - 1;
@@ -146,7 +144,6 @@ module ICacheCntrl #(parameter BLOCKLEN = 256)
   logic [`PA_BITS-1:OFFSETWIDTH] PCPTrunkF;
 
   
-  logic [15:0] 			 SpillDataBlock0;
   
   localparam [31:0]  	     NOP = 32'h13;
 
@@ -181,9 +178,6 @@ module ICacheCntrl #(parameter BLOCKLEN = 256)
   // truncate the offset from PCPF for memory address generation
   assign PCPTrunkF = PCTagF[`PA_BITS-1:OFFSETWIDTH];
   
-  // Detect if the instruction is compressed
-  assign CompressedF = FinalInstrRawF[1:0] != 2'b11;
-
 
   // the FSM is always runing, do not stall.
   always_ff @(posedge clk, posedge reset)
@@ -436,23 +430,5 @@ module ICacheCntrl #(parameter BLOCKLEN = 256)
   // what address is used to write the SRAM?
   
 
-  // spills require storing the first cache block so it can merged
-  // with the second
-  // can optimize size, for now just make it the size of the data
-  // leaving the cache memory. 
-  flopenr #(16) SpillInstrReg(.clk(clk),
-			      .en(spillSave),
-			      .reset(reset),
-			      .d(ICacheMemReadData[15:0]),
-			      .q(SpillDataBlock0));
-
-  // use the not quite final PC to do the final selection.
-  logic [1:1] PCPreFinalF_q;
-  flopenr #(1) PCFReg(.clk(clk),
-		      .reset(reset),
-		      .en(~StallF),
-		      .d(PCPreFinalF[1]),
-		      .q(PCPreFinalF_q[1]));
-  assign FinalInstrRawF = spill ? {ICacheMemReadData[15:0], SpillDataBlock0} : ICacheMemReadData;
   
 endmodule
diff --git a/wally-pipelined/src/cache/icache.sv b/wally-pipelined/src/cache/icache.sv
index 954828fd5..1d034f711 100644
--- a/wally-pipelined/src/cache/icache.sv
+++ b/wally-pipelined/src/cache/icache.sv
@@ -68,6 +68,11 @@ module icache
   logic 		    ICacheReadEn;
   logic [BLOCKLEN-1:0] 	    ReadLineF;
   
+
+  logic [15:0] 			 SpillDataBlock0;
+  logic 			 spill;
+  logic 			 spillSave;
+ 
   
   ICacheMem #(.BLOCKLEN(BLOCKLEN), .NUMLINES(NUMLINES)) 
   cachemem(.clk,
@@ -104,6 +109,21 @@ module icache
     endcase
   end
 
+  // spills require storing the first cache block so it can merged
+  // with the second
+  // can optimize size, for now just make it the size of the data
+  // leaving the cache memory. 
+  flopenr #(16) SpillInstrReg(.clk(clk),
+			      .en(spillSave),
+			      .reset(reset),
+			      .d(ICacheMemReadData[15:0]),
+			      .q(SpillDataBlock0));
+
+  assign FinalInstrRawF = spill ? {ICacheMemReadData[15:0], SpillDataBlock0} : ICacheMemReadData;
+
+  // Detect if the instruction is compressed
+  assign CompressedF = FinalInstrRawF[1:0] != 2'b11;
+  
 
   ICacheCntrl #(.BLOCKLEN(BLOCKLEN)) 
   controller(.clk,
@@ -120,8 +140,6 @@ module icache
 	     .ICacheReadEn,
 	     .ICacheMemWriteEnable,
 	     .ICacheMemWriteData,
-	     .CompressedF,
-	     .FinalInstrRawF,
 	     .ICacheStallF,
 	     . EndFetchState,
 	     .ITLBMissF,
@@ -130,7 +148,9 @@ module icache
 	     .InstrInF,
 	     .InstrAckF,
 	     .InstrPAdrF,
-	     .InstrReadF);
+	     .InstrReadF,
+	     .spill,
+	     .spillSave);
 
   // For now, assume no writes to executable memory
   assign FlushMem = 1'b0;
diff --git a/wally-pipelined/testbench/testbench-imperas.sv b/wally-pipelined/testbench/testbench-imperas.sv
index 8cc37aff3..38436752e 100644
--- a/wally-pipelined/testbench/testbench-imperas.sv
+++ b/wally-pipelined/testbench/testbench-imperas.sv
@@ -577,7 +577,7 @@ string tests32f[] = '{
 
   // Track names of instructions
   instrTrackerTB it(clk, reset, dut.hart.ieu.dp.FlushE,
-                dut.hart.ifu.icache.controller.FinalInstrRawF,
+                dut.hart.ifu.icache.FinalInstrRawF,
                 dut.hart.ifu.InstrD, dut.hart.ifu.InstrE,
                 dut.hart.ifu.InstrM,  dut.hart.ifu.InstrW,
                 InstrFName, InstrDName, InstrEName, InstrMName, InstrWName);
diff --git a/wally-pipelined/testbench/testbench-linux.sv b/wally-pipelined/testbench/testbench-linux.sv
index 44446098d..aa7de3801 100644
--- a/wally-pipelined/testbench/testbench-linux.sv
+++ b/wally-pipelined/testbench/testbench-linux.sv
@@ -619,7 +619,7 @@ module testbench();
   string InstrFName, InstrDName, InstrEName, InstrMName, InstrWName;
   logic [31:0] InstrW;
   instrTrackerTB it(clk, reset, dut.hart.ieu.dp.FlushE,
-                dut.hart.ifu.icache.controller.FinalInstrRawF,
+                dut.hart.ifu.icache.FinalInstrRawF,
                 dut.hart.ifu.InstrD, dut.hart.ifu.InstrE,
                 dut.hart.ifu.InstrM,  dut.hart.ifu.InstrW,
                 InstrFName, InstrDName, InstrEName, InstrMName, InstrWName);

From b230d4daec8363b0b968c732dbcf6b95db61377b Mon Sep 17 00:00:00 2001
From: Ross Thompson <ross1728@gmail.com>
Date: Thu, 26 Aug 2021 13:00:21 -0500
Subject: [PATCH 04/16] Finished moving data path logic from the ICacheCntrl.sv
 to icache.sv.

---
 wally-pipelined/src/cache/ICacheCntrl.sv | 172 ++++-------------------
 wally-pipelined/src/cache/icache.sv      | 118 ++++++++++++++--
 2 files changed, 130 insertions(+), 160 deletions(-)

diff --git a/wally-pipelined/src/cache/ICacheCntrl.sv b/wally-pipelined/src/cache/ICacheCntrl.sv
index ea3257aa8..711461253 100644
--- a/wally-pipelined/src/cache/ICacheCntrl.sv
+++ b/wally-pipelined/src/cache/ICacheCntrl.sv
@@ -28,45 +28,38 @@
 module ICacheCntrl #(parameter BLOCKLEN = 256) 
   (
    // Inputs from pipeline
-   input logic 		       clk, reset,
-   input logic 		       StallF, StallD,
-   input logic 		       FlushD,
+   input logic 	      clk, reset,
 
-   // Input the address to read
-   // The upper bits of the physical pc
-   input logic [`PA_BITS-1:0]  PCNextF,
-   input logic [`PA_BITS-1:0]  PCPF,
-   // Signals to/from cache memory
-   // The read coming out of it
-   input logic [31:0] 	       ICacheMemReadData,
-   input logic 		       ICacheMemReadValid,
-   // The address at which we want to search the cache memory
-   output logic [`PA_BITS-1:0] PCTagF,
-   output logic [`PA_BITS-1:0] PCNextIndexF, 
-   output logic 	       ICacheReadEn,
+   // inputs from mmu
+   input logic 	      ITLBMissF,
+   input logic 	      ITLBWriteF,
+   input logic 	      WalkerInstrPageFaultF,
+
+   // BUS interface
+   input logic 	      InstrAckF,
+
+   // icache internal inputs
+   input logic 	      hit,
+   input logic 	      FetchCountFlag,
+   input logic 	      spill,
+
+   // icache internal outputs
+   output logic       ICacheReadEn,
    // Load data into the cache
-   output logic 	       ICacheMemWriteEnable,
-   output logic [BLOCKLEN-1:0] ICacheMemWriteData,
-
-   // The instruction that was requested
-   // If this instruction is compressed, upper 16 bits may be the next 16 bits or may be zeros
+   output logic       ICacheMemWriteEnable,
 
    // Outputs to pipeline control stuff
-   output logic 	       ICacheStallF, EndFetchState,
-   input logic 		       ITLBMissF,
-   input logic 		       ITLBWriteF,
-   input logic 		       WalkerInstrPageFaultF,
+   output logic       ICacheStallF,
 
-   // Signals to/from ahblite interface
-   // A read containing the requested data
-   input logic [`XLEN-1:0]     InstrInF,
-   input logic 		       InstrAckF,
-   // The read we request from main memory
-   output logic [`PA_BITS-1:0] InstrPAdrF,
-   output logic 	       InstrReadF,
+   // Bus interface outputs
+   output logic       InstrReadF,
 
-   output logic 	       spill,
-   output logic 	       spillSave
+   // icache internal outputs
+   output logic       spillSave,
+   output logic       CntEn,
+   output logic       CntReset,
+   output logic [1:0] PCMux,
+   output logic       SavePC
    );
 
   // FSM states
@@ -114,80 +107,15 @@ module ICacheCntrl #(parameter BLOCKLEN = 256)
 		STATE_TLB_MISS_DONE
 		} statetype;
   
-
-  
-  localparam AHBByteLength = `XLEN / 8;
-  localparam AHBOFFETWIDTH = $clog2(AHBByteLength);
-  
-  
-  localparam BlockByteLength = BLOCKLEN / 8;
-  localparam OFFSETWIDTH = $clog2(BlockByteLength);
-  
-  localparam WORDSPERLINE = BLOCKLEN/`XLEN;
-  localparam LOGWPL = $clog2(WORDSPERLINE);
-  localparam integer 	       PA_WIDTH = `PA_BITS - 2;
-  
-
   statetype CurrState, NextState;
-  logic 		       hit;
-  logic 		       SavePC;
-  logic [1:0] 		       PCMux;
-  logic 		       CntReset;
-  logic 		       PreCntEn, CntEn;
+  logic 		       PreCntEn;
   logic 		       UnalignedSelect;
-  logic 		       FetchCountFlag;
-  localparam FetchCountThreshold = WORDSPERLINE - 1;
-  
-  logic [LOGWPL-1:0] 	       FetchCount, NextFetchCount;
-
-  logic [`PA_BITS-1:0] 	       PCPreFinalF, PCPSpillF;
-  logic [`PA_BITS-1:OFFSETWIDTH] PCPTrunkF;
-
-  
-  
-  localparam [31:0]  	     NOP = 32'h13;
-
-  logic [1:0] 			 PCMux_q;
-  
-  
-  // Misaligned signals
-  //logic [`XLEN:0] MisalignedInstrRawF;
-  //logic           MisalignedStall;
-  // Cache fault signals
-  //logic           FaultStall;
-  
-  // on spill we want to get the first 2 bytes of the next cache block.
-  // the spill only occurs if the PCPF mod BlockByteLength == -2.  Therefore we can
-  // simply add 2 to land on the next cache block.
-  assign PCPSpillF = PCPF + {{{PA_WIDTH}{1'b0}}, 2'b10}; // *** modelsim does not allow the use of PA_BITS for literal width.
-
-  // now we have to select between these three PCs
-  assign PCPreFinalF = PCMux[0] | StallF ? PCPF : PCNextF; // *** don't like the stallf, but it is necessary
-  assign PCNextIndexF = PCMux[1] ? PCPSpillF : PCPreFinalF;
-
-  // this mux needs to be delayed 1 cycle as it occurs 1 pipeline stage later.
-  // *** read enable may not be necessary.
-  flopenr #(2) PCMuxReg(.clk(clk),
-			.reset(reset),
-			.en(ICacheReadEn),
-			.d(PCMux),
-			.q(PCMux_q));
-  
-  assign PCTagF = PCMux_q[1] ? PCPSpillF : PCPF;
-  
-  // truncate the offset from PCPF for memory address generation
-  assign PCPTrunkF = PCTagF[`PA_BITS-1:OFFSETWIDTH];
-  
 
   // the FSM is always runing, do not stall.
   always_ff @(posedge clk, posedge reset)
     if (reset)    CurrState <= #1 STATE_READY;
     else CurrState <= #1 NextState;
 
-  assign spill = PCPF[4:1] == 4'b1111 ? 1'b1 : 1'b0;
-  assign hit = ICacheMemReadValid; // note ICacheMemReadValid is hit.
-  assign FetchCountFlag = (FetchCount == FetchCountThreshold[LOGWPL-1:0]);
-  
   // Next state logic
   always_comb begin
     UnalignedSelect = 1'b0;
@@ -384,51 +312,5 @@ module ICacheCntrl #(parameter BLOCKLEN = 256)
 		      (CurrState == STATE_MISS_FETCH_WDV) ||
 		      (CurrState == STATE_MISS_SPILL_FETCH_WDV) ||
 		      (CurrState == STATE_MISS_SPILL_MISS_FETCH_WDV);
-
-  // to compute the fetch address we need to add the bit shifted
-  // counter output to the address.
-
-  flopenr #(LOGWPL) 
-  FetchCountReg(.clk(clk),
-		.reset(reset | CntReset),
-		.en(CntEn),
-		.d(NextFetchCount),
-		.q(FetchCount));
-
-  assign NextFetchCount = FetchCount + 1'b1;
-  
-  // This part is confusing.
-  // *** Ross Thompson reduce the complexity. This is just dumb.
-  // we need to remove the offset bits (PCPTrunkF).  Because the AHB interface is XLEN wide
-  // we need to address on that number of bits so the PC is extended to the right by AHBByteLength with zeros.
-  // fetch count is already aligned to AHBByteLength, but we need to extend back to the full address width with
-  // more zeros after the addition.  This will be the number of offset bits less the AHBByteLength.
-  logic [`PA_BITS-1:OFFSETWIDTH-LOGWPL] PCPTrunkExtF, InstrPAdrTrunkF ;
-
-  assign PCPTrunkExtF = {PCPTrunkF, {{LOGWPL}{1'b0}}};
-  // verilator lint_off WIDTH
-  assign InstrPAdrTrunkF = PCPTrunkExtF + FetchCount;
-  // verilator lint_on WIDTH
-  
-  //assign InstrPAdrF = {{PCPTrunkF, {{LOGWPL}{1'b0}}} + FetchCount, {{OFFSETWIDTH-LOGWPL}{1'b0}}};
-  assign InstrPAdrF = {InstrPAdrTrunkF, {{OFFSETWIDTH-LOGWPL}{1'b0}}};
-  
-
-
-  // store read data from memory interface before writing into SRAM.
-  genvar 				i;
-  generate
-    for (i = 0; i < WORDSPERLINE; i++) begin:storebuffer
-      flopenr #(`XLEN) sb(.clk(clk),
-			    .reset(reset), 
-			    .en(InstrAckF & (i == FetchCount)),
-			    .d(InstrInF),
-			    .q(ICacheMemWriteData[(i+1)*`XLEN-1:i*`XLEN]));
-    end
-  endgenerate
-
-  // what address is used to write the SRAM?
-  
-
   
 endmodule
diff --git a/wally-pipelined/src/cache/icache.sv b/wally-pipelined/src/cache/icache.sv
index 1d034f711..c73d9cc6c 100644
--- a/wally-pipelined/src/cache/icache.sv
+++ b/wally-pipelined/src/cache/icache.sv
@@ -56,11 +56,20 @@ module icache
   localparam integer 	    BLOCKLEN = `ICACHE_BLOCKLENINBITS;
   localparam integer 	    NUMLINES = `ICACHE_WAYSIZEINBYTES*8/`ICACHE_BLOCKLENINBITS;
 
+  localparam WORDSPERLINE = BLOCKLEN/`XLEN;
+  localparam LOGWPL = $clog2(WORDSPERLINE);
+
+  localparam FetchCountThreshold = WORDSPERLINE - 1;
+  localparam BlockByteLength = BLOCKLEN / 8;
+
+  localparam OFFSETWIDTH = $clog2(BlockByteLength);
+
+  localparam integer 	       PA_WIDTH = `PA_BITS - 2;
+
   // Input signals to cache memory
   logic 		    FlushMem;
   logic 		    ICacheMemWriteEnable;
   logic [BLOCKLEN-1:0] 	    ICacheMemWriteData;
-  logic 		    EndFetchState;
   logic [`PA_BITS-1:0] 	    PCTagF, PCNextIndexF;  
   // Output signals from cache memory
   logic [31:0] 		    ICacheMemReadData;
@@ -72,7 +81,23 @@ module icache
   logic [15:0] 			 SpillDataBlock0;
   logic 			 spill;
   logic 			 spillSave;
+
+  logic 			 FetchCountFlag;
+  logic 			 CntEn;
+  
+  logic [1:0] 			 PCMux_q;
+  
+  
+  logic [LOGWPL-1:0] 	       FetchCount, NextFetchCount;
  
+  logic [`PA_BITS-1:0] 	       PCPreFinalF, PCPSpillF;
+  logic [`PA_BITS-1:OFFSETWIDTH] PCPTrunkF;
+
+  logic 		       CntReset;
+  logic [1:0] 		       PCMux;
+  logic 		       SavePC;
+  
+
   
   ICacheMem #(.BLOCKLEN(BLOCKLEN), .NUMLINES(NUMLINES)) 
   cachemem(.clk,
@@ -125,32 +150,95 @@ module icache
   assign CompressedF = FinalInstrRawF[1:0] != 2'b11;
   
 
+  assign spill = PCPF[4:1] == 4'b1111 ? 1'b1 : 1'b0;
+  assign hit = ICacheMemReadValid; // note ICacheMemReadValid is hit.
+  assign FetchCountFlag = (FetchCount == FetchCountThreshold[LOGWPL-1:0]);
+
+  // to compute the fetch address we need to add the bit shifted
+  // counter output to the address.
+
+  flopenr #(LOGWPL) 
+  FetchCountReg(.clk(clk),
+		.reset(reset | CntReset),
+		.en(CntEn),
+		.d(NextFetchCount),
+		.q(FetchCount));
+
+  assign NextFetchCount = FetchCount + 1'b1;
+
+  // This part is confusing.
+  // *** Ross Thompson reduce the complexity. This is just dumb.
+  // we need to remove the offset bits (PCPTrunkF).  Because the AHB interface is XLEN wide
+  // we need to address on that number of bits so the PC is extended to the right by AHBByteLength with zeros.
+  // fetch count is already aligned to AHBByteLength, but we need to extend back to the full address width with
+  // more zeros after the addition.  This will be the number of offset bits less the AHBByteLength.
+  logic [`PA_BITS-1:OFFSETWIDTH-LOGWPL] PCPTrunkExtF, InstrPAdrTrunkF ;
+
+  assign PCPTrunkExtF = {PCPTrunkF, {{LOGWPL}{1'b0}}};
+  // verilator lint_off WIDTH
+  assign InstrPAdrTrunkF = PCPTrunkExtF + FetchCount;
+  // verilator lint_on WIDTH
+  
+  //assign InstrPAdrF = {{PCPTrunkF, {{LOGWPL}{1'b0}}} + FetchCount, {{OFFSETWIDTH-LOGWPL}{1'b0}}};
+  assign InstrPAdrF = {InstrPAdrTrunkF, {{OFFSETWIDTH-LOGWPL}{1'b0}}};
+  
+
+
+  // store read data from memory interface before writing into SRAM.
+  genvar 				i;
+  generate
+    for (i = 0; i < WORDSPERLINE; i++) begin:storebuffer
+      flopenr #(`XLEN) sb(.clk(clk),
+			    .reset(reset), 
+			    .en(InstrAckF & (i == FetchCount)),
+			    .d(InstrInF),
+			    .q(ICacheMemWriteData[(i+1)*`XLEN-1:i*`XLEN]));
+    end
+  endgenerate
+
+  // on spill we want to get the first 2 bytes of the next cache block.
+  // the spill only occurs if the PCPF mod BlockByteLength == -2.  Therefore we can
+  // simply add 2 to land on the next cache block.
+  assign PCPSpillF = PCPF + {{{PA_WIDTH}{1'b0}}, 2'b10}; // *** modelsim does not allow the use of PA_BITS for literal width.
+
+  // now we have to select between these three PCs
+  assign PCPreFinalF = PCMux[0] | StallF ? PCPF : PCNextF; // *** don't like the stallf, but it is necessary
+  assign PCNextIndexF = PCMux[1] ? PCPSpillF : PCPreFinalF;
+
+  // this mux needs to be delayed 1 cycle as it occurs 1 pipeline stage later.
+  // *** read enable may not be necessary.
+  flopenr #(2) PCMuxReg(.clk(clk),
+			.reset(reset),
+			.en(ICacheReadEn),
+			.d(PCMux),
+			.q(PCMux_q));
+  
+  assign PCTagF = PCMux_q[1] ? PCPSpillF : PCPF;
+  
+  // truncate the offset from PCPF for memory address generation
+  assign PCPTrunkF = PCTagF[`PA_BITS-1:OFFSETWIDTH];
+  
+
   ICacheCntrl #(.BLOCKLEN(BLOCKLEN)) 
   controller(.clk,
 	     .reset,
-	     .StallF,
-	     .StallD,
-	     .FlushD,
-	     .PCNextF,
-	     .PCPF,
-	     .ICacheMemReadData,
-	     .ICacheMemReadValid,
-	     .PCTagF,
-	     .PCNextIndexF, 
 	     .ICacheReadEn,
 	     .ICacheMemWriteEnable,
-	     .ICacheMemWriteData,
 	     .ICacheStallF,
-	     . EndFetchState,
 	     .ITLBMissF,
 	     .ITLBWriteF,
 	     .WalkerInstrPageFaultF,
-	     .InstrInF,
 	     .InstrAckF,
-	     .InstrPAdrF,
 	     .InstrReadF,
+	     .hit(ICacheMemReadValid),
+	     .FetchCountFlag,
 	     .spill,
-	     .spillSave);
+	     .spillSave,
+	     .CntEn,
+	     .CntReset,
+	     .PCMux,
+    	     .SavePC
+	     );
 
   // For now, assume no writes to executable memory
   assign FlushMem = 1'b0;

From cbb47956cbbe9829e47965cdf33e211e75ef3cb8 Mon Sep 17 00:00:00 2001
From: Ross Thompson <ross1728@gmail.com>
Date: Thu, 26 Aug 2021 15:43:02 -0500
Subject: [PATCH 05/16] Swapped out the icachemem for cacheway.  cacheway is
 modified to optionally support dirty bits.

---
 wally-pipelined/regression/wave.do    | 372 ++++++++++++--------------
 wally-pipelined/src/cache/cacheway.sv |  41 +--
 wally-pipelined/src/cache/dcache.sv   |   4 +-
 wally-pipelined/src/cache/icache.sv   |  30 +++
 4 files changed, 233 insertions(+), 214 deletions(-)

diff --git a/wally-pipelined/regression/wave.do b/wally-pipelined/regression/wave.do
index 48efbe285..925198962 100644
--- a/wally-pipelined/regression/wave.do
+++ b/wally-pipelined/regression/wave.do
@@ -185,43 +185,31 @@ add wave -noupdate -group divider /testbench/dut/hart/mdu/genblk1/div/N
 add wave -noupdate -group divider /testbench/dut/hart/mdu/genblk1/div/D
 add wave -noupdate -group divider /testbench/dut/hart/mdu/genblk1/div/Q
 add wave -noupdate -group divider /testbench/dut/hart/mdu/genblk1/div/rem0
-add wave -noupdate -group icache -color Orange /testbench/dut/hart/ifu/icache/controller/CurrState
-add wave -noupdate -group icache /testbench/dut/hart/ifu/icache/controller/NextState
-add wave -noupdate -group icache /testbench/dut/hart/ifu/ITLBMissF
-add wave -noupdate -group icache /testbench/dut/hart/ifu/icache/ITLBWriteF
-add wave -noupdate -group icache -group {tag read} /testbench/dut/hart/ifu/icache/cachemem/DataValidBit
-add wave -noupdate -group icache -group {tag read} /testbench/dut/hart/ifu/icache/cachemem/cachetags/ReadData
-add wave -noupdate -group icache -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/hit
-add wave -noupdate -group icache -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/spill
-add wave -noupdate -group icache -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/ICacheStallF
-add wave -noupdate -group icache -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/SavePC
-add wave -noupdate -group icache -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/spillSave
-add wave -noupdate -group icache -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/UnalignedSelect
-add wave -noupdate -group icache -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/PCMux
-add wave -noupdate -group icache -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/spillSave
-add wave -noupdate -group icache -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/CntReset
-add wave -noupdate -group icache -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/PreCntEn
-add wave -noupdate -group icache -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/CntEn
-add wave -noupdate -group icache -group {icache parameters} -radix unsigned /testbench/dut/hart/ifu/icache/cachemem/NUMLINES
-add wave -noupdate -group icache -group {icache parameters} -radix unsigned /testbench/dut/hart/ifu/icache/cachemem/BLOCKLEN
-add wave -noupdate -group icache -group {icache parameters} -radix unsigned /testbench/dut/hart/ifu/icache/cachemem/BLOCKBYTELEN
-add wave -noupdate -group icache -group {icache parameters} -radix unsigned /testbench/dut/hart/ifu/icache/cachemem/OFFSETLEN
-add wave -noupdate -group icache -group {icache parameters} -radix unsigned /testbench/dut/hart/ifu/icache/cachemem/INDEXLEN
-add wave -noupdate -group icache -group {icache parameters} -radix unsigned /testbench/dut/hart/ifu/icache/cachemem/TAGLEN
-add wave -noupdate -group icache -expand -group memory /testbench/dut/hart/ifu/icache/controller/FetchCountFlag
-add wave -noupdate -group icache -expand -group memory /testbench/dut/hart/ifu/icache/controller/FetchCount
-add wave -noupdate -group icache -expand -group memory /testbench/dut/hart/ifu/icache/controller/InstrPAdrF
-add wave -noupdate -group icache -expand -group memory /testbench/dut/hart/ifu/icache/controller/InstrReadF
-add wave -noupdate -group icache -expand -group memory /testbench/dut/hart/ifu/icache/controller/InstrAckF
-add wave -noupdate -group icache -expand -group memory /testbench/dut/hart/ifu/icache/controller/InstrInF
-add wave -noupdate -group icache -expand -group memory /testbench/dut/hart/ifu/icache/controller/ICacheMemWriteEnable
-add wave -noupdate -group icache -expand -group memory /testbench/dut/hart/ifu/icache/controller/ICacheMemWriteData
-add wave -noupdate -group icache -expand -group memory -group {tag write} /testbench/dut/hart/ifu/icache/cachemem/WriteEnable
-add wave -noupdate -group icache -expand -group memory -group {tag write} /testbench/dut/hart/ifu/icache/cachemem/WriteLine
-add wave -noupdate -group icache -expand -group memory -group {tag write} /testbench/dut/hart/ifu/icache/cachemem/cachetags/StoredData
-add wave -noupdate -group icache -expand -group {instr to cpu} /testbench/dut/hart/ifu/icache/controller/FinalInstrRawF
-add wave -noupdate -group icache -expand -group pc /testbench/dut/hart/ifu/icache/controller/PCPF
-add wave -noupdate -group icache -expand -group pc /testbench/dut/hart/ifu/icache/controller/PCPreFinalF
+add wave -noupdate -expand -group icache -color Orange /testbench/dut/hart/ifu/icache/controller/CurrState
+add wave -noupdate -expand -group icache /testbench/dut/hart/ifu/icache/controller/NextState
+add wave -noupdate -expand -group icache /testbench/dut/hart/ifu/ITLBMissF
+add wave -noupdate -expand -group icache /testbench/dut/hart/ifu/icache/ITLBWriteF
+add wave -noupdate -expand -group icache /testbench/dut/hart/ifu/icache/ReadLineF
+add wave -noupdate -expand -group icache {/testbench/dut/hart/ifu/icache/icachemem/word[0]/CacheDataMem/Addr}
+add wave -noupdate -expand -group icache {/testbench/dut/hart/ifu/icache/icachemem/word[0]/CacheDataMem/WriteData}
+add wave -noupdate -expand -group icache {/testbench/dut/hart/ifu/icache/icachemem/word[0]/CacheDataMem/WriteEnable}
+add wave -noupdate -expand -group icache -expand -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/hit
+add wave -noupdate -expand -group icache -expand -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/spill
+add wave -noupdate -expand -group icache -expand -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/ICacheStallF
+add wave -noupdate -expand -group icache -expand -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/SavePC
+add wave -noupdate -expand -group icache -expand -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/spillSave
+add wave -noupdate -expand -group icache -expand -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/UnalignedSelect
+add wave -noupdate -expand -group icache -expand -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/PCMux
+add wave -noupdate -expand -group icache -expand -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/spillSave
+add wave -noupdate -expand -group icache -expand -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/CntReset
+add wave -noupdate -expand -group icache -expand -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/PreCntEn
+add wave -noupdate -expand -group icache -expand -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/CntEn
+add wave -noupdate -expand -group icache -expand -group memory /testbench/dut/hart/ifu/icache/controller/FetchCountFlag
+add wave -noupdate -expand -group icache -expand -group memory /testbench/dut/hart/ifu/icache/FetchCount
+add wave -noupdate -expand -group icache -expand -group memory /testbench/dut/hart/ifu/icache/controller/InstrReadF
+add wave -noupdate -expand -group icache -expand -group memory /testbench/dut/hart/ifu/icache/controller/InstrAckF
+add wave -noupdate -expand -group icache -expand -group memory /testbench/dut/hart/ifu/icache/controller/ICacheMemWriteEnable
+add wave -noupdate -expand -group icache -expand -group memory /testbench/dut/hart/ifu/icache/ICacheMemWriteData
 add wave -noupdate -group AHB -color Gold /testbench/dut/hart/ebu/BusState
 add wave -noupdate -group AHB /testbench/dut/hart/ebu/NextBusState
 add wave -noupdate -group AHB -expand -group {input requests} /testbench/dut/hart/ebu/AtomicMaskedM
@@ -244,160 +232,156 @@ add wave -noupdate -group AHB /testbench/dut/hart/ebu/HADDRD
 add wave -noupdate -group AHB /testbench/dut/hart/ebu/HSIZED
 add wave -noupdate -group AHB /testbench/dut/hart/ebu/HWRITED
 add wave -noupdate -group AHB /testbench/dut/hart/ebu/StallW
-add wave -noupdate -expand -group lsu -expand -group {LSU ARB} /testbench/dut/hart/lsu/arbiter/SelPTW
-add wave -noupdate -expand -group lsu -expand -group dcache -color Gold /testbench/dut/hart/lsu/dcache/dcachefsm/CurrState
-add wave -noupdate -expand -group lsu -expand -group dcache /testbench/dut/hart/lsu/dcache/WalkerPageFaultM
-add wave -noupdate -expand -group lsu -expand -group dcache /testbench/dut/hart/lsu/dcache/WriteDataM
-add wave -noupdate -expand -group lsu -expand -group dcache /testbench/dut/hart/lsu/dcache/SRAMBlockWriteEnableM
-add wave -noupdate -expand -group lsu -expand -group dcache /testbench/dut/hart/lsu/dcache/SRAMWordWriteEnableM
-add wave -noupdate -expand -group lsu -expand -group dcache /testbench/dut/hart/lsu/dcache/SRAMWayWriteEnable
-add wave -noupdate -expand -group lsu -expand -group dcache /testbench/dut/hart/lsu/dcache/SRAMWordEnable
-add wave -noupdate -expand -group lsu -expand -group dcache /testbench/dut/hart/lsu/dcache/SRAMBlockWayWriteEnableM
-add wave -noupdate -expand -group lsu -expand -group dcache /testbench/dut/hart/lsu/dcache/SelAdrM
-add wave -noupdate -expand -group lsu -expand -group dcache /testbench/dut/hart/lsu/dcache/DCacheMemWriteData
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 {/testbench/dut/hart/lsu/dcache/MemWay[0]/WriteEnable}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 {/testbench/dut/hart/lsu/dcache/MemWay[0]/SetValid}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 {/testbench/dut/hart/lsu/dcache/MemWay[0]/SetDirty}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 -label TAG {/testbench/dut/hart/lsu/dcache/MemWay[0]/CacheTagMem/StoredData}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 {/testbench/dut/hart/lsu/dcache/MemWay[0]/DirtyBits}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 {/testbench/dut/hart/lsu/dcache/MemWay[0]/ValidBits}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 -expand -group Way0Word0 {/testbench/dut/hart/lsu/dcache/MemWay[0]/word[0]/CacheDataMem/StoredData}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 -expand -group Way0Word0 {/testbench/dut/hart/lsu/dcache/MemWay[0]/word[0]/CacheDataMem/WriteEnable}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 -expand -group Way0Word1 {/testbench/dut/hart/lsu/dcache/MemWay[0]/word[1]/CacheDataMem/StoredData}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 -expand -group Way0Word1 {/testbench/dut/hart/lsu/dcache/MemWay[0]/word[1]/CacheDataMem/WriteEnable}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 -expand -group Way0Word2 {/testbench/dut/hart/lsu/dcache/MemWay[0]/word[2]/CacheDataMem/WriteEnable}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 -expand -group Way0Word2 {/testbench/dut/hart/lsu/dcache/MemWay[0]/word[2]/CacheDataMem/StoredData}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 -expand -group Way0Word3 {/testbench/dut/hart/lsu/dcache/MemWay[0]/word[3]/CacheDataMem/WriteEnable}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 -expand -group Way0Word3 {/testbench/dut/hart/lsu/dcache/MemWay[0]/word[3]/CacheDataMem/StoredData}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 {/testbench/dut/hart/lsu/dcache/MemWay[1]/DirtyBits}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 {/testbench/dut/hart/lsu/dcache/MemWay[1]/SetDirty}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 {/testbench/dut/hart/lsu/dcache/MemWay[1]/WriteEnable}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 {/testbench/dut/hart/lsu/dcache/MemWay[1]/WriteWordEnable}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 -label TAG {/testbench/dut/hart/lsu/dcache/MemWay[1]/CacheTagMem/StoredData}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 -expand -group Way1Word0 {/testbench/dut/hart/lsu/dcache/MemWay[1]/word[0]/CacheDataMem/WriteEnable}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 -expand -group Way1Word0 {/testbench/dut/hart/lsu/dcache/MemWay[1]/word[0]/CacheDataMem/StoredData}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 -expand -group Way1Word1 {/testbench/dut/hart/lsu/dcache/MemWay[1]/word[1]/CacheDataMem/WriteEnable}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 -expand -group Way1Word1 {/testbench/dut/hart/lsu/dcache/MemWay[1]/word[1]/CacheDataMem/StoredData}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 -expand -group Way1Word2 {/testbench/dut/hart/lsu/dcache/MemWay[1]/word[2]/CacheDataMem/WriteEnable}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 -expand -group Way1Word2 {/testbench/dut/hart/lsu/dcache/MemWay[1]/word[2]/CacheDataMem/StoredData}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 -expand -group Way1Word3 {/testbench/dut/hart/lsu/dcache/MemWay[1]/word[3]/CacheDataMem/WriteEnable}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 -expand -group Way1Word3 {/testbench/dut/hart/lsu/dcache/MemWay[1]/word[3]/CacheDataMem/StoredData}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 {/testbench/dut/hart/lsu/dcache/MemWay[2]/WriteEnable}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 {/testbench/dut/hart/lsu/dcache/MemWay[2]/SetValid}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 {/testbench/dut/hart/lsu/dcache/MemWay[2]/SetDirty}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 -label TAG {/testbench/dut/hart/lsu/dcache/MemWay[2]/CacheTagMem/StoredData}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 {/testbench/dut/hart/lsu/dcache/MemWay[2]/DirtyBits}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 {/testbench/dut/hart/lsu/dcache/MemWay[2]/ValidBits}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 -expand -group Way2Word0 {/testbench/dut/hart/lsu/dcache/MemWay[2]/word[0]/CacheDataMem/StoredData}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 -expand -group Way2Word0 {/testbench/dut/hart/lsu/dcache/MemWay[2]/word[0]/CacheDataMem/WriteEnable}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 -expand -group Way2Word1 {/testbench/dut/hart/lsu/dcache/MemWay[2]/word[1]/CacheDataMem/StoredData}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 -expand -group Way2Word1 {/testbench/dut/hart/lsu/dcache/MemWay[2]/word[1]/CacheDataMem/WriteEnable}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 -expand -group Way2Word2 {/testbench/dut/hart/lsu/dcache/MemWay[2]/word[2]/CacheDataMem/WriteEnable}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 -expand -group Way2Word2 {/testbench/dut/hart/lsu/dcache/MemWay[2]/word[2]/CacheDataMem/StoredData}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 -expand -group Way2Word3 {/testbench/dut/hart/lsu/dcache/MemWay[2]/word[3]/CacheDataMem/WriteEnable}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 -expand -group Way2Word3 {/testbench/dut/hart/lsu/dcache/MemWay[2]/word[3]/CacheDataMem/StoredData}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 {/testbench/dut/hart/lsu/dcache/MemWay[3]/WriteEnable}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 {/testbench/dut/hart/lsu/dcache/MemWay[3]/SetValid}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 {/testbench/dut/hart/lsu/dcache/MemWay[3]/SetDirty}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 -label TAG {/testbench/dut/hart/lsu/dcache/MemWay[3]/CacheTagMem/StoredData}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 {/testbench/dut/hart/lsu/dcache/MemWay[3]/DirtyBits}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 {/testbench/dut/hart/lsu/dcache/MemWay[3]/ValidBits}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 -expand -group Way3Word0 {/testbench/dut/hart/lsu/dcache/MemWay[3]/word[0]/CacheDataMem/StoredData}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 -expand -group Way3Word0 {/testbench/dut/hart/lsu/dcache/MemWay[3]/word[0]/CacheDataMem/WriteEnable}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 -expand -group Way3Word1 {/testbench/dut/hart/lsu/dcache/MemWay[3]/word[1]/CacheDataMem/StoredData}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 -expand -group Way3Word1 {/testbench/dut/hart/lsu/dcache/MemWay[3]/word[1]/CacheDataMem/WriteEnable}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 -expand -group Way3Word2 {/testbench/dut/hart/lsu/dcache/MemWay[3]/word[2]/CacheDataMem/WriteEnable}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 -expand -group Way3Word2 {/testbench/dut/hart/lsu/dcache/MemWay[3]/word[2]/CacheDataMem/StoredData}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 -expand -group Way3Word3 {/testbench/dut/hart/lsu/dcache/MemWay[3]/word[3]/CacheDataMem/WriteEnable}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 -expand -group Way3Word3 {/testbench/dut/hart/lsu/dcache/MemWay[3]/word[3]/CacheDataMem/StoredData}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group valid/dirty /testbench/dut/hart/lsu/dcache/SetValid
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group valid/dirty /testbench/dut/hart/lsu/dcache/ClearValid
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group valid/dirty /testbench/dut/hart/lsu/dcache/SetDirty
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group valid/dirty /testbench/dut/hart/lsu/dcache/ClearDirty
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM read} -group way0 {/testbench/dut/hart/lsu/dcache/MemWay[0]/ReadDataBlockWayMaskedM}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM read} -group way0 {/testbench/dut/hart/lsu/dcache/MemWay[0]/WayHit}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM read} -group way0 {/testbench/dut/hart/lsu/dcache/MemWay[0]/Valid}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM read} -group way0 {/testbench/dut/hart/lsu/dcache/MemWay[0]/Dirty}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM read} -group way0 {/testbench/dut/hart/lsu/dcache/MemWay[0]/ReadTag}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way1 {/testbench/dut/hart/lsu/dcache/MemWay[1]/ReadDataBlockWayMaskedM}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way1 {/testbench/dut/hart/lsu/dcache/MemWay[1]/WayHit}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way1 {/testbench/dut/hart/lsu/dcache/MemWay[1]/Valid}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way1 {/testbench/dut/hart/lsu/dcache/MemWay[1]/Dirty}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way1 {/testbench/dut/hart/lsu/dcache/MemWay[1]/ReadTag}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way2 {/testbench/dut/hart/lsu/dcache/MemWay[2]/ReadDataBlockWayMaskedM}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way2 {/testbench/dut/hart/lsu/dcache/MemWay[2]/WayHit}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way2 {/testbench/dut/hart/lsu/dcache/MemWay[2]/Valid}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way2 {/testbench/dut/hart/lsu/dcache/MemWay[2]/Dirty}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way2 {/testbench/dut/hart/lsu/dcache/MemWay[2]/ReadTag}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way3 {/testbench/dut/hart/lsu/dcache/MemWay[3]/ReadDataBlockWayMaskedM}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way3 {/testbench/dut/hart/lsu/dcache/MemWay[3]/WayHit}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way3 {/testbench/dut/hart/lsu/dcache/MemWay[3]/Valid}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way3 {/testbench/dut/hart/lsu/dcache/MemWay[3]/Dirty}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way3 {/testbench/dut/hart/lsu/dcache/MemWay[3]/ReadTag}
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM read} /testbench/dut/hart/lsu/dcache/WayHit
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM read} /testbench/dut/hart/lsu/dcache/ReadDataBlockWayMaskedM
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM read} /testbench/dut/hart/lsu/dcache/ReadDataWordM
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Cache SRAM read} /testbench/dut/hart/lsu/dcache/ReadDataWordMuxM
-add wave -noupdate -expand -group lsu -expand -group dcache -group Victim /testbench/dut/hart/lsu/dcache/VictimTag
-add wave -noupdate -expand -group lsu -expand -group dcache -group Victim /testbench/dut/hart/lsu/dcache/VictimWay
-add wave -noupdate -expand -group lsu -expand -group dcache -group Victim /testbench/dut/hart/lsu/dcache/VictimDirtyWay
-add wave -noupdate -expand -group lsu -expand -group dcache -group Victim /testbench/dut/hart/lsu/dcache/VictimDirty
-add wave -noupdate -expand -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/MemRWM
-add wave -noupdate -expand -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/MemAdrE
-add wave -noupdate -expand -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/MemPAdrM
-add wave -noupdate -expand -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/Funct3M
-add wave -noupdate -expand -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/Funct7M
-add wave -noupdate -expand -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/AtomicM
-add wave -noupdate -expand -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/CacheableM
-add wave -noupdate -expand -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/WriteDataM
-add wave -noupdate -expand -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/ReadDataM
-add wave -noupdate -expand -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/DCacheStall
-add wave -noupdate -expand -group lsu -expand -group dcache -group status /testbench/dut/hart/lsu/dcache/WayHit
-add wave -noupdate -expand -group lsu -expand -group dcache -group status -color {Medium Orchid} /testbench/dut/hart/lsu/dcache/CacheHit
-add wave -noupdate -expand -group lsu -expand -group dcache -group status /testbench/dut/hart/lsu/dcache/FetchCount
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Memory Side} /testbench/dut/hart/lsu/dcache/AHBPAdr
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Memory Side} /testbench/dut/hart/lsu/dcache/AHBRead
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Memory Side} /testbench/dut/hart/lsu/dcache/AHBWrite
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Memory Side} /testbench/dut/hart/lsu/dcache/AHBAck
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Memory Side} /testbench/dut/hart/lsu/dcache/HRDATA
-add wave -noupdate -expand -group lsu -expand -group dcache -group {Memory Side} /testbench/dut/hart/lsu/dcache/HWDATA
-add wave -noupdate -expand -group lsu -group dtlb /testbench/dut/hart/lsu/dmmu/genblk1/tlb/tlbcontrol/EffectivePrivilegeMode
-add wave -noupdate -expand -group lsu -group dtlb /testbench/dut/hart/lsu/dmmu/genblk1/tlb/tlbcontrol/Translate
-add wave -noupdate -expand -group lsu -group dtlb /testbench/dut/hart/lsu/dmmu/genblk1/tlb/tlbcontrol/DisableTranslation
-add wave -noupdate -expand -group lsu -group dtlb /testbench/dut/hart/lsu/dmmu/TLBMiss
-add wave -noupdate -expand -group lsu -group dtlb /testbench/dut/hart/lsu/dmmu/TLBHit
-add wave -noupdate -expand -group lsu -group dtlb /testbench/dut/hart/lsu/dmmu/PhysicalAddress
-add wave -noupdate -expand -group lsu -group dtlb -expand -group faults /testbench/dut/hart/lsu/dmmu/TLBPageFault
-add wave -noupdate -expand -group lsu -group dtlb -expand -group faults /testbench/dut/hart/lsu/dmmu/LoadAccessFaultM
-add wave -noupdate -expand -group lsu -group dtlb -expand -group faults /testbench/dut/hart/lsu/dmmu/StoreAccessFaultM
-add wave -noupdate -expand -group lsu -group dtlb /testbench/dut/hart/lsu/dmmu/genblk1/tlb/TLBPAdr
-add wave -noupdate -expand -group lsu -group dtlb -expand -group write /testbench/dut/hart/lsu/dmmu/genblk1/tlb/PTE
-add wave -noupdate -expand -group lsu -group dtlb -expand -group write /testbench/dut/hart/lsu/dmmu/genblk1/tlb/TLBWrite
-add wave -noupdate -expand -group lsu -group pma /testbench/dut/hart/lsu/dmmu/pmachecker/PhysicalAddress
-add wave -noupdate -expand -group lsu -group pma /testbench/dut/hart/lsu/dmmu/pmachecker/SelRegions
-add wave -noupdate -expand -group lsu -group pma /testbench/dut/hart/lsu/dmmu/Cacheable
-add wave -noupdate -expand -group lsu -group pma /testbench/dut/hart/lsu/dmmu/Idempotent
-add wave -noupdate -expand -group lsu -group pma /testbench/dut/hart/lsu/dmmu/AtomicAllowed
-add wave -noupdate -expand -group lsu -group pma /testbench/dut/hart/lsu/dmmu/pmachecker/PMAAccessFault
-add wave -noupdate -expand -group lsu -group pma /testbench/dut/hart/lsu/dmmu/PMAInstrAccessFaultF
-add wave -noupdate -expand -group lsu -group pma /testbench/dut/hart/lsu/dmmu/PMALoadAccessFaultM
-add wave -noupdate -expand -group lsu -group pma /testbench/dut/hart/lsu/dmmu/PMAStoreAccessFaultM
-add wave -noupdate -expand -group lsu -group pmp /testbench/dut/hart/lsu/dmmu/PMPInstrAccessFaultF
-add wave -noupdate -expand -group lsu -group pmp /testbench/dut/hart/lsu/dmmu/PMPLoadAccessFaultM
-add wave -noupdate -expand -group lsu -group pmp /testbench/dut/hart/lsu/dmmu/PMPStoreAccessFaultM
-add wave -noupdate -expand -group lsu -group ptwalker -color Gold /testbench/dut/hart/lsu/hptw/genblk1/WalkerState
-add wave -noupdate -expand -group lsu -group ptwalker /testbench/dut/hart/lsu/hptw/PCF
-add wave -noupdate -expand -group lsu -group ptwalker /testbench/dut/hart/lsu/hptw/genblk1/TranslationVAdr
-add wave -noupdate -expand -group lsu -group ptwalker /testbench/dut/hart/lsu/hptw/TranslationPAdr
-add wave -noupdate -expand -group lsu -group ptwalker /testbench/dut/hart/lsu/hptw/HPTWReadPTE
-add wave -noupdate -expand -group lsu -group ptwalker /testbench/dut/hart/lsu/hptw/PTE
-add wave -noupdate -expand -group lsu -group ptwalker -expand -group types /testbench/dut/hart/lsu/hptw/ITLBMissF
-add wave -noupdate -expand -group lsu -group ptwalker -expand -group types /testbench/dut/hart/lsu/hptw/DTLBMissM
-add wave -noupdate -expand -group lsu -group ptwalker -expand -group types /testbench/dut/hart/lsu/hptw/ITLBWriteF
-add wave -noupdate -expand -group lsu -group ptwalker -expand -group types /testbench/dut/hart/lsu/hptw/DTLBWriteM
-add wave -noupdate -expand -group lsu -group ptwalker -expand -group types /testbench/dut/hart/lsu/hptw/WalkerInstrPageFaultF
-add wave -noupdate -expand -group lsu -group ptwalker -expand -group types /testbench/dut/hart/lsu/hptw/WalkerLoadPageFaultM
-add wave -noupdate -expand -group lsu -group ptwalker -expand -group types /testbench/dut/hart/lsu/hptw/WalkerStorePageFaultM
+add wave -noupdate -group lsu -expand -group {LSU ARB} /testbench/dut/hart/lsu/arbiter/SelPTW
+add wave -noupdate -group lsu -expand -group dcache -color Gold /testbench/dut/hart/lsu/dcache/dcachefsm/CurrState
+add wave -noupdate -group lsu -expand -group dcache /testbench/dut/hart/lsu/dcache/WalkerPageFaultM
+add wave -noupdate -group lsu -expand -group dcache /testbench/dut/hart/lsu/dcache/WriteDataM
+add wave -noupdate -group lsu -expand -group dcache /testbench/dut/hart/lsu/dcache/SRAMBlockWriteEnableM
+add wave -noupdate -group lsu -expand -group dcache /testbench/dut/hart/lsu/dcache/SRAMWordWriteEnableM
+add wave -noupdate -group lsu -expand -group dcache /testbench/dut/hart/lsu/dcache/SRAMWayWriteEnable
+add wave -noupdate -group lsu -expand -group dcache /testbench/dut/hart/lsu/dcache/SRAMWordEnable
+add wave -noupdate -group lsu -expand -group dcache /testbench/dut/hart/lsu/dcache/SRAMBlockWayWriteEnableM
+add wave -noupdate -group lsu -expand -group dcache /testbench/dut/hart/lsu/dcache/SelAdrM
+add wave -noupdate -group lsu -expand -group dcache /testbench/dut/hart/lsu/dcache/DCacheMemWriteData
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 {/testbench/dut/hart/lsu/dcache/MemWay[0]/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 {/testbench/dut/hart/lsu/dcache/MemWay[0]/SetValid}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 {/testbench/dut/hart/lsu/dcache/MemWay[0]/SetDirty}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 -label TAG {/testbench/dut/hart/lsu/dcache/MemWay[0]/CacheTagMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 {/testbench/dut/hart/lsu/dcache/MemWay[0]/DirtyBits}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 {/testbench/dut/hart/lsu/dcache/MemWay[0]/ValidBits}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 -expand -group Way0Word0 {/testbench/dut/hart/lsu/dcache/MemWay[0]/word[0]/CacheDataMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 -expand -group Way0Word0 {/testbench/dut/hart/lsu/dcache/MemWay[0]/word[0]/CacheDataMem/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 -expand -group Way0Word1 {/testbench/dut/hart/lsu/dcache/MemWay[0]/word[1]/CacheDataMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 -expand -group Way0Word1 {/testbench/dut/hart/lsu/dcache/MemWay[0]/word[1]/CacheDataMem/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 -expand -group Way0Word2 {/testbench/dut/hart/lsu/dcache/MemWay[0]/word[2]/CacheDataMem/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 -expand -group Way0Word2 {/testbench/dut/hart/lsu/dcache/MemWay[0]/word[2]/CacheDataMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 -expand -group Way0Word3 {/testbench/dut/hart/lsu/dcache/MemWay[0]/word[3]/CacheDataMem/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 -expand -group Way0Word3 {/testbench/dut/hart/lsu/dcache/MemWay[0]/word[3]/CacheDataMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 {/testbench/dut/hart/lsu/dcache/MemWay[1]/DirtyBits}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 {/testbench/dut/hart/lsu/dcache/MemWay[1]/SetDirty}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 {/testbench/dut/hart/lsu/dcache/MemWay[1]/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 {/testbench/dut/hart/lsu/dcache/MemWay[1]/WriteWordEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 -label TAG {/testbench/dut/hart/lsu/dcache/MemWay[1]/CacheTagMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 -expand -group Way1Word0 {/testbench/dut/hart/lsu/dcache/MemWay[1]/word[0]/CacheDataMem/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 -expand -group Way1Word0 {/testbench/dut/hart/lsu/dcache/MemWay[1]/word[0]/CacheDataMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 -expand -group Way1Word1 {/testbench/dut/hart/lsu/dcache/MemWay[1]/word[1]/CacheDataMem/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 -expand -group Way1Word1 {/testbench/dut/hart/lsu/dcache/MemWay[1]/word[1]/CacheDataMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 -expand -group Way1Word2 {/testbench/dut/hart/lsu/dcache/MemWay[1]/word[2]/CacheDataMem/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 -expand -group Way1Word2 {/testbench/dut/hart/lsu/dcache/MemWay[1]/word[2]/CacheDataMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 -expand -group Way1Word3 {/testbench/dut/hart/lsu/dcache/MemWay[1]/word[3]/CacheDataMem/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 -expand -group Way1Word3 {/testbench/dut/hart/lsu/dcache/MemWay[1]/word[3]/CacheDataMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 {/testbench/dut/hart/lsu/dcache/MemWay[2]/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 {/testbench/dut/hart/lsu/dcache/MemWay[2]/SetValid}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 {/testbench/dut/hart/lsu/dcache/MemWay[2]/SetDirty}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 -label TAG {/testbench/dut/hart/lsu/dcache/MemWay[2]/CacheTagMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 {/testbench/dut/hart/lsu/dcache/MemWay[2]/DirtyBits}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 {/testbench/dut/hart/lsu/dcache/MemWay[2]/ValidBits}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 -expand -group Way2Word0 {/testbench/dut/hart/lsu/dcache/MemWay[2]/word[0]/CacheDataMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 -expand -group Way2Word0 {/testbench/dut/hart/lsu/dcache/MemWay[2]/word[0]/CacheDataMem/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 -expand -group Way2Word1 {/testbench/dut/hart/lsu/dcache/MemWay[2]/word[1]/CacheDataMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 -expand -group Way2Word1 {/testbench/dut/hart/lsu/dcache/MemWay[2]/word[1]/CacheDataMem/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 -expand -group Way2Word2 {/testbench/dut/hart/lsu/dcache/MemWay[2]/word[2]/CacheDataMem/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 -expand -group Way2Word2 {/testbench/dut/hart/lsu/dcache/MemWay[2]/word[2]/CacheDataMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 -expand -group Way2Word3 {/testbench/dut/hart/lsu/dcache/MemWay[2]/word[3]/CacheDataMem/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 -expand -group Way2Word3 {/testbench/dut/hart/lsu/dcache/MemWay[2]/word[3]/CacheDataMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 {/testbench/dut/hart/lsu/dcache/MemWay[3]/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 {/testbench/dut/hart/lsu/dcache/MemWay[3]/SetValid}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 {/testbench/dut/hart/lsu/dcache/MemWay[3]/SetDirty}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 -label TAG {/testbench/dut/hart/lsu/dcache/MemWay[3]/CacheTagMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 {/testbench/dut/hart/lsu/dcache/MemWay[3]/DirtyBits}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 {/testbench/dut/hart/lsu/dcache/MemWay[3]/ValidBits}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 -expand -group Way3Word0 {/testbench/dut/hart/lsu/dcache/MemWay[3]/word[0]/CacheDataMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 -expand -group Way3Word0 {/testbench/dut/hart/lsu/dcache/MemWay[3]/word[0]/CacheDataMem/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 -expand -group Way3Word1 {/testbench/dut/hart/lsu/dcache/MemWay[3]/word[1]/CacheDataMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 -expand -group Way3Word1 {/testbench/dut/hart/lsu/dcache/MemWay[3]/word[1]/CacheDataMem/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 -expand -group Way3Word2 {/testbench/dut/hart/lsu/dcache/MemWay[3]/word[2]/CacheDataMem/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 -expand -group Way3Word2 {/testbench/dut/hart/lsu/dcache/MemWay[3]/word[2]/CacheDataMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 -expand -group Way3Word3 {/testbench/dut/hart/lsu/dcache/MemWay[3]/word[3]/CacheDataMem/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 -expand -group Way3Word3 {/testbench/dut/hart/lsu/dcache/MemWay[3]/word[3]/CacheDataMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group valid/dirty /testbench/dut/hart/lsu/dcache/SetValid
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group valid/dirty /testbench/dut/hart/lsu/dcache/ClearValid
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group valid/dirty /testbench/dut/hart/lsu/dcache/SetDirty
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group valid/dirty /testbench/dut/hart/lsu/dcache/ClearDirty
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} -group way0 {/testbench/dut/hart/lsu/dcache/MemWay[0]/WayHit}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} -group way0 {/testbench/dut/hart/lsu/dcache/MemWay[0]/Valid}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} -group way0 {/testbench/dut/hart/lsu/dcache/MemWay[0]/Dirty}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} -group way0 {/testbench/dut/hart/lsu/dcache/MemWay[0]/ReadTag}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way1 {/testbench/dut/hart/lsu/dcache/MemWay[1]/WayHit}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way1 {/testbench/dut/hart/lsu/dcache/MemWay[1]/Valid}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way1 {/testbench/dut/hart/lsu/dcache/MemWay[1]/Dirty}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way1 {/testbench/dut/hart/lsu/dcache/MemWay[1]/ReadTag}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way2 {/testbench/dut/hart/lsu/dcache/MemWay[2]/WayHit}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way2 {/testbench/dut/hart/lsu/dcache/MemWay[2]/Valid}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way2 {/testbench/dut/hart/lsu/dcache/MemWay[2]/Dirty}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way2 {/testbench/dut/hart/lsu/dcache/MemWay[2]/ReadTag}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way3 {/testbench/dut/hart/lsu/dcache/MemWay[3]/WayHit}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way3 {/testbench/dut/hart/lsu/dcache/MemWay[3]/Valid}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way3 {/testbench/dut/hart/lsu/dcache/MemWay[3]/Dirty}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way3 {/testbench/dut/hart/lsu/dcache/MemWay[3]/ReadTag}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} /testbench/dut/hart/lsu/dcache/WayHit
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} /testbench/dut/hart/lsu/dcache/ReadDataBlockWayMaskedM
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} /testbench/dut/hart/lsu/dcache/ReadDataWordM
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} /testbench/dut/hart/lsu/dcache/ReadDataWordMuxM
+add wave -noupdate -group lsu -expand -group dcache -group Victim /testbench/dut/hart/lsu/dcache/VictimTag
+add wave -noupdate -group lsu -expand -group dcache -group Victim /testbench/dut/hart/lsu/dcache/VictimWay
+add wave -noupdate -group lsu -expand -group dcache -group Victim /testbench/dut/hart/lsu/dcache/VictimDirtyWay
+add wave -noupdate -group lsu -expand -group dcache -group Victim /testbench/dut/hart/lsu/dcache/VictimDirty
+add wave -noupdate -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/MemRWM
+add wave -noupdate -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/MemAdrE
+add wave -noupdate -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/MemPAdrM
+add wave -noupdate -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/Funct3M
+add wave -noupdate -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/Funct7M
+add wave -noupdate -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/AtomicM
+add wave -noupdate -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/CacheableM
+add wave -noupdate -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/WriteDataM
+add wave -noupdate -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/ReadDataM
+add wave -noupdate -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/DCacheStall
+add wave -noupdate -group lsu -expand -group dcache -group status /testbench/dut/hart/lsu/dcache/WayHit
+add wave -noupdate -group lsu -expand -group dcache -group status -color {Medium Orchid} /testbench/dut/hart/lsu/dcache/CacheHit
+add wave -noupdate -group lsu -expand -group dcache -group status /testbench/dut/hart/lsu/dcache/FetchCount
+add wave -noupdate -group lsu -expand -group dcache -group {Memory Side} /testbench/dut/hart/lsu/dcache/AHBPAdr
+add wave -noupdate -group lsu -expand -group dcache -group {Memory Side} /testbench/dut/hart/lsu/dcache/AHBRead
+add wave -noupdate -group lsu -expand -group dcache -group {Memory Side} /testbench/dut/hart/lsu/dcache/AHBWrite
+add wave -noupdate -group lsu -expand -group dcache -group {Memory Side} /testbench/dut/hart/lsu/dcache/AHBAck
+add wave -noupdate -group lsu -expand -group dcache -group {Memory Side} /testbench/dut/hart/lsu/dcache/HRDATA
+add wave -noupdate -group lsu -expand -group dcache -group {Memory Side} /testbench/dut/hart/lsu/dcache/HWDATA
+add wave -noupdate -group lsu -group dtlb /testbench/dut/hart/lsu/dmmu/genblk1/tlb/tlbcontrol/EffectivePrivilegeMode
+add wave -noupdate -group lsu -group dtlb /testbench/dut/hart/lsu/dmmu/genblk1/tlb/tlbcontrol/Translate
+add wave -noupdate -group lsu -group dtlb /testbench/dut/hart/lsu/dmmu/genblk1/tlb/tlbcontrol/DisableTranslation
+add wave -noupdate -group lsu -group dtlb /testbench/dut/hart/lsu/dmmu/TLBMiss
+add wave -noupdate -group lsu -group dtlb /testbench/dut/hart/lsu/dmmu/TLBHit
+add wave -noupdate -group lsu -group dtlb /testbench/dut/hart/lsu/dmmu/PhysicalAddress
+add wave -noupdate -group lsu -group dtlb -expand -group faults /testbench/dut/hart/lsu/dmmu/TLBPageFault
+add wave -noupdate -group lsu -group dtlb -expand -group faults /testbench/dut/hart/lsu/dmmu/LoadAccessFaultM
+add wave -noupdate -group lsu -group dtlb -expand -group faults /testbench/dut/hart/lsu/dmmu/StoreAccessFaultM
+add wave -noupdate -group lsu -group dtlb /testbench/dut/hart/lsu/dmmu/genblk1/tlb/TLBPAdr
+add wave -noupdate -group lsu -group dtlb -expand -group write /testbench/dut/hart/lsu/dmmu/genblk1/tlb/PTE
+add wave -noupdate -group lsu -group dtlb -expand -group write /testbench/dut/hart/lsu/dmmu/genblk1/tlb/TLBWrite
+add wave -noupdate -group lsu -group pma /testbench/dut/hart/lsu/dmmu/pmachecker/PhysicalAddress
+add wave -noupdate -group lsu -group pma /testbench/dut/hart/lsu/dmmu/pmachecker/SelRegions
+add wave -noupdate -group lsu -group pma /testbench/dut/hart/lsu/dmmu/Cacheable
+add wave -noupdate -group lsu -group pma /testbench/dut/hart/lsu/dmmu/Idempotent
+add wave -noupdate -group lsu -group pma /testbench/dut/hart/lsu/dmmu/AtomicAllowed
+add wave -noupdate -group lsu -group pma /testbench/dut/hart/lsu/dmmu/pmachecker/PMAAccessFault
+add wave -noupdate -group lsu -group pma /testbench/dut/hart/lsu/dmmu/PMAInstrAccessFaultF
+add wave -noupdate -group lsu -group pma /testbench/dut/hart/lsu/dmmu/PMALoadAccessFaultM
+add wave -noupdate -group lsu -group pma /testbench/dut/hart/lsu/dmmu/PMAStoreAccessFaultM
+add wave -noupdate -group lsu -group pmp /testbench/dut/hart/lsu/dmmu/PMPInstrAccessFaultF
+add wave -noupdate -group lsu -group pmp /testbench/dut/hart/lsu/dmmu/PMPLoadAccessFaultM
+add wave -noupdate -group lsu -group pmp /testbench/dut/hart/lsu/dmmu/PMPStoreAccessFaultM
+add wave -noupdate -group lsu -group ptwalker -color Gold /testbench/dut/hart/lsu/hptw/genblk1/WalkerState
+add wave -noupdate -group lsu -group ptwalker /testbench/dut/hart/lsu/hptw/PCF
+add wave -noupdate -group lsu -group ptwalker /testbench/dut/hart/lsu/hptw/genblk1/TranslationVAdr
+add wave -noupdate -group lsu -group ptwalker /testbench/dut/hart/lsu/hptw/TranslationPAdr
+add wave -noupdate -group lsu -group ptwalker /testbench/dut/hart/lsu/hptw/HPTWReadPTE
+add wave -noupdate -group lsu -group ptwalker /testbench/dut/hart/lsu/hptw/PTE
+add wave -noupdate -group lsu -group ptwalker -expand -group types /testbench/dut/hart/lsu/hptw/ITLBMissF
+add wave -noupdate -group lsu -group ptwalker -expand -group types /testbench/dut/hart/lsu/hptw/DTLBMissM
+add wave -noupdate -group lsu -group ptwalker -expand -group types /testbench/dut/hart/lsu/hptw/ITLBWriteF
+add wave -noupdate -group lsu -group ptwalker -expand -group types /testbench/dut/hart/lsu/hptw/DTLBWriteM
+add wave -noupdate -group lsu -group ptwalker -expand -group types /testbench/dut/hart/lsu/hptw/WalkerInstrPageFaultF
+add wave -noupdate -group lsu -group ptwalker -expand -group types /testbench/dut/hart/lsu/hptw/WalkerLoadPageFaultM
+add wave -noupdate -group lsu -group ptwalker -expand -group types /testbench/dut/hart/lsu/hptw/WalkerStorePageFaultM
 add wave -noupdate -group csr /testbench/dut/hart/priv/csr/MIP_REGW
 add wave -noupdate -group itlb /testbench/dut/hart/ifu/immu/TLBWrite
 add wave -noupdate -group itlb /testbench/dut/hart/ifu/ITLBMissF
@@ -480,8 +464,8 @@ add wave -noupdate /testbench/dut/hart/ExceptionM
 add wave -noupdate /testbench/dut/hart/PendingInterruptM
 add wave -noupdate /testbench/dut/hart/TrapM
 TreeUpdate [SetDefaultTree]
-WaveRestoreCursors {{Cursor 6} {32245 ns} 0} {{Cursor 2} {32581 ns} 0} {{Cursor 3} {25666 ns} 0} {{Cursor 4} {2221 ns} 0}
-quietly wave cursor active 4
+WaveRestoreCursors {{Cursor 6} {32245 ns} 0} {{Cursor 2} {176 ns} 0} {{Cursor 3} {25666 ns} 0} {{Cursor 4} {2221 ns} 0}
+quietly wave cursor active 2
 configure wave -namecolwidth 250
 configure wave -valuecolwidth 297
 configure wave -justifyvalue left
@@ -496,4 +480,4 @@ configure wave -griddelta 40
 configure wave -timeline 0
 configure wave -timelineunits ns
 update
-WaveRestoreZoom {0 ns} {325837 ns}
+WaveRestoreZoom {0 ns} {1124571 ns}
diff --git a/wally-pipelined/src/cache/cacheway.sv b/wally-pipelined/src/cache/cacheway.sv
index 7ad9fa980..4d3641693 100644
--- a/wally-pipelined/src/cache/cacheway.sv
+++ b/wally-pipelined/src/cache/cacheway.sv
@@ -26,12 +26,12 @@
 `include "wally-config.vh"
 
 module cacheway #(parameter NUMLINES=512, parameter BLOCKLEN = 256, TAGLEN = 26,
-		   parameter OFFSETLEN, parameter INDEXLEN) 
+		   parameter OFFSETLEN, parameter INDEXLEN, parameter DIRTY_BITS = 1) 
   (input logic 		       clk,
    input logic 			      reset,
 
    input logic [$clog2(NUMLINES)-1:0] RAdr,
-   input logic [`PA_BITS-1:0] 	      MemPAdrM,
+   input logic [`PA_BITS-1:0] 	      PAdr,
    input logic 			      WriteEnable,
    input logic [BLOCKLEN/`XLEN-1:0]   WriteWordEnable,
    input logic 			      TagWriteEnable,
@@ -43,14 +43,14 @@ module cacheway #(parameter NUMLINES=512, parameter BLOCKLEN = 256, TAGLEN = 26,
    input logic 			      SelEvict,
    input logic 			      VictimWay,
 
-   output logic [BLOCKLEN-1:0] 	      ReadDataBlockWayMaskedM,
+   output logic [BLOCKLEN-1:0] 	      ReadDataBlockWayMasked,
    output logic 		      WayHit,
    output logic 		      VictimDirtyWay,
    output logic [TAGLEN-1:0] 	      VictimTagWay
    );
 
   logic [NUMLINES-1:0] 		      ValidBits, DirtyBits;
-  logic [BLOCKLEN-1:0] 		      ReadDataBlockWayM;
+  logic [BLOCKLEN-1:0] 		      ReadDataBlockWay;
   logic [TAGLEN-1:0] 		      ReadTag;
   logic 			      Valid;
   logic 			      Dirty;
@@ -63,7 +63,7 @@ module cacheway #(parameter NUMLINES=512, parameter BLOCKLEN = 256, TAGLEN = 26,
 		.WIDTH(NUMLINES))
       CacheDataMem(.clk(clk),
 		   .Addr(RAdr),
-		   .ReadData(ReadDataBlockWayM[(words+1)*`XLEN-1:words*`XLEN]),
+		   .ReadData(ReadDataBlockWay[(words+1)*`XLEN-1:words*`XLEN]),
 		   .WriteData(WriteData[(words+1)*`XLEN-1:words*`XLEN]),
 		   .WriteEnable(WriteEnable & WriteWordEnable[words]));
     end
@@ -74,12 +74,12 @@ module cacheway #(parameter NUMLINES=512, parameter BLOCKLEN = 256, TAGLEN = 26,
   CacheTagMem(.clk(clk),
 	      .Addr(RAdr),
 	      .ReadData(ReadTag),
-	      .WriteData(MemPAdrM[`PA_BITS-1:OFFSETLEN+INDEXLEN]),
+	      .WriteData(PAdr[`PA_BITS-1:OFFSETLEN+INDEXLEN]),
 	      .WriteEnable(TagWriteEnable));
 
-  assign WayHit = Valid & (ReadTag == MemPAdrM[`PA_BITS-1:OFFSETLEN+INDEXLEN]);
+  assign WayHit = Valid & (ReadTag == PAdr[`PA_BITS-1:OFFSETLEN+INDEXLEN]);
   assign SelectedWay = SelEvict ? VictimWay : WayHit;  
-  assign ReadDataBlockWayMaskedM = SelectedWay ? ReadDataBlockWayM : '0;  // first part of AO mux.
+  assign ReadDataBlockWayMasked = SelectedWay ? ReadDataBlockWay : '0;  // first part of AO mux.
 
   assign VictimDirtyWay = VictimWay & Dirty & Valid;
   assign VictimTagWay = VictimWay ? ReadTag : '0;
@@ -87,20 +87,25 @@ module cacheway #(parameter NUMLINES=512, parameter BLOCKLEN = 256, TAGLEN = 26,
   always_ff @(posedge clk, posedge reset) begin
     if (reset) 
   	ValidBits <= {NUMLINES{1'b0}};
-    else if (SetValid & WriteEnable) ValidBits[MemPAdrM[INDEXLEN+OFFSETLEN-1:OFFSETLEN]] <= 1'b1;
-    else if (ClearValid & WriteEnable) ValidBits[MemPAdrM[INDEXLEN+OFFSETLEN-1:OFFSETLEN]] <= 1'b0;
+    else if (SetValid & WriteEnable) ValidBits[PAdr[INDEXLEN+OFFSETLEN-1:OFFSETLEN]] <= 1'b1;
+    else if (ClearValid & WriteEnable) ValidBits[PAdr[INDEXLEN+OFFSETLEN-1:OFFSETLEN]] <= 1'b0;
     Valid <= ValidBits[RAdr];
   end
 
-  always_ff @(posedge clk, posedge reset) begin
-    if (reset) 
-  	DirtyBits <= {NUMLINES{1'b0}};
-    else if (SetDirty & WriteEnable) DirtyBits[MemPAdrM[INDEXLEN+OFFSETLEN-1:OFFSETLEN]] <= 1'b1;
-    else if (ClearDirty & WriteEnable) DirtyBits[MemPAdrM[INDEXLEN+OFFSETLEN-1:OFFSETLEN]] <= 1'b0;
-    Dirty <= DirtyBits[RAdr];
-  end
+  generate
+    if(DIRTY_BITS) begin
+      always_ff @(posedge clk, posedge reset) begin
+	if (reset) 
+  	  DirtyBits <= {NUMLINES{1'b0}};
+	else if (SetDirty & WriteEnable) DirtyBits[PAdr[INDEXLEN+OFFSETLEN-1:OFFSETLEN]] <= 1'b1;
+	else if (ClearDirty & WriteEnable) DirtyBits[PAdr[INDEXLEN+OFFSETLEN-1:OFFSETLEN]] <= 1'b0;
+	Dirty <= DirtyBits[RAdr];
+      end
+    end else begin
+      assign Dirty = 1'b0;
+    end
+  endgenerate
   
-
 endmodule // DCacheMemWay
 
 
diff --git a/wally-pipelined/src/cache/dcache.sv b/wally-pipelined/src/cache/dcache.sv
index d9620cf98..2af4cfe90 100644
--- a/wally-pipelined/src/cache/dcache.sv
+++ b/wally-pipelined/src/cache/dcache.sv
@@ -149,7 +149,7 @@ module dcache
   MemWay[NUMWAYS-1:0](.clk,
 		      .reset,
 		      .RAdr,
-		      .MemPAdrM(MemPAdrM[`PA_BITS-1:0]),
+		      .PAdr(MemPAdrM[`PA_BITS-1:0]),
 		      .WriteEnable(SRAMWayWriteEnable),
 		      .WriteWordEnable(SRAMWordEnable),
 		      .TagWriteEnable(SRAMBlockWayWriteEnableM), 
@@ -160,7 +160,7 @@ module dcache
 		      .ClearDirty,
 		      .SelEvict,
 		      .VictimWay,
-		      .ReadDataBlockWayMaskedM,
+		      .ReadDataBlockWayMasked(ReadDataBlockWayMaskedM),
 		      .WayHit,
 		      .VictimDirtyWay,
 		      .VictimTagWay);
diff --git a/wally-pipelined/src/cache/icache.sv b/wally-pipelined/src/cache/icache.sv
index c73d9cc6c..8717d5b7b 100644
--- a/wally-pipelined/src/cache/icache.sv
+++ b/wally-pipelined/src/cache/icache.sv
@@ -55,6 +55,11 @@ module icache
   // Configuration parameters
   localparam integer 	    BLOCKLEN = `ICACHE_BLOCKLENINBITS;
   localparam integer 	    NUMLINES = `ICACHE_WAYSIZEINBYTES*8/`ICACHE_BLOCKLENINBITS;
+  localparam integer 	    BLOCKBYTELEN = BLOCKLEN/8;
+
+  localparam integer 	    OFFSETLEN = $clog2(BLOCKBYTELEN);
+  localparam integer 	    INDEXLEN = $clog2(NUMLINES);
+  localparam integer 	    TAGLEN = `PA_BITS - OFFSETLEN - INDEXLEN;
 
   localparam WORDSPERLINE = BLOCKLEN/`XLEN;
   localparam LOGWPL = $clog2(WORDSPERLINE);
@@ -99,6 +104,7 @@ module icache
   
 
   
+/* -----\/----- EXCLUDED -----\/-----
   ICacheMem #(.BLOCKLEN(BLOCKLEN), .NUMLINES(NUMLINES)) 
   cachemem(.clk,
 	   .reset,
@@ -109,6 +115,30 @@ module icache
            .WriteLine(ICacheMemWriteData),
 	   .ReadLineF,
            .HitF(ICacheMemReadValid));
+ -----/\----- EXCLUDED -----/\----- */
+
+  cacheway #(.NUMLINES(NUMLINES), .BLOCKLEN(BLOCKLEN), .TAGLEN(TAGLEN), .OFFSETLEN(OFFSETLEN), .INDEXLEN(INDEXLEN),
+	     .DIRTY_BITS(0))
+  icachemem(.clk,
+	    .reset,
+	    .RAdr(PCNextIndexF[INDEXLEN+OFFSETLEN-1:OFFSETLEN]),
+	    .PAdr(PCTagF),
+	    .WriteEnable(ICacheMemWriteEnable),
+	    .WriteWordEnable('1),
+	    .TagWriteEnable(ICacheMemWriteEnable),
+	    .WriteData(ICacheMemWriteData),
+	    .SetValid(ICacheMemWriteEnable),
+	    .ClearValid(1'b0),
+	    .SetDirty(1'b0),
+	    .ClearDirty(1'b0),
+	    .SelEvict(1'b0),
+	    .VictimWay(1'b0),
+	    .ReadDataBlockWayMasked(ReadLineF),
+	    .WayHit(ICacheMemReadValid),
+	    .VictimDirtyWay(),
+	    .VictimTagWay()
+	    );
+  
 
   always_comb begin
     case (PCTagF[4:1])

From 62d91e9ea1572e0ba4aa7993a557aaa22929ac1d Mon Sep 17 00:00:00 2001
From: Ross Thompson <ross1728@gmail.com>
Date: Thu, 26 Aug 2021 15:57:17 -0500
Subject: [PATCH 06/16] Renamed ICacheCntrl to icachefsm.

---
 wally-pipelined/src/cache/icache.sv           | 35 +++++++------------
 .../cache/{ICacheCntrl.sv => icachefsm.sv}    |  2 +-
 2 files changed, 14 insertions(+), 23 deletions(-)
 rename wally-pipelined/src/cache/{ICacheCntrl.sv => icachefsm.sv} (99%)

diff --git a/wally-pipelined/src/cache/icache.sv b/wally-pipelined/src/cache/icache.sv
index 8717d5b7b..7fc1e19b9 100644
--- a/wally-pipelined/src/cache/icache.sv
+++ b/wally-pipelined/src/cache/icache.sv
@@ -103,19 +103,18 @@ module icache
   logic 		       SavePC;
   
 
-  
-/* -----\/----- EXCLUDED -----\/-----
-  ICacheMem #(.BLOCKLEN(BLOCKLEN), .NUMLINES(NUMLINES)) 
-  cachemem(.clk,
-	   .reset,
-           .flush(FlushMem),
-	   .PCTagF,
-	   .PCNextIndexF,
-           .WriteEnable(ICacheMemWriteEnable),
-           .WriteLine(ICacheMemWriteData),
-	   .ReadLineF,
-           .HitF(ICacheMemReadValid));
- -----/\----- EXCLUDED -----/\----- */
+  // on spill we want to get the first 2 bytes of the next cache block.
+  // the spill only occurs if the PCPF mod BlockByteLength == -2.  Therefore we can
+  // simply add 2 to land on the next cache block.
+  assign PCPSpillF = PCPF + {{{PA_WIDTH}{1'b0}}, 2'b10}; // *** modelsim does not allow the use of PA_BITS for literal width.
+
+  // now we have to select between these three PCs
+  assign PCPreFinalF = PCMux[0] | StallF ? PCPF : PCNextF; // *** don't like the stallf, but it is necessary
+  // for the data cache i used a cpu busy state which is triggered by StallW.  In the case of the icache I
+  // modified the select on this address mux. Both are not ideal; however the cpu_busy state is required for the
+  // dcache as a write would repeatedly update the sram or worse for an uncached write multiple times.
+  // I like reducing some complexity of the fsm; however I weight commonality between the i/d cache more.
+  assign PCNextIndexF = PCMux[1] ? PCPSpillF : PCPreFinalF;
 
   cacheway #(.NUMLINES(NUMLINES), .BLOCKLEN(BLOCKLEN), .TAGLEN(TAGLEN), .OFFSETLEN(OFFSETLEN), .INDEXLEN(INDEXLEN),
 	     .DIRTY_BITS(0))
@@ -226,14 +225,6 @@ module icache
     end
   endgenerate
 
-  // on spill we want to get the first 2 bytes of the next cache block.
-  // the spill only occurs if the PCPF mod BlockByteLength == -2.  Therefore we can
-  // simply add 2 to land on the next cache block.
-  assign PCPSpillF = PCPF + {{{PA_WIDTH}{1'b0}}, 2'b10}; // *** modelsim does not allow the use of PA_BITS for literal width.
-
-  // now we have to select between these three PCs
-  assign PCPreFinalF = PCMux[0] | StallF ? PCPF : PCNextF; // *** don't like the stallf, but it is necessary
-  assign PCNextIndexF = PCMux[1] ? PCPSpillF : PCPreFinalF;
 
   // this mux needs to be delayed 1 cycle as it occurs 1 pipeline stage later.
   // *** read enable may not be necessary.
@@ -249,7 +240,7 @@ module icache
   assign PCPTrunkF = PCTagF[`PA_BITS-1:OFFSETWIDTH];
   
 
-  ICacheCntrl #(.BLOCKLEN(BLOCKLEN)) 
+  icachefsm #(.BLOCKLEN(BLOCKLEN)) 
   controller(.clk,
 	     .reset,
 	     .ICacheReadEn,
diff --git a/wally-pipelined/src/cache/ICacheCntrl.sv b/wally-pipelined/src/cache/icachefsm.sv
similarity index 99%
rename from wally-pipelined/src/cache/ICacheCntrl.sv
rename to wally-pipelined/src/cache/icachefsm.sv
index 711461253..c0c2e003f 100644
--- a/wally-pipelined/src/cache/ICacheCntrl.sv
+++ b/wally-pipelined/src/cache/icachefsm.sv
@@ -25,7 +25,7 @@
 
 `include "wally-config.vh"
 
-module ICacheCntrl #(parameter BLOCKLEN = 256) 
+module icachefsm #(parameter BLOCKLEN = 256) 
   (
    // Inputs from pipeline
    input logic 	      clk, reset,

From de9e234ffa337ab23de65c590f07ae6bcabb0803 Mon Sep 17 00:00:00 2001
From: Ross Thompson <ross1728@gmail.com>
Date: Fri, 27 Aug 2021 11:03:36 -0500
Subject: [PATCH 07/16] Modified icache to no longer need StallF in the PCMux
 logic.  Instead this is handled in the icachefsm. One downside is it
 increases the icache complexity.  However it also fixes an untested bug.  If
 a region was uncacheable it would have been possible for the request to be
 made multiple times.  Now that is not possible.  Additionally spills were
 oscillating between the spill hit states without this change. The impact was
 'benign' as the final spilled instruction always had the correct upper 16
 bits.

---
 wally-pipelined/regression/wave.do            | 127 +++++++++++-------
 .../cache/dcache_ptw_interaction_README.txt   |   2 +-
 wally-pipelined/src/cache/icache.sv           |   3 +-
 wally-pipelined/src/cache/icachefsm.sv        |  90 +++++++++++--
 4 files changed, 159 insertions(+), 63 deletions(-)

diff --git a/wally-pipelined/regression/wave.do b/wally-pipelined/regression/wave.do
index 925198962..82fb8d3c2 100644
--- a/wally-pipelined/regression/wave.do
+++ b/wally-pipelined/regression/wave.do
@@ -16,41 +16,41 @@ add wave -noupdate -expand -group {Memory Stage} /testbench/dut/hart/InstrM
 add wave -noupdate -expand -group {Memory Stage} /testbench/dut/hart/lsu/MemAdrM
 add wave -noupdate /testbench/dut/hart/ieu/dp/ResultM
 add wave -noupdate /testbench/dut/hart/ieu/dp/ResultW
-add wave -noupdate -group HDU -group traps /testbench/dut/hart/priv/trap/InstrMisalignedFaultM
-add wave -noupdate -group HDU -group traps /testbench/dut/hart/priv/trap/InstrAccessFaultM
-add wave -noupdate -group HDU -group traps /testbench/dut/hart/priv/trap/IllegalInstrFaultM
-add wave -noupdate -group HDU -group traps /testbench/dut/hart/priv/trap/BreakpointFaultM
-add wave -noupdate -group HDU -group traps /testbench/dut/hart/priv/trap/LoadMisalignedFaultM
-add wave -noupdate -group HDU -group traps /testbench/dut/hart/priv/trap/StoreMisalignedFaultM
-add wave -noupdate -group HDU -group traps /testbench/dut/hart/priv/trap/LoadAccessFaultM
-add wave -noupdate -group HDU -group traps /testbench/dut/hart/priv/trap/StoreAccessFaultM
-add wave -noupdate -group HDU -group traps /testbench/dut/hart/priv/trap/EcallFaultM
-add wave -noupdate -group HDU -group traps /testbench/dut/hart/priv/trap/InstrPageFaultM
-add wave -noupdate -group HDU -group traps /testbench/dut/hart/priv/trap/LoadPageFaultM
-add wave -noupdate -group HDU -group traps /testbench/dut/hart/priv/trap/StorePageFaultM
-add wave -noupdate -group HDU -group traps /testbench/dut/hart/priv/trap/InterruptM
-add wave -noupdate -group HDU -group interrupts /testbench/dut/hart/priv/trap/PendingIntsM
-add wave -noupdate -group HDU -group interrupts /testbench/dut/hart/priv/trap/CommittedM
-add wave -noupdate -group HDU -group interrupts /testbench/dut/hart/priv/trap/InstrValidM
-add wave -noupdate -group HDU -group hazards /testbench/dut/hart/hzu/BPPredWrongE
-add wave -noupdate -group HDU -group hazards /testbench/dut/hart/hzu/CSRWritePendingDEM
-add wave -noupdate -group HDU -group hazards /testbench/dut/hart/hzu/RetM
-add wave -noupdate -group HDU -group hazards /testbench/dut/hart/hzu/TrapM
-add wave -noupdate -group HDU -group hazards /testbench/dut/hart/hzu/LoadStallD
-add wave -noupdate -group HDU -group hazards /testbench/dut/hart/hzu/StoreStallD
-add wave -noupdate -group HDU -group hazards /testbench/dut/hart/hzu/ICacheStallF
-add wave -noupdate -group HDU -group hazards /testbench/dut/hart/hzu/LSUStall
-add wave -noupdate -group HDU -group hazards /testbench/dut/hart/MulDivStallD
-add wave -noupdate -group HDU -expand -group Flush -color Yellow /testbench/dut/hart/hzu/FlushF
-add wave -noupdate -group HDU -expand -group Flush -color Yellow /testbench/dut/hart/FlushD
-add wave -noupdate -group HDU -expand -group Flush -color Yellow /testbench/dut/hart/FlushE
-add wave -noupdate -group HDU -expand -group Flush -color Yellow /testbench/dut/hart/FlushM
-add wave -noupdate -group HDU -expand -group Flush -color Yellow /testbench/dut/hart/FlushW
-add wave -noupdate -group HDU -expand -group Stall -color Orange /testbench/dut/hart/StallF
-add wave -noupdate -group HDU -expand -group Stall -color Orange /testbench/dut/hart/StallD
-add wave -noupdate -group HDU -expand -group Stall -color Orange /testbench/dut/hart/StallE
-add wave -noupdate -group HDU -expand -group Stall -color Orange /testbench/dut/hart/StallM
-add wave -noupdate -group HDU -expand -group Stall -color Orange /testbench/dut/hart/StallW
+add wave -noupdate -expand -group HDU -group traps /testbench/dut/hart/priv/trap/InstrMisalignedFaultM
+add wave -noupdate -expand -group HDU -group traps /testbench/dut/hart/priv/trap/InstrAccessFaultM
+add wave -noupdate -expand -group HDU -group traps /testbench/dut/hart/priv/trap/IllegalInstrFaultM
+add wave -noupdate -expand -group HDU -group traps /testbench/dut/hart/priv/trap/BreakpointFaultM
+add wave -noupdate -expand -group HDU -group traps /testbench/dut/hart/priv/trap/LoadMisalignedFaultM
+add wave -noupdate -expand -group HDU -group traps /testbench/dut/hart/priv/trap/StoreMisalignedFaultM
+add wave -noupdate -expand -group HDU -group traps /testbench/dut/hart/priv/trap/LoadAccessFaultM
+add wave -noupdate -expand -group HDU -group traps /testbench/dut/hart/priv/trap/StoreAccessFaultM
+add wave -noupdate -expand -group HDU -group traps /testbench/dut/hart/priv/trap/EcallFaultM
+add wave -noupdate -expand -group HDU -group traps /testbench/dut/hart/priv/trap/InstrPageFaultM
+add wave -noupdate -expand -group HDU -group traps /testbench/dut/hart/priv/trap/LoadPageFaultM
+add wave -noupdate -expand -group HDU -group traps /testbench/dut/hart/priv/trap/StorePageFaultM
+add wave -noupdate -expand -group HDU -group traps /testbench/dut/hart/priv/trap/InterruptM
+add wave -noupdate -expand -group HDU -group interrupts /testbench/dut/hart/priv/trap/PendingIntsM
+add wave -noupdate -expand -group HDU -group interrupts /testbench/dut/hart/priv/trap/CommittedM
+add wave -noupdate -expand -group HDU -group interrupts /testbench/dut/hart/priv/trap/InstrValidM
+add wave -noupdate -expand -group HDU -group hazards /testbench/dut/hart/hzu/BPPredWrongE
+add wave -noupdate -expand -group HDU -group hazards /testbench/dut/hart/hzu/CSRWritePendingDEM
+add wave -noupdate -expand -group HDU -group hazards /testbench/dut/hart/hzu/RetM
+add wave -noupdate -expand -group HDU -group hazards /testbench/dut/hart/hzu/TrapM
+add wave -noupdate -expand -group HDU -group hazards /testbench/dut/hart/hzu/LoadStallD
+add wave -noupdate -expand -group HDU -group hazards /testbench/dut/hart/hzu/StoreStallD
+add wave -noupdate -expand -group HDU -group hazards /testbench/dut/hart/hzu/ICacheStallF
+add wave -noupdate -expand -group HDU -group hazards /testbench/dut/hart/hzu/LSUStall
+add wave -noupdate -expand -group HDU -group hazards /testbench/dut/hart/MulDivStallD
+add wave -noupdate -expand -group HDU -expand -group Flush -color Yellow /testbench/dut/hart/hzu/FlushF
+add wave -noupdate -expand -group HDU -expand -group Flush -color Yellow /testbench/dut/hart/FlushD
+add wave -noupdate -expand -group HDU -expand -group Flush -color Yellow /testbench/dut/hart/FlushE
+add wave -noupdate -expand -group HDU -expand -group Flush -color Yellow /testbench/dut/hart/FlushM
+add wave -noupdate -expand -group HDU -expand -group Flush -color Yellow /testbench/dut/hart/FlushW
+add wave -noupdate -expand -group HDU -expand -group Stall -color Orange /testbench/dut/hart/StallF
+add wave -noupdate -expand -group HDU -expand -group Stall -color Orange /testbench/dut/hart/StallD
+add wave -noupdate -expand -group HDU -expand -group Stall -color Orange /testbench/dut/hart/StallE
+add wave -noupdate -expand -group HDU -expand -group Stall -color Orange /testbench/dut/hart/StallM
+add wave -noupdate -expand -group HDU -expand -group Stall -color Orange /testbench/dut/hart/StallW
 add wave -noupdate -group Bpred -color Orange /testbench/dut/hart/ifu/bpred/bpred/Predictor/DirPredictor/GHR
 add wave -noupdate -group Bpred -expand -group {branch update selection inputs} /testbench/dut/hart/ifu/bpred/bpred/Predictor/DirPredictor/BPPredF
 add wave -noupdate -group Bpred -expand -group {branch update selection inputs} {/testbench/dut/hart/ifu/bpred/bpred/Predictor/DirPredictor/InstrClassE[0]}
@@ -98,11 +98,12 @@ add wave -noupdate -group Bpred -expand -group {bp wrong} /testbench/dut/hart/if
 add wave -noupdate -group Bpred -expand -group {bp wrong} /testbench/dut/hart/ifu/bpred/bpred/BPPredClassNonCFIWrongE
 add wave -noupdate -group Bpred -expand -group {bp wrong} /testbench/dut/hart/ifu/bpred/bpred/BPPredWrongE
 add wave -noupdate -group Bpred /testbench/dut/hart/ifu/bpred/bpred/BPPredWrongE
-add wave -noupdate -group {instruction pipeline} /testbench/InstrFName
-add wave -noupdate -group {instruction pipeline} /testbench/dut/hart/ifu/InstrD
-add wave -noupdate -group {instruction pipeline} /testbench/dut/hart/ifu/InstrE
-add wave -noupdate -group {instruction pipeline} /testbench/dut/hart/ifu/InstrM
-add wave -noupdate -group {instruction pipeline} /testbench/InstrW
+add wave -noupdate -expand -group {instruction pipeline} /testbench/InstrFName
+add wave -noupdate -expand -group {instruction pipeline} /testbench/dut/hart/ifu/icache/FinalInstrRawF
+add wave -noupdate -expand -group {instruction pipeline} /testbench/dut/hart/ifu/InstrD
+add wave -noupdate -expand -group {instruction pipeline} /testbench/dut/hart/ifu/InstrE
+add wave -noupdate -expand -group {instruction pipeline} /testbench/dut/hart/ifu/InstrM
+add wave -noupdate -expand -group {instruction pipeline} /testbench/InstrW
 add wave -noupdate -group {PCNext Generation} /testbench/dut/hart/ifu/PCNextF
 add wave -noupdate -group {PCNext Generation} /testbench/dut/hart/ifu/PCF
 add wave -noupdate -group {PCNext Generation} /testbench/dut/hart/ifu/PCPlus2or4F
@@ -160,12 +161,12 @@ add wave -noupdate -group {alu execution stage} /testbench/dut/hart/ieu/dp/Write
 add wave -noupdate -group {alu execution stage} /testbench/dut/hart/ieu/dp/ALUResultE
 add wave -noupdate -group {alu execution stage} /testbench/dut/hart/ieu/dp/SrcAE
 add wave -noupdate -group {alu execution stage} /testbench/dut/hart/ieu/dp/SrcBE
-add wave -noupdate -group PCS /testbench/dut/hart/ifu/PCNextF
-add wave -noupdate -group PCS /testbench/dut/hart/PCF
-add wave -noupdate -group PCS /testbench/dut/hart/ifu/PCD
-add wave -noupdate -group PCS /testbench/dut/hart/PCE
-add wave -noupdate -group PCS /testbench/dut/hart/PCM
-add wave -noupdate -group PCS /testbench/PCW
+add wave -noupdate -expand -group PCS /testbench/dut/hart/ifu/PCNextF
+add wave -noupdate -expand -group PCS /testbench/dut/hart/PCF
+add wave -noupdate -expand -group PCS /testbench/dut/hart/ifu/PCD
+add wave -noupdate -expand -group PCS /testbench/dut/hart/PCE
+add wave -noupdate -expand -group PCS /testbench/dut/hart/PCM
+add wave -noupdate -expand -group PCS /testbench/PCW
 add wave -noupdate -group muldiv /testbench/dut/hart/mdu/InstrD
 add wave -noupdate -group muldiv /testbench/dut/hart/mdu/SrcAE
 add wave -noupdate -group muldiv /testbench/dut/hart/mdu/SrcBE
@@ -185,7 +186,7 @@ add wave -noupdate -group divider /testbench/dut/hart/mdu/genblk1/div/N
 add wave -noupdate -group divider /testbench/dut/hart/mdu/genblk1/div/D
 add wave -noupdate -group divider /testbench/dut/hart/mdu/genblk1/div/Q
 add wave -noupdate -group divider /testbench/dut/hart/mdu/genblk1/div/rem0
-add wave -noupdate -expand -group icache -color Orange /testbench/dut/hart/ifu/icache/controller/CurrState
+add wave -noupdate -expand -group icache -color Gold /testbench/dut/hart/ifu/icache/controller/CurrState
 add wave -noupdate -expand -group icache /testbench/dut/hart/ifu/icache/controller/NextState
 add wave -noupdate -expand -group icache /testbench/dut/hart/ifu/ITLBMissF
 add wave -noupdate -expand -group icache /testbench/dut/hart/ifu/icache/ITLBWriteF
@@ -193,6 +194,8 @@ add wave -noupdate -expand -group icache /testbench/dut/hart/ifu/icache/ReadLine
 add wave -noupdate -expand -group icache {/testbench/dut/hart/ifu/icache/icachemem/word[0]/CacheDataMem/Addr}
 add wave -noupdate -expand -group icache {/testbench/dut/hart/ifu/icache/icachemem/word[0]/CacheDataMem/WriteData}
 add wave -noupdate -expand -group icache {/testbench/dut/hart/ifu/icache/icachemem/word[0]/CacheDataMem/WriteEnable}
+add wave -noupdate -expand -group icache /testbench/dut/hart/ifu/icache/PCNextIndexF
+add wave -noupdate -expand -group icache /testbench/dut/hart/ifu/icache/ReadLineF
 add wave -noupdate -expand -group icache -expand -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/hit
 add wave -noupdate -expand -group icache -expand -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/spill
 add wave -noupdate -expand -group icache -expand -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/ICacheStallF
@@ -456,16 +459,38 @@ add wave -noupdate -group UART /testbench/dut/uncore/uart/uart/HSELUART
 add wave -noupdate -group UART /testbench/dut/uncore/uart/uart/HADDR
 add wave -noupdate -group UART /testbench/dut/uncore/uart/uart/HWRITE
 add wave -noupdate -group UART /testbench/dut/uncore/uart/uart/HWDATA
-add wave -noupdate /testbench/dut/hart/lsu/dcache/subwordread/offset0
+add wave -noupdate -color Gold /testbench/dut/hart/lsu/dcache/subwordread/offset0
 add wave -noupdate /testbench/dut/hart/lsu/dcache/subwordread/offset1
 add wave -noupdate /testbench/dut/hart/lsu/dcache/subwordread/offset2
 add wave -noupdate /testbench/dut/hart/lsu/dcache/subwordread/offset3
 add wave -noupdate /testbench/dut/hart/ExceptionM
 add wave -noupdate /testbench/dut/hart/PendingInterruptM
 add wave -noupdate /testbench/dut/hart/TrapM
+add wave -noupdate /testbench/dut/hart/ifu/icache/CompressedF
+add wave -noupdate /testbench/dut/hart/ifu/icache/PCPF
+add wave -noupdate /testbench/dut/hart/ifu/PCPFmmu
+add wave -noupdate /testbench/dut/hart/ifu/PCPF
+add wave -noupdate /testbench/dut/hart/ifu/PCF
+add wave -noupdate /testbench/dut/hart/ifu/immu/Translate
+add wave -noupdate /testbench/dut/hart/ifu/icache/FinalInstrRawF
+add wave -noupdate /testbench/dut/hart/ifu/icache/StallF
+add wave -noupdate /testbench/dut/hart/ifu/icache/ICacheMemReadData
+add wave -noupdate {/testbench/dut/hart/ifu/icache/icachemem/word[0]/CacheDataMem/Addr}
+add wave -noupdate {/testbench/dut/hart/ifu/icache/icachemem/word[0]/CacheDataMem/ReadData}
+add wave -noupdate {/testbench/dut/hart/ifu/icache/icachemem/word[1]/CacheDataMem/Addr}
+add wave -noupdate {/testbench/dut/hart/ifu/icache/icachemem/word[1]/CacheDataMem/ReadData}
+add wave -noupdate {/testbench/dut/hart/ifu/icache/icachemem/word[2]/CacheDataMem/Addr}
+add wave -noupdate {/testbench/dut/hart/ifu/icache/icachemem/word[2]/CacheDataMem/ReadData}
+add wave -noupdate {/testbench/dut/hart/ifu/icache/icachemem/word[3]/CacheDataMem/Addr}
+add wave -noupdate {/testbench/dut/hart/ifu/icache/icachemem/word[3]/CacheDataMem/ReadData}
+add wave -noupdate /testbench/dut/hart/ifu/icache/icachemem/WayHit
+add wave -noupdate /testbench/dut/hart/ifu/icache/PCTagF
+add wave -noupdate /testbench/dut/hart/ifu/icache/PCPSpillF
+add wave -noupdate /testbench/dut/hart/ifu/icache/ICacheReadEn
+add wave -noupdate /testbench/dut/hart/ifu/icache/PCMux_q
 TreeUpdate [SetDefaultTree]
-WaveRestoreCursors {{Cursor 6} {32245 ns} 0} {{Cursor 2} {176 ns} 0} {{Cursor 3} {25666 ns} 0} {{Cursor 4} {2221 ns} 0}
-quietly wave cursor active 2
+WaveRestoreCursors {{Cursor 6} {53451 ns} 0}
+quietly wave cursor active 1
 configure wave -namecolwidth 250
 configure wave -valuecolwidth 297
 configure wave -justifyvalue left
@@ -480,4 +505,4 @@ configure wave -griddelta 40
 configure wave -timeline 0
 configure wave -timelineunits ns
 update
-WaveRestoreZoom {0 ns} {1124571 ns}
+WaveRestoreZoom {53011 ns} {53891 ns}
diff --git a/wally-pipelined/src/cache/dcache_ptw_interaction_README.txt b/wally-pipelined/src/cache/dcache_ptw_interaction_README.txt
index b52987eda..59d768fce 100644
--- a/wally-pipelined/src/cache/dcache_ptw_interaction_README.txt
+++ b/wally-pipelined/src/cache/dcache_ptw_interaction_README.txt
@@ -9,7 +9,7 @@ should continue.
 It is important to note ITLB misses and faults DO NOT flush a memory operation
 in the memory stage.  This is the core reason for the complexity.
 
-| Type | ITLB miss | DTLB miss | mem op |              |
+| Type | ITLB miss | DTLB miss | mem op |               |
 |-------+-----------+-----------+--------+--------------|
 |     0 |         0 |         0 |      0 |              |
 |     1 |         0 |         0 |      1 |              |
diff --git a/wally-pipelined/src/cache/icache.sv b/wally-pipelined/src/cache/icache.sv
index 7fc1e19b9..7bbeccc60 100644
--- a/wally-pipelined/src/cache/icache.sv
+++ b/wally-pipelined/src/cache/icache.sv
@@ -109,7 +109,7 @@ module icache
   assign PCPSpillF = PCPF + {{{PA_WIDTH}{1'b0}}, 2'b10}; // *** modelsim does not allow the use of PA_BITS for literal width.
 
   // now we have to select between these three PCs
-  assign PCPreFinalF = PCMux[0] | StallF ? PCPF : PCNextF; // *** don't like the stallf, but it is necessary
+  assign PCPreFinalF = PCMux[0] ? PCPF : PCNextF; // *** don't like the stallf, but it is necessary
   // for the data cache i used a cpu busy state which is triggered by StallW.  In the case of the icache I
   // modified the select on this address mux. Both are not ideal; however the cpu_busy state is required for the
   // dcache as a write would repeatedly update the sram or worse for an uncached write multiple times.
@@ -243,6 +243,7 @@ module icache
   icachefsm #(.BLOCKLEN(BLOCKLEN)) 
   controller(.clk,
 	     .reset,
+	     .StallF,
 	     .ICacheReadEn,
 	     .ICacheMemWriteEnable,
 	     .ICacheStallF,
diff --git a/wally-pipelined/src/cache/icachefsm.sv b/wally-pipelined/src/cache/icachefsm.sv
index c0c2e003f..ddf9b50ca 100644
--- a/wally-pipelined/src/cache/icachefsm.sv
+++ b/wally-pipelined/src/cache/icachefsm.sv
@@ -30,6 +30,8 @@ module icachefsm #(parameter BLOCKLEN = 256)
    // Inputs from pipeline
    input logic 	      clk, reset,
 
+   input logic 	      StallF,
+
    // inputs from mmu
    input logic 	      ITLBMissF,
    input logic 	      ITLBWriteF,
@@ -88,7 +90,8 @@ module icachefsm #(parameter BLOCKLEN = 256)
   
 		STATE_MISS_FETCH_WDV, // aligned miss, issue read to AHB and wait for data.
 		STATE_MISS_FETCH_DONE, // write data into SRAM/LUT
-		STATE_MISS_READ, // read block 1 from SRAM/LUT  
+		STATE_MISS_READ, // read block 1 from SRAM/LUT
+		STATE_MISS_READ_DELAY, // read block 1 from SRAM/LUT  		
 
 		STATE_MISS_SPILL_FETCH_WDV, // spill, miss on block 0, issue read to AHB and wait
 		STATE_MISS_SPILL_FETCH_DONE, // write data into SRAM/LUT
@@ -104,7 +107,10 @@ module icachefsm #(parameter BLOCKLEN = 256)
 
 		STATE_INVALIDATE, // *** not sure if invalidate or evict? invalidate by cache block or address?
 		STATE_TLB_MISS,
-		STATE_TLB_MISS_DONE
+		STATE_TLB_MISS_DONE,
+
+		STATE_CPU_BUSY,
+		STATE_CPU_BUSY_SPILL		
 		} statetype;
   
   statetype CurrState, NextState;
@@ -138,7 +144,12 @@ module icachefsm #(parameter BLOCKLEN = 256)
         end else if (hit & ~spill) begin
           SavePC = 1'b1;
           ICacheStallF = 1'b0;
-          NextState = STATE_READY;
+	  if(StallF) begin
+	    NextState = STATE_CPU_BUSY;
+	    PCMux = 2'b01;
+	  end else begin
+            NextState = STATE_READY;
+	  end
         end else if (hit & spill) begin
           spillSave = 1'b1;
           PCMux = 2'b10;
@@ -151,7 +162,12 @@ module icachefsm #(parameter BLOCKLEN = 256)
           PCMux = 2'b01;
           NextState = STATE_MISS_SPILL_FETCH_WDV;
         end else begin
-          NextState = STATE_READY;
+	  if(StallF) begin
+	    NextState = STATE_CPU_BUSY;
+	    PCMux = 2'b01;
+	  end else begin
+            NextState = STATE_READY;
+	  end
         end
       end
       // branch 1,  hit spill and 2, miss spill hit
@@ -192,8 +208,14 @@ module icachefsm #(parameter BLOCKLEN = 256)
         PCMux = 2'b00;
         UnalignedSelect = 1'b1;
         SavePC = 1'b1;
-        NextState = STATE_READY;
-        ICacheStallF = 1'b0;	
+        ICacheStallF = 1'b0;
+	if(StallF) begin
+	  NextState = STATE_CPU_BUSY_SPILL;
+	  PCMux = 2'b10;
+	end else begin
+          NextState = STATE_READY;
+	end
+	
       end
       // branch 3 miss no spill
       STATE_MISS_FETCH_WDV: begin
@@ -214,7 +236,19 @@ module icachefsm #(parameter BLOCKLEN = 256)
       STATE_MISS_READ: begin
         PCMux = 2'b01;
         ICacheReadEn = 1'b1;
-        NextState = STATE_READY;
+        NextState = STATE_MISS_READ_DELAY;
+      end
+      STATE_MISS_READ_DELAY: begin
+        //PCMux = 2'b01;
+        ICacheReadEn = 1'b1;
+	ICacheStallF = 1'b0;
+	if(StallF) begin
+	  PCMux = 2'b01;
+	  NextState = STATE_CPU_BUSY;
+	  PCMux = 2'b01;
+	end else begin
+          NextState = STATE_READY;
+	end
       end
       // branch 4 miss spill hit, and 5 miss spill miss
       STATE_MISS_SPILL_FETCH_WDV: begin
@@ -249,12 +283,17 @@ module icachefsm #(parameter BLOCKLEN = 256)
           CntReset = 1'b1;
           NextState = STATE_MISS_SPILL_MISS_FETCH_WDV;
         end else begin
-          NextState = STATE_READY;
           ICacheReadEn = 1'b1;
           PCMux = 2'b00;
           UnalignedSelect = 1'b1;
           SavePC = 1'b1;
-          ICacheStallF = 1'b0;	
+          ICacheStallF = 1'b0;
+	  if(StallF) begin
+	    NextState = STATE_CPU_BUSY;
+	    PCMux = 2'b01;
+	  end else begin
+            NextState = STATE_READY;
+	  end
         end
       end
       STATE_MISS_SPILL_MISS_FETCH_WDV: begin
@@ -284,7 +323,12 @@ module icachefsm #(parameter BLOCKLEN = 256)
         UnalignedSelect = 1'b1;
         SavePC = 1'b1;
         ICacheStallF = 1'b0;	
-        NextState = STATE_READY;
+	if(StallF) begin
+	  NextState = STATE_CPU_BUSY;
+	  PCMux = 2'b01;
+	end else begin
+          NextState = STATE_READY;
+	end
       end
       STATE_TLB_MISS: begin
         if (WalkerInstrPageFaultF) begin
@@ -297,8 +341,34 @@ module icachefsm #(parameter BLOCKLEN = 256)
         end
       end
       STATE_TLB_MISS_DONE: begin
+	PCMux = 2'b01;
         NextState = STATE_READY;
       end
+      STATE_CPU_BUSY: begin
+	ICacheStallF = 1'b0;
+	if (ITLBMissF) begin
+          NextState = STATE_TLB_MISS;
+	end else if(StallF) begin
+	  NextState = STATE_CPU_BUSY;
+	  PCMux = 2'b01;
+	end
+	else begin
+	  NextState = STATE_READY;
+	end
+      end
+      STATE_CPU_BUSY_SPILL: begin
+	ICacheStallF = 1'b0;
+	ICacheReadEn = 1'b1;
+	if (ITLBMissF) begin
+          NextState = STATE_TLB_MISS;
+	end else if(StallF) begin
+	  NextState = STATE_CPU_BUSY_SPILL;
+	  PCMux = 2'b10;
+	end
+	else begin
+	  NextState = STATE_READY;
+	end
+      end
       default: begin
         PCMux = 2'b01;
         NextState = STATE_READY;

From 2dff72d9e95169e59e75325c15c33e5b12924697 Mon Sep 17 00:00:00 2001
From: Ross Thompson <ross1728@gmail.com>
Date: Fri, 27 Aug 2021 11:14:10 -0500
Subject: [PATCH 08/16] Renamed PCMux (icache) to SelAdr to match dcache.
 Removed unused cache files.

---
 wally-pipelined/src/cache/ICacheMem.sv  |  59 ------
 wally-pipelined/src/cache/cache-sram.sv |  22 ---
 wally-pipelined/src/cache/dmapped.sv    | 234 ------------------------
 wally-pipelined/src/cache/icache.sv     |  32 ++--
 wally-pipelined/src/cache/icachefsm.sv  |  68 +++----
 5 files changed, 51 insertions(+), 364 deletions(-)
 delete mode 100644 wally-pipelined/src/cache/ICacheMem.sv
 delete mode 100644 wally-pipelined/src/cache/cache-sram.sv
 delete mode 100644 wally-pipelined/src/cache/dmapped.sv

diff --git a/wally-pipelined/src/cache/ICacheMem.sv b/wally-pipelined/src/cache/ICacheMem.sv
deleted file mode 100644
index ce3507ba1..000000000
--- a/wally-pipelined/src/cache/ICacheMem.sv
+++ /dev/null
@@ -1,59 +0,0 @@
-`include "wally-config.vh"
-
-module ICacheMem #(parameter NUMLINES=512, parameter BLOCKLEN = 256) 
-  (
-   // Pipeline stuff
-   input logic 		       clk,
-   input logic 		       reset,
-   // If flush is high, invalidate the entire cache
-   input logic 		       flush,
-
-   input logic [`PA_BITS-1:0]  PCTagF, // physical address
-   input logic [`PA_BITS-1:0]  PCNextIndexF, // virtual address
-   input logic 		       WriteEnable,
-   input logic [BLOCKLEN-1:0]  WriteLine,
-   output logic [BLOCKLEN-1:0] ReadLineF,
-   output logic 	       HitF
-   );
-
-  // divide the address bus into sections; tag, index, and offset
-  localparam BLOCKBYTELEN = BLOCKLEN/8;
-  localparam OFFSETLEN = $clog2(BLOCKBYTELEN);
-  localparam INDEXLEN = $clog2(NUMLINES);
-  // *** BUG. `XLEN needs to be replaced with the virtual address width, S32, S39, or S48
-  localparam TAGLEN = `PA_BITS - OFFSETLEN - INDEXLEN;
-
-  logic [TAGLEN-1:0] 	       LookupTag;
-  logic [NUMLINES-1:0] 	       ValidOut;
-  logic 		       DataValidBit;
-
-  // Depth is number of bits in one "word" of the memory, width is number of such words
-  sram1rw #(.DEPTH(BLOCKLEN), .WIDTH(NUMLINES)) 
-  cachemem (.*,
-	    .Addr(PCNextIndexF[INDEXLEN+OFFSETLEN-1:OFFSETLEN]),
-	    .ReadData(ReadLineF),
-	    .WriteData(WriteLine)
-	    );
-  
-  sram1rw #(.DEPTH(TAGLEN), .WIDTH(NUMLINES)) 
-  cachetags (.*,
-	     .Addr(PCNextIndexF[INDEXLEN+OFFSETLEN-1:OFFSETLEN]),
-	     .ReadData(LookupTag),
-	     .WriteData(PCTagF[`PA_BITS-1:INDEXLEN+OFFSETLEN])
-	     );
-
-  // Correctly handle the valid bits
-  always_ff @(posedge clk, posedge reset) begin
-    if (reset) begin
-      ValidOut <= {NUMLINES{1'b0}};
-  end else if (flush) begin
-    ValidOut <= {NUMLINES{1'b0}};
-         end else begin
-           if (WriteEnable) begin
-             ValidOut[PCNextIndexF[INDEXLEN+OFFSETLEN-1:OFFSETLEN]] <= 1;
-           end
-         end
-    DataValidBit <= ValidOut[PCNextIndexF[INDEXLEN+OFFSETLEN-1:OFFSETLEN]];
-  end
-  assign HitF = DataValidBit && (LookupTag == PCTagF[`PA_BITS-1:INDEXLEN+OFFSETLEN]);
-endmodule
diff --git a/wally-pipelined/src/cache/cache-sram.sv b/wally-pipelined/src/cache/cache-sram.sv
deleted file mode 100644
index 0ba0efa57..000000000
--- a/wally-pipelined/src/cache/cache-sram.sv
+++ /dev/null
@@ -1,22 +0,0 @@
-// Depth is number of bits in one "word" of the memory, width is number of such words
-module Sram1Read1Write #(parameter DEPTH=128, WIDTH=256) (
-    input logic clk,
-    // port 1 is read only
-    input  logic [$clog2(WIDTH)-1:0] ReadAddr,
-    output logic [DEPTH-1:0] ReadData,
-  
-    // port 2 is write only
-    input logic [$clog2(WIDTH)-1:0]  WriteAddr,
-    input logic [DEPTH-1:0]  WriteData,
-    input logic 		     WriteEnable
-);
-
-    logic [WIDTH-1:0][DEPTH-1:0] StoredData;
-
-    always_ff @(posedge clk) begin
-        ReadData <= StoredData[ReadAddr];
-        if (WriteEnable) begin
-            StoredData[WriteAddr] <= WriteData;
-        end
-    end
-endmodule
diff --git a/wally-pipelined/src/cache/dmapped.sv b/wally-pipelined/src/cache/dmapped.sv
deleted file mode 100644
index 426697529..000000000
--- a/wally-pipelined/src/cache/dmapped.sv
+++ /dev/null
@@ -1,234 +0,0 @@
-///////////////////////////////////////////
-// dmapped.sv
-//
-// Written: jaallen@g.hmc.edu 2021-03-23
-// Modified: 
-//
-// Purpose: An implementation of a direct-mapped cache memory, with read-only and write-through versions
-// 
-// A component of the Wally configurable RISC-V project.
-// 
-// Copyright (C) 2021 Harvey Mudd College & Oklahoma State University
-//
-// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation
-// files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, 
-// modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software 
-// is furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 
-// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 
-// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT 
-// OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-///////////////////////////////////////////
-
-`include "wally-config.vh"
-
-// Read-only direct-mapped memory
-module rodirectmappedmem #(parameter NUMLINES=512, parameter LINESIZE = 256, parameter WORDSIZE = `XLEN) (
-    // Pipeline stuff
-    input  logic clk,
-    input  logic reset,
-    input  logic stall,
-    // If flush is high, invalidate the entire cache
-    input  logic flush,
-    // Select which address to read (broken for efficiency's sake)
-    input  logic [`XLEN-1:12]   ReadUpperPAdr,
-    input  logic [11:0]         ReadLowerAdr,
-    // Write new data to the cache
-    input  logic                WriteEnable,
-    input  logic [LINESIZE-1:0] WriteLine,
-    input  logic [`XLEN-1:0]    WritePAdr,
-    // Output the word, as well as if it is valid
-    output logic [WORDSIZE-1:0] DataWord,
-    output logic                DataValid
-);
-
-    // Various compile-time constants
-    localparam integer WORDWIDTH = $clog2(WORDSIZE/8);
-    localparam integer OFFSETWIDTH = $clog2(LINESIZE/WORDSIZE);
-    localparam integer SETWIDTH = $clog2(NUMLINES);
-    localparam integer TAGWIDTH = `XLEN - OFFSETWIDTH - SETWIDTH - WORDWIDTH;
-
-    localparam integer OFFSETBEGIN = WORDWIDTH;
-    localparam integer OFFSETEND = OFFSETBEGIN+OFFSETWIDTH-1;
-    localparam integer SETBEGIN = OFFSETEND+1;
-    localparam integer SETEND = SETBEGIN + SETWIDTH - 1;
-    localparam integer TAGBEGIN = SETEND + 1;
-    localparam integer TAGEND = TAGBEGIN + TAGWIDTH - 1;
-
-    // Machinery to read from and write to the correct addresses in memory
-    logic [`XLEN-1:0]       ReadPAdr;
-    logic [`XLEN-1:0]       OldReadPAdr;
-    logic [OFFSETWIDTH-1:0] ReadOffset, WriteOffset;
-    logic [SETWIDTH-1:0]    ReadSet, WriteSet;
-    logic [TAGWIDTH-1:0]    ReadTag, WriteTag;
-    logic [LINESIZE-1:0]    ReadLine;
-    logic [LINESIZE/WORDSIZE-1:0][WORDSIZE-1:0] ReadLineTransformed;
-
-    // Machinery to check if a given read is valid and is the desired value
-    logic [TAGWIDTH-1:0]    DataTag;
-    logic [NUMLINES-1:0]    ValidOut;
-    logic                   DataValidBit;
-
-    flopenr #(`XLEN) ReadPAdrFlop(clk, reset, ~stall, ReadPAdr, OldReadPAdr);
-
-    // Assign the read and write addresses in cache memory
-    always_comb begin
-        ReadOffset = OldReadPAdr[OFFSETEND:OFFSETBEGIN];
-        ReadPAdr = {ReadUpperPAdr, ReadLowerAdr};
-        ReadSet = ReadPAdr[SETEND:SETBEGIN];
-        ReadTag = OldReadPAdr[TAGEND:TAGBEGIN];
-
-        WriteOffset = WritePAdr[OFFSETEND:OFFSETBEGIN];
-        WriteSet = WritePAdr[SETEND:SETBEGIN];
-        WriteTag = WritePAdr[TAGEND:TAGBEGIN];
-    end
-
-    // Depth is number of bits in one "word" of the memory, width is number of such words
-    Sram1Read1Write #(.DEPTH(LINESIZE), .WIDTH(NUMLINES)) cachemem (
-        .*,
-        .ReadAddr(ReadSet),
-        .ReadData(ReadLine),
-        .WriteAddr(WriteSet),
-        .WriteData(WriteLine)
-    );
-    Sram1Read1Write #(.DEPTH(TAGWIDTH), .WIDTH(NUMLINES)) cachetags (
-        .*,
-        .ReadAddr(ReadSet),
-        .ReadData(DataTag),
-        .WriteAddr(WriteSet),
-        .WriteData(WriteTag)
-    );
-
-    // Pick the right bits coming out the read line
-    assign DataWord = ReadLineTransformed[ReadOffset];
-    genvar i;
-    generate
-        for (i=0; i < LINESIZE/WORDSIZE; i++) begin:readline
-            assign ReadLineTransformed[i] = ReadLine[(i+1)*WORDSIZE-1:i*WORDSIZE];
-        end
-    endgenerate
-
-    // Correctly handle the valid bits
-    always_ff @(posedge clk, posedge reset) begin
-        if (reset || flush) begin
-            ValidOut <= {NUMLINES{1'b0}};
-        end else begin
-            if (WriteEnable) begin
-                ValidOut[WriteSet] <= 1;
-            end
-        end
-        DataValidBit <= ValidOut[ReadSet];
-    end
-    assign DataValid = DataValidBit && (DataTag == ReadTag);
-endmodule
-
-
-// Write-through direct-mapped memory
-module wtdirectmappedmem #(parameter NUMLINES=512, parameter LINESIZE = 256, parameter WORDSIZE = `XLEN) (
-    // Pipeline stuff
-    input  logic clk,
-    input  logic reset,
-    input  logic stall,
-    // If flush is high, invalidate the entire cache
-    input  logic flush,
-    // Select which address to read (broken for efficiency's sake)
-    input  logic [`XLEN-1:12]   ReadUpperPAdr,
-    input  logic [11:0]         ReadLowerAdr,
-    // Load new data into the cache (from main memory)
-    input  logic                LoadEnable,
-    input  logic [LINESIZE-1:0] LoadLine,
-    input  logic [`XLEN-1:0]    LoadPAdr,
-    // Write data to the cache (like from a store instruction)
-    input  logic                WriteEnable,
-    input  logic [WORDSIZE-1:0] WriteWord,
-    input  logic [`XLEN-1:0]    WritePAdr,
-    input  logic [1:0]          WriteSize, // Specify size of the write (non-written bits should be preserved)
-    // Output the word, as well as if it is valid
-    output logic [WORDSIZE-1:0] DataWord,
-    output logic                DataValid
-);
-
-    // Various compile-time constants
-    localparam integer WORDWIDTH = $clog2(WORDSIZE/8);
-    localparam integer OFFSETWIDTH = $clog2(LINESIZE/WORDSIZE);
-    localparam integer SETWIDTH = $clog2(NUMLINES);
-    localparam integer TAGWIDTH = `XLEN - OFFSETWIDTH - SETWIDTH - WORDWIDTH;
-
-    localparam integer OFFSETBEGIN = WORDWIDTH;
-    localparam integer OFFSETEND = OFFSETBEGIN+OFFSETWIDTH-1;
-    localparam integer SETBEGIN = OFFSETEND+1;
-    localparam integer SETEND = SETBEGIN + SETWIDTH - 1;
-    localparam integer TAGBEGIN = SETEND + 1;
-    localparam integer TAGEND = TAGBEGIN + TAGWIDTH - 1;
-
-    // Machinery to read from and write to the correct addresses in memory
-    logic [`XLEN-1:0]       ReadPAdr;
-    logic [`XLEN-1:0]       OldReadPAdr;
-    logic [OFFSETWIDTH-1:0] ReadOffset, LoadOffset;
-    logic [SETWIDTH-1:0]    ReadSet, LoadSet;
-    logic [TAGWIDTH-1:0]    ReadTag, LoadTag;
-    logic [LINESIZE-1:0]    ReadLine;
-    logic [LINESIZE/WORDSIZE-1:0][WORDSIZE-1:0] ReadLineTransformed;
-
-    // Machinery to check if a given read is valid and is the desired value
-    logic [TAGWIDTH-1:0]    DataTag;
-    logic [NUMLINES-1:0]    ValidOut;
-    logic                   DataValidBit;
-
-    flopenr #(`XLEN) ReadPAdrFlop(clk, reset, ~stall, ReadPAdr, OldReadPAdr);
-
-    // Assign the read and write addresses in cache memory
-    always_comb begin
-        ReadOffset = OldReadPAdr[OFFSETEND:OFFSETBEGIN];
-        ReadPAdr = {ReadUpperPAdr, ReadLowerAdr};
-        ReadSet = ReadPAdr[SETEND:SETBEGIN];
-        ReadTag = OldReadPAdr[TAGEND:TAGBEGIN];
-
-        LoadOffset = LoadPAdr[OFFSETEND:OFFSETBEGIN];
-        LoadSet = LoadPAdr[SETEND:SETBEGIN];
-        LoadTag = LoadPAdr[TAGEND:TAGBEGIN];
-    end
-
-    // Depth is number of bits in one "word" of the memory, width is number of such words
-    Sram1Read1Write #(.DEPTH(LINESIZE), .WIDTH(NUMLINES)) cachemem (
-        .*,
-        .ReadAddr(ReadSet),
-        .ReadData(ReadLine),
-        .WriteAddr(LoadSet),
-        .WriteData(LoadLine),
-        .WriteEnable(LoadEnable)
-    );
-    Sram1Read1Write #(.DEPTH(TAGWIDTH), .WIDTH(NUMLINES)) cachetags (
-        .*,
-        .ReadAddr(ReadSet),
-        .ReadData(DataTag),
-        .WriteAddr(LoadSet),
-        .WriteData(LoadTag),
-        .WriteEnable(LoadEnable)
-    );
-
-    // Pick the right bits coming out the read line
-    assign DataWord = ReadLineTransformed[ReadOffset];
-    genvar i;
-    generate
-        for (i=0; i < LINESIZE/WORDSIZE; i++) begin:readline
-            assign ReadLineTransformed[i] = ReadLine[(i+1)*WORDSIZE-1:i*WORDSIZE];
-        end
-    endgenerate
-
-    // Correctly handle the valid bits
-    always_ff @(posedge clk, posedge reset) begin
-        if (reset || flush) begin
-            ValidOut <= {NUMLINES{1'b0}};
-        end else begin
-            if (LoadEnable) begin
-                ValidOut[LoadSet] <= 1;
-            end
-        end
-        DataValidBit <= ValidOut[ReadSet];
-    end
-    assign DataValid = DataValidBit && (DataTag == ReadTag);
-endmodule
diff --git a/wally-pipelined/src/cache/icache.sv b/wally-pipelined/src/cache/icache.sv
index 7bbeccc60..839504547 100644
--- a/wally-pipelined/src/cache/icache.sv
+++ b/wally-pipelined/src/cache/icache.sv
@@ -90,7 +90,7 @@ module icache
   logic 			 FetchCountFlag;
   logic 			 CntEn;
   
-  logic [1:0] 			 PCMux_q;
+  logic [1:0] 			 SelAdr_q;
   
   
   logic [LOGWPL-1:0] 	       FetchCount, NextFetchCount;
@@ -99,8 +99,9 @@ module icache
   logic [`PA_BITS-1:OFFSETWIDTH] PCPTrunkF;
 
   logic 		       CntReset;
-  logic [1:0] 		       PCMux;
+  logic [1:0] 		       SelAdr;
   logic 		       SavePC;
+  logic [INDEXLEN-1:0]	       RAdr;
   
 
   // on spill we want to get the first 2 bytes of the next cache block.
@@ -108,19 +109,20 @@ module icache
   // simply add 2 to land on the next cache block.
   assign PCPSpillF = PCPF + {{{PA_WIDTH}{1'b0}}, 2'b10}; // *** modelsim does not allow the use of PA_BITS for literal width.
 
-  // now we have to select between these three PCs
-  assign PCPreFinalF = PCMux[0] ? PCPF : PCNextF; // *** don't like the stallf, but it is necessary
-  // for the data cache i used a cpu busy state which is triggered by StallW.  In the case of the icache I
-  // modified the select on this address mux. Both are not ideal; however the cpu_busy state is required for the
-  // dcache as a write would repeatedly update the sram or worse for an uncached write multiple times.
-  // I like reducing some complexity of the fsm; however I weight commonality between the i/d cache more.
-  assign PCNextIndexF = PCMux[1] ? PCPSpillF : PCPreFinalF;
+  mux3 #(INDEXLEN)
+  AdrSelMux(.d0(PCNextF[INDEXLEN+OFFSETLEN-1:OFFSETLEN]),
+	    .d1(PCPF[INDEXLEN+OFFSETLEN-1:OFFSETLEN]),
+	    .d2(PCPSpillF[INDEXLEN+OFFSETLEN-1:OFFSETLEN]),
+	    .s(SelAdr),
+	    .y(RAdr));
+
+  
 
   cacheway #(.NUMLINES(NUMLINES), .BLOCKLEN(BLOCKLEN), .TAGLEN(TAGLEN), .OFFSETLEN(OFFSETLEN), .INDEXLEN(INDEXLEN),
 	     .DIRTY_BITS(0))
   icachemem(.clk,
 	    .reset,
-	    .RAdr(PCNextIndexF[INDEXLEN+OFFSETLEN-1:OFFSETLEN]),
+	    .RAdr(RAdr),
 	    .PAdr(PCTagF),
 	    .WriteEnable(ICacheMemWriteEnable),
 	    .WriteWordEnable('1),
@@ -228,13 +230,13 @@ module icache
 
   // this mux needs to be delayed 1 cycle as it occurs 1 pipeline stage later.
   // *** read enable may not be necessary.
-  flopenr #(2) PCMuxReg(.clk(clk),
+  flopenr #(2) SelAdrReg(.clk(clk),
 			.reset(reset),
 			.en(ICacheReadEn),
-			.d(PCMux),
-			.q(PCMux_q));
+			.d(SelAdr),
+			.q(SelAdr_q));
   
-  assign PCTagF = PCMux_q[1] ? PCPSpillF : PCPF;
+  assign PCTagF = SelAdr_q[1] ? PCPSpillF : PCPF;
   
   // truncate the offset from PCPF for memory address generation
   assign PCPTrunkF = PCTagF[`PA_BITS-1:OFFSETWIDTH];
@@ -258,7 +260,7 @@ module icache
 	     .spillSave,
 	     .CntEn,
 	     .CntReset,
-	     .PCMux,
+	     .SelAdr,
     	     .SavePC
 	     );
 
diff --git a/wally-pipelined/src/cache/icachefsm.sv b/wally-pipelined/src/cache/icachefsm.sv
index ddf9b50ca..7ec4acfbe 100644
--- a/wally-pipelined/src/cache/icachefsm.sv
+++ b/wally-pipelined/src/cache/icachefsm.sv
@@ -60,7 +60,7 @@ module icachefsm #(parameter BLOCKLEN = 256)
    output logic       spillSave,
    output logic       CntEn,
    output logic       CntReset,
-   output logic [1:0] PCMux,
+   output logic [1:0] SelAdr,
    output logic       SavePC
    );
 
@@ -130,14 +130,14 @@ module icachefsm #(parameter BLOCKLEN = 256)
     //InstrReadF = 1'b0;
     ICacheMemWriteEnable = 1'b0;
     spillSave = 1'b0;
-    PCMux = 2'b00;
+    SelAdr = 2'b00;
     ICacheReadEn = 1'b0;
     SavePC = 1'b0;
     ICacheStallF = 1'b1;
     
     case (CurrState)
       STATE_READY: begin
-        PCMux = 2'b00;
+        SelAdr = 2'b00;
         ICacheReadEn = 1'b1;
         if (ITLBMissF) begin
           NextState = STATE_TLB_MISS;
@@ -146,25 +146,25 @@ module icachefsm #(parameter BLOCKLEN = 256)
           ICacheStallF = 1'b0;
 	  if(StallF) begin
 	    NextState = STATE_CPU_BUSY;
-	    PCMux = 2'b01;
+	    SelAdr = 2'b01;
 	  end else begin
             NextState = STATE_READY;
 	  end
         end else if (hit & spill) begin
           spillSave = 1'b1;
-          PCMux = 2'b10;
+          SelAdr = 2'b10;
           NextState = STATE_HIT_SPILL;
         end else if (~hit & ~spill) begin
           CntReset = 1'b1;
           NextState = STATE_MISS_FETCH_WDV;
         end else if (~hit & spill) begin
           CntReset = 1'b1;
-          PCMux = 2'b01;
+          SelAdr = 2'b01;
           NextState = STATE_MISS_SPILL_FETCH_WDV;
         end else begin
 	  if(StallF) begin
 	    NextState = STATE_CPU_BUSY;
-	    PCMux = 2'b01;
+	    SelAdr = 2'b01;
 	  end else begin
             NextState = STATE_READY;
 	  end
@@ -172,7 +172,7 @@ module icachefsm #(parameter BLOCKLEN = 256)
       end
       // branch 1,  hit spill and 2, miss spill hit
       STATE_HIT_SPILL: begin
-        PCMux = 2'b10;
+        SelAdr = 2'b10;
         UnalignedSelect = 1'b1;
         ICacheReadEn = 1'b1;
         if (hit) begin
@@ -183,7 +183,7 @@ module icachefsm #(parameter BLOCKLEN = 256)
         end
       end
       STATE_HIT_SPILL_MISS_FETCH_WDV: begin
-        PCMux = 2'b10;
+        SelAdr = 2'b10;
         //InstrReadF = 1'b1;
         PreCntEn = 1'b1;
         if (FetchCountFlag & InstrAckF) begin
@@ -193,25 +193,25 @@ module icachefsm #(parameter BLOCKLEN = 256)
         end
       end
       STATE_HIT_SPILL_MISS_FETCH_DONE: begin
-        PCMux = 2'b10;
+        SelAdr = 2'b10;
         ICacheMemWriteEnable = 1'b1;
         NextState = STATE_HIT_SPILL_MERGE;
       end
       STATE_HIT_SPILL_MERGE: begin
-        PCMux = 2'b10;
+        SelAdr = 2'b10;
         UnalignedSelect = 1'b1;
         ICacheReadEn = 1'b1;
         NextState = STATE_HIT_SPILL_FINAL;
       end
       STATE_HIT_SPILL_FINAL: begin
         ICacheReadEn = 1'b1;
-        PCMux = 2'b00;
+        SelAdr = 2'b00;
         UnalignedSelect = 1'b1;
         SavePC = 1'b1;
         ICacheStallF = 1'b0;
 	if(StallF) begin
 	  NextState = STATE_CPU_BUSY_SPILL;
-	  PCMux = 2'b10;
+	  SelAdr = 2'b10;
 	end else begin
           NextState = STATE_READY;
 	end
@@ -219,7 +219,7 @@ module icachefsm #(parameter BLOCKLEN = 256)
       end
       // branch 3 miss no spill
       STATE_MISS_FETCH_WDV: begin
-        PCMux = 2'b01;
+        SelAdr = 2'b01;
         //InstrReadF = 1'b1;
         PreCntEn = 1'b1;
         if (FetchCountFlag & InstrAckF) begin
@@ -229,30 +229,30 @@ module icachefsm #(parameter BLOCKLEN = 256)
         end
       end
       STATE_MISS_FETCH_DONE: begin
-        PCMux = 2'b01;
+        SelAdr = 2'b01;
         ICacheMemWriteEnable = 1'b1;
         NextState = STATE_MISS_READ;
       end
       STATE_MISS_READ: begin
-        PCMux = 2'b01;
+        SelAdr = 2'b01;
         ICacheReadEn = 1'b1;
         NextState = STATE_MISS_READ_DELAY;
       end
       STATE_MISS_READ_DELAY: begin
-        //PCMux = 2'b01;
+        //SelAdr = 2'b01;
         ICacheReadEn = 1'b1;
 	ICacheStallF = 1'b0;
 	if(StallF) begin
-	  PCMux = 2'b01;
+	  SelAdr = 2'b01;
 	  NextState = STATE_CPU_BUSY;
-	  PCMux = 2'b01;
+	  SelAdr = 2'b01;
 	end else begin
           NextState = STATE_READY;
 	end
       end
       // branch 4 miss spill hit, and 5 miss spill miss
       STATE_MISS_SPILL_FETCH_WDV: begin
-        PCMux = 2'b01;
+        SelAdr = 2'b01;
         PreCntEn = 1'b1;
         //InstrReadF = 1'b1;	
         if (FetchCountFlag & InstrAckF) begin 
@@ -262,17 +262,17 @@ module icachefsm #(parameter BLOCKLEN = 256)
         end
       end
       STATE_MISS_SPILL_FETCH_DONE: begin
-        PCMux = 2'b01;	
+        SelAdr = 2'b01;	
         ICacheMemWriteEnable = 1'b1;
         NextState = STATE_MISS_SPILL_READ1;
       end
       STATE_MISS_SPILL_READ1: begin // always be a hit as we just wrote that cache block.
-        PCMux = 2'b01;	 // there is a 1 cycle delay after setting the address before the date arrives.
+        SelAdr = 2'b01;	 // there is a 1 cycle delay after setting the address before the date arrives.
         ICacheReadEn = 1'b1;	
         NextState = STATE_MISS_SPILL_2;
       end
       STATE_MISS_SPILL_2: begin
-        PCMux = 2'b10;
+        SelAdr = 2'b10;
         UnalignedSelect = 1'b1;
         spillSave = 1'b1; /// *** Could pipeline these to make it clearer in the fsm.
         ICacheReadEn = 1'b1;
@@ -284,20 +284,20 @@ module icachefsm #(parameter BLOCKLEN = 256)
           NextState = STATE_MISS_SPILL_MISS_FETCH_WDV;
         end else begin
           ICacheReadEn = 1'b1;
-          PCMux = 2'b00;
+          SelAdr = 2'b00;
           UnalignedSelect = 1'b1;
           SavePC = 1'b1;
           ICacheStallF = 1'b0;
 	  if(StallF) begin
 	    NextState = STATE_CPU_BUSY;
-	    PCMux = 2'b01;
+	    SelAdr = 2'b01;
 	  end else begin
             NextState = STATE_READY;
 	  end
         end
       end
       STATE_MISS_SPILL_MISS_FETCH_WDV: begin
-        PCMux = 2'b10;
+        SelAdr = 2'b10;
         PreCntEn = 1'b1;
         //InstrReadF = 1'b1;	
         if (FetchCountFlag & InstrAckF) begin
@@ -307,25 +307,25 @@ module icachefsm #(parameter BLOCKLEN = 256)
         end
       end
       STATE_MISS_SPILL_MISS_FETCH_DONE: begin
-        PCMux = 2'b10;
+        SelAdr = 2'b10;
         ICacheMemWriteEnable = 1'b1;
         NextState = STATE_MISS_SPILL_MERGE;
       end
       STATE_MISS_SPILL_MERGE: begin
-        PCMux = 2'b10;
+        SelAdr = 2'b10;
         UnalignedSelect = 1'b1;
         ICacheReadEn = 1'b1;	
         NextState = STATE_MISS_SPILL_FINAL;
       end
       STATE_MISS_SPILL_FINAL: begin
         ICacheReadEn = 1'b1;
-        PCMux = 2'b00;
+        SelAdr = 2'b00;
         UnalignedSelect = 1'b1;
         SavePC = 1'b1;
         ICacheStallF = 1'b0;	
 	if(StallF) begin
 	  NextState = STATE_CPU_BUSY;
-	  PCMux = 2'b01;
+	  SelAdr = 2'b01;
 	end else begin
           NextState = STATE_READY;
 	end
@@ -341,7 +341,7 @@ module icachefsm #(parameter BLOCKLEN = 256)
         end
       end
       STATE_TLB_MISS_DONE: begin
-	PCMux = 2'b01;
+	SelAdr = 2'b01;
         NextState = STATE_READY;
       end
       STATE_CPU_BUSY: begin
@@ -350,7 +350,7 @@ module icachefsm #(parameter BLOCKLEN = 256)
           NextState = STATE_TLB_MISS;
 	end else if(StallF) begin
 	  NextState = STATE_CPU_BUSY;
-	  PCMux = 2'b01;
+	  SelAdr = 2'b01;
 	end
 	else begin
 	  NextState = STATE_READY;
@@ -363,14 +363,14 @@ module icachefsm #(parameter BLOCKLEN = 256)
           NextState = STATE_TLB_MISS;
 	end else if(StallF) begin
 	  NextState = STATE_CPU_BUSY_SPILL;
-	  PCMux = 2'b10;
+	  SelAdr = 2'b10;
 	end
 	else begin
 	  NextState = STATE_READY;
 	end
       end
       default: begin
-        PCMux = 2'b01;
+        SelAdr = 2'b01;
         NextState = STATE_READY;
       end
       // *** add in error handling and invalidate/evict

From 4b0344898b7545992a5aa63757d4c34df486a261 Mon Sep 17 00:00:00 2001
From: Ross Thompson <ross1728@gmail.com>
Date: Fri, 27 Aug 2021 15:00:40 -0500
Subject: [PATCH 09/16] Fixed bugs I introduced to the icache.

---
 wally-pipelined/regression/linux-wave.do | 299 ++++++++++++-----------
 wally-pipelined/src/cache/icachefsm.sv   |   4 +-
 wally-pipelined/src/ifu/bpred.sv         |   2 +-
 3 files changed, 165 insertions(+), 140 deletions(-)

diff --git a/wally-pipelined/regression/linux-wave.do b/wally-pipelined/regression/linux-wave.do
index 44ade6092..c6d6ce3cc 100644
--- a/wally-pipelined/regression/linux-wave.do
+++ b/wally-pipelined/regression/linux-wave.do
@@ -96,11 +96,11 @@ add wave -noupdate -group Bpred -expand -group {bp wrong} /testbench/dut/hart/if
 add wave -noupdate -group Bpred -expand -group {bp wrong} /testbench/dut/hart/ifu/bpred/bpred/BPPredClassNonCFIWrongE
 add wave -noupdate -group Bpred -expand -group {bp wrong} /testbench/dut/hart/ifu/bpred/bpred/BPPredWrongE
 add wave -noupdate -group Bpred /testbench/dut/hart/ifu/bpred/bpred/BPPredWrongE
-add wave -noupdate -group {instruction pipeline} /testbench/InstrFName
-add wave -noupdate -group {instruction pipeline} /testbench/dut/hart/ifu/InstrD
-add wave -noupdate -group {instruction pipeline} /testbench/dut/hart/ifu/InstrE
-add wave -noupdate -group {instruction pipeline} /testbench/dut/hart/ifu/InstrM
-add wave -noupdate -group {instruction pipeline} /testbench/InstrW
+add wave -noupdate -expand -group {instruction pipeline} /testbench/InstrFName
+add wave -noupdate -expand -group {instruction pipeline} /testbench/dut/hart/ifu/icache/FinalInstrRawF
+add wave -noupdate -expand -group {instruction pipeline} /testbench/dut/hart/ifu/InstrD
+add wave -noupdate -expand -group {instruction pipeline} /testbench/dut/hart/ifu/InstrE
+add wave -noupdate -expand -group {instruction pipeline} /testbench/dut/hart/ifu/InstrM
 add wave -noupdate -group {PCNext Generation} /testbench/dut/hart/ifu/PCNextF
 add wave -noupdate -group {PCNext Generation} /testbench/dut/hart/ifu/PCF
 add wave -noupdate -group {PCNext Generation} /testbench/dut/hart/ifu/PCPlus2or4F
@@ -129,18 +129,18 @@ add wave -noupdate -group RegFile -group {write regfile mux} /testbench/dut/hart
 add wave -noupdate -group RegFile -group {write regfile mux} /testbench/dut/hart/ieu/dp/CSRReadValW
 add wave -noupdate -group RegFile -group {write regfile mux} /testbench/dut/hart/ieu/dp/ResultSrcW
 add wave -noupdate -group RegFile -group {write regfile mux} /testbench/dut/hart/ieu/dp/ResultW
-add wave -noupdate -group alu /testbench/dut/hart/ieu/dp/alu/a
-add wave -noupdate -group alu /testbench/dut/hart/ieu/dp/alu/b
-add wave -noupdate -group alu /testbench/dut/hart/ieu/dp/alu/alucontrol
-add wave -noupdate -group alu /testbench/dut/hart/ieu/dp/alu/result
-add wave -noupdate -group alu /testbench/dut/hart/ieu/dp/alu/flags
-add wave -noupdate -group alu -divider internals
-add wave -noupdate -group alu /testbench/dut/hart/ieu/dp/alu/overflow
-add wave -noupdate -group alu /testbench/dut/hart/ieu/dp/alu/carry
-add wave -noupdate -group alu /testbench/dut/hart/ieu/dp/alu/zero
-add wave -noupdate -group alu /testbench/dut/hart/ieu/dp/alu/neg
-add wave -noupdate -group alu /testbench/dut/hart/ieu/dp/alu/lt
-add wave -noupdate -group alu /testbench/dut/hart/ieu/dp/alu/ltu
+add wave -noupdate -expand -group alu /testbench/dut/hart/ieu/dp/alu/a
+add wave -noupdate -expand -group alu /testbench/dut/hart/ieu/dp/alu/b
+add wave -noupdate -expand -group alu /testbench/dut/hart/ieu/dp/alu/alucontrol
+add wave -noupdate -expand -group alu /testbench/dut/hart/ieu/dp/alu/result
+add wave -noupdate -expand -group alu /testbench/dut/hart/ieu/dp/alu/flags
+add wave -noupdate -expand -group alu -divider internals
+add wave -noupdate -expand -group alu /testbench/dut/hart/ieu/dp/alu/overflow
+add wave -noupdate -expand -group alu /testbench/dut/hart/ieu/dp/alu/carry
+add wave -noupdate -expand -group alu /testbench/dut/hart/ieu/dp/alu/zero
+add wave -noupdate -expand -group alu /testbench/dut/hart/ieu/dp/alu/neg
+add wave -noupdate -expand -group alu /testbench/dut/hart/ieu/dp/alu/lt
+add wave -noupdate -expand -group alu /testbench/dut/hart/ieu/dp/alu/ltu
 add wave -noupdate -group Forward /testbench/dut/hart/ieu/fw/Rs1D
 add wave -noupdate -group Forward /testbench/dut/hart/ieu/fw/Rs2D
 add wave -noupdate -group Forward /testbench/dut/hart/ieu/fw/Rs1E
@@ -158,12 +158,12 @@ add wave -noupdate -group {alu execution stage} /testbench/dut/hart/ieu/dp/Write
 add wave -noupdate -group {alu execution stage} /testbench/dut/hart/ieu/dp/ALUResultE
 add wave -noupdate -group {alu execution stage} /testbench/dut/hart/ieu/dp/SrcAE
 add wave -noupdate -group {alu execution stage} /testbench/dut/hart/ieu/dp/SrcBE
-add wave -noupdate -group PCS /testbench/dut/hart/ifu/PCNextF
-add wave -noupdate -group PCS /testbench/dut/hart/PCF
-add wave -noupdate -group PCS /testbench/dut/hart/ifu/PCD
-add wave -noupdate -group PCS /testbench/dut/hart/PCE
-add wave -noupdate -group PCS /testbench/dut/hart/PCM
-add wave -noupdate -group PCS /testbench/PCW
+add wave -noupdate -expand -group PCS /testbench/dut/hart/ifu/PCNextF
+add wave -noupdate -expand -group PCS /testbench/dut/hart/PCF
+add wave -noupdate -expand -group PCS /testbench/dut/hart/ifu/PCD
+add wave -noupdate -expand -group PCS /testbench/dut/hart/PCE
+add wave -noupdate -expand -group PCS /testbench/dut/hart/PCM
+add wave -noupdate -expand -group PCS /testbench/PCW
 add wave -noupdate -group muldiv /testbench/dut/hart/mdu/InstrD
 add wave -noupdate -group muldiv /testbench/dut/hart/mdu/SrcAE
 add wave -noupdate -group muldiv /testbench/dut/hart/mdu/SrcBE
@@ -183,43 +183,32 @@ add wave -noupdate -group divider /testbench/dut/hart/mdu/genblk1/div/N
 add wave -noupdate -group divider /testbench/dut/hart/mdu/genblk1/div/D
 add wave -noupdate -group divider /testbench/dut/hart/mdu/genblk1/div/Q
 add wave -noupdate -group divider /testbench/dut/hart/mdu/genblk1/div/rem0
-add wave -noupdate -group icache -color Orange /testbench/dut/hart/ifu/icache/controller/CurrState
-add wave -noupdate -group icache /testbench/dut/hart/ifu/icache/controller/NextState
-add wave -noupdate -group icache /testbench/dut/hart/ifu/ITLBMissF
-add wave -noupdate -group icache /testbench/dut/hart/ifu/icache/ITLBWriteF
-add wave -noupdate -group icache -group {tag read} /testbench/dut/hart/ifu/icache/cachemem/DataValidBit
-add wave -noupdate -group icache -group {tag read} /testbench/dut/hart/ifu/icache/cachemem/cachetags/ReadData
-add wave -noupdate -group icache -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/hit
-add wave -noupdate -group icache -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/spill
-add wave -noupdate -group icache -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/ICacheStallF
-add wave -noupdate -group icache -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/SavePC
-add wave -noupdate -group icache -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/spillSave
-add wave -noupdate -group icache -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/UnalignedSelect
-add wave -noupdate -group icache -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/PCMux
-add wave -noupdate -group icache -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/spillSave
-add wave -noupdate -group icache -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/CntReset
-add wave -noupdate -group icache -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/PreCntEn
-add wave -noupdate -group icache -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/CntEn
-add wave -noupdate -group icache -group {icache parameters} -radix unsigned /testbench/dut/hart/ifu/icache/cachemem/NUMLINES
-add wave -noupdate -group icache -group {icache parameters} -radix unsigned /testbench/dut/hart/ifu/icache/cachemem/BLOCKLEN
-add wave -noupdate -group icache -group {icache parameters} -radix unsigned /testbench/dut/hart/ifu/icache/cachemem/BLOCKBYTELEN
-add wave -noupdate -group icache -group {icache parameters} -radix unsigned /testbench/dut/hart/ifu/icache/cachemem/OFFSETLEN
-add wave -noupdate -group icache -group {icache parameters} -radix unsigned /testbench/dut/hart/ifu/icache/cachemem/INDEXLEN
-add wave -noupdate -group icache -group {icache parameters} -radix unsigned /testbench/dut/hart/ifu/icache/cachemem/TAGLEN
-add wave -noupdate -group icache -expand -group memory /testbench/dut/hart/ifu/icache/controller/FetchCountFlag
-add wave -noupdate -group icache -expand -group memory /testbench/dut/hart/ifu/icache/controller/FetchCount
-add wave -noupdate -group icache -expand -group memory /testbench/dut/hart/ifu/icache/controller/InstrPAdrF
-add wave -noupdate -group icache -expand -group memory /testbench/dut/hart/ifu/icache/controller/InstrReadF
-add wave -noupdate -group icache -expand -group memory /testbench/dut/hart/ifu/icache/controller/InstrAckF
-add wave -noupdate -group icache -expand -group memory /testbench/dut/hart/ifu/icache/controller/InstrInF
-add wave -noupdate -group icache -expand -group memory /testbench/dut/hart/ifu/icache/controller/ICacheMemWriteEnable
-add wave -noupdate -group icache -expand -group memory /testbench/dut/hart/ifu/icache/controller/ICacheMemWriteData
-add wave -noupdate -group icache -expand -group memory -group {tag write} /testbench/dut/hart/ifu/icache/cachemem/WriteEnable
-add wave -noupdate -group icache -expand -group memory -group {tag write} /testbench/dut/hart/ifu/icache/cachemem/WriteLine
-add wave -noupdate -group icache -expand -group memory -group {tag write} /testbench/dut/hart/ifu/icache/cachemem/cachetags/StoredData
-add wave -noupdate -group icache -expand -group {instr to cpu} /testbench/dut/hart/ifu/icache/controller/FinalInstrRawF
-add wave -noupdate -group icache -expand -group pc /testbench/dut/hart/ifu/icache/controller/PCPF
-add wave -noupdate -group icache -expand -group pc /testbench/dut/hart/ifu/icache/controller/PCPreFinalF
+add wave -noupdate -expand -group icache -color Gold /testbench/dut/hart/ifu/icache/controller/CurrState
+add wave -noupdate -expand -group icache /testbench/dut/hart/ifu/icache/controller/NextState
+add wave -noupdate -expand -group icache /testbench/dut/hart/ifu/ITLBMissF
+add wave -noupdate -expand -group icache /testbench/dut/hart/ifu/icache/ITLBWriteF
+add wave -noupdate -expand -group icache /testbench/dut/hart/ifu/icache/ReadLineF
+add wave -noupdate -expand -group icache {/testbench/dut/hart/ifu/icache/icachemem/word[0]/CacheDataMem/Addr}
+add wave -noupdate -expand -group icache {/testbench/dut/hart/ifu/icache/icachemem/word[0]/CacheDataMem/WriteData}
+add wave -noupdate -expand -group icache {/testbench/dut/hart/ifu/icache/icachemem/word[0]/CacheDataMem/WriteEnable}
+add wave -noupdate -expand -group icache /testbench/dut/hart/ifu/icache/PCNextIndexF
+add wave -noupdate -expand -group icache /testbench/dut/hart/ifu/icache/ReadLineF
+add wave -noupdate -expand -group icache -expand -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/hit
+add wave -noupdate -expand -group icache -expand -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/spill
+add wave -noupdate -expand -group icache -expand -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/ICacheStallF
+add wave -noupdate -expand -group icache -expand -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/SavePC
+add wave -noupdate -expand -group icache -expand -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/spillSave
+add wave -noupdate -expand -group icache -expand -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/UnalignedSelect
+add wave -noupdate -expand -group icache -expand -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/spillSave
+add wave -noupdate -expand -group icache -expand -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/CntReset
+add wave -noupdate -expand -group icache -expand -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/PreCntEn
+add wave -noupdate -expand -group icache -expand -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/CntEn
+add wave -noupdate -expand -group icache -expand -group memory /testbench/dut/hart/ifu/icache/controller/FetchCountFlag
+add wave -noupdate -expand -group icache -expand -group memory /testbench/dut/hart/ifu/icache/FetchCount
+add wave -noupdate -expand -group icache -expand -group memory /testbench/dut/hart/ifu/icache/controller/InstrReadF
+add wave -noupdate -expand -group icache -expand -group memory /testbench/dut/hart/ifu/icache/controller/InstrAckF
+add wave -noupdate -expand -group icache -expand -group memory /testbench/dut/hart/ifu/icache/controller/ICacheMemWriteEnable
+add wave -noupdate -expand -group icache -expand -group memory /testbench/dut/hart/ifu/icache/ICacheMemWriteData
 add wave -noupdate -group AHB -color Gold /testbench/dut/hart/ebu/BusState
 add wave -noupdate -group AHB /testbench/dut/hart/ebu/NextBusState
 add wave -noupdate -group AHB -expand -group {input requests} /testbench/dut/hart/ebu/AtomicMaskedM
@@ -243,7 +232,7 @@ add wave -noupdate -group AHB /testbench/dut/hart/ebu/HSIZED
 add wave -noupdate -group AHB /testbench/dut/hart/ebu/HWRITED
 add wave -noupdate -group AHB /testbench/dut/hart/ebu/StallW
 add wave -noupdate -group lsu -expand -group {LSU ARB} /testbench/dut/hart/lsu/arbiter/SelPTW
-add wave -noupdate -group lsu -expand -group dcache -color Gold /testbench/dut/hart/lsu/dcache/CurrState
+add wave -noupdate -group lsu -expand -group dcache -color Gold /testbench/dut/hart/lsu/dcache/dcachefsm/CurrState
 add wave -noupdate -group lsu -expand -group dcache /testbench/dut/hart/lsu/dcache/WalkerPageFaultM
 add wave -noupdate -group lsu -expand -group dcache /testbench/dut/hart/lsu/dcache/WriteDataM
 add wave -noupdate -group lsu -expand -group dcache /testbench/dut/hart/lsu/dcache/SRAMBlockWriteEnableM
@@ -253,72 +242,102 @@ add wave -noupdate -group lsu -expand -group dcache /testbench/dut/hart/lsu/dcac
 add wave -noupdate -group lsu -expand -group dcache /testbench/dut/hart/lsu/dcache/SRAMBlockWayWriteEnableM
 add wave -noupdate -group lsu -expand -group dcache /testbench/dut/hart/lsu/dcache/SelAdrM
 add wave -noupdate -group lsu -expand -group dcache /testbench/dut/hart/lsu/dcache/DCacheMemWriteData
-add wave -noupdate -group lsu -expand -group dcache -group replacement /testbench/dut/hart/lsu/dcache/genblk2/cacheLRU/LRUIn
-add wave -noupdate -group lsu -expand -group dcache -group replacement /testbench/dut/hart/lsu/dcache/genblk2/cacheLRU/WayIn
-add wave -noupdate -group lsu -expand -group dcache -group replacement /testbench/dut/hart/lsu/dcache/genblk2/cacheLRU/LRUEn
-add wave -noupdate -group lsu -expand -group dcache -group replacement /testbench/dut/hart/lsu/dcache/genblk2/cacheLRU/LRUMask
-add wave -noupdate -group lsu -expand -group dcache -group replacement /testbench/dut/hart/lsu/dcache/genblk2/cacheLRU/LRUOut
-add wave -noupdate -group lsu -expand -group dcache -group replacement /testbench/dut/hart/lsu/dcache/genblk2/cacheLRU/VictimWay
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} /testbench/dut/hart/lsu/dcache/ReplacementBits
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} /testbench/dut/hart/lsu/dcache/NewReplacement
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} /testbench/dut/hart/lsu/dcache/LRUWriteEn
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way0 {/testbench/dut/hart/lsu/dcache/CacheWays[0]/MemWay/WriteEnable}
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way0 {/testbench/dut/hart/lsu/dcache/CacheWays[0]/MemWay/SetValid}
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way0 {/testbench/dut/hart/lsu/dcache/CacheWays[0]/MemWay/SetDirty}
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way0 {/testbench/dut/hart/lsu/dcache/CacheWays[0]/MemWay/Adr}
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way0 -label TAG {/testbench/dut/hart/lsu/dcache/CacheWays[0]/MemWay/CacheTagMem/StoredData}
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way0 {/testbench/dut/hart/lsu/dcache/CacheWays[0]/MemWay/DirtyBits}
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way0 {/testbench/dut/hart/lsu/dcache/CacheWays[0]/MemWay/ValidBits}
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way0 -expand -group Way0Word0 {/testbench/dut/hart/lsu/dcache/CacheWays[0]/MemWay/word[0]/CacheDataMem/StoredData}
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way0 -expand -group Way0Word0 {/testbench/dut/hart/lsu/dcache/CacheWays[0]/MemWay/word[0]/CacheDataMem/WriteEnable}
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way0 -expand -group Way0Word1 {/testbench/dut/hart/lsu/dcache/CacheWays[0]/MemWay/word[1]/CacheDataMem/StoredData}
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way0 -expand -group Way0Word1 {/testbench/dut/hart/lsu/dcache/CacheWays[0]/MemWay/word[1]/CacheDataMem/WriteEnable}
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way0 -expand -group Way0Word2 {/testbench/dut/hart/lsu/dcache/CacheWays[0]/MemWay/word[2]/CacheDataMem/WriteEnable}
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way0 -expand -group Way0Word2 {/testbench/dut/hart/lsu/dcache/CacheWays[0]/MemWay/word[2]/CacheDataMem/StoredData}
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way0 -expand -group Way0Word3 {/testbench/dut/hart/lsu/dcache/CacheWays[0]/MemWay/word[3]/CacheDataMem/WriteEnable}
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way0 -expand -group Way0Word3 {/testbench/dut/hart/lsu/dcache/CacheWays[0]/MemWay/word[3]/CacheDataMem/StoredData}
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way1 {/testbench/dut/hart/lsu/dcache/CacheWays[1]/MemWay/DirtyBits}
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way1 {/testbench/dut/hart/lsu/dcache/CacheWays[1]/MemWay/SetDirty}
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way1 {/testbench/dut/hart/lsu/dcache/CacheWays[1]/MemWay/WriteEnable}
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way1 {/testbench/dut/hart/lsu/dcache/CacheWays[1]/MemWay/WriteWordEnable}
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way1 -label TAG {/testbench/dut/hart/lsu/dcache/CacheWays[1]/MemWay/CacheTagMem/StoredData}
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way1 -expand -group Way1Word0 {/testbench/dut/hart/lsu/dcache/CacheWays[1]/MemWay/word[0]/CacheDataMem/WriteEnable}
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way1 -expand -group Way1Word0 {/testbench/dut/hart/lsu/dcache/CacheWays[1]/MemWay/word[0]/CacheDataMem/StoredData}
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way1 -expand -group Way1Word1 {/testbench/dut/hart/lsu/dcache/CacheWays[1]/MemWay/word[1]/CacheDataMem/WriteEnable}
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way1 -expand -group Way1Word1 {/testbench/dut/hart/lsu/dcache/CacheWays[1]/MemWay/word[1]/CacheDataMem/StoredData}
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way1 -expand -group Way1Word2 {/testbench/dut/hart/lsu/dcache/CacheWays[1]/MemWay/word[2]/CacheDataMem/WriteEnable}
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way1 -expand -group Way1Word2 {/testbench/dut/hart/lsu/dcache/CacheWays[1]/MemWay/word[2]/CacheDataMem/StoredData}
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way1 -expand -group Way1Word3 {/testbench/dut/hart/lsu/dcache/CacheWays[1]/MemWay/word[3]/CacheDataMem/WriteEnable}
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way1 -expand -group Way1Word3 {/testbench/dut/hart/lsu/dcache/CacheWays[1]/MemWay/word[3]/CacheDataMem/StoredData}
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} /testbench/dut/hart/lsu/dcache/SRAMAdr
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} /testbench/dut/hart/lsu/dcache/ReadDataBlockWayM
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} /testbench/dut/hart/lsu/dcache/ReadDataBlockWayMaskedM
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} /testbench/dut/hart/lsu/dcache/ReadDataBlockM
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} /testbench/dut/hart/lsu/dcache/ReadDataBlockSetsM
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} /testbench/dut/hart/lsu/dcache/ReadDataWordM
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} /testbench/dut/hart/lsu/dcache/ReadTag
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} /testbench/dut/hart/lsu/dcache/BlockReplacementBits
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 {/testbench/dut/hart/lsu/dcache/MemWay[0]/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 {/testbench/dut/hart/lsu/dcache/MemWay[0]/SetValid}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 {/testbench/dut/hart/lsu/dcache/MemWay[0]/SetDirty}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 -label TAG {/testbench/dut/hart/lsu/dcache/MemWay[0]/CacheTagMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 {/testbench/dut/hart/lsu/dcache/MemWay[0]/DirtyBits}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 {/testbench/dut/hart/lsu/dcache/MemWay[0]/ValidBits}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 -expand -group Way0Word0 {/testbench/dut/hart/lsu/dcache/MemWay[0]/word[0]/CacheDataMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 -expand -group Way0Word0 {/testbench/dut/hart/lsu/dcache/MemWay[0]/word[0]/CacheDataMem/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 -expand -group Way0Word1 {/testbench/dut/hart/lsu/dcache/MemWay[0]/word[1]/CacheDataMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 -expand -group Way0Word1 {/testbench/dut/hart/lsu/dcache/MemWay[0]/word[1]/CacheDataMem/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 -expand -group Way0Word2 {/testbench/dut/hart/lsu/dcache/MemWay[0]/word[2]/CacheDataMem/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 -expand -group Way0Word2 {/testbench/dut/hart/lsu/dcache/MemWay[0]/word[2]/CacheDataMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 -expand -group Way0Word3 {/testbench/dut/hart/lsu/dcache/MemWay[0]/word[3]/CacheDataMem/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way0 -expand -group Way0Word3 {/testbench/dut/hart/lsu/dcache/MemWay[0]/word[3]/CacheDataMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 {/testbench/dut/hart/lsu/dcache/MemWay[1]/DirtyBits}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 {/testbench/dut/hart/lsu/dcache/MemWay[1]/SetDirty}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 {/testbench/dut/hart/lsu/dcache/MemWay[1]/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 {/testbench/dut/hart/lsu/dcache/MemWay[1]/WriteWordEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 -label TAG {/testbench/dut/hart/lsu/dcache/MemWay[1]/CacheTagMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 -expand -group Way1Word0 {/testbench/dut/hart/lsu/dcache/MemWay[1]/word[0]/CacheDataMem/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 -expand -group Way1Word0 {/testbench/dut/hart/lsu/dcache/MemWay[1]/word[0]/CacheDataMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 -expand -group Way1Word1 {/testbench/dut/hart/lsu/dcache/MemWay[1]/word[1]/CacheDataMem/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 -expand -group Way1Word1 {/testbench/dut/hart/lsu/dcache/MemWay[1]/word[1]/CacheDataMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 -expand -group Way1Word2 {/testbench/dut/hart/lsu/dcache/MemWay[1]/word[2]/CacheDataMem/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 -expand -group Way1Word2 {/testbench/dut/hart/lsu/dcache/MemWay[1]/word[2]/CacheDataMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 -expand -group Way1Word3 {/testbench/dut/hart/lsu/dcache/MemWay[1]/word[3]/CacheDataMem/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way1 -expand -group Way1Word3 {/testbench/dut/hart/lsu/dcache/MemWay[1]/word[3]/CacheDataMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 {/testbench/dut/hart/lsu/dcache/MemWay[2]/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 {/testbench/dut/hart/lsu/dcache/MemWay[2]/SetValid}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 {/testbench/dut/hart/lsu/dcache/MemWay[2]/SetDirty}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 -label TAG {/testbench/dut/hart/lsu/dcache/MemWay[2]/CacheTagMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 {/testbench/dut/hart/lsu/dcache/MemWay[2]/DirtyBits}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 {/testbench/dut/hart/lsu/dcache/MemWay[2]/ValidBits}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 -expand -group Way2Word0 {/testbench/dut/hart/lsu/dcache/MemWay[2]/word[0]/CacheDataMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 -expand -group Way2Word0 {/testbench/dut/hart/lsu/dcache/MemWay[2]/word[0]/CacheDataMem/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 -expand -group Way2Word1 {/testbench/dut/hart/lsu/dcache/MemWay[2]/word[1]/CacheDataMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 -expand -group Way2Word1 {/testbench/dut/hart/lsu/dcache/MemWay[2]/word[1]/CacheDataMem/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 -expand -group Way2Word2 {/testbench/dut/hart/lsu/dcache/MemWay[2]/word[2]/CacheDataMem/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 -expand -group Way2Word2 {/testbench/dut/hart/lsu/dcache/MemWay[2]/word[2]/CacheDataMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 -expand -group Way2Word3 {/testbench/dut/hart/lsu/dcache/MemWay[2]/word[3]/CacheDataMem/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -group way2 -expand -group Way2Word3 {/testbench/dut/hart/lsu/dcache/MemWay[2]/word[3]/CacheDataMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 {/testbench/dut/hart/lsu/dcache/MemWay[3]/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 {/testbench/dut/hart/lsu/dcache/MemWay[3]/SetValid}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 {/testbench/dut/hart/lsu/dcache/MemWay[3]/SetDirty}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 -label TAG {/testbench/dut/hart/lsu/dcache/MemWay[3]/CacheTagMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 {/testbench/dut/hart/lsu/dcache/MemWay[3]/DirtyBits}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 {/testbench/dut/hart/lsu/dcache/MemWay[3]/ValidBits}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 -expand -group Way3Word0 {/testbench/dut/hart/lsu/dcache/MemWay[3]/word[0]/CacheDataMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 -expand -group Way3Word0 {/testbench/dut/hart/lsu/dcache/MemWay[3]/word[0]/CacheDataMem/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 -expand -group Way3Word1 {/testbench/dut/hart/lsu/dcache/MemWay[3]/word[1]/CacheDataMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 -expand -group Way3Word1 {/testbench/dut/hart/lsu/dcache/MemWay[3]/word[1]/CacheDataMem/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 -expand -group Way3Word2 {/testbench/dut/hart/lsu/dcache/MemWay[3]/word[2]/CacheDataMem/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 -expand -group Way3Word2 {/testbench/dut/hart/lsu/dcache/MemWay[3]/word[2]/CacheDataMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 -expand -group Way3Word3 {/testbench/dut/hart/lsu/dcache/MemWay[3]/word[3]/CacheDataMem/WriteEnable}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group way3 -expand -group Way3Word3 {/testbench/dut/hart/lsu/dcache/MemWay[3]/word[3]/CacheDataMem/StoredData}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group valid/dirty /testbench/dut/hart/lsu/dcache/SetValid
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group valid/dirty /testbench/dut/hart/lsu/dcache/ClearValid
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group valid/dirty /testbench/dut/hart/lsu/dcache/SetDirty
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM writes} -expand -group valid/dirty /testbench/dut/hart/lsu/dcache/ClearDirty
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} -group way0 {/testbench/dut/hart/lsu/dcache/MemWay[0]/WayHit}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} -group way0 {/testbench/dut/hart/lsu/dcache/MemWay[0]/Valid}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} -group way0 {/testbench/dut/hart/lsu/dcache/MemWay[0]/Dirty}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} -group way0 {/testbench/dut/hart/lsu/dcache/MemWay[0]/ReadTag}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way1 {/testbench/dut/hart/lsu/dcache/MemWay[1]/WayHit}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way1 {/testbench/dut/hart/lsu/dcache/MemWay[1]/Valid}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way1 {/testbench/dut/hart/lsu/dcache/MemWay[1]/Dirty}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way1 {/testbench/dut/hart/lsu/dcache/MemWay[1]/ReadTag}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way2 {/testbench/dut/hart/lsu/dcache/MemWay[2]/WayHit}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way2 {/testbench/dut/hart/lsu/dcache/MemWay[2]/Valid}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way2 {/testbench/dut/hart/lsu/dcache/MemWay[2]/Dirty}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way2 {/testbench/dut/hart/lsu/dcache/MemWay[2]/ReadTag}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way3 {/testbench/dut/hart/lsu/dcache/MemWay[3]/WayHit}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way3 {/testbench/dut/hart/lsu/dcache/MemWay[3]/Valid}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way3 {/testbench/dut/hart/lsu/dcache/MemWay[3]/Dirty}
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} -expand -group way3 {/testbench/dut/hart/lsu/dcache/MemWay[3]/ReadTag}
 add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} /testbench/dut/hart/lsu/dcache/WayHit
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} /testbench/dut/hart/lsu/dcache/Dirty
-add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} /testbench/dut/hart/lsu/dcache/Valid
-add wave -noupdate -group lsu -expand -group dcache -group Victim /testbench/dut/hart/lsu/dcache/VictimReadDataBlockM
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} /testbench/dut/hart/lsu/dcache/ReadDataBlockWayMaskedM
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} /testbench/dut/hart/lsu/dcache/ReadDataWordM
+add wave -noupdate -group lsu -expand -group dcache -group {Cache SRAM read} /testbench/dut/hart/lsu/dcache/ReadDataWordMuxM
 add wave -noupdate -group lsu -expand -group dcache -group Victim /testbench/dut/hart/lsu/dcache/VictimTag
 add wave -noupdate -group lsu -expand -group dcache -group Victim /testbench/dut/hart/lsu/dcache/VictimWay
 add wave -noupdate -group lsu -expand -group dcache -group Victim /testbench/dut/hart/lsu/dcache/VictimDirtyWay
 add wave -noupdate -group lsu -expand -group dcache -group Victim /testbench/dut/hart/lsu/dcache/VictimDirty
-add wave -noupdate -group lsu -expand -group dcache -expand -group {CPU side} /testbench/dut/hart/lsu/dcache/MemRWM
-add wave -noupdate -group lsu -expand -group dcache -expand -group {CPU side} /testbench/dut/hart/lsu/dcache/MemAdrE
-add wave -noupdate -group lsu -expand -group dcache -expand -group {CPU side} /testbench/dut/hart/lsu/MemAdrM
-add wave -noupdate -group lsu -expand -group dcache -expand -group {CPU side} /testbench/dut/hart/lsu/dcache/MemPAdrM
-add wave -noupdate -group lsu -expand -group dcache -expand -group {CPU side} /testbench/dut/hart/lsu/dcache/Funct3M
-add wave -noupdate -group lsu -expand -group dcache -expand -group {CPU side} /testbench/dut/hart/lsu/dcache/Funct7M
-add wave -noupdate -group lsu -expand -group dcache -expand -group {CPU side} /testbench/dut/hart/lsu/dcache/AtomicM
-add wave -noupdate -group lsu -expand -group dcache -expand -group {CPU side} /testbench/dut/hart/lsu/dcache/CacheableM
-add wave -noupdate -group lsu -expand -group dcache -expand -group {CPU side} /testbench/dut/hart/lsu/dcache/WriteDataM
-add wave -noupdate -group lsu -expand -group dcache -expand -group {CPU side} /testbench/dut/hart/lsu/dcache/ReadDataM
-add wave -noupdate -group lsu -expand -group dcache -expand -group {CPU side} /testbench/dut/hart/lsu/dcache/DCacheStall
+add wave -noupdate -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/MemRWM
+add wave -noupdate -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/MemAdrE
+add wave -noupdate -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/MemPAdrM
+add wave -noupdate -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/Funct3M
+add wave -noupdate -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/Funct7M
+add wave -noupdate -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/AtomicM
+add wave -noupdate -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/CacheableM
+add wave -noupdate -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/WriteDataM
+add wave -noupdate -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/ReadDataM
+add wave -noupdate -group lsu -expand -group dcache -group {CPU side} /testbench/dut/hart/lsu/dcache/DCacheStall
 add wave -noupdate -group lsu -expand -group dcache -group status /testbench/dut/hart/lsu/dcache/WayHit
 add wave -noupdate -group lsu -expand -group dcache -group status -color {Medium Orchid} /testbench/dut/hart/lsu/dcache/CacheHit
+add wave -noupdate -group lsu -expand -group dcache -group status /testbench/dut/hart/lsu/dcache/FetchCount
 add wave -noupdate -group lsu -expand -group dcache -group {Memory Side} /testbench/dut/hart/lsu/dcache/AHBPAdr
 add wave -noupdate -group lsu -expand -group dcache -group {Memory Side} /testbench/dut/hart/lsu/dcache/AHBRead
 add wave -noupdate -group lsu -expand -group dcache -group {Memory Side} /testbench/dut/hart/lsu/dcache/AHBWrite
@@ -450,29 +469,35 @@ add wave -noupdate -group UART /testbench/dut/uncore/uart/uart/HSELUART
 add wave -noupdate -group UART /testbench/dut/uncore/uart/uart/HADDR
 add wave -noupdate -group UART /testbench/dut/uncore/uart/uart/HWRITE
 add wave -noupdate -group UART /testbench/dut/uncore/uart/uart/HWDATA
-add wave -noupdate -group {debug trace} -expand -group mem -color Yellow /testbench/dut/hart/FlushW
-add wave -noupdate -group {debug trace} -expand -group mem /testbench/dut/hart/priv/trap/InstrValidM
-add wave -noupdate -group {debug trace} -expand -group mem /testbench/checkInstrM
-add wave -noupdate -group {debug trace} -expand -group mem /testbench/dut/hart/PCM
-add wave -noupdate -group {debug trace} -expand -group mem /testbench/ExpectedPCM
-add wave -noupdate -group {debug trace} -expand -group mem /testbench/line
-add wave -noupdate -group {debug trace} -expand -group mem /testbench/textM
-add wave -noupdate -group {debug trace} -expand -group mem -color Brown /testbench/dut/hart/hzu/TrapM
-add wave -noupdate -group {debug trace} -expand -group wb /testbench/dut/hart/ieu/c/InstrValidW
-add wave -noupdate -group {debug trace} -expand -group wb /testbench/checkInstrW
-add wave -noupdate -group {debug trace} -expand -group wb /testbench/PCW
-add wave -noupdate -group {debug trace} -expand -group wb /testbench/ExpectedPCW
-add wave -noupdate -group {debug trace} -expand -group wb /testbench/TrapW
-add wave -noupdate -group {debug trace} -expand -group wb /testbench/textW
+add wave -noupdate -expand -group {debug trace} -expand -group mem -color Yellow /testbench/dut/hart/FlushW
+add wave -noupdate -expand -group {debug trace} -expand -group mem /testbench/dut/hart/priv/trap/InstrValidM
+add wave -noupdate -expand -group {debug trace} -expand -group mem /testbench/checkInstrM
+add wave -noupdate -expand -group {debug trace} -expand -group mem /testbench/dut/hart/PCM
+add wave -noupdate -expand -group {debug trace} -expand -group mem /testbench/ExpectedPCM
+add wave -noupdate -expand -group {debug trace} -expand -group mem /testbench/line
+add wave -noupdate -expand -group {debug trace} -expand -group mem /testbench/textM
+add wave -noupdate -expand -group {debug trace} -expand -group mem -color Brown /testbench/dut/hart/hzu/TrapM
+add wave -noupdate -expand -group {debug trace} -expand -group wb /testbench/dut/hart/ieu/c/InstrValidW
+add wave -noupdate -expand -group {debug trace} -expand -group wb /testbench/checkInstrW
+add wave -noupdate -expand -group {debug trace} -expand -group wb /testbench/PCW
+add wave -noupdate -expand -group {debug trace} -expand -group wb /testbench/ExpectedPCW
+add wave -noupdate -expand -group {debug trace} -expand -group wb /testbench/TrapW
+add wave -noupdate -expand -group {debug trace} -expand -group wb /testbench/textW
 add wave -noupdate -group {pc selection} /testbench/dut/hart/ifu/PCNext2F
 add wave -noupdate -group {pc selection} /testbench/dut/hart/ifu/PrivilegedNextPCM
 add wave -noupdate -group {pc selection} /testbench/dut/hart/ifu/PrivilegedChangePCM
 add wave -noupdate -group {pc selection} /testbench/dut/hart/priv/PrivilegedNextPCM
 add wave -noupdate -group {pc selection} /testbench/dut/hart/priv/trap/PrivilegedVectoredTrapVector
 add wave -noupdate -group {pc selection} /testbench/dut/hart/priv/trap/PrivilegedTrapVector
+add wave -noupdate /testbench/dut/hart/ifu/PCCorrectE
+add wave -noupdate /testbench/dut/hart/ifu/PCSrcE
+add wave -noupdate /testbench/dut/hart/ieu/c/BranchTakenE
+add wave -noupdate /testbench/dut/hart/ieu/c/BranchE
+add wave -noupdate /testbench/dut/hart/ifu/PCTargetE
+add wave -noupdate /testbench/dut/hart/ifu/PCLinkE
 TreeUpdate [SetDefaultTree]
-WaveRestoreCursors {{Cursor 6} {161370956 ns} 0} {{Cursor 21} {161371350 ns} 0} {{Cursor 22} {72228581 ns} 0} {{Cursor 23} {11297671 ns} 0}
-quietly wave cursor active 3
+WaveRestoreCursors {{Cursor 6} {160470606 ns} 0} {{Cursor 21} {161371350 ns} 0} {{Cursor 22} {72228581 ns} 0} {{Cursor 23} {11297671 ns} 0}
+quietly wave cursor active 1
 configure wave -namecolwidth 250
 configure wave -valuecolwidth 354
 configure wave -justifyvalue left
@@ -487,4 +512,4 @@ configure wave -griddelta 40
 configure wave -timeline 0
 configure wave -timelineunits ns
 update
-WaveRestoreZoom {72227242 ns} {72229920 ns}
+WaveRestoreZoom {160470352 ns} {160470676 ns}
diff --git a/wally-pipelined/src/cache/icachefsm.sv b/wally-pipelined/src/cache/icachefsm.sv
index 7ec4acfbe..d020d6967 100644
--- a/wally-pipelined/src/cache/icachefsm.sv
+++ b/wally-pipelined/src/cache/icachefsm.sv
@@ -324,8 +324,8 @@ module icachefsm #(parameter BLOCKLEN = 256)
         SavePC = 1'b1;
         ICacheStallF = 1'b0;	
 	if(StallF) begin
-	  NextState = STATE_CPU_BUSY;
-	  SelAdr = 2'b01;
+	  NextState = STATE_CPU_BUSY_SPILL;
+	  SelAdr = 2'b10;
 	end else begin
           NextState = STATE_READY;
 	end
diff --git a/wally-pipelined/src/ifu/bpred.sv b/wally-pipelined/src/ifu/bpred.sv
index 92471c574..22c9ec336 100644
--- a/wally-pipelined/src/ifu/bpred.sv
+++ b/wally-pipelined/src/ifu/bpred.sv
@@ -230,7 +230,7 @@ module bpred
   assign FallThroughWrongE = PCLinkE != PCD;
   // If the target is taken check the target rather than fallthrough.  The instruction needs to be a branch if PCSrcE is selected
   // Remember the bpred can incorrectly predict a non cfi instruction as a branch taken.  If the real instruction is non cfi
-  // it must have selected teh fall through.
+  // it must have selected the fall through.
   assign PredictionPCWrongE = (PCSrcE  & (|InstrClassE) ? TargetWrongE : FallThroughWrongE);
 
   // The branch direction also need to checked.

From 5c2deab4e4c3afe95995c4833844a53427807006 Mon Sep 17 00:00:00 2001
From: Ross Thompson <ross1728@gmail.com>
Date: Mon, 30 Aug 2021 10:49:24 -0500
Subject: [PATCH 10/16] Partial multiway set associative icache.

---
 wally-pipelined/regression/wave.do  |  21 ++---
 wally-pipelined/src/cache/icache.sv | 122 ++++++++++++++++++----------
 2 files changed, 82 insertions(+), 61 deletions(-)

diff --git a/wally-pipelined/regression/wave.do b/wally-pipelined/regression/wave.do
index 82fb8d3c2..b3a31db16 100644
--- a/wally-pipelined/regression/wave.do
+++ b/wally-pipelined/regression/wave.do
@@ -191,22 +191,21 @@ add wave -noupdate -expand -group icache /testbench/dut/hart/ifu/icache/controll
 add wave -noupdate -expand -group icache /testbench/dut/hart/ifu/ITLBMissF
 add wave -noupdate -expand -group icache /testbench/dut/hart/ifu/icache/ITLBWriteF
 add wave -noupdate -expand -group icache /testbench/dut/hart/ifu/icache/ReadLineF
-add wave -noupdate -expand -group icache {/testbench/dut/hart/ifu/icache/icachemem/word[0]/CacheDataMem/Addr}
-add wave -noupdate -expand -group icache {/testbench/dut/hart/ifu/icache/icachemem/word[0]/CacheDataMem/WriteData}
-add wave -noupdate -expand -group icache {/testbench/dut/hart/ifu/icache/icachemem/word[0]/CacheDataMem/WriteEnable}
 add wave -noupdate -expand -group icache /testbench/dut/hart/ifu/icache/PCNextIndexF
 add wave -noupdate -expand -group icache /testbench/dut/hart/ifu/icache/ReadLineF
+add wave -noupdate -expand -group icache /testbench/dut/hart/ifu/icache/BasePAdrF
 add wave -noupdate -expand -group icache -expand -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/hit
 add wave -noupdate -expand -group icache -expand -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/spill
 add wave -noupdate -expand -group icache -expand -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/ICacheStallF
 add wave -noupdate -expand -group icache -expand -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/SavePC
 add wave -noupdate -expand -group icache -expand -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/spillSave
 add wave -noupdate -expand -group icache -expand -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/UnalignedSelect
-add wave -noupdate -expand -group icache -expand -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/PCMux
 add wave -noupdate -expand -group icache -expand -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/spillSave
 add wave -noupdate -expand -group icache -expand -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/CntReset
 add wave -noupdate -expand -group icache -expand -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/PreCntEn
 add wave -noupdate -expand -group icache -expand -group {fsm out and control} /testbench/dut/hart/ifu/icache/controller/CntEn
+add wave -noupdate -expand -group icache -expand -group memory /testbench/dut/hart/ifu/icache/InstrPAdrF
+add wave -noupdate -expand -group icache -expand -group memory /testbench/dut/hart/ifu/icache/InstrInF
 add wave -noupdate -expand -group icache -expand -group memory /testbench/dut/hart/ifu/icache/controller/FetchCountFlag
 add wave -noupdate -expand -group icache -expand -group memory /testbench/dut/hart/ifu/icache/FetchCount
 add wave -noupdate -expand -group icache -expand -group memory /testbench/dut/hart/ifu/icache/controller/InstrReadF
@@ -475,21 +474,11 @@ add wave -noupdate /testbench/dut/hart/ifu/immu/Translate
 add wave -noupdate /testbench/dut/hart/ifu/icache/FinalInstrRawF
 add wave -noupdate /testbench/dut/hart/ifu/icache/StallF
 add wave -noupdate /testbench/dut/hart/ifu/icache/ICacheMemReadData
-add wave -noupdate {/testbench/dut/hart/ifu/icache/icachemem/word[0]/CacheDataMem/Addr}
-add wave -noupdate {/testbench/dut/hart/ifu/icache/icachemem/word[0]/CacheDataMem/ReadData}
-add wave -noupdate {/testbench/dut/hart/ifu/icache/icachemem/word[1]/CacheDataMem/Addr}
-add wave -noupdate {/testbench/dut/hart/ifu/icache/icachemem/word[1]/CacheDataMem/ReadData}
-add wave -noupdate {/testbench/dut/hart/ifu/icache/icachemem/word[2]/CacheDataMem/Addr}
-add wave -noupdate {/testbench/dut/hart/ifu/icache/icachemem/word[2]/CacheDataMem/ReadData}
-add wave -noupdate {/testbench/dut/hart/ifu/icache/icachemem/word[3]/CacheDataMem/Addr}
-add wave -noupdate {/testbench/dut/hart/ifu/icache/icachemem/word[3]/CacheDataMem/ReadData}
-add wave -noupdate /testbench/dut/hart/ifu/icache/icachemem/WayHit
 add wave -noupdate /testbench/dut/hart/ifu/icache/PCTagF
 add wave -noupdate /testbench/dut/hart/ifu/icache/PCPSpillF
 add wave -noupdate /testbench/dut/hart/ifu/icache/ICacheReadEn
-add wave -noupdate /testbench/dut/hart/ifu/icache/PCMux_q
 TreeUpdate [SetDefaultTree]
-WaveRestoreCursors {{Cursor 6} {53451 ns} 0}
+WaveRestoreCursors {{Cursor 6} {183 ns} 0}
 quietly wave cursor active 1
 configure wave -namecolwidth 250
 configure wave -valuecolwidth 297
@@ -505,4 +494,4 @@ configure wave -griddelta 40
 configure wave -timeline 0
 configure wave -timelineunits ns
 update
-WaveRestoreZoom {53011 ns} {53891 ns}
+WaveRestoreZoom {0 ns} {456 ns}
diff --git a/wally-pipelined/src/cache/icache.sv b/wally-pipelined/src/cache/icache.sv
index 839504547..c60a1344f 100644
--- a/wally-pipelined/src/cache/icache.sv
+++ b/wally-pipelined/src/cache/icache.sv
@@ -70,6 +70,8 @@ module icache
   localparam OFFSETWIDTH = $clog2(BlockByteLength);
 
   localparam integer 	       PA_WIDTH = `PA_BITS - 2;
+  localparam integer 	       NUMWAYS = 4;
+  
 
   // Input signals to cache memory
   logic 		    FlushMem;
@@ -95,13 +97,26 @@ module icache
   
   logic [LOGWPL-1:0] 	       FetchCount, NextFetchCount;
  
-  logic [`PA_BITS-1:0] 	       PCPreFinalF, PCPSpillF;
-  logic [`PA_BITS-1:OFFSETWIDTH] PCPTrunkF;
+  logic [`PA_BITS-1:0] 	       PCPSpillF;
 
   logic 		       CntReset;
   logic [1:0] 		       SelAdr;
   logic 		       SavePC;
   logic [INDEXLEN-1:0]	       RAdr;
+  logic [NUMWAYS-1:0] 	       VictimWay;
+  logic 		       LRUWriteEn;
+  logic [NUMWAYS-1:0] 	       WayHit;
+  logic 		       hit;
+  
+  
+  logic [BLOCKLEN-1:0] 	       ReadDataBlockWayMasked [NUMWAYS-1:0];
+
+
+  logic 		       CacheableF;
+  
+  logic [`PA_BITS-1:0] 	       BasePAdrF, BasePAdrMaskedF;
+  logic [OFFSETLEN-1:0]        BasePAdrOffsetF;
+  
   
 
   // on spill we want to get the first 2 bytes of the next cache block.
@@ -120,27 +135,48 @@ module icache
 
   cacheway #(.NUMLINES(NUMLINES), .BLOCKLEN(BLOCKLEN), .TAGLEN(TAGLEN), .OFFSETLEN(OFFSETLEN), .INDEXLEN(INDEXLEN),
 	     .DIRTY_BITS(0))
-  icachemem(.clk,
-	    .reset,
-	    .RAdr(RAdr),
-	    .PAdr(PCTagF),
-	    .WriteEnable(ICacheMemWriteEnable),
-	    .WriteWordEnable('1),
-	    .TagWriteEnable(ICacheMemWriteEnable),
-	    .WriteData(ICacheMemWriteData),
-	    .SetValid(ICacheMemWriteEnable),
-	    .ClearValid(1'b0),
-	    .SetDirty(1'b0),
-	    .ClearDirty(1'b0),
-	    .SelEvict(1'b0),
-	    .VictimWay(1'b0),
-	    .ReadDataBlockWayMasked(ReadLineF),
-	    .WayHit(ICacheMemReadValid),
-	    .VictimDirtyWay(),
-	    .VictimTagWay()
-	    );
+  icachemem[NUMWAYS-1:0](.clk,
+			 .reset,
+			 .RAdr(RAdr),
+			 .PAdr(PCTagF),
+			 .WriteEnable(ICacheMemWriteEnable), // *** connect
+			 .WriteWordEnable('1),
+			 .TagWriteEnable(ICacheMemWriteEnable), // *** connect
+			 .WriteData(ICacheMemWriteData),
+			 .SetValid(ICacheMemWriteEnable),
+			 .ClearValid(1'b0),
+			 .SetDirty(1'b0),
+			 .ClearDirty(1'b0),
+			 .SelEvict(1'b0),
+			 .VictimWay,
+			 .ReadDataBlockWayMasked,
+			 .WayHit,
+			 .VictimDirtyWay(),
+			 .VictimTagWay()
+			 );
   
+  generate
+    if(NUMWAYS > 1) begin
+      cachereplacementpolicy #(NUMWAYS, INDEXLEN, OFFSETLEN, NUMLINES)
+      cachereplacementpolicy(.clk, .reset,
+			     .WayHit,
+			     .VictimWay,
+			     .MemPAdrM(PCTagF[INDEXLEN+OFFSETLEN-1:OFFSETLEN]),
+			     .RAdr,
+			     .LRUWriteEn); // *** connect
+    end else begin
+      assign VictimWay = 1'b1; // one hot.
+    end
+  endgenerate
 
+  assign hit = | WayHit;
+
+  // ReadDataBlockWayMasked is a 2d array of cache block len by number of ways.
+  // Need to OR together each way in a bitwise manner.
+  // Final part of the AO Mux.  First is the AND in the cacheway.
+  or_rows #(NUMWAYS, BLOCKLEN) ReadDataAOMux(.a(ReadDataBlockWayMasked), .y(ReadLineF));
+  
+  
   always_comb begin
     case (PCTagF[4:1])
       0: ICacheMemReadData = ReadLineF[31:0];
@@ -179,14 +215,12 @@ module icache
 
   // Detect if the instruction is compressed
   assign CompressedF = FinalInstrRawF[1:0] != 2'b11;
-  
-
   assign spill = PCPF[4:1] == 4'b1111 ? 1'b1 : 1'b0;
-  assign hit = ICacheMemReadValid; // note ICacheMemReadValid is hit.
-  assign FetchCountFlag = (FetchCount == FetchCountThreshold[LOGWPL-1:0]);
+
 
   // to compute the fetch address we need to add the bit shifted
   // counter output to the address.
+  assign FetchCountFlag = (FetchCount == FetchCountThreshold[LOGWPL-1:0]);
 
   flopenr #(LOGWPL) 
   FetchCountReg(.clk(clk),
@@ -196,24 +230,7 @@ module icache
 		.q(FetchCount));
 
   assign NextFetchCount = FetchCount + 1'b1;
-
-  // This part is confusing.
-  // *** Ross Thompson reduce the complexity. This is just dumb.
-  // we need to remove the offset bits (PCPTrunkF).  Because the AHB interface is XLEN wide
-  // we need to address on that number of bits so the PC is extended to the right by AHBByteLength with zeros.
-  // fetch count is already aligned to AHBByteLength, but we need to extend back to the full address width with
-  // more zeros after the addition.  This will be the number of offset bits less the AHBByteLength.
-  logic [`PA_BITS-1:OFFSETWIDTH-LOGWPL] PCPTrunkExtF, InstrPAdrTrunkF ;
-
-  assign PCPTrunkExtF = {PCPTrunkF, {{LOGWPL}{1'b0}}};
-  // verilator lint_off WIDTH
-  assign InstrPAdrTrunkF = PCPTrunkExtF + FetchCount;
-  // verilator lint_on WIDTH
   
-  //assign InstrPAdrF = {{PCPTrunkF, {{LOGWPL}{1'b0}}} + FetchCount, {{OFFSETWIDTH-LOGWPL}{1'b0}}};
-  assign InstrPAdrF = {InstrPAdrTrunkF, {{OFFSETWIDTH-LOGWPL}{1'b0}}};
-  
-
 
   // store read data from memory interface before writing into SRAM.
   genvar 				i;
@@ -237,10 +254,25 @@ module icache
 			.q(SelAdr_q));
   
   assign PCTagF = SelAdr_q[1] ? PCPSpillF : PCPF;
+
+  // unlike the dcache the victim is never dirty so no eviction is necessary.
+/* -----\/----- EXCLUDED -----\/-----
+  mux2 #(`PA_BITS) BaseAdrMux(.d0(PCTagF),
+			      .d1({VictimTag, PCTagF[INDEXLEN+OFFSETLEN-1:OFFSETLEN], {{OFFSETLEN}{1'b0}}}),
+			      .s(SelEvict),
+			      .y(BasePAdrF));
+ -----/\----- EXCLUDED -----/\----- */
+  assign BasePAdrF = PCTagF;
+
+  // if not cacheable the offset bits needs to be sent to the EBU.
+  // if cacheable the offset bits are discarded.  $ FSM will fetch the whole block.
+  assign CacheableF = 1'b1; // *** BUG needs to be an input from MMU.
+  assign BasePAdrOffsetF = CacheableF ? {{OFFSETLEN}{1'b0}} : BasePAdrF[OFFSETLEN-1:0];
+  assign BasePAdrMaskedF = {BasePAdrF[`PA_BITS-1:OFFSETLEN], BasePAdrOffsetF};
+  
+  assign InstrPAdrF = ({{`PA_BITS-LOGWPL{1'b0}}, FetchCount} << $clog2(`XLEN/8)) + BasePAdrMaskedF;
   
   // truncate the offset from PCPF for memory address generation
-  assign PCPTrunkF = PCTagF[`PA_BITS-1:OFFSETWIDTH];
-  
 
   icachefsm #(.BLOCKLEN(BLOCKLEN)) 
   controller(.clk,
@@ -254,7 +286,7 @@ module icache
 	     .WalkerInstrPageFaultF,
 	     .InstrAckF,
 	     .InstrReadF,
-	     .hit(ICacheMemReadValid),
+	     .hit,
 	     .FetchCountFlag,
 	     .spill,
 	     .spillSave,

From 00f50184d8e5dda75afea44873647e85834eb08d Mon Sep 17 00:00:00 2001
From: Ross Thompson <ross1728@gmail.com>
Date: Mon, 6 Sep 2021 20:54:52 -0500
Subject: [PATCH 11/16] Changed name of memory in icache.

---
 wally-pipelined/regression/wave.do  | 51 +++++++++++++++++++++++++++++
 wally-pipelined/src/cache/icache.sv |  2 +-
 2 files changed, 52 insertions(+), 1 deletion(-)

diff --git a/wally-pipelined/regression/wave.do b/wally-pipelined/regression/wave.do
index b3a31db16..042178cd2 100644
--- a/wally-pipelined/regression/wave.do
+++ b/wally-pipelined/regression/wave.do
@@ -187,6 +187,57 @@ add wave -noupdate -group divider /testbench/dut/hart/mdu/genblk1/div/D
 add wave -noupdate -group divider /testbench/dut/hart/mdu/genblk1/div/Q
 add wave -noupdate -group divider /testbench/dut/hart/mdu/genblk1/div/rem0
 add wave -noupdate -expand -group icache -color Gold /testbench/dut/hart/ifu/icache/controller/CurrState
+add wave -noupdate -expand -group icache /testbench/dut/hart/ifu/icache/BasePAdrF
+add wave -noupdate -expand -group icache /testbench/dut/hart/ifu/icache/WayHit
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} /testbench/dut/hart/ifu/icache/VictimWay
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way0 {/testbench/dut/hart/ifu/icache/MemWay[0]/WriteEnable}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way0 {/testbench/dut/hart/ifu/icache/MemWay[0]/SetValid}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way0 -label TAG {/testbench/dut/hart/ifu/icache/MemWay[0]/CacheTagMem/StoredData}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way0 {/testbench/dut/hart/ifu/icache/MemWay[0]/ValidBits}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way0 -expand -group Way0Word0 {/testbench/dut/hart/ifu/icache/MemWay[0]/word[0]/CacheDataMem/StoredData}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way0 -expand -group Way0Word0 {/testbench/dut/hart/ifu/icache/MemWay[0]/word[0]/CacheDataMem/WriteEnable}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way0 -group Way0Word1 {/testbench/dut/hart/ifu/icache/MemWay[0]/word[1]/CacheDataMem/StoredData}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way0 -group Way0Word1 {/testbench/dut/hart/ifu/icache/MemWay[0]/word[1]/CacheDataMem/WriteEnable}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way0 -group Way0Word2 {/testbench/dut/hart/ifu/icache/MemWay[0]/word[2]/CacheDataMem/WriteEnable}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way0 -group Way0Word2 {/testbench/dut/hart/ifu/icache/MemWay[0]/word[2]/CacheDataMem/StoredData}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way0 -group Way0Word3 {/testbench/dut/hart/ifu/icache/MemWay[0]/word[3]/CacheDataMem/WriteEnable}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way0 -group Way0Word3 {/testbench/dut/hart/ifu/icache/MemWay[0]/word[3]/CacheDataMem/StoredData}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way1 {/testbench/dut/hart/ifu/icache/MemWay[1]/WriteEnable}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way1 {/testbench/dut/hart/ifu/icache/MemWay[1]/WriteWordEnable}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way1 -label TAG {/testbench/dut/hart/ifu/icache/MemWay[1]/CacheTagMem/StoredData}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way1 -expand -group Way1Word0 {/testbench/dut/hart/ifu/icache/MemWay[1]/word[0]/CacheDataMem/WriteEnable}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way1 -expand -group Way1Word0 {/testbench/dut/hart/ifu/icache/MemWay[1]/word[0]/CacheDataMem/StoredData}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way1 -group Way1Word1 {/testbench/dut/hart/ifu/icache/MemWay[1]/word[1]/CacheDataMem/WriteEnable}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way1 -group Way1Word1 {/testbench/dut/hart/ifu/icache/MemWay[1]/word[1]/CacheDataMem/StoredData}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way1 -group Way1Word2 {/testbench/dut/hart/ifu/icache/MemWay[1]/word[2]/CacheDataMem/WriteEnable}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way1 -group Way1Word2 {/testbench/dut/hart/ifu/icache/MemWay[1]/word[2]/CacheDataMem/StoredData}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way1 -group Way1Word3 {/testbench/dut/hart/ifu/icache/MemWay[1]/word[3]/CacheDataMem/WriteEnable}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way1 -group Way1Word3 {/testbench/dut/hart/ifu/icache/MemWay[1]/word[3]/CacheDataMem/StoredData}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way2 {/testbench/dut/hart/ifu/icache/MemWay[2]/WriteEnable}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way2 {/testbench/dut/hart/ifu/icache/MemWay[2]/SetValid}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way2 -label TAG {/testbench/dut/hart/ifu/icache/MemWay[2]/CacheTagMem/StoredData}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way2 {/testbench/dut/hart/ifu/icache/MemWay[2]/ValidBits}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way2 -expand -group Way2Word0 {/testbench/dut/hart/ifu/icache/MemWay[2]/word[0]/CacheDataMem/StoredData}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way2 -expand -group Way2Word0 {/testbench/dut/hart/ifu/icache/MemWay[2]/word[0]/CacheDataMem/WriteEnable}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way2 -group Way2Word1 {/testbench/dut/hart/ifu/icache/MemWay[2]/word[1]/CacheDataMem/StoredData}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way2 -group Way2Word1 {/testbench/dut/hart/ifu/icache/MemWay[2]/word[1]/CacheDataMem/WriteEnable}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way2 -group Way2Word2 {/testbench/dut/hart/ifu/icache/MemWay[2]/word[2]/CacheDataMem/WriteEnable}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way2 -group Way2Word2 {/testbench/dut/hart/ifu/icache/MemWay[2]/word[2]/CacheDataMem/StoredData}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way2 -group Way2Word3 {/testbench/dut/hart/ifu/icache/MemWay[2]/word[3]/CacheDataMem/WriteEnable}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way2 -group Way2Word3 {/testbench/dut/hart/ifu/icache/MemWay[2]/word[3]/CacheDataMem/StoredData}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way3 {/testbench/dut/hart/ifu/icache/MemWay[3]/WriteEnable}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way3 {/testbench/dut/hart/ifu/icache/MemWay[3]/SetValid}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way3 -label TAG {/testbench/dut/hart/ifu/icache/MemWay[3]/CacheTagMem/StoredData}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way3 {/testbench/dut/hart/ifu/icache/MemWay[3]/DirtyBits}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way3 {/testbench/dut/hart/ifu/icache/MemWay[3]/ValidBits}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way3 -expand -group Way3Word0 {/testbench/dut/hart/ifu/icache/MemWay[3]/word[0]/CacheDataMem/StoredData}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way3 -expand -group Way3Word0 {/testbench/dut/hart/ifu/icache/MemWay[3]/word[0]/CacheDataMem/WriteEnable}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way3 -group Way3Word1 {/testbench/dut/hart/ifu/icache/MemWay[3]/word[1]/CacheDataMem/StoredData}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way3 -group Way3Word1 {/testbench/dut/hart/ifu/icache/MemWay[3]/word[1]/CacheDataMem/WriteEnable}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way3 -group Way3Word2 {/testbench/dut/hart/ifu/icache/MemWay[3]/word[2]/CacheDataMem/WriteEnable}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way3 -group Way3Word2 {/testbench/dut/hart/ifu/icache/MemWay[3]/word[2]/CacheDataMem/StoredData}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way3 -group Way3Word3 {/testbench/dut/hart/ifu/icache/MemWay[3]/word[3]/CacheDataMem/WriteEnable}
+add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way3 -group Way3Word3 {/testbench/dut/hart/ifu/icache/MemWay[3]/word[3]/CacheDataMem/StoredData}
 add wave -noupdate -expand -group icache /testbench/dut/hart/ifu/icache/controller/NextState
 add wave -noupdate -expand -group icache /testbench/dut/hart/ifu/ITLBMissF
 add wave -noupdate -expand -group icache /testbench/dut/hart/ifu/icache/ITLBWriteF
diff --git a/wally-pipelined/src/cache/icache.sv b/wally-pipelined/src/cache/icache.sv
index c60a1344f..4d4d9fc87 100644
--- a/wally-pipelined/src/cache/icache.sv
+++ b/wally-pipelined/src/cache/icache.sv
@@ -135,7 +135,7 @@ module icache
 
   cacheway #(.NUMLINES(NUMLINES), .BLOCKLEN(BLOCKLEN), .TAGLEN(TAGLEN), .OFFSETLEN(OFFSETLEN), .INDEXLEN(INDEXLEN),
 	     .DIRTY_BITS(0))
-  icachemem[NUMWAYS-1:0](.clk,
+  MemWay[NUMWAYS-1:0](.clk,
 			 .reset,
 			 .RAdr(RAdr),
 			 .PAdr(PCTagF),

From 150a73d6cff5bf6572afd53fd38b25847ca955af Mon Sep 17 00:00:00 2001
From: Ross Thompson <ross1728@gmail.com>
Date: Tue, 7 Sep 2021 12:46:16 -0500
Subject: [PATCH 12/16] Set associate icache working, but way 0 is never
 written.

---
 wally-pipelined/config/rv64ic/wally-config.vh |  4 ++--
 wally-pipelined/src/cache/icache.sv           | 13 +++++++++----
 wally-pipelined/src/cache/icachefsm.sv        | 17 +++++++++++++----
 3 files changed, 24 insertions(+), 10 deletions(-)

diff --git a/wally-pipelined/config/rv64ic/wally-config.vh b/wally-pipelined/config/rv64ic/wally-config.vh
index af932dc4a..5b3eddb60 100644
--- a/wally-pipelined/config/rv64ic/wally-config.vh
+++ b/wally-pipelined/config/rv64ic/wally-config.vh
@@ -57,10 +57,10 @@
 // Cache configuration.  Sizes should be a power of two
 // typical configuration 4 ways, 4096 bytes per way, 256 bit or more blocks
 `define DCACHE_NUMWAYS 4
-`define DCACHE_WAYSIZEINBYTES 2048
+`define DCACHE_WAYSIZEINBYTES 4096
 `define DCACHE_BLOCKLENINBITS 256
 `define DCACHE_REPLBITS 3
-`define ICACHE_NUMWAYS 1
+`define ICACHE_NUMWAYS 4
 `define ICACHE_WAYSIZEINBYTES 4096
 `define ICACHE_BLOCKLENINBITS 256
 
diff --git a/wally-pipelined/src/cache/icache.sv b/wally-pipelined/src/cache/icache.sv
index 4d4d9fc87..7992c56c5 100644
--- a/wally-pipelined/src/cache/icache.sv
+++ b/wally-pipelined/src/cache/icache.sv
@@ -70,7 +70,7 @@ module icache
   localparam OFFSETWIDTH = $clog2(BlockByteLength);
 
   localparam integer 	       PA_WIDTH = `PA_BITS - 2;
-  localparam integer 	       NUMWAYS = 4;
+  localparam integer 	       NUMWAYS = `ICACHE_NUMWAYS;
   
 
   // Input signals to cache memory
@@ -118,6 +118,8 @@ module icache
   logic [OFFSETLEN-1:0]        BasePAdrOffsetF;
   
   
+  logic [NUMWAYS-1:0] 	       SRAMWayWriteEnable;
+
 
   // on spill we want to get the first 2 bytes of the next cache block.
   // the spill only occurs if the PCPF mod BlockByteLength == -2.  Therefore we can
@@ -139,9 +141,9 @@ module icache
 			 .reset,
 			 .RAdr(RAdr),
 			 .PAdr(PCTagF),
-			 .WriteEnable(ICacheMemWriteEnable), // *** connect
+			 .WriteEnable(SRAMWayWriteEnable), 
 			 .WriteWordEnable('1),
-			 .TagWriteEnable(ICacheMemWriteEnable), // *** connect
+			 .TagWriteEnable(SRAMWayWriteEnable),
 			 .WriteData(ICacheMemWriteData),
 			 .SetValid(ICacheMemWriteEnable),
 			 .ClearValid(1'b0),
@@ -274,6 +276,8 @@ module icache
   
   // truncate the offset from PCPF for memory address generation
 
+  assign SRAMWayWriteEnable = ICacheMemWriteEnable ? VictimWay : '0;
+
   icachefsm #(.BLOCKLEN(BLOCKLEN)) 
   controller(.clk,
 	     .reset,
@@ -293,7 +297,8 @@ module icache
 	     .CntEn,
 	     .CntReset,
 	     .SelAdr,
-    	     .SavePC
+    	     .SavePC,
+	     .LRUWriteEn
 	     );
 
   // For now, assume no writes to executable memory
diff --git a/wally-pipelined/src/cache/icachefsm.sv b/wally-pipelined/src/cache/icachefsm.sv
index d020d6967..49cbcb434 100644
--- a/wally-pipelined/src/cache/icachefsm.sv
+++ b/wally-pipelined/src/cache/icachefsm.sv
@@ -61,7 +61,8 @@ module icachefsm #(parameter BLOCKLEN = 256)
    output logic       CntEn,
    output logic       CntReset,
    output logic [1:0] SelAdr,
-   output logic       SavePC
+   output logic       SavePC,
+   output logic       LRUWriteEn
    );
 
   // FSM states
@@ -134,7 +135,7 @@ module icachefsm #(parameter BLOCKLEN = 256)
     ICacheReadEn = 1'b0;
     SavePC = 1'b0;
     ICacheStallF = 1'b1;
-    
+    LRUWriteEn = 1'b0;
     case (CurrState)
       STATE_READY: begin
         SelAdr = 2'b00;
@@ -144,6 +145,7 @@ module icachefsm #(parameter BLOCKLEN = 256)
         end else if (hit & ~spill) begin
           SavePC = 1'b1;
           ICacheStallF = 1'b0;
+	  LRUWriteEn = 1'b1;
 	  if(StallF) begin
 	    NextState = STATE_CPU_BUSY;
 	    SelAdr = 2'b01;
@@ -153,7 +155,8 @@ module icachefsm #(parameter BLOCKLEN = 256)
         end else if (hit & spill) begin
           spillSave = 1'b1;
           SelAdr = 2'b10;
-          NextState = STATE_HIT_SPILL;
+          LRUWriteEn = 1'b1;
+	  NextState = STATE_HIT_SPILL;
         end else if (~hit & ~spill) begin
           CntReset = 1'b1;
           NextState = STATE_MISS_FETCH_WDV;
@@ -209,6 +212,8 @@ module icachefsm #(parameter BLOCKLEN = 256)
         UnalignedSelect = 1'b1;
         SavePC = 1'b1;
         ICacheStallF = 1'b0;
+	LRUWriteEn = 1'b1;
+	
 	if(StallF) begin
 	  NextState = STATE_CPU_BUSY_SPILL;
 	  SelAdr = 2'b10;
@@ -242,6 +247,7 @@ module icachefsm #(parameter BLOCKLEN = 256)
         //SelAdr = 2'b01;
         ICacheReadEn = 1'b1;
 	ICacheStallF = 1'b0;
+	LRUWriteEn = 1'b1;
 	if(StallF) begin
 	  SelAdr = 2'b01;
 	  NextState = STATE_CPU_BUSY;
@@ -268,7 +274,8 @@ module icachefsm #(parameter BLOCKLEN = 256)
       end
       STATE_MISS_SPILL_READ1: begin // always be a hit as we just wrote that cache block.
         SelAdr = 2'b01;	 // there is a 1 cycle delay after setting the address before the date arrives.
-        ICacheReadEn = 1'b1;	
+        ICacheReadEn = 1'b1;
+	LRUWriteEn = 1'b1;
         NextState = STATE_MISS_SPILL_2;
       end
       STATE_MISS_SPILL_2: begin
@@ -288,6 +295,7 @@ module icachefsm #(parameter BLOCKLEN = 256)
           UnalignedSelect = 1'b1;
           SavePC = 1'b1;
           ICacheStallF = 1'b0;
+	  LRUWriteEn = 1'b1;
 	  if(StallF) begin
 	    NextState = STATE_CPU_BUSY;
 	    SelAdr = 2'b01;
@@ -323,6 +331,7 @@ module icachefsm #(parameter BLOCKLEN = 256)
         UnalignedSelect = 1'b1;
         SavePC = 1'b1;
         ICacheStallF = 1'b0;	
+	LRUWriteEn = 1'b1;
 	if(StallF) begin
 	  NextState = STATE_CPU_BUSY_SPILL;
 	  SelAdr = 2'b10;

From 5bc90ef32fdf952a933bfb34be309a26f81072f4 Mon Sep 17 00:00:00 2001
From: Ross Thompson <ross1728@gmail.com>
Date: Wed, 8 Sep 2021 10:40:46 -0500
Subject: [PATCH 13/16] Slight modification to wave file.

---
 wally-pipelined/regression/wave.do | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/wally-pipelined/regression/wave.do b/wally-pipelined/regression/wave.do
index 042178cd2..6d38b8853 100644
--- a/wally-pipelined/regression/wave.do
+++ b/wally-pipelined/regression/wave.do
@@ -189,7 +189,9 @@ add wave -noupdate -group divider /testbench/dut/hart/mdu/genblk1/div/rem0
 add wave -noupdate -expand -group icache -color Gold /testbench/dut/hart/ifu/icache/controller/CurrState
 add wave -noupdate -expand -group icache /testbench/dut/hart/ifu/icache/BasePAdrF
 add wave -noupdate -expand -group icache /testbench/dut/hart/ifu/icache/WayHit
-add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} /testbench/dut/hart/ifu/icache/VictimWay
+add wave -noupdate -expand -group icache /testbench/dut/hart/ifu/icache/genblk1/cachereplacementpolicy/BlockReplacementBits
+add wave -noupdate -expand -group icache /testbench/dut/hart/ifu/icache/genblk1/cachereplacementpolicy/EncVicWay
+add wave -noupdate -expand -group icache /testbench/dut/hart/ifu/icache/VictimWay
 add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way0 {/testbench/dut/hart/ifu/icache/MemWay[0]/WriteEnable}
 add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way0 {/testbench/dut/hart/ifu/icache/MemWay[0]/SetValid}
 add wave -noupdate -expand -group icache -expand -group {Cache SRAM writes} -expand -group way0 -label TAG {/testbench/dut/hart/ifu/icache/MemWay[0]/CacheTagMem/StoredData}

From 8141a515bbe66a5e3337b9bb5213b522c5c15342 Mon Sep 17 00:00:00 2001
From: Ross Thompson <ross1728@gmail.com>
Date: Wed, 8 Sep 2021 12:52:49 -0500
Subject: [PATCH 14/16] Changed configs to support 4 ways set associative
 caches.

---
 wally-pipelined/config/buildroot/wally-config.vh | 4 ++--
 wally-pipelined/config/rv32ic/wally-config.vh    | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/wally-pipelined/config/buildroot/wally-config.vh b/wally-pipelined/config/buildroot/wally-config.vh
index 6b7b8655b..f6373e1a4 100644
--- a/wally-pipelined/config/buildroot/wally-config.vh
+++ b/wally-pipelined/config/buildroot/wally-config.vh
@@ -58,10 +58,10 @@
 // Cache configuration.  Sizes should be a power of two
 // typical configuration 4 ways, 4096 bytes per way, 256 bit or more blocks
 `define DCACHE_NUMWAYS 4
-`define DCACHE_WAYSIZEINBYTES 2048
+`define DCACHE_WAYSIZEINBYTES 4096
 `define DCACHE_BLOCKLENINBITS 256
 `define DCACHE_REPLBITS 3
-`define ICACHE_NUMWAYS 1
+`define ICACHE_NUMWAYS 4
 `define ICACHE_WAYSIZEINBYTES 4096
 `define ICACHE_BLOCKLENINBITS 256
 
diff --git a/wally-pipelined/config/rv32ic/wally-config.vh b/wally-pipelined/config/rv32ic/wally-config.vh
index d04afaef1..6325a2363 100644
--- a/wally-pipelined/config/rv32ic/wally-config.vh
+++ b/wally-pipelined/config/rv32ic/wally-config.vh
@@ -56,10 +56,10 @@
 // Cache configuration.  Sizes should be a power of two
 // typical configuration 4 ways, 4096 bytes per way, 256 bit or more blocks
 `define DCACHE_NUMWAYS 4
-`define DCACHE_WAYSIZEINBYTES 2048
+`define DCACHE_WAYSIZEINBYTES 4096
 `define DCACHE_BLOCKLENINBITS 256
 `define DCACHE_REPLBITS 3
-`define ICACHE_NUMWAYS 1
+`define ICACHE_NUMWAYS 4
 `define ICACHE_WAYSIZEINBYTES 4096
 `define ICACHE_BLOCKLENINBITS 256
 

From 30e2ec39872f3e5ecde6923434219994004c3b62 Mon Sep 17 00:00:00 2001
From: David Harris <david_harris@hmc.edu>
Date: Wed, 8 Sep 2021 15:59:40 -0400
Subject: [PATCH 15/16] Added testbench-arch for riscv-arch-test suite

---
 .../{exe2memfile0.pl => exe2memfile.pl.old}   |   0
 wally-pipelined/regression/wally-arch.do      |  50 +
 wally-pipelined/src/generic/or_rows.sv        |  15 +-
 wally-pipelined/testbench/testbench-arch.sv   | 930 ++++++++++++++++++
 4 files changed, 983 insertions(+), 12 deletions(-)
 rename wally-pipelined/bin/{exe2memfile0.pl => exe2memfile.pl.old} (100%)
 create mode 100644 wally-pipelined/regression/wally-arch.do
 create mode 100644 wally-pipelined/testbench/testbench-arch.sv

diff --git a/wally-pipelined/bin/exe2memfile0.pl b/wally-pipelined/bin/exe2memfile.pl.old
similarity index 100%
rename from wally-pipelined/bin/exe2memfile0.pl
rename to wally-pipelined/bin/exe2memfile.pl.old
diff --git a/wally-pipelined/regression/wally-arch.do b/wally-pipelined/regression/wally-arch.do
new file mode 100644
index 000000000..06b50079f
--- /dev/null
+++ b/wally-pipelined/regression/wally-arch.do
@@ -0,0 +1,50 @@
+# wally-arch.do 
+#
+# Modification by Oklahoma State University & Harvey Mudd College
+# Use with Testbench 
+# James Stine, 2008; David Harris 2021
+# Go Cowboys!!!!!!
+#
+# Takes 1:10 to run RV64IC tests using gui
+
+# Use this wally-arch.do file to run this example.
+# Either bring up ModelSim and type the following at the "ModelSim>" prompt:
+#     do wally-arch.do
+# or, to run from a shell, type the following at the shell prompt:
+#     vsim -do wally-arch.do -c
+# (omit the "-c" to see the GUI while running from the shell)
+
+onbreak {resume}
+
+# create library
+if [file exists work] {
+    vdel -all
+}
+vlib work
+
+# compile source files
+# suppress spurious warnngs about 
+# "Extra checking for conflicts with always_comb done at vopt time"
+# because vsim will run vopt
+
+# default to config/rv64ic, but allow this to be overridden at the command line.  For example:
+# do wally-pipelined.do ../config/rv32ic
+switch $argc {
+    0 {vlog +incdir+../config/rv64ic +incdir+../config/shared ../testbench/testbench-arch.sv ../testbench/common/*.sv ../src/*/*.sv -suppress 2583}
+    1 {vlog +incdir+$1  +incdir+../config/shared ../testbench/testbench-arch.sv ../testbench/common/*.sv ../src/*/*.sv -suppress 2583}
+}
+# start and run simulation
+# remove +acc flag for faster sim during regressions if there is no need to access internal signals
+vopt +acc work.testbench -o workopt 
+vsim workopt
+
+view wave
+-- display input and output signals as hexidecimal values
+do ./wave-dos/peripheral-waves.do
+
+-- Run the Simulation 
+#run 5000 
+run -all
+#quit
+noview ../testbench/testbench-arch.sv
+view wave
diff --git a/wally-pipelined/src/generic/or_rows.sv b/wally-pipelined/src/generic/or_rows.sv
index 4c74f62e9..c29528e48 100644
--- a/wally-pipelined/src/generic/or_rows.sv
+++ b/wally-pipelined/src/generic/or_rows.sv
@@ -33,23 +33,14 @@ module or_rows #(parameter ROWS = 8, COLS=2) (
   input  var logic [COLS-1:0] a[ROWS-1:0],
   output logic [COLS-1:0] y); 
 
-  logic [COLS-1:0] mid[ROWS-1:0];
+  logic [COLS-1:0] mid[ROWS-1:1];
   genvar row, col;
   generate
     assign mid[1] = a[0] | a[1];
     for (row=2; row < ROWS; row++)
       assign mid[row] = mid[row-1] | a[row];
-    assign y = mid[ROWS-1];
-    
-    /*
-      for (col = 0; col < COLS; col++) begin
-          assign mid[1][col] = a[0][col] | a[1][col];
-          for (row=2; row < ROWS; row++)
-            assign mid[row][col] = mid[row-1][col] | a[row][col];
-          assign y[col] = mid[ROWS-1][col];
-      end
-      */
-  endgenerate
+    assign y = mid[ROWS-1];   
+   endgenerate
 endmodule
 
 /* verilator lint_on UNOPTFLAT */
diff --git a/wally-pipelined/testbench/testbench-arch.sv b/wally-pipelined/testbench/testbench-arch.sv
new file mode 100644
index 000000000..22ed0bef9
--- /dev/null
+++ b/wally-pipelined/testbench/testbench-arch.sv
@@ -0,0 +1,930 @@
+///////////////////////////////////////////
+// testbench-imperas.sv
+//
+// Written: David_Harris@hmc.edu 9 January 2021
+// Modified: 
+//
+// Purpose: Wally Testbench and helper modules
+//          Applies test programs from the Imperas suite
+// 
+// A component of the Wally configurable RISC-V project.
+// 
+// Copyright (C) 2021 Harvey Mudd College & Oklahoma State University
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, 
+// modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software 
+// is furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 
+// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT 
+// OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+///////////////////////////////////////////
+
+`include "wally-config.vh"
+
+module testbench();
+  parameter DEBUG = 0;
+  parameter TESTSPERIPH = 0; // set to 0 for regression
+  parameter TESTSPRIV = 0; // set to 0 for regression
+  
+  logic        clk;
+  logic        reset;
+
+  parameter SIGNATURESIZE = 5000000;
+
+  int test, i, errors, totalerrors;
+  logic [31:0] sig32[0:SIGNATURESIZE];
+  logic [`XLEN-1:0] signature[0:SIGNATURESIZE];
+  logic [`XLEN-1:0] testadr;
+  string InstrFName, InstrDName, InstrEName, InstrMName, InstrWName;
+  logic [31:0] InstrW;
+  logic [`XLEN-1:0] meminit;
+
+  string tests32mmu[] = '{
+    "rv32mmu/WALLY-MMU-SV32", "3000"
+    //"rv32mmu/WALLY-PMA", "3000",
+    //"rv32mmu/WALLY-PMA", "3000"
+    };
+
+  string tests64mmu[] = '{
+    "rv64mmu/WALLY-MMU-SV48", "3000",
+    "rv64mmu/WALLY-MMU-SV39", "3000"
+    //"rv64mmu/WALLY-PMA", "3000",
+    //"rv64mmu/WALLY-PMA", "3000"
+  };
+
+  
+string tests32f[] = '{
+    "rv32f/I-FADD-S-01", "2000",
+    "rv32f/I-FCLASS-S-01", "2000",
+    "rv32f/I-FCVT-S-W-01", "2000",
+    "rv32f/I-FCVT-S-WU-01", "2000",
+    "rv32f/I-FCVT-W-S-01", "2000",
+    "rv32f/I-FCVT-WU-S-01", "2000",
+    "rv32f/I-FDIV-S-01", "2000",
+    "rv32f/I-FEQ-S-01", "2000",
+    "rv32f/I-FLE-S-01", "2000",
+    "rv32f/I-FLT-S-01", "2000",
+    "rv32f/I-FMADD-S-01", "2000",
+    "rv32f/I-FMAX-S-01", "2000",
+    "rv32f/I-FMIN-S-01", "2000",
+    "rv32f/I-FMSUB-S-01", "2000",
+    "rv32f/I-FMUL-S-01", "2000",
+    "rv32f/I-FMV-W-X-01", "2000",
+    "rv32f/I-FMV-X-W-01", "2000",
+    "rv32f/I-FNMADD-S-01", "2000",
+    "rv32f/I-FNMSUB-S-01", "2000",
+    "rv32f/I-FSGNJ-S-01", "2000",
+    "rv32f/I-FSGNJN-S-01", "2000",
+    "rv32f/I-FSGNJX-S-01", "2000",
+    "rv32f/I-FSQRT-S-01", "2000",
+    "rv32f/I-FSW-01", "2000",
+    "rv32f/I-FLW-01", "2110",
+    "rv32f/I-FSUB-S-01", "2000"
+  };
+
+  string tests64f[] = '{
+    "rv64f/I-FLW-01", "2110",
+    "rv64f/I-FMV-W-X-01", "2000",
+    "rv64f/I-FMV-X-W-01", "2000",
+    "rv64f/I-FSW-01", "2000",
+    "rv64f/I-FCLASS-S-01", "2000",
+    "rv64f/I-FADD-S-01", "2000",
+//    "rv64f/I-FCVT-S-L-01", "2000",
+//    "rv64f/I-FCVT-S-LU-01", "2000",
+//    "rv64f/I-FCVT-S-W-01", "2000",
+//    "rv64f/I-FCVT-S-WU-01", "2000",
+    "rv64f/I-FCVT-L-S-01", "2000",
+    "rv64f/I-FCVT-LU-S-01", "2000",
+    "rv64f/I-FCVT-W-S-01", "2000",
+    "rv64f/I-FCVT-WU-S-01", "2000",
+    "rv64f/I-FDIV-S-01", "2000",
+    "rv64f/I-FEQ-S-01", "2000",
+    "rv64f/I-FLE-S-01", "2000",
+    "rv64f/I-FLT-S-01", "2000",
+    "rv64f/I-FMADD-S-01", "2000",
+    "rv64f/I-FMAX-S-01", "2000",
+    "rv64f/I-FMIN-S-01", "2000",
+    "rv64f/I-FMSUB-S-01", "2000",
+    "rv64f/I-FMUL-S-01", "2000",
+    "rv64f/I-FNMADD-S-01", "2000",
+    "rv64f/I-FNMSUB-S-01", "2000",
+    "rv64f/I-FSGNJ-S-01", "2000",
+    "rv64f/I-FSGNJN-S-01", "2000",
+    "rv64f/I-FSGNJX-S-01", "2000",
+    "rv64f/I-FSQRT-S-01", "2000",
+    "rv64f/I-FSUB-S-01", "2000"
+  };
+
+  string tests64d[] = '{
+    "rv64d/I-FSD-01", "2000",
+    "rv64d/I-FLD-01", "2420",
+    "rv64d/I-FMV-X-D-01", "2000",
+    "rv64d/I-FMV-D-X-01", "2000",
+    "rv64d/I-FDIV-D-01", "2000",
+    "rv64d/I-FNMADD-D-01", "2000",
+    "rv64d/I-FNMSUB-D-01", "2000",
+    "rv64d/I-FMSUB-D-01", "2000",
+    "rv64d/I-FMAX-D-01", "2000",
+    "rv64d/I-FMIN-D-01", "2000",
+    "rv64d/I-FLE-D-01", "2000",
+    "rv64d/I-FLT-D-01", "2000",
+    "rv64d/I-FEQ-D-01", "2000",
+    "rv64d/I-FADD-D-01", "2000",
+    "rv64d/I-FCLASS-D-01", "2000",
+    "rv64d/I-FMADD-D-01", "2000",
+    "rv64d/I-FMUL-D-01", "2000",
+    "rv64d/I-FSGNJ-D-01", "2000",
+    "rv64d/I-FSGNJN-D-01", "2000",
+    "rv64d/I-FSGNJX-D-01", "2000",
+    "rv64d/I-FSQRT-D-01", "2000",
+    "rv64d/I-FSUB-D-01", "2000",
+//    "rv64d/I-FCVT-D-L-01", "2000",
+//    "rv64d/I-FCVT-D-LU-01", "2000",
+    "rv64d/I-FCVT-D-S-01", "2000", 
+//    "rv64d/I-FCVT-D-W-01", "2000",
+//    "rv64d/I-FCVT-D-WU-01", "2000",
+    "rv64d/I-FCVT-L-D-01", "2000",
+    "rv64d/I-FCVT-LU-D-01", "2000",
+    "rv64d/I-FCVT-S-D-01", "2000", 
+    "rv64d/I-FCVT-W-D-01", "2000",
+    "rv64d/I-FCVT-WU-D-01", "2000"
+};
+
+  string tests64a[] = '{
+    "rv64a/WALLY-AMO", "2110",
+    "rv64a/WALLY-LRSC", "2110"
+  };
+
+  string tests64m[] = '{
+    "rv64m/I-MUL-01", "3000",
+    "rv64m/I-MULH-01", "3000",
+    "rv64m/I-MULHSU-01", "3000",
+    "rv64m/I-MULHU-01", "3000",
+    "rv64m/I-MULW-01", "3000",
+    "rv64m/I-DIV-01", "3000",
+    "rv64m/I-DIVU-01", "3000",
+    "rv64m/I-DIVUW-01", "3000",
+    "rv64m/I-DIVW-01", "3000",
+    "rv64m/I-REM-01", "3000",
+    "rv64m/I-REMU-01", "3000",
+    "rv64m/I-REMUW-01", "3000",
+    "rv64m/I-REMW-01", "3000"
+  };
+
+  string tests64ic[] = '{
+    "rv64ic/I-C-ADD-01", "3000",
+    "rv64ic/I-C-ADDI-01", "3000",
+    "rv64ic/I-C-ADDIW-01", "3000",
+    "rv64ic/I-C-ADDW-01", "3000",
+    "rv64ic/I-C-AND-01", "3000",
+    "rv64ic/I-C-ANDI-01", "3000",
+    "rv64ic/I-C-BEQZ-01", "3000",
+    "rv64ic/I-C-BNEZ-01", "3000",
+    "rv64ic/I-C-EBREAK-01", "2000",
+    "rv64ic/I-C-J-01", "3000",
+    "rv64ic/I-C-JALR-01", "4000",
+    "rv64ic/I-C-JR-01", "4000",
+    "rv64ic/I-C-LD-01", "3420",
+    "rv64ic/I-C-LDSP-01", "3420",
+    "rv64ic/I-C-LI-01", "3000",
+    "rv64ic/I-C-LUI-01", "2000",
+    "rv64ic/I-C-LW-01", "3110",
+    "rv64ic/I-C-LWSP-01", "3110",
+    "rv64ic/I-C-MV-01", "3000",
+    "rv64ic/I-C-NOP-01", "2000",
+    "rv64ic/I-C-OR-01", "3000",
+    "rv64ic/I-C-SD-01", "3000",
+    "rv64ic/I-C-SDSP-01", "3000",
+    "rv64ic/I-C-SLLI-01", "3000",
+    "rv64ic/I-C-SRAI-01", "3000",
+    "rv64ic/I-C-SRLI-01", "3000",
+    "rv64ic/I-C-SUB-01", "3000",
+    "rv64ic/I-C-SUBW-01", "3000",
+    "rv64ic/I-C-SW-01", "3000",
+    "rv64ic/I-C-SWSP-01", "3000",
+    "rv64ic/I-C-XOR-01", "3000"
+  };
+
+  string tests64iNOc[] = {
+    "rv64i/I-MISALIGN_JMP-01","2000"
+  };
+
+  string tests64i[] = '{
+    "rv64i_m/I/add-01", "9010",
+    "rv64i_m/I/addi-01", "6010",
+    "rv64i_m/I/addiw-01", "6010",
+    "rv64i_m/I/addw-01", "9010",
+    "rv64i_m/I/and-01", "9010",
+    "rv64i_m/I/andi-01", "6010",
+    "rv64i_m/I/auipc-01", "2010",
+    "rv64i_m/I/beq-01", "47010",
+    "rv64i_m/I/bge-01", "47010",
+    "rv64i_m/I/bgeu-01", "56010",
+    "rv64i_m/I/blt-01", "4d010",
+    "rv64i_m/I/bltu-01", "57010",
+    "rv64i_m/I/bne-01", "43010",
+    "rv64i_m/I/fence-01", "2010",
+    "rv64i_m/I/jal-01", "122010",
+    "rv64i_m/I/jalr-01", "2010",
+    "rv64i_m/I/lb-align-01", "2010",
+    "rv64i_m/I/lbu-align-01", "2010",
+    "rv64i_m/I/ld-align-01", "2010",
+    "rv64i_m/I/lh-align-01", "2010",
+    "rv64i_m/I/lhu-align-01", "2010",
+    "rv64i_m/I/lui-01", "2010",
+    "rv64i_m/I/lw-align-01", "2010",
+    "rv64i_m/I/lwu-align-01", "2010",
+    "rv64i_m/I/or-01", "9010",
+    "rv64i_m/I/ori-01", "6010",
+    "rv64i_m/I/sb-align-01", "3010",
+    "rv64i_m/I/sd-align-01", "3010",
+    "rv64i_m/I/sh-align-01", "3010",
+    "rv64i_m/I/sll-01", "3010",
+    "rv64i_m/I/slli-01", "2010",
+    "rv64i_m/I/slliw-01", "2010",
+    "rv64i_m/I/sllw-01", "3010",
+    "rv64i_m/I/slt-01", "9010",
+    "rv64i_m/I/slti-01", "6010",
+    "rv64i_m/I/sltiu-01", "6010",
+    "rv64i_m/I/sltu-01", "a010",
+    "rv64i_m/I/sra-01", "3010",
+    "rv64i_m/I/srai-01", "2010",
+    "rv64i_m/I/sraiw-01", "2010",
+    "rv64i_m/I/sraw-01", "3010",
+    "rv64i_m/I/srl-01", "3010",
+    "rv64i_m/I/srli-01", "2010",
+    "rv64i_m/I/srliw-01", "2010",
+    "rv64i_m/I/srlw-01", "3010",
+    "rv64i_m/I/sub-01", "9010",
+    "rv64i_m/I/subw-01", "9010",
+    "rv64i_m/I/sw-align-01", "3010",
+    "rv64i_m/I/xor-01", "9010",
+    "rv64i_m/I/xori-01", "6010"
+    
+ /*   //"rv64i/WALLY-PIPELINE-100K", "f7ff0",
+    "rv64i/I-ADD-01", "3000",
+    "rv64i/I-ADDI-01", "3000",
+    "rv64i/I-ADDIW-01", "3000",
+    "rv64i/I-ADDW-01", "3000",
+    "rv64i/I-AND-01", "3000",
+    "rv64i/I-ANDI-01", "3000",
+    "rv64i/I-AUIPC-01", "3000",
+    "rv64i/I-BEQ-01", "4000",
+    "rv64i/I-BGE-01", "4000",
+    "rv64i/I-BGEU-01", "4000",
+    "rv64i/I-BLT-01", "4000",
+    "rv64i/I-BLTU-01", "4000",
+    "rv64i/I-BNE-01", "4000",
+    "rv64i/I-DELAY_SLOTS-01", "2000",
+    "rv64i/I-EBREAK-01", "2000",
+    "rv64i/I-ECALL-01", "2000",
+    "rv64i/I-ENDIANESS-01", "2010",
+    "rv64i/I-IO-01", "2050",
+    "rv64i/I-JAL-01", "3000",
+    "rv64i/I-JALR-01", "4000",
+    "rv64i/I-LB-01", "4020",
+    "rv64i/I-LBU-01", "4020",
+    "rv64i/I-LD-01", "4420",
+    "rv64i/I-LH-01", "4050",
+    "rv64i/I-LHU-01", "4050",
+    "rv64i/I-LUI-01", "2000",
+    "rv64i/I-LW-01", "4110",
+    "rv64i/I-LWU-01", "4110", 
+    "rv64i/I-MISALIGN_LDST-01", "2010",
+    "rv64i/I-NOP-01", "2000",
+    "rv64i/I-OR-01", "3000",
+    "rv64i/I-ORI-01", "3000",
+    "rv64i/I-RF_size-01", "2000",
+    "rv64i/I-RF_width-01", "2000",
+    "rv64i/I-RF_x0-01", "2010",
+    "rv64i/I-SB-01", "4000",
+    "rv64i/I-SD-01", "4000",
+    "rv64i/I-SH-01", "4000",
+    "rv64i/I-SLL-01", "3000",
+    "rv64i/I-SLLI-01", "3000",
+    "rv64i/I-SLLIW-01", "3000",
+    "rv64i/I-SLLW-01", "3000",
+    "rv64i/I-SLT-01", "3000",
+    "rv64i/I-SLTI-01", "3000",
+    "rv64i/I-SLTIU-01", "3000",
+    "rv64i/I-SLTU-01", "3000",
+    "rv64i/I-SRA-01", "3000",
+    "rv64i/I-SRAI-01", "3000",
+    "rv64i/I-SRAIW-01", "3000",
+    "rv64i/I-SRAW-01", "3000",
+    "rv64i/I-SRL-01", "3000",
+    "rv64i/I-SRLI-01", "3000",
+    "rv64i/I-SRLIW-01", "3000",
+    "rv64i/I-SRLW-01", "3000",
+    "rv64i/I-SUB-01", "3000",
+    "rv64i/I-SUBW-01", "3000",
+    "rv64i/I-SW-01", "4000",
+    "rv64i/I-XOR-01", "3000",
+    "rv64i/I-XORI-01", "3000",
+    "rv64i/WALLY-ADD", "4000",
+    "rv64i/WALLY-SUB", "4000",
+    "rv64i/WALLY-ADDI", "3000",
+    "rv64i/WALLY-ANDI", "3000",
+    "rv64i/WALLY-ORI", "3000",
+    "rv64i/WALLY-XORI", "3000",
+    "rv64i/WALLY-SLTI", "3000",
+    "rv64i/WALLY-SLTIU", "3000",
+    "rv64i/WALLY-SLLI", "3000",
+    "rv64i/WALLY-SRLI", "3000",
+    "rv64i/WALLY-SRAI", "3000",
+    "rv64i/WALLY-JAL", "4000",
+    "rv64i/WALLY-JALR", "3000",
+    "rv64i/WALLY-STORE", "3000",
+    "rv64i/WALLY-ADDIW", "3000",
+    "rv64i/WALLY-SLLIW", "3000",
+    "rv64i/WALLY-SRLIW", "3000",
+    "rv64i/WALLY-SRAIW", "3000",
+    "rv64i/WALLY-ADDW", "4000",
+    "rv64i/WALLY-SUBW", "4000",
+    "rv64i/WALLY-SLLW", "3000",
+    "rv64i/WALLY-SRLW", "3000",
+    "rv64i/WALLY-SRAW", "3000",
+    "rv64i/WALLY-BEQ" ,"5000",
+    "rv64i/WALLY-BNE", "5000 ",
+    "rv64i/WALLY-BLTU", "5000 ",
+    "rv64i/WALLY-BLT", "5000",
+    "rv64i/WALLY-BGE", "5000 ",
+    "rv64i/WALLY-BGEU", "5000 ",
+    "rv64i/WALLY-CSRRW", "4000",
+    "rv64i/WALLY-CSRRS", "4000",
+    "rv64i/WALLY-CSRRC", "5000",
+    "rv64i/WALLY-CSRRWI", "4000",
+    "rv64i/WALLY-CSRRSI", "4000",
+    "rv64i/WALLY-CSRRCI", "4000" */
+  };
+
+  string tests32a[] = '{
+    "rv32a/WALLY-AMO", "2110",
+    "rv32a/WALLY-LRSC", "2110"
+  };
+
+  string tests32m[] = '{
+    "rv32m/I-MUL-01", "2000",
+    "rv32m/I-MULH-01", "2000",
+    "rv32m/I-MULHSU-01", "2000",
+    "rv32m/I-MULHU-01", "2000",
+    "rv32m/I-DIV-01", "2000",
+    "rv32m/I-DIVU-01", "2000",
+    "rv32m/I-REM-01", "2000",
+    "rv32m/I-REMU-01", "2000"
+  };
+
+  string tests32ic[] = '{
+    "rv32ic/I-C-ADD-01", "2000",
+    "rv32ic/I-C-ADDI-01", "2000",
+    "rv32ic/I-C-AND-01", "2000",
+    "rv32ic/I-C-ANDI-01", "2000",
+    "rv32ic/I-C-BEQZ-01", "2000",
+    "rv32ic/I-C-BNEZ-01", "2000",
+    "rv32ic/I-C-EBREAK-01", "2000",
+    "rv32ic/I-C-J-01", "2000",
+    "rv32ic/I-C-JALR-01", "3000",
+    "rv32ic/I-C-JR-01", "3000",
+    "rv32ic/I-C-LI-01", "2000",
+    "rv32ic/I-C-LUI-01", "2000",
+    "rv32ic/I-C-LW-01", "2110",
+    "rv32ic/I-C-LWSP-01", "2110",
+    "rv32ic/I-C-MV-01", "2000",
+    "rv32ic/I-C-NOP-01", "2000",
+    "rv32ic/I-C-OR-01", "2000",
+    "rv32ic/I-C-SLLI-01", "2000",
+    "rv32ic/I-C-SRAI-01", "2000",
+    "rv32ic/I-C-SRLI-01", "2000",
+    "rv32ic/I-C-SUB-01", "2000",
+    "rv32ic/I-C-SW-01", "2000",
+    "rv32ic/I-C-SWSP-01", "2000",
+    "rv32ic/I-C-XOR-01", "2000"
+  };
+
+  string tests32iNOc[] = {
+    "rv32i/I-MISALIGN_JMP-01","2000"
+  };
+
+  string tests32i[] = {
+    //"rv32i/WALLY-PIPELINE-100K", "10a800",
+    "rv32i/I-ADD-01", "2000",
+    "rv32i/I-ADDI-01","2000",
+    "rv32i/I-AND-01","2000",
+    "rv32i/I-ANDI-01","2000",
+    "rv32i/I-AUIPC-01","2000",
+    "rv32i/I-BEQ-01","3000",
+    "rv32i/I-BGE-01","3000",
+    "rv32i/I-BGEU-01","3000",
+    "rv32i/I-BLT-01","3000",
+    "rv32i/I-BLTU-01","3000",
+    "rv32i/I-BNE-01","3000",
+    "rv32i/I-DELAY_SLOTS-01","2000",
+    "rv32i/I-EBREAK-01","2000",
+    "rv32i/I-ECALL-01","2000",
+    "rv32i/I-ENDIANESS-01","2010",
+    "rv32i/I-IO-01","2030rv",
+    "rv32i/I-JAL-01","3000",
+    "rv32i/I-JALR-01","3000",
+    "rv32i/I-LB-01","3020",
+    "rv32i/I-LBU-01","3020",
+    "rv32i/I-LH-01","3050",
+    "rv32i/I-LHU-01","3050",
+    "rv32i/I-LUI-01","2000",
+    "rv32i/I-LW-01","3110",
+    "rv32i/I-MISALIGN_LDST-01","2010",
+    "rv32i/I-NOP-01","2000",
+    "rv32i/I-OR-01","2000",
+    "rv32i/I-ORI-01","2000",
+    "rv32i/I-RF_size-01","2000",
+    "rv32i/I-RF_width-01","2000",
+    "rv32i/I-RF_x0-01","2010",
+    "rv32i/I-SB-01","3000",
+    "rv32i/I-SH-01","3000",
+    "rv32i/I-SLL-01","2000",
+    "rv32i/I-SLLI-01","2000",
+    "rv32i/I-SLT-01","2000",
+    "rv32i/I-SLTI-01","2000",
+    "rv32i/I-SLTIU-01","2000",
+    "rv32i/I-SLTU-01","2000",
+    "rv32i/I-SRA-01","2000",
+    "rv32i/I-SRAI-01","2000",
+    "rv32i/I-SRL-01","2000",
+    "rv32i/I-SRLI-01","2000",
+    "rv32i/I-SUB-01","2000",
+    "rv32i/I-SW-01","3000",
+    "rv32i/I-XOR-01","2000",
+    "rv32i/I-XORI-01","2000",
+    "rv32i/WALLY-ADD", "3000",
+    "rv32i/WALLY-SUB", "3000",
+    "rv32i/WALLY-ADDI", "2000",
+    "rv32i/WALLY-ANDI", "2000",
+    "rv32i/WALLY-ORI", "2000",
+    "rv32i/WALLY-XORI", "2000",
+    "rv32i/WALLY-SLTI", "2000",
+    "rv32i/WALLY-SLTIU", "2000",
+    "rv32i/WALLY-SLLI", "2000",
+    "rv32i/WALLY-SRLI", "2000",
+    "rv32i/WALLY-SRAI", "2000",
+    "rv32i/WALLY-LOAD", "11c00",
+    "rv32i/WALLY-SUB", "3000",
+    "rv32i/WALLY-STORE", "2000",
+    "rv32i/WALLY-JAL", "3000",
+    "rv32i/WALLY-JALR", "2000",
+    "rv32i/WALLY-BEQ" ,"4000",
+    "rv32i/WALLY-BNE", "4000 ",
+    "rv32i/WALLY-BLTU", "4000 ",
+    "rv32i/WALLY-BLT", "4000",
+    "rv32i/WALLY-BGE", "4000 ",
+    "rv32i/WALLY-BGEU", "4000 ",
+    "rv32i/WALLY-CSRRW", "3000",
+    "rv32i/WALLY-CSRRS", "3000",
+    "rv32i/WALLY-CSRRC", "4000",
+    "rv32i/WALLY-CSRRWI", "3000",
+    "rv32i/WALLY-CSRRSI", "3000",
+    "rv32i/WALLY-CSRRCI", "3000"
+  };
+
+  string testsBP64[] = '{
+    "rv64BP/simple", "10000",
+    "rv64BP/mmm", "1000000",
+    "rv64BP/linpack_bench", "1000000",
+    "rv64BP/sieve", "1000000",
+    "rv64BP/qsort", "1000000",
+    "rv64BP/dhrystone", "1000000"
+  };
+
+  string tests64p[] = '{
+    "rv64p/WALLY-MSTATUS", "2000",
+    "rv64p/WALLY-MCAUSE", "3000",
+    "rv64p/WALLY-SCAUSE", "2000",
+    "rv64p/WALLY-MEPC", "5000",
+    "rv64p/WALLY-SEPC", "4000",
+    "rv64p/WALLY-MTVAL", "6000",
+    "rv64p/WALLY-STVAL", "4000",
+    "rv64p/WALLY-MTVEC", "2000",
+    "rv64p/WALLY-STVEC", "2000",
+    "rv64p/WALLY-MARCHID", "4000",
+    "rv64p/WALLY-MIMPID", "4000",
+    "rv64p/WALLY-MHARTID", "4000",
+    "rv64p/WALLY-MVENDORID", "4000",
+    "rv64p/WALLY-MIE", "3000",
+    "rv64p/WALLY-MEDELEG", "4000",
+    "rv64p/WALLY-IP", "2000",
+    "rv64p/WALLY-CSR-PERMISSIONS-M", "5000",
+    "rv64p/WALLY-CSR-PERMISSIONS-S", "3000"
+  };
+
+  string tests32p[] = '{
+    "rv32p/WALLY-MSTATUS", "2000",
+    "rv32p/WALLY-MCAUSE", "3000",
+    "rv32p/WALLY-SCAUSE", "2000",
+    "rv32p/WALLY-MEPC", "5000",
+    "rv32p/WALLY-SEPC", "4000",
+    "rv32p/WALLY-MTVAL", "5000",
+    "rv32p/WALLY-STVAL", "4000",
+    "rv32p/WALLY-MARCHID", "4000",
+    "rv32p/WALLY-MIMPID", "4000",
+    "rv32p/WALLY-MHARTID", "4000",
+    "rv32p/WALLY-MVENDORID", "4000",
+    "rv32p/WALLY-MTVEC", "2000",
+    "rv32p/WALLY-STVEC", "2000",
+    "rv32p/WALLY-MIE", "3000",
+    "rv32p/WALLY-MEDELEG", "4000",
+    "rv32p/WALLY-IP", "3000",
+    "rv32p/WALLY-CSR-PERMISSIONS-M", "5000",
+    "rv32p/WALLY-CSR-PERMISSIONS-S", "3000"
+  };
+
+  string tests64periph[] = '{
+    "rv64i-periph/WALLY-PERIPH", "2000"
+  };
+
+  string tests32periph[] = '{
+    "rv32i-periph/WALLY-PLIC", "2080"
+  };
+
+   string tests[];
+  string ProgramAddrMapFile, ProgramLabelMapFile;
+  logic [`AHBW-1:0] HRDATAEXT;
+  logic             HREADYEXT, HRESPEXT;
+  logic [31:0]      HADDR;
+  logic [`AHBW-1:0] HWDATA;
+  logic             HWRITE;
+  logic [2:0]       HSIZE;
+  logic [2:0]       HBURST;
+  logic [3:0]       HPROT;
+  logic [1:0]       HTRANS;
+  logic             HMASTLOCK;
+  logic             HCLK, HRESETn;
+  logic [`XLEN-1:0] PCW;
+
+  logic 	    DCacheFlushDone, DCacheFlushStart;
+    
+  flopenr #(`XLEN) PCWReg(clk, reset, ~dut.hart.ieu.dp.StallW, dut.hart.ifu.PCM, PCW);
+  flopenr  #(32)   InstrWReg(clk, reset, ~dut.hart.ieu.dp.StallW,  dut.hart.ifu.InstrM, InstrW);
+
+  // check assertions for a legal configuration
+  riscvassertions riscvassertions();
+  logging logging(clk, reset, dut.uncore.HADDR, dut.uncore.HTRANS);
+
+  // pick tests based on modes supported
+  initial begin
+    if (`XLEN == 64) begin // RV64
+      if (`TESTSBP) begin
+        tests = testsBP64;
+	// testsbp should not run the other tests. It starts at address 0 rather than
+	// 0x8000_0000, the next if must remain an else if.	
+      end else if (TESTSPERIPH)
+        tests = tests64periph;
+      else if (TESTSPRIV)
+        tests = tests64p;
+      else begin
+        tests = {tests64i};
+/*        tests = {tests64p,tests64i, tests64periph};
+        if (`C_SUPPORTED) tests = {tests, tests64ic};
+        else              tests = {tests, tests64iNOc};
+        if (`M_SUPPORTED) tests = {tests, tests64m};
+        if (`F_SUPPORTED) tests = {tests64f, tests};
+        if (`D_SUPPORTED) tests = {tests64d, tests};
+        if (`MEM_VIRTMEM) tests = {tests64mmu, tests};
+        if (`A_SUPPORTED) tests = {tests64a, tests}; */
+      end
+      //tests = {tests64a, tests};
+    end else begin // RV32
+      // *** add the 32 bit bp tests
+      if (TESTSPERIPH)
+        tests = tests32periph;
+      else if (TESTSPRIV)
+        tests = tests32p;
+      else begin
+          tests = {tests32i, tests32p};//,tests32periph}; *** broken at the moment
+          if (`C_SUPPORTED % 2 == 1) tests = {tests, tests32ic};    
+          else                       tests = {tests, tests32iNOc};
+          if (`M_SUPPORTED % 2 == 1) tests = {tests, tests32m};
+          if (`F_SUPPORTED) tests = {tests32f, tests};
+          if (`MEM_VIRTMEM) tests = {tests32mmu, tests};
+          if (`A_SUPPORTED) tests = {tests32a, tests};
+     end
+    end
+  end
+
+  string tvroot, bootroot, signame, memfilename, romfilename;
+
+  logic [31:0] GPIOPinsIn, GPIOPinsOut, GPIOPinsEn;
+  logic UARTSin, UARTSout;
+
+  // instantiate device to be tested
+  assign GPIOPinsIn = 0;
+  assign UARTSin = 1;
+  assign HREADYEXT = 1;
+  assign HRESPEXT = 0;
+  assign HRDATAEXT = 0;
+
+  wallypipelinedsoc dut(.*); 
+
+  // Track names of instructions
+  instrTrackerTB it(clk, reset, dut.hart.ieu.dp.FlushE,
+                dut.hart.ifu.icache.controller.FinalInstrRawF,
+                dut.hart.ifu.InstrD, dut.hart.ifu.InstrE,
+                dut.hart.ifu.InstrM,  dut.hart.ifu.InstrW,
+                InstrFName, InstrDName, InstrEName, InstrMName, InstrWName);
+
+  // initialize tests
+  localparam integer 	   MemStartAddr = `TIM_BASE>>(1+`XLEN/32);
+  localparam integer 	   MemEndAddr = (`TIM_RANGE+`TIM_BASE)>>1+(`XLEN/32);
+
+  initial
+    begin
+      test = 0;
+      totalerrors = 0;
+      testadr = 0;
+      // fill memory with defined values to reduce Xs in simulation
+      // Quick note the memory will need to be initialized.  The C library does not
+      //  guarantee the  initialized reads.  For example a strcmp can read 6 byte
+      //  strings, but uses a load double to read them in.  If the last 2 bytes are
+      //  not initialized the compare results in an 'x' which propagates through 
+      // the design.
+      if (`XLEN == 32) meminit = 32'hFEDC0123;
+      else meminit = 64'hFEDCBA9876543210;
+      // *** broken because DTIM also drives RAM
+      if (`TESTSBP) begin
+	for (i=MemStartAddr; i<MemEndAddr; i = i+1) begin
+	  dut.uncore.dtim.RAM[i] = meminit;
+	end
+      end
+      // read test vectors into memory
+      bootroot = "../../imperas-riscv-tests/";
+      tvroot = "/home/harris/github/riscv-arch-test/";
+      memfilename = {tvroot, "work/", tests[test], ".elf.memfile"};
+      romfilename = {bootroot, "imperas-boottim.txt"};
+      $readmemh(memfilename, dut.uncore.dtim.RAM);
+      $readmemh(romfilename, dut.uncore.bootdtim.bootdtim.RAM);
+      ProgramAddrMapFile = {tvroot, "work/", tests[test], ".elf.objdump.addr"};
+      ProgramLabelMapFile = {tvroot, "work/", tests[test], ".elf.objdump.lab"};
+      $display("Read memfile %s", memfilename);
+      reset = 1; # 42; reset = 0;
+    end
+
+  // generate clock to sequence tests
+  always
+    begin
+      clk = 1; # 5; clk = 0; # 5;
+    end
+   
+  // check results
+  always @(negedge clk)
+    begin    
+/* -----\/----- EXCLUDED -----\/-----
+      if (dut.hart.priv.EcallFaultM && 
+			    (dut.hart.ieu.dp.regf.rf[3] == 1 || 
+			     (dut.hart.ieu.dp.regf.we3 && 
+			      dut.hart.ieu.dp.regf.a3 == 3 && 
+			      dut.hart.ieu.dp.regf.wd3 == 1))) begin
+ -----/\----- EXCLUDED -----/\----- */
+      if (DCacheFlushDone) begin
+        //$display("Code ended with ecall with gp = 1");
+
+        #600; // give time for instructions in pipeline to finish
+        // clear signature to prevent contamination from previous tests
+        for(i=0; i<SIGNATURESIZE; i=i+1) begin
+          sig32[i] = 'bx;
+        end
+
+        // read signature, reformat in 64 bits if necessary
+        signame = {tvroot, "work/", tests[test], ".signature.output"};
+        $readmemh(signame, sig32);
+        i = 0;
+        while (i < SIGNATURESIZE) begin
+          if (`XLEN == 32) begin
+            signature[i] = sig32[i];
+            i = i+1;
+          end else begin
+            signature[i/2] = {sig32[i+1], sig32[i]};
+            i = i + 2;
+          end
+          if (sig32[i-1] === 'bx) begin
+            if (i == 1) begin
+              i = SIGNATURESIZE+1; // flag empty file
+              $display("  Error: empty test file");
+            end else i = SIGNATURESIZE; // skip over the rest of the x's for efficiency
+          end
+        end
+
+        // Check errors
+        errors = (i == SIGNATURESIZE+1); // error if file is empty
+        i = 0;
+        testadr = (`TIM_BASE+tests[test+1].atohex())/(`XLEN/8);
+        /* verilator lint_off INFINITELOOP */
+        while (signature[i] !== 'bx) begin
+          //$display("signature[%h] = %h", i, signature[i]);
+          if (signature[i] !== dut.uncore.dtim.RAM[testadr+i] &&
+	      (signature[i] !== DCacheFlushFSM.ShadowRAM[testadr+i])) begin
+            if (signature[i+4] !== 'bx || signature[i] !== 32'hFFFFFFFF) begin
+              // report errors unless they are garbage at the end of the sim
+              // kind of hacky test for garbage right now
+              errors = errors+1;
+              $display("  Error on test %s result %d: adr = %h sim (D$) %h sim (TIM) = %h, signature = %h", 
+                    tests[test], i, (testadr+i)*(`XLEN/8), DCacheFlushFSM.ShadowRAM[testadr+i], dut.uncore.dtim.RAM[testadr+i], signature[i]);
+              $stop;//***debug
+            end
+          end
+          i = i + 1;
+        end
+        /* verilator lint_on INFINITELOOP */
+        if (errors == 0) begin
+          $display("%s succeeded.  Brilliant!!!", tests[test]);
+        end
+        else begin
+          $display("%s failed with %d errors. :(", tests[test], errors);
+          totalerrors = totalerrors+1;
+        end
+        test = test + 2;
+        if (test == tests.size()) begin
+          if (totalerrors == 0) $display("SUCCESS! All tests ran without failures.");
+          else $display("FAIL: %d test programs had errors", totalerrors);
+          $stop;
+        end
+        else begin
+          memfilename = {tvroot, "work/",tests[test], ".elf.memfile"};
+          $readmemh(memfilename, dut.uncore.dtim.RAM);
+          $display("Read memfile %s", memfilename);
+	  ProgramAddrMapFile = {tvroot, "work/", tests[test], ".elf.objdump.addr"};
+	  ProgramLabelMapFile = {tvroot, "work/", tests[test], ".elf.objdump.lab"};
+          reset = 1; # 17; reset = 0;
+        end
+      end
+    end // always @ (negedge clk)
+
+  // track the current function or global label
+  if (DEBUG == 1) begin : FunctionName
+    FunctionName FunctionName(.reset(reset),
+			      .clk(clk),
+			      .ProgramAddrMapFile(ProgramAddrMapFile),
+			      .ProgramLabelMapFile(ProgramLabelMapFile));
+  end
+
+    // flush cache and halt on infinite loop at end of riscv-arch-test
+    assign DCacheFlushStart = dut.hart.ifu.InstrM == 32'h6f && dut.hart.ieu.c.InstrValidM;
+//  assign DCacheFlushStart = dut.hart.lsu.dcache.MemPAdrM[31:0] == 32'h80008000 & 
+//            dut.hart.lsu.dcache.MemRWM == 2'b01;
+
+/*  assign DCacheFlushStart = dut.hart.priv.EcallFaultM && 
+			    (dut.hart.ieu.dp.regf.rf[3] == 1 || 
+			     (dut.hart.ieu.dp.regf.we3 && 
+			      dut.hart.ieu.dp.regf.a3 == 3 && 
+			      dut.hart.ieu.dp.regf.wd3 == 1));
+ */ 
+  DCacheFlushFSM DCacheFlushFSM(.clk(clk),
+				.reset(reset),
+				.start(DCacheFlushStart),
+				.done(DCacheFlushDone));
+  
+
+  generate
+    // initialize the branch predictor
+    if (`BPRED_ENABLED == 1) begin : bpred
+      
+      initial begin
+	$readmemb(`TWO_BIT_PRELOAD, dut.hart.ifu.bpred.bpred.Predictor.DirPredictor.PHT.memory);
+	$readmemb(`BTB_PRELOAD, dut.hart.ifu.bpred.bpred.TargetPredictor.memory.memory);    
+      end
+    end
+  endgenerate
+  
+endmodule
+
+module riscvassertions();
+  // Legal number of PMP entries are 0, 16, or 64
+  initial begin
+    assert (`PMP_ENTRIES == 0 || `PMP_ENTRIES==16 || `PMP_ENTRIES==64) else $error("Illegal number of PMP entries: PMP_ENTRIES must be 0, 16, or 64");
+    assert (`F_SUPPORTED || ~`D_SUPPORTED) else $error("Can't support double without supporting float");
+    assert (`XLEN == 64 || ~`D_SUPPORTED) else $error("Wally does not yet support D extensions on RV32");
+    assert (`DCACHE_WAYSIZEINBYTES <= 4096 || `MEM_DCACHE == 0 || `MEM_VIRTMEM == 0) else $error("DCACHE_WAYSIZEINBYTES cannot exceed 4 KiB when caches and vitual memory is enabled (to prevent aliasing)");
+    assert (`DCACHE_BLOCKLENINBITS >= 128 || `MEM_DCACHE == 0) else $error("DCACHE_BLOCKLENINBITS must be at least 128 when caches are enabled");
+    assert (`DCACHE_BLOCKLENINBITS < `DCACHE_WAYSIZEINBYTES*8) else $error("DCACHE_BLOCKLENINBITS must be smaller than way size");
+    assert (`ICACHE_WAYSIZEINBYTES <= 4096 || `MEM_ICACHE == 0 || `MEM_VIRTMEM == 0) else $error("ICACHE_WAYSIZEINBYTES cannot exceed 4 KiB when caches and vitual memory is enabled (to prevent aliasing)");
+    assert (`ICACHE_BLOCKLENINBITS >= 32 || `MEM_ICACHE == 0) else $error("ICACHE_BLOCKLENINBITS must be at least 32 when caches are enabled");
+    assert (`ICACHE_BLOCKLENINBITS < `ICACHE_WAYSIZEINBYTES*8) else $error("ICACHE_BLOCKLENINBITS must be smaller than way size");
+    assert (2**$clog2(`DCACHE_BLOCKLENINBITS) == `DCACHE_BLOCKLENINBITS) else $error("DCACHE_BLOCKLENINBITS must be a power of 2");
+    assert (2**$clog2(`DCACHE_WAYSIZEINBYTES) == `DCACHE_WAYSIZEINBYTES) else $error("DCACHE_WAYSIZEINBYTES must be a power of 2");
+    assert (2**$clog2(`ICACHE_BLOCKLENINBITS) == `ICACHE_BLOCKLENINBITS) else $error("ICACHE_BLOCKLENINBITS must be a power of 2");
+    assert (2**$clog2(`ICACHE_WAYSIZEINBYTES) == `ICACHE_WAYSIZEINBYTES) else $error("ICACHE_WAYSIZEINBYTES must be a power of 2");
+    assert (`ICACHE_NUMWAYS == 1 || `MEM_ICACHE == 0) else $error("Multiple Instruction Cache ways not yet implemented");
+    assert (2**$clog2(`ITLB_ENTRIES) == `ITLB_ENTRIES) else $error("ITLB_ENTRIES must be a power of 2");
+    assert (2**$clog2(`DTLB_ENTRIES) == `DTLB_ENTRIES) else $error("DTLB_ENTRIES must be a power of 2");
+    assert (`TIM_RANGE >= 56'h07FFFFFF) else $error("Some regression tests will fail if TIM_RANGE is less than 56'h07FFFFFF");
+  end
+endmodule
+
+
+/* verilator lint_on STMTDLY */
+/* verilator lint_on WIDTH */
+
+module DCacheFlushFSM
+  (input logic clk,
+   input logic reset,
+   input logic start,
+   output logic done);
+
+  localparam integer numlines = testbench.dut.hart.lsu.dcache.NUMLINES;
+  localparam integer numways = testbench.dut.hart.lsu.dcache.NUMWAYS;
+  localparam integer blockbytelen = testbench.dut.hart.lsu.dcache.BLOCKBYTELEN;
+  localparam integer numwords = testbench.dut.hart.lsu.dcache.BLOCKLEN/`XLEN;  
+  localparam integer lognumlines = $clog2(numlines);
+  localparam integer logblockbytelen = $clog2(blockbytelen);
+  localparam integer lognumways = $clog2(numways);
+  localparam integer tagstart = lognumlines + logblockbytelen;
+
+
+
+  genvar index, way, cacheWord;
+  logic [`XLEN-1:0]  CacheData [numways-1:0] [numlines-1:0] [numwords-1:0];
+  logic [`XLEN-1:0]  CacheTag [numways-1:0] [numlines-1:0] [numwords-1:0];
+  logic CacheValid  [numways-1:0] [numlines-1:0] [numwords-1:0];
+  logic CacheDirty  [numways-1:0] [numlines-1:0] [numwords-1:0];
+  logic [`PA_BITS-1:0] CacheAdr [numways-1:0] [numlines-1:0] [numwords-1:0];
+  genvar adr;
+
+  logic [`XLEN-1:0] ShadowRAM[`TIM_BASE>>(1+`XLEN/32):(`TIM_RANGE+`TIM_BASE)>>1+(`XLEN/32)];
+  
+  generate
+    for(index = 0; index < numlines; index++) begin
+      for(way = 0; way < numways; way++) begin
+	for(cacheWord = 0; cacheWord < numwords; cacheWord++) begin
+	  copyShadow #(.tagstart(tagstart),
+		       .logblockbytelen(logblockbytelen))
+	  copyShadow(.clk,
+		     .start,
+		     .tag(testbench.dut.hart.lsu.dcache.MemWay[way].CacheTagMem.StoredData[index]),
+		     .valid(testbench.dut.hart.lsu.dcache.MemWay[way].ValidBits[index]),
+		     .dirty(testbench.dut.hart.lsu.dcache.MemWay[way].DirtyBits[index]),
+		     .data(testbench.dut.hart.lsu.dcache.MemWay[way].word[cacheWord].CacheDataMem.StoredData[index]),
+		     .index(index),
+		     .cacheWord(cacheWord),
+		     .CacheData(CacheData[way][index][cacheWord]),
+		     .CacheAdr(CacheAdr[way][index][cacheWord]),
+		     .CacheTag(CacheTag[way][index][cacheWord]),
+		     .CacheValid(CacheValid[way][index][cacheWord]),
+		     .CacheDirty(CacheDirty[way][index][cacheWord]));
+	end
+      end
+    end
+  endgenerate
+
+  integer i, j, k;
+  
+  always @(posedge clk) begin
+    if (start) begin #1
+      #1
+      for(i = 0; i < numlines; i++) begin
+	for(j = 0; j < numways; j++) begin
+	  for(k = 0; k < numwords; k++) begin
+	  if (CacheValid[j][i][k] && CacheDirty[j][i][k]) begin
+	    ShadowRAM[CacheAdr[j][i][k] >> $clog2(`XLEN/8)] = CacheData[j][i][k];
+	    end
+	  end	
+	end
+      end
+    end
+  end
+
+
+  flop #(1) doneReg(.clk(clk),
+		    .d(start),
+		    .q(done));
+		    
+endmodule
+
+module copyShadow
+  #(parameter tagstart, logblockbytelen)
+  (input logic clk,
+   input logic 			     start,
+   input logic [`PA_BITS-1:tagstart] tag,
+   input logic 			     valid, dirty,
+   input logic [`XLEN-1:0] 	     data,
+   input logic [32-1:0] 	     index,
+   input logic [32-1:0] 	     cacheWord,
+   output logic [`XLEN-1:0] 	     CacheData,
+   output logic [`PA_BITS-1:0] 	     CacheAdr,
+   output logic [`XLEN-1:0] 	     CacheTag,
+   output logic 		     CacheValid,
+   output logic 		     CacheDirty);
+  
+
+  always_ff @(posedge clk) begin
+    if(start) begin
+      CacheTag = tag;
+      CacheValid = valid;
+      CacheDirty = dirty;
+      CacheData = data;
+      CacheAdr = (tag << tagstart) + (index << logblockbytelen) + (cacheWord << $clog2(`XLEN/8));
+    end
+  end
+  
+endmodule		      
+

From cb624fe679818d4c029d29ec763eee9bbadd3672 Mon Sep 17 00:00:00 2001
From: David Harris <david_harris@hmc.edu>
Date: Thu, 9 Sep 2021 11:05:12 -0400
Subject: [PATCH 16/16] Lint cleaning, riscv-arch-test testing

---
 .../regression/regression-wally.py            |   5 +
 .../regression/sim-wally-arch-batch           |   3 +
 wally-pipelined/regression/wally-arch.do      |   6 +-
 .../src/cache/cachereplacementpolicy.sv       |   6 +-
 wally-pipelined/src/cache/cacheway.sv         |   3 +-
 wally-pipelined/src/cache/icache.sv           |   2 +-
 wally-pipelined/src/mmu/priorityonehot.sv     |   6 +-
 .../src/mmu/prioritythermometer.sv            |   4 +
 wally-pipelined/testbench/testbench-arch.sv   | 200 +++++-------------
 .../testbench/testbench-imperas.sv            |   2 +-
 10 files changed, 80 insertions(+), 157 deletions(-)
 create mode 100755 wally-pipelined/regression/sim-wally-arch-batch

diff --git a/wally-pipelined/regression/regression-wally.py b/wally-pipelined/regression/regression-wally.py
index 2cd2d77ec..4d7baf8e5 100755
--- a/wally-pipelined/regression/regression-wally.py
+++ b/wally-pipelined/regression/regression-wally.py
@@ -33,6 +33,11 @@ configs = [
         cmd="vsim -do wally-buildroot-batch.do -c > {}",
         grepstr="loaded 6000 instructions"
     ),
+    TestCase(
+        name="arch64",
+        cmd="vsim > {} -c <<!\ndo wally-arch.do ../config/rv64ic rv64ic\n!",
+        grepstr="All tests ran without failures"
+    ),
     TestCase(
         name="rv32ic",
         cmd="vsim > {} -c <<!\ndo wally-pipelined-batch.do ../config/rv32ic rv32ic\n!",
diff --git a/wally-pipelined/regression/sim-wally-arch-batch b/wally-pipelined/regression/sim-wally-arch-batch
new file mode 100755
index 000000000..5b22b95e3
--- /dev/null
+++ b/wally-pipelined/regression/sim-wally-arch-batch
@@ -0,0 +1,3 @@
+vsim -c <<!
+do wally-arch.do ../config/rv64ic rv64ic
+!
diff --git a/wally-pipelined/regression/wally-arch.do b/wally-pipelined/regression/wally-arch.do
index 06b50079f..bfa93c254 100644
--- a/wally-pipelined/regression/wally-arch.do
+++ b/wally-pipelined/regression/wally-arch.do
@@ -17,10 +17,10 @@
 onbreak {resume}
 
 # create library
-if [file exists work] {
+if [file exists work-arch] {
     vdel -all
 }
-vlib work
+vlib work-arch
 
 # compile source files
 # suppress spurious warnngs about 
@@ -35,7 +35,7 @@ switch $argc {
 }
 # start and run simulation
 # remove +acc flag for faster sim during regressions if there is no need to access internal signals
-vopt +acc work.testbench -o workopt 
+vopt +acc work-arch.testbench -o workopt 
 vsim workopt
 
 view wave
diff --git a/wally-pipelined/src/cache/cachereplacementpolicy.sv b/wally-pipelined/src/cache/cachereplacementpolicy.sv
index a0b37cec8..0e508ca11 100644
--- a/wally-pipelined/src/cache/cachereplacementpolicy.sv
+++ b/wally-pipelined/src/cache/cachereplacementpolicy.sv
@@ -46,11 +46,11 @@ module cachereplacementpolicy
   always_ff @(posedge clk, posedge reset) begin
     if (reset) begin
       for(int index = 0; index < NUMLINES; index++)
-	ReplacementBits[index] <= '0;
+	      ReplacementBits[index] = '0;
     end else begin
-      BlockReplacementBits <= ReplacementBits[RAdr];
+      BlockReplacementBits = ReplacementBits[RAdr];
       if (LRUWriteEn) begin
-	ReplacementBits[MemPAdrM[INDEXLEN+OFFSETLEN-1:OFFSETLEN]] <= NewReplacement;
+	      ReplacementBits[MemPAdrM[INDEXLEN+OFFSETLEN-1:OFFSETLEN]] = NewReplacement;
       end
     end
   end
diff --git a/wally-pipelined/src/cache/cacheway.sv b/wally-pipelined/src/cache/cacheway.sv
index 4d3641693..a50fdb241 100644
--- a/wally-pipelined/src/cache/cacheway.sv
+++ b/wally-pipelined/src/cache/cacheway.sv
@@ -41,7 +41,7 @@ module cacheway #(parameter NUMLINES=512, parameter BLOCKLEN = 256, TAGLEN = 26,
    input logic 			      SetDirty,
    input logic 			      ClearDirty,
    input logic 			      SelEvict,
-   input logic 			      VictimWay,
+   input logic 			      VictimWay, 
 
    output logic [BLOCKLEN-1:0] 	      ReadDataBlockWayMasked,
    output logic 		      WayHit,
@@ -54,6 +54,7 @@ module cacheway #(parameter NUMLINES=512, parameter BLOCKLEN = 256, TAGLEN = 26,
   logic [TAGLEN-1:0] 		      ReadTag;
   logic 			      Valid;
   logic 			      Dirty;
+  logic             SelectedWay; // dh: *** do ways need to have more than 1 bit?
 
   genvar 			      words;
 
diff --git a/wally-pipelined/src/cache/icache.sv b/wally-pipelined/src/cache/icache.sv
index 7992c56c5..681e34976 100644
--- a/wally-pipelined/src/cache/icache.sv
+++ b/wally-pipelined/src/cache/icache.sv
@@ -142,7 +142,7 @@ module icache
 			 .RAdr(RAdr),
 			 .PAdr(PCTagF),
 			 .WriteEnable(SRAMWayWriteEnable), 
-			 .WriteWordEnable('1),
+			 .WriteWordEnable({{(BLOCKLEN/`XLEN)-1{1'b0}}, 1'b1}),
 			 .TagWriteEnable(SRAMWayWriteEnable),
 			 .WriteData(ICacheMemWriteData),
 			 .SetValid(ICacheMemWriteEnable),
diff --git a/wally-pipelined/src/mmu/priorityonehot.sv b/wally-pipelined/src/mmu/priorityonehot.sv
index 75825dc40..7a17f8d28 100644
--- a/wally-pipelined/src/mmu/priorityonehot.sv
+++ b/wally-pipelined/src/mmu/priorityonehot.sv
@@ -35,6 +35,8 @@ module priorityonehot #(parameter ENTRIES = 8) (
   output logic  [ENTRIES-1:0] y
 );
 
+  /* verilator lint_off UNOPTFLAT */
+
   logic [ENTRIES-1:0] nolower;
 
   // generate thermometer code mask
@@ -47,5 +49,7 @@ module priorityonehot #(parameter ENTRIES = 8) (
   endgenerate
 
   assign y = a & nolower;
-  
+
+  /* verilator lint_on UNOPTFLAT */
+
 endmodule
diff --git a/wally-pipelined/src/mmu/prioritythermometer.sv b/wally-pipelined/src/mmu/prioritythermometer.sv
index 132a489c1..a6374579c 100644
--- a/wally-pipelined/src/mmu/prioritythermometer.sv
+++ b/wally-pipelined/src/mmu/prioritythermometer.sv
@@ -30,11 +30,14 @@
 
 `include "wally-config.vh"
 
+/* verilator lint_off UNOPTFLAT */
+
 module prioritythemometer #(parameter N = 8) (
   input  logic  [N-1:0] a,
   output logic  [N-1:0] y
 );
 
+
   // generate thermometer code mask
   genvar i;
   generate
@@ -47,4 +50,5 @@ module prioritythemometer #(parameter N = 8) (
 endmodule
 
 
+/* verilator lint_on UNOPTFLAT */
 
diff --git a/wally-pipelined/testbench/testbench-arch.sv b/wally-pipelined/testbench/testbench-arch.sv
index 22ed0bef9..f200c7189 100644
--- a/wally-pipelined/testbench/testbench-arch.sv
+++ b/wally-pipelined/testbench/testbench-arch.sv
@@ -161,53 +161,55 @@ string tests32f[] = '{
   };
 
   string tests64m[] = '{
-    "rv64m/I-MUL-01", "3000",
-    "rv64m/I-MULH-01", "3000",
-    "rv64m/I-MULHSU-01", "3000",
-    "rv64m/I-MULHU-01", "3000",
-    "rv64m/I-MULW-01", "3000",
-    "rv64m/I-DIV-01", "3000",
-    "rv64m/I-DIVU-01", "3000",
-    "rv64m/I-DIVUW-01", "3000",
-    "rv64m/I-DIVW-01", "3000",
-    "rv64m/I-REM-01", "3000",
-    "rv64m/I-REMU-01", "3000",
-    "rv64m/I-REMUW-01", "3000",
-    "rv64m/I-REMW-01", "3000"
-  };
+    "rv64i_m/M/div-01", "9010",
+    "rv64i_m/M/divu-01", "a010",
+    "rv64i_m/M/divuw-01", "a010",
+    "rv64i_m/M/divw-01", "9010",
+    "rv64i_m/M/mul-01", "9010",
+    "rv64i_m/M/mulh-01", "9010",
+    "rv64i_m/M/mulhsu-01", "9010",
+    "rv64i_m/M/mulhu-01", "a010",
+    "rv64i_m/M/mulw-01", "9010",
+    "rv64i_m/M/rem-01", "9010",
+    "rv64i_m/M/remu-01", "a010",
+    "rv64i_m/M/remuw-01", "a010",
+    "rv64i_m/M/remw-01", "9010"
+   };
 
   string tests64ic[] = '{
-    "rv64ic/I-C-ADD-01", "3000",
-    "rv64ic/I-C-ADDI-01", "3000",
-    "rv64ic/I-C-ADDIW-01", "3000",
-    "rv64ic/I-C-ADDW-01", "3000",
-    "rv64ic/I-C-AND-01", "3000",
-    "rv64ic/I-C-ANDI-01", "3000",
-    "rv64ic/I-C-BEQZ-01", "3000",
-    "rv64ic/I-C-BNEZ-01", "3000",
-    "rv64ic/I-C-EBREAK-01", "2000",
-    "rv64ic/I-C-J-01", "3000",
-    "rv64ic/I-C-JALR-01", "4000",
-    "rv64ic/I-C-JR-01", "4000",
-    "rv64ic/I-C-LD-01", "3420",
-    "rv64ic/I-C-LDSP-01", "3420",
-    "rv64ic/I-C-LI-01", "3000",
-    "rv64ic/I-C-LUI-01", "2000",
-    "rv64ic/I-C-LW-01", "3110",
-    "rv64ic/I-C-LWSP-01", "3110",
-    "rv64ic/I-C-MV-01", "3000",
-    "rv64ic/I-C-NOP-01", "2000",
-    "rv64ic/I-C-OR-01", "3000",
-    "rv64ic/I-C-SD-01", "3000",
-    "rv64ic/I-C-SDSP-01", "3000",
-    "rv64ic/I-C-SLLI-01", "3000",
-    "rv64ic/I-C-SRAI-01", "3000",
-    "rv64ic/I-C-SRLI-01", "3000",
-    "rv64ic/I-C-SUB-01", "3000",
-    "rv64ic/I-C-SUBW-01", "3000",
-    "rv64ic/I-C-SW-01", "3000",
-    "rv64ic/I-C-SWSP-01", "3000",
-    "rv64ic/I-C-XOR-01", "3000"
+    "rv64i_m/C/cadd-01", "8010",
+    "rv64i_m/C/caddi-01", "4010",
+    "rv64i_m/C/caddi16sp-01", "2010",
+    "rv64i_m/C/caddi4spn-01", "2010",
+    "rv64i_m/C/caddiw-01", "4010",
+    "rv64i_m/C/caddw-01", "8010",
+    "rv64i_m/C/cand-01", "8010",
+    "rv64i_m/C/candi-01", "4010",
+    "rv64i_m/C/cbeqz-01", "4010",
+    "rv64i_m/C/cbnez-01", "5010",
+    "rv64i_m/C/cebreak-01", "2070",
+    "rv64i_m/C/cj-01", "3010",
+    "rv64i_m/C/cjalr-01", "2010",
+    "rv64i_m/C/cjr-01", "2010",
+    "rv64i_m/C/cld-01", "2010",
+    "rv64i_m/C/cldsp-01", "2010",
+    "rv64i_m/C/cli-01", "2010",
+    "rv64i_m/C/clui-01", "2010",
+    "rv64i_m/C/clw-01", "2010",
+    "rv64i_m/C/clwsp-01", "2010",
+    "rv64i_m/C/cmv-01", "2010",
+    "rv64i_m/C/cnop-01", "2010",
+    "rv64i_m/C/cor-01", "8010",
+    "rv64i_m/C/csd-01", "3010",
+    "rv64i_m/C/csdsp-01", "3010",
+    "rv64i_m/C/cslli-01", "2010",
+    "rv64i_m/C/csrai-01", "2010",
+    "rv64i_m/C/csrli-01", "2010",
+    "rv64i_m/C/csub-01", "8010",
+    "rv64i_m/C/csubw-01", "8010",
+    "rv64i_m/C/csw-01", "3010",
+    "rv64i_m/C/cswsp-01", "3010",
+    "rv64i_m/C/cxor-01", "8010"
   };
 
   string tests64iNOc[] = {
@@ -265,102 +267,6 @@ string tests32f[] = '{
     "rv64i_m/I/sw-align-01", "3010",
     "rv64i_m/I/xor-01", "9010",
     "rv64i_m/I/xori-01", "6010"
-    
- /*   //"rv64i/WALLY-PIPELINE-100K", "f7ff0",
-    "rv64i/I-ADD-01", "3000",
-    "rv64i/I-ADDI-01", "3000",
-    "rv64i/I-ADDIW-01", "3000",
-    "rv64i/I-ADDW-01", "3000",
-    "rv64i/I-AND-01", "3000",
-    "rv64i/I-ANDI-01", "3000",
-    "rv64i/I-AUIPC-01", "3000",
-    "rv64i/I-BEQ-01", "4000",
-    "rv64i/I-BGE-01", "4000",
-    "rv64i/I-BGEU-01", "4000",
-    "rv64i/I-BLT-01", "4000",
-    "rv64i/I-BLTU-01", "4000",
-    "rv64i/I-BNE-01", "4000",
-    "rv64i/I-DELAY_SLOTS-01", "2000",
-    "rv64i/I-EBREAK-01", "2000",
-    "rv64i/I-ECALL-01", "2000",
-    "rv64i/I-ENDIANESS-01", "2010",
-    "rv64i/I-IO-01", "2050",
-    "rv64i/I-JAL-01", "3000",
-    "rv64i/I-JALR-01", "4000",
-    "rv64i/I-LB-01", "4020",
-    "rv64i/I-LBU-01", "4020",
-    "rv64i/I-LD-01", "4420",
-    "rv64i/I-LH-01", "4050",
-    "rv64i/I-LHU-01", "4050",
-    "rv64i/I-LUI-01", "2000",
-    "rv64i/I-LW-01", "4110",
-    "rv64i/I-LWU-01", "4110", 
-    "rv64i/I-MISALIGN_LDST-01", "2010",
-    "rv64i/I-NOP-01", "2000",
-    "rv64i/I-OR-01", "3000",
-    "rv64i/I-ORI-01", "3000",
-    "rv64i/I-RF_size-01", "2000",
-    "rv64i/I-RF_width-01", "2000",
-    "rv64i/I-RF_x0-01", "2010",
-    "rv64i/I-SB-01", "4000",
-    "rv64i/I-SD-01", "4000",
-    "rv64i/I-SH-01", "4000",
-    "rv64i/I-SLL-01", "3000",
-    "rv64i/I-SLLI-01", "3000",
-    "rv64i/I-SLLIW-01", "3000",
-    "rv64i/I-SLLW-01", "3000",
-    "rv64i/I-SLT-01", "3000",
-    "rv64i/I-SLTI-01", "3000",
-    "rv64i/I-SLTIU-01", "3000",
-    "rv64i/I-SLTU-01", "3000",
-    "rv64i/I-SRA-01", "3000",
-    "rv64i/I-SRAI-01", "3000",
-    "rv64i/I-SRAIW-01", "3000",
-    "rv64i/I-SRAW-01", "3000",
-    "rv64i/I-SRL-01", "3000",
-    "rv64i/I-SRLI-01", "3000",
-    "rv64i/I-SRLIW-01", "3000",
-    "rv64i/I-SRLW-01", "3000",
-    "rv64i/I-SUB-01", "3000",
-    "rv64i/I-SUBW-01", "3000",
-    "rv64i/I-SW-01", "4000",
-    "rv64i/I-XOR-01", "3000",
-    "rv64i/I-XORI-01", "3000",
-    "rv64i/WALLY-ADD", "4000",
-    "rv64i/WALLY-SUB", "4000",
-    "rv64i/WALLY-ADDI", "3000",
-    "rv64i/WALLY-ANDI", "3000",
-    "rv64i/WALLY-ORI", "3000",
-    "rv64i/WALLY-XORI", "3000",
-    "rv64i/WALLY-SLTI", "3000",
-    "rv64i/WALLY-SLTIU", "3000",
-    "rv64i/WALLY-SLLI", "3000",
-    "rv64i/WALLY-SRLI", "3000",
-    "rv64i/WALLY-SRAI", "3000",
-    "rv64i/WALLY-JAL", "4000",
-    "rv64i/WALLY-JALR", "3000",
-    "rv64i/WALLY-STORE", "3000",
-    "rv64i/WALLY-ADDIW", "3000",
-    "rv64i/WALLY-SLLIW", "3000",
-    "rv64i/WALLY-SRLIW", "3000",
-    "rv64i/WALLY-SRAIW", "3000",
-    "rv64i/WALLY-ADDW", "4000",
-    "rv64i/WALLY-SUBW", "4000",
-    "rv64i/WALLY-SLLW", "3000",
-    "rv64i/WALLY-SRLW", "3000",
-    "rv64i/WALLY-SRAW", "3000",
-    "rv64i/WALLY-BEQ" ,"5000",
-    "rv64i/WALLY-BNE", "5000 ",
-    "rv64i/WALLY-BLTU", "5000 ",
-    "rv64i/WALLY-BLT", "5000",
-    "rv64i/WALLY-BGE", "5000 ",
-    "rv64i/WALLY-BGEU", "5000 ",
-    "rv64i/WALLY-CSRRW", "4000",
-    "rv64i/WALLY-CSRRS", "4000",
-    "rv64i/WALLY-CSRRC", "5000",
-    "rv64i/WALLY-CSRRWI", "4000",
-    "rv64i/WALLY-CSRRSI", "4000",
-    "rv64i/WALLY-CSRRCI", "4000" */
   };
 
   string tests32a[] = '{
@@ -585,11 +491,11 @@ string tests32f[] = '{
         tests = tests64p;
       else begin
         tests = {tests64i};
-/*        tests = {tests64p,tests64i, tests64periph};
+//        tests = {tests64p,tests64i, tests64periph};
         if (`C_SUPPORTED) tests = {tests, tests64ic};
-        else              tests = {tests, tests64iNOc};
+//        else              tests = {tests, tests64iNOc};
         if (`M_SUPPORTED) tests = {tests, tests64m};
-        if (`F_SUPPORTED) tests = {tests64f, tests};
+/*        if (`F_SUPPORTED) tests = {tests64f, tests};
         if (`D_SUPPORTED) tests = {tests64d, tests};
         if (`MEM_VIRTMEM) tests = {tests64mmu, tests};
         if (`A_SUPPORTED) tests = {tests64a, tests}; */
@@ -629,7 +535,7 @@ string tests32f[] = '{
 
   // Track names of instructions
   instrTrackerTB it(clk, reset, dut.hart.ieu.dp.FlushE,
-                dut.hart.ifu.icache.controller.FinalInstrRawF,
+                dut.hart.ifu.icache.FinalInstrRawF,
                 dut.hart.ifu.InstrD, dut.hart.ifu.InstrE,
                 dut.hart.ifu.InstrM,  dut.hart.ifu.InstrW,
                 InstrFName, InstrDName, InstrEName, InstrMName, InstrWName);
@@ -661,9 +567,9 @@ string tests32f[] = '{
       bootroot = "../../imperas-riscv-tests/";
       tvroot = "/home/harris/github/riscv-arch-test/";
       memfilename = {tvroot, "work/", tests[test], ".elf.memfile"};
-      romfilename = {bootroot, "imperas-boottim.txt"};
+//      romfilename = {bootroot, "imperas-boottim.txt"};
       $readmemh(memfilename, dut.uncore.dtim.RAM);
-      $readmemh(romfilename, dut.uncore.bootdtim.bootdtim.RAM);
+//      $readmemh(romfilename, dut.uncore.bootdtim.bootdtim.RAM);
       ProgramAddrMapFile = {tvroot, "work/", tests[test], ".elf.objdump.addr"};
       ProgramLabelMapFile = {tvroot, "work/", tests[test], ".elf.objdump.lab"};
       $display("Read memfile %s", memfilename);
diff --git a/wally-pipelined/testbench/testbench-imperas.sv b/wally-pipelined/testbench/testbench-imperas.sv
index 38436752e..318140769 100644
--- a/wally-pipelined/testbench/testbench-imperas.sv
+++ b/wally-pipelined/testbench/testbench-imperas.sv
@@ -755,7 +755,7 @@ module riscvassertions();
     assert (2**$clog2(`DCACHE_WAYSIZEINBYTES) == `DCACHE_WAYSIZEINBYTES) else $error("DCACHE_WAYSIZEINBYTES must be a power of 2");
     assert (2**$clog2(`ICACHE_BLOCKLENINBITS) == `ICACHE_BLOCKLENINBITS) else $error("ICACHE_BLOCKLENINBITS must be a power of 2");
     assert (2**$clog2(`ICACHE_WAYSIZEINBYTES) == `ICACHE_WAYSIZEINBYTES) else $error("ICACHE_WAYSIZEINBYTES must be a power of 2");
-    assert (`ICACHE_NUMWAYS == 1 || `MEM_ICACHE == 0) else $error("Multiple Instruction Cache ways not yet implemented");
+    assert (`ICACHE_NUMWAYS == 1 || `MEM_ICACHE == 0) else $warning("Multiple Instruction Cache ways not yet implemented");
     assert (2**$clog2(`ITLB_ENTRIES) == `ITLB_ENTRIES) else $error("ITLB_ENTRIES must be a power of 2");
     assert (2**$clog2(`DTLB_ENTRIES) == `DTLB_ENTRIES) else $error("DTLB_ENTRIES must be a power of 2");
     assert (`TIM_RANGE >= 56'h07FFFFFF) else $error("Some regression tests will fail if TIM_RANGE is less than 56'h07FFFFFF");