diff --git a/src/lsu/atomic.sv b/src/lsu/atomic.sv index 9c37b636c..8ad9159ce 100644 --- a/src/lsu/atomic.sv +++ b/src/lsu/atomic.sv @@ -39,7 +39,7 @@ module atomic import cvw::*; #(parameter cvw_t P) ( input logic [2:0] LSUFunct3M, // IEU or HPTW memory operation size input logic [1:0] LSUAtomicM, // 10: AMO operation, select AMOResultM as the writedata output, 01: LR/SC operation input logic [1:0] PreLSURWM, // IEU or HPTW Read/Write signal - input logic IgnoreRequest, // On FlushM or TLB miss ignore memory operation + input logic LSUFlushW, // On FlushM or TLB miss ignore memory operation output logic [P.XLEN-1:0] IMAWriteDataM, // IEU, HPTW, or AMO write data output logic SquashSCW, // Store conditional failed disable write to GPR output logic [1:0] LSURWM // IEU or HPTW Read/Write signal gated by LR/SC @@ -57,7 +57,7 @@ module atomic import cvw::*; #(parameter cvw_t P) ( // LRSC unit if (P.ZALRSC_SUPPORTED) begin - assign MemReadM = PreLSURWM[1] & ~IgnoreRequest; + assign MemReadM = PreLSURWM[1] & ~LSUFlushW; lrsc #(P) lrsc(.clk, .reset, .StallW, .MemReadM, .PreLSURWM, .LSUAtomicM, .PAdrM, .SquashSCW, .LSURWM); end else begin assign SquashSCW = 0; diff --git a/src/lsu/lsu.sv b/src/lsu/lsu.sv index bc8852cf4..08e62f9ad 100644 --- a/src/lsu/lsu.sv +++ b/src/lsu/lsu.sv @@ -110,8 +110,7 @@ module lsu import cvw::*; #(parameter cvw_t P) ( logic GatedStallW; // Hazard unit StallW gated when SelHPTW = 1 - logic BusStall; // Bus interface busy with multicycle operation - logic LSUBusStallM; // Bus interface busy with multicycle operation masked by IgnoreRequestTLB + logic LSUBusStallM; // Bus interface busy with multicycle operation masked by HPTWFlushW logic HPTWStall; // HPTW busy with multicycle operation logic DCacheBusStallM; // Cache or bus stall logic CacheBusHPWTStall; // Cache, bus, or hptw is requesting a stall @@ -146,8 +145,8 @@ module lsu import cvw::*; #(parameter cvw_t P) ( logic DTLBWriteM; // Writes PTE and PageType to DTLB logic LSULoadAccessFaultM; // Load acces fault logic LSUStoreAmoAccessFaultM; // Store access fault - logic IgnoreRequestTLB; // On either ITLB or DTLB miss, ignore miss so HPTW can handle - logic IgnoreRequest; // On FlushM or TLB miss ignore memory operation + logic HPTWFlushW; // HPTW needs to flush operation + logic LSUFlushW; // HPTW or hazard unit flushes operation logic SelDTIM; // Select DTIM rather than bus or D$ logic [P.XLEN-1:0] WriteDataZM; logic LSULoadPageFaultM, LSUStoreAmoPageFaultM; @@ -200,7 +199,7 @@ module lsu import cvw::*; #(parameter cvw_t P) ( .WriteDataM(WriteDataZM), .Funct3M, .LSUFunct3M, .Funct7M, .LSUFunct7M, .IEUAdrExtM, .PTE, .IHWriteDataM, .PageType, .PreLSURWM, .LSUAtomicM, .IHAdrM, .HPTWStall, .SelHPTW, - .IgnoreRequestTLB, .LSULoadAccessFaultM, .LSUStoreAmoAccessFaultM, + .HPTWFlushW, .LSULoadAccessFaultM, .LSUStoreAmoAccessFaultM, .LoadAccessFaultM, .StoreAmoAccessFaultM, .HPTWInstrAccessFaultF, .LoadPageFaultM, .StoreAmoPageFaultM, .LSULoadPageFaultM, .LSUStoreAmoPageFaultM, .HPTWInstrPageFaultF ); @@ -215,7 +214,7 @@ module lsu import cvw::*; #(parameter cvw_t P) ( assign StoreAmoAccessFaultM = LSUStoreAmoAccessFaultM; assign LoadPageFaultM = LSULoadPageFaultM; assign StoreAmoPageFaultM = LSUStoreAmoPageFaultM; - assign {HPTWStall, SelHPTW, PTE, PageType, DTLBWriteM, ITLBWriteF, IgnoreRequestTLB} = '0; + assign {HPTWStall, SelHPTW, PTE, PageType, DTLBWriteM, ITLBWriteF, HPTWFlushW} = '0; assign {HPTWInstrAccessFaultF, HPTWInstrPageFaultF} = '0; end @@ -274,7 +273,7 @@ module lsu import cvw::*; #(parameter cvw_t P) ( // Pause IEU memory request if TLB miss. After TLB fill, replay request. // Discard memory request on pipeline flush - assign IgnoreRequest = IgnoreRequestTLB | FlushW; + assign LSUFlushW = HPTWFlushW | FlushW; if (P.DTIM_SUPPORTED) begin : dtim logic [P.PA_BITS-1:0] DTIMAdr; @@ -285,7 +284,7 @@ module lsu import cvw::*; #(parameter cvw_t P) ( assign DTIMMemRWM = SelDTIM ? LSURWM : 0; dtim #(P) dtim(.clk, .reset, .ce(~GatedStallW), .MemRWM(DTIMMemRWM), - .DTIMAdr, .FlushW(IgnoreRequest), .WriteDataM(LSUWriteDataM), + .DTIMAdr, .FlushW(LSUFlushW), .WriteDataM(LSUWriteDataM), .ReadDataWordM(DTIMReadDataWordM[P.LLEN-1:0]), .ByteMaskM(ByteMaskM)); end else assign DTIMReadDataWordM = '0; @@ -309,8 +308,6 @@ module lsu import cvw::*; #(parameter cvw_t P) ( logic CacheableOrFlushCacheM; // Memory address is cacheable or operation is a cache flush logic [1:0] CacheRWM; // Cache read (10), write (01), AMO (11) logic FlushDCache; // Suppress d cache flush if there is an ITLB miss. - logic CacheStall; - logic [1:0] CacheBusRWTemp; logic BusCMOZero; logic [3:0] CacheCMOpM; logic BusAtomic; @@ -331,29 +328,26 @@ module lsu import cvw::*; #(parameter cvw_t P) ( cache #(.P(P), .PA_BITS(P.PA_BITS), .XLEN(P.XLEN), .LINELEN(P.DCACHE_LINELENINBITS), .NUMSETS(P.DCACHE_WAYSIZEINBYTES*8/LINELEN), .NUMWAYS(P.DCACHE_NUMWAYS), .LOGBWPL(LLENLOGBWPL), .WORDLEN(CACHEWORDLEN), .MUXINTERVAL(P.LLEN), .READ_ONLY_CACHE(0)) dcache( - .clk, .reset, .Stall(GatedStallW & ~SelSpillE), .SelBusBeat, .FlushStage(IgnoreRequest), + .clk, .reset, .Stall(GatedStallW & ~SelSpillE), .SelBusBeat, .FlushStage(LSUFlushW), .CacheRW(CacheRWM), .FlushCache(FlushDCache), .NextSet(IEUAdrExtE[11:0]), .PAdr(PAdrM), .ByteMask(ByteMaskSpillM), .BeatCount(BeatCount[AHBWLOGBWPL-1:AHBWLOGBWPL-LLENLOGBWPL]), .WriteData(LSUWriteDataSpillM), .SelHPTW, - .CacheStall, .CacheMiss(DCacheMiss), .CacheAccess(DCacheAccess), + .CacheStall(DCacheStallM), .CacheMiss(DCacheMiss), .CacheAccess(DCacheAccess), .CacheCommitted(DCacheCommittedM), .CacheBusAdr(DCacheBusAdr), .ReadDataWord(DCacheReadDataWordM), - .FetchBuffer, .CacheBusRW(CacheBusRWTemp), + .FetchBuffer, .CacheBusRW(CacheBusRW), .CacheBusAck(DCacheBusAck), .InvalidateCache(1'b0), .CMOpM(CacheCMOpM)); - assign DCacheStallM = CacheStall & ~IgnoreRequestTLB; - assign CacheBusRW = CacheBusRWTemp; - ahbcacheinterface #(.P(P), .BEATSPERLINE(BEATSPERLINE), .AHBWLOGBWPL(AHBWLOGBWPL), .LINELEN(LINELEN), .LLENPOVERAHBW(LLENPOVERAHBW), .READ_ONLY_CACHE(0)) ahbcacheinterface( - .HCLK(clk), .HRESETn(~reset), .Flush(IgnoreRequest), + .HCLK(clk), .HRESETn(~reset), .Flush(LSUFlushW), .HRDATA, .HWDATA(LSUHWDATA), .HWSTRB(LSUHWSTRB), .HSIZE(LSUHSIZE), .HBURST(LSUHBURST), .HTRANS(LSUHTRANS), .HWRITE(LSUHWRITE), .HREADY(LSUHREADY), .BeatCount, .SelBusBeat, .CacheReadDataWordM(DCacheReadDataWordM[P.LLEN-1:0]), .WriteDataM(LSUWriteDataM), .Funct3(LSUFunct3M), .HADDR(LSUHADDR), .CacheBusAdr(DCacheBusAdr), .CacheBusRW, .BusAtomic, .BusCMOZero, .CacheableOrFlushCacheM, .CacheBusAck(DCacheBusAck), .FetchBuffer, .PAdr(PAdrM), .Cacheable(CacheableOrFlushCacheM), .BusRW, .Stall(GatedStallW), - .BusStall, .BusCommitted(BusCommittedM)); + .BusStall(LSUBusStallM), .BusCommitted(BusCommittedM)); mux3 #(P.LLEN) UnCachedDataMux(.d0(DCacheReadDataWordSpillM), .d1({LLENPOVERAHBW{FetchBuffer[P.XLEN-1:0]}}), .d2({{P.LLEN-P.XLEN{1'b0}}, DTIMReadDataWordM[P.XLEN-1:0]}), @@ -366,10 +360,10 @@ module lsu import cvw::*; #(parameter cvw_t P) ( assign LSUHADDR = PAdrM; assign LSUHSIZE = LSUFunct3M; - ahbinterface #(P.XLEN, 1'b1) ahbinterface(.HCLK(clk), .HRESETn(~reset), .Flush(IgnoreRequest), .HREADY(LSUHREADY), + ahbinterface #(P.XLEN, 1'b1) ahbinterface(.HCLK(clk), .HRESETn(~reset), .Flush(LSUFlushW), .HREADY(LSUHREADY), .HRDATA(HRDATA), .HTRANS(LSUHTRANS), .HWRITE(LSUHWRITE), .HWDATA(LSUHWDATA), .HWSTRB(LSUHWSTRB), .BusRW, .BusAtomic(AtomicM[1]), .ByteMask(ByteMaskM[P.XLEN/8-1:0]), .WriteData(LSUWriteDataM[P.XLEN-1:0]), - .Stall(GatedStallW), .BusStall, .BusCommitted(BusCommittedM), .FetchBuffer(FetchBuffer)); + .Stall(GatedStallW), .BusStall(LSUBusStallM), .BusCommitted(BusCommittedM), .FetchBuffer(FetchBuffer)); // Mux between the 2 sources of read data, 0: Bus, 1: DTIM if(P.DTIM_SUPPORTED) mux2 #(P.XLEN) ReadDataMux2(FetchBuffer, DTIMReadDataWordM[P.XLEN-1:0], SelDTIM, ReadDataWordMuxM[P.XLEN-1:0]); @@ -381,12 +375,10 @@ module lsu import cvw::*; #(parameter cvw_t P) ( assign {LSUHWDATA, LSUHADDR, LSUHWRITE, LSUHSIZE, LSUHBURST, LSUHTRANS, LSUHWSTRB} = '0; assign DCacheReadDataWordM = '0; assign ReadDataWordMuxM = DTIMReadDataWordM; - assign {BusStall, BusCommittedM} = '0; + assign {LSUBusStallM, BusCommittedM} = '0; assign {DCacheMiss, DCacheAccess} = '0; assign {DCacheStallM, DCacheCommittedM} = '0; end - - assign LSUBusStallM = BusStall & ~IgnoreRequestTLB; ///////////////////////////////////////////////////////////////////////////////////////////// // Atomic operations @@ -394,7 +386,7 @@ module lsu import cvw::*; #(parameter cvw_t P) ( if (P.ZAAMO_SUPPORTED | P.ZALRSC_SUPPORTED) begin:atomic atomic #(P) atomic(.clk, .reset, .StallW, .ReadDataM(ReadDataM[P.XLEN-1:0]), .IHWriteDataM, .PAdrM, - .LSUFunct7M, .LSUFunct3M, .LSUAtomicM, .PreLSURWM, .IgnoreRequest, + .LSUFunct7M, .LSUFunct3M, .LSUAtomicM, .PreLSURWM, .LSUFlushW, .IMAWriteDataM, .SquashSCW, .LSURWM); end else begin:lrsc assign SquashSCW = 1'b0; diff --git a/src/mmu/hptw.sv b/src/mmu/hptw.sv index 7a0d2c4a6..4e9003dc3 100644 --- a/src/mmu/hptw.sv +++ b/src/mmu/hptw.sv @@ -58,7 +58,7 @@ module hptw import cvw::*; #(parameter cvw_t P) ( output logic [1:0] LSUAtomicM, output logic [2:0] LSUFunct3M, output logic [6:0] LSUFunct7M, - output logic IgnoreRequestTLB, + output logic HPTWFlushW, output logic SelHPTW, output logic HPTWStall, input logic LSULoadAccessFaultM, LSUStoreAmoAccessFaultM, @@ -105,6 +105,7 @@ module hptw import cvw::*; #(parameter cvw_t P) ( logic TakeHPTWFault; logic PBMTFaultM; logic HPTWFaultM; + logic ResetPTE; // map hptw access faults onto either the original LSU load/store fault or instruction access fault assign LSUAccessFaultM = LSULoadAccessFaultM | LSUStoreAmoAccessFaultM; @@ -143,7 +144,7 @@ module hptw import cvw::*; #(parameter cvw_t P) ( // State flops flopenr #(1) TLBMissMReg(clk, reset, StartWalk, DTLBMissOrUpdateDAM, DTLBWalk); // when walk begins, record whether it was for DTLB (or record 0 for ITLB) assign PRegEn = HPTWRW[1] & ~DCacheBusStallM | UpdatePTE; - flopenr #(P.XLEN) PTEReg(clk, reset, PRegEn, NextPTE, PTE); // Capture page table entry from data cache + flopenr #(P.XLEN) PTEReg(clk, ResetPTE, PRegEn, NextPTE, PTE); // Capture page table entry from data cache // Assign PTE descriptors common across all XLEN values // For non-leaf PTEs, D, A, U bits are reserved and ignored. They do not cause faults while walking the page table @@ -274,23 +275,26 @@ module hptw import cvw::*; #(parameter cvw_t P) ( IDLE: if (TLBMissOrUpdateDA) NextWalkerState = InitialWalkerState; else NextWalkerState = IDLE; L3_ADR: NextWalkerState = L3_RD; // First access in SV48 - L3_RD: if (DCacheBusStallM) NextWalkerState = L3_RD; - else if (HPTWFaultM) NextWalkerState = FAULT; + L3_RD: if (HPTWFaultM) NextWalkerState = FAULT; + else if (DCacheBusStallM) NextWalkerState = L3_RD; else NextWalkerState = L2_ADR; - L2_ADR: if (InitialWalkerState == L2_ADR | ValidNonLeafPTE) NextWalkerState = L2_RD; // First access in SV39 + L2_ADR: if (HPTWFaultM) NextWalkerState = FAULT; + else if (InitialWalkerState == L2_ADR | ValidNonLeafPTE) NextWalkerState = L2_RD; // First access in SV39 else NextWalkerState = LEAF; - L2_RD: if (DCacheBusStallM) NextWalkerState = L2_RD; - else if (HPTWFaultM) NextWalkerState = FAULT; + L2_RD: if (HPTWFaultM) NextWalkerState = FAULT; + else if (DCacheBusStallM) NextWalkerState = L2_RD; else NextWalkerState = L1_ADR; - L1_ADR: if (InitialWalkerState == L1_ADR | ValidNonLeafPTE) NextWalkerState = L1_RD; // First access in SV32 + L1_ADR: if (HPTWFaultM) NextWalkerState = FAULT; + else if (InitialWalkerState == L1_ADR | ValidNonLeafPTE) NextWalkerState = L1_RD; // First access in SV32 else NextWalkerState = LEAF; - L1_RD: if (DCacheBusStallM) NextWalkerState = L1_RD; - else if (HPTWFaultM) NextWalkerState = FAULT; + L1_RD: if (HPTWFaultM) NextWalkerState = FAULT; + else if (DCacheBusStallM) NextWalkerState = L1_RD; else NextWalkerState = L0_ADR; - L0_ADR: if (ValidNonLeafPTE) NextWalkerState = L0_RD; + L0_ADR: if (HPTWFaultM) NextWalkerState = FAULT; + else if (ValidNonLeafPTE) NextWalkerState = L0_RD; else NextWalkerState = LEAF; - L0_RD: if (DCacheBusStallM) NextWalkerState = L0_RD; - else if (HPTWFaultM) NextWalkerState = FAULT; + L0_RD: if (HPTWFaultM) NextWalkerState = FAULT; + else if (DCacheBusStallM) NextWalkerState = L0_RD; else NextWalkerState = LEAF; LEAF: if (P.SVADU_SUPPORTED & HPTWUpdateDA) NextWalkerState = UPDATE_PTE; else NextWalkerState = IDLE; @@ -300,7 +304,9 @@ module hptw import cvw::*; #(parameter cvw_t P) ( default: NextWalkerState = IDLE; // Should never be reached endcase // case (WalkerState) - assign IgnoreRequestTLB = (WalkerState == IDLE & TLBMissOrUpdateDA) | (HPTWFaultM); // If hptw request has pmp/a fault suppress bus access. + assign HPTWFlushW = (WalkerState == IDLE & TLBMissOrUpdateDA) | (WalkerState != IDLE & HPTWFaultM); + + assign ResetPTE = reset | (NextWalkerState == IDLE); assign SelHPTW = WalkerState != IDLE; assign HPTWStall = (WalkerState != IDLE & WalkerState != FAULT) | (WalkerState == IDLE & TLBMissOrUpdateDA); diff --git a/testbench/tests.vh b/testbench/tests.vh index a91aeda15..bef41ef22 100644 --- a/testbench/tests.vh +++ b/testbench/tests.vh @@ -60,6 +60,7 @@ string coverage64gc[] = '{ "tlbTP", "tlbMisaligned", "hptwAccessFault", + "nonleafpbmtfault", "amoAccessFault", "floatmisc", "ifuCamlineWrite", diff --git a/tests/coverage/nonleafpbmtfault.S b/tests/coverage/nonleafpbmtfault.S new file mode 100644 index 000000000..8a580ce02 --- /dev/null +++ b/tests/coverage/nonleafpbmtfault.S @@ -0,0 +1,143 @@ +/////////////////////////////////////////// +// hptwAccessFault.S +// +// Written: Rose Thompson rose@rosethompson.net +// +// Purpose: Force the HPTW to walk a page table with non-leaf non-zero PBMT bits. This will generate +// a load or store/amo page fault based on the original access type. +// +// A component of the CORE-V-WALLY configurable RISC-V project. +// https://github.com/openhwgroup/cvw +// +// Copyright (C) 2021-24 Harvey Mudd College & Oklahoma State University +// +// SPDX-License-Identifier: Apache-2.0 WITH SHL-2.1 +// +// Licensed under the Solderpad Hardware License v 2.1 (the “License”); you may not use this file +// except in compliance with the License, or, at your option, the Apache License version 2.0. You +// may obtain a copy of the License at +// +// https://solderpad.org/licenses/SHL-2.1/ +// +// Unless required by applicable law or agreed to in writing, any work distributed under the +// License is distributed on an “AS IS” BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +// either express or implied. See the License for the specific language governing permissions +// and limitations under the License. +//////////////////////////////////////////////////////////////////////////////////////////////// + +// load code to initalize stack, handle interrupts, terminate + +#include "WALLY-init-lib.h" + +# run-elf.bash find this in project description +main: + + # Page table root address at 0x80010000 + li t5, 0x9000000000080010 + csrw satp, t5 + + # sfence.vma x0, x0 + + # switch to supervisor mode + li a0, 1 + ecall + li t5, 0 + li t2, 0x1000 + li t0, 0x8000001000 + + lw t1, 0(t0) # valid virtual address, valid physical address, but invalid PBMT in middle of page table. + li t1, 0x00008067 + add t0, t0, t2 + sw t1, 0(t0) # valid virtual address, valid physical address, but invalid PBMT in middle of page table. + + fence.I + +finished: + j done + +.data + +.align 16 +# Page table situated at 0x80010000 +pagetable: + .8byte 0x200044C1 + .8byte 0x200044C1 + +.align 12 + .8byte 0x40000040200048C1 + .8byte 0x00000000200048C1 + .8byte 0x00000000200048C1 + + +.align 12 + .8byte 0x0000000020004CC1 + +.align 12 + #80000000 + .8byte 0x200000CF + .8byte 0x200004CF + .8byte 0x200008CF + .8byte 0x20000CCF + + .8byte 0x200010CF + .8byte 0x200014CF + .8byte 0x200018CF + .8byte 0x20001CCF + + .8byte 0x200020CF + .8byte 0x200024CF + .8byte 0x200028CF + .8byte 0x20002CCF + + .8byte 0x200030CF + .8byte 0x200034CF + .8byte 0x200038CF + .8byte 0x20003CCF + + .8byte 0x200040CF + .8byte 0x200044CF + .8byte 0x200048CF + .8byte 0x20004CCF + + .8byte 0x200050CF + .8byte 0x200054CF + .8byte 0x200058CF + .8byte 0x20005CCF + + .8byte 0x200060CF + .8byte 0x200064CF + .8byte 0x200068CF + .8byte 0x20006CCF + + .8byte 0x200070CF + .8byte 0x200074CF + .8byte 0x200078CF + .8byte 0x20007CCF + + .8byte 0x200080CF + .8byte 0x200084CF + .8byte 0x200088CF + .8byte 0x20008CCF + + .8byte 0x200090CF + .8byte 0x200094CF + .8byte 0x200098CF + .8byte 0x20009CCF + + .8byte 0x2000A0CF + .8byte 0x2000A4CF + .8byte 0x2000A8CF + .8byte 0x2000ACCF + + .8byte 0x2000B0CF + .8byte 0x2000B4CF + .8byte 0x2000B8CF + .8byte 0x2000BCCF + + .8byte 0x2000C0CF + .8byte 0x2000C4CF + .8byte 0x2000C8CF + .8byte 0x2000CCCF + + .8byte 0x2000D0CF + .8byte 0x2000D4CF