MMU now allow $ to match tag against tlb pyhsical values directly

D$ retiming
D$ directTlbHit feature added for better timings

src/main/scala/vexriscv/Services.scala

@@ -68,17 +68,24 @@ case class MemoryTranslatorCmd() extends Bundle{
   val virtualAddress  = UInt(32 bits)
   val bypassTranslation = Bool
 }
-case class MemoryTranslatorRsp() extends Bundle{
+case class MemoryTranslatorRsp(wayCount : Int) extends Bundle{
   val physicalAddress = UInt(32 bits)
   val isIoAccess = Bool
   val allowRead, allowWrite, allowExecute = Bool
   val exception = Bool
   val refilling = Bool
+  val bypassTranslation = Bool
+  val ways = Vec(MemoryTranslatorRspWay(), wayCount)
+}
+case class MemoryTranslatorRspWay() extends Bundle{
+  val sel = Bool()
+  val physical = UInt(32 bits)
 }
 
-case class MemoryTranslatorBus() extends Bundle with IMasterSlave{
+
+case class MemoryTranslatorBus(wayCount : Int) extends Bundle with IMasterSlave{
   val cmd = MemoryTranslatorCmd()
-  val rsp = MemoryTranslatorRsp()
+  val rsp = MemoryTranslatorRsp(wayCount)
   val end = Bool
   val busy = Bool
 

src/main/scala/vexriscv/ip/DataCache.scala

@@ -28,6 +28,7 @@ case class DataCacheConfig(cacheSize : Int,
                            withExclusive : Boolean = false,
                            withInvalidate : Boolean = false,
                            pendingMax : Int = 32,
+                           directTlbHit : Boolean = false,
                            mergeExecuteMemory : Boolean = false){
   assert(!(mergeExecuteMemory && (earlyDataMux || earlyWaysHits)))
   assert(!(earlyDataMux && !earlyWaysHits))
@@ -124,13 +125,13 @@ case class DataCacheCpuExecuteArgs(p : DataCacheConfig) extends Bundle{
   val totalyConsistent = Bool() //Only for AMO/LRSC
 }
 
-case class DataCacheCpuMemory(p : DataCacheConfig) extends Bundle with IMasterSlave{
+case class DataCacheCpuMemory(p : DataCacheConfig, tlbWayCount : Int) extends Bundle with IMasterSlave{
   val isValid = Bool
   val isStuck = Bool
   val isRemoved = Bool
   val isWrite = Bool
   val address = UInt(p.addressWidth bit)
-  val mmuBus  = MemoryTranslatorBus()
+  val mmuBus  = MemoryTranslatorBus(tlbWayCount)
 
   override def asMaster(): Unit = {
     out(isValid, isStuck, isRemoved, address)
@@ -174,9 +175,9 @@ case class DataCacheCpuWriteBack(p : DataCacheConfig) extends Bundle with IMaste
   }
 }
 
-case class DataCacheCpuBus(p : DataCacheConfig) extends Bundle with IMasterSlave{
+case class DataCacheCpuBus(p : DataCacheConfig, tlbWayCount : Int) extends Bundle with IMasterSlave{
   val execute   = DataCacheCpuExecute(p)
-  val memory    = DataCacheCpuMemory(p)
+  val memory    = DataCacheCpuMemory(p, tlbWayCount)
   val writeBack = DataCacheCpuWriteBack(p)
 
   val redo = Bool()
@@ -422,11 +423,11 @@ object DataCacheExternalAmoStates extends SpinalEnum{
 }
 
 //If external amo, mem rsp should stay
-class DataCache(val p : DataCacheConfig) extends Component{
+class DataCache(val p : DataCacheConfig, tlbWayCount : Int) extends Component{
   import p._
 
   val io = new Bundle{
-    val cpu = slave(DataCacheCpuBus(p))
+    val cpu = slave(DataCacheCpuBus(p, tlbWayCount))
     val mem = master(DataCacheMemBus(p))
   }
 
@@ -537,11 +538,12 @@ class DataCache(val p : DataCacheConfig) extends Component{
   val memCmdSent = RegInit(False) setWhen (io.mem.cmd.ready) clearWhen (!io.cpu.writeBack.isStuck)
   val pending = withExclusive generate new Area{
     val counter = Reg(UInt(log2Up(pendingMax) + 1 bits)) init(0)
-    counter := counter + U(io.mem.cmd.fire && io.mem.cmd.last) - U(io.mem.rsp.valid  && io.mem.rsp.last)
+    val counterNext = counter + U(io.mem.cmd.fire && io.mem.cmd.last) - U(io.mem.rsp.valid  && io.mem.rsp.last)
+    counter := counterNext
 
-    val done = counter === 0
+    val done = RegNext(counterNext === 0)
-    val full = RegNext(counter.msb)
+    val full = RegNext(counter.msb)       //Has margin
-    val last = counter === 1
+    val last = RegNext(counterNext === 1) //Equivalent to counter === 1 but pipelined
 
     if(!withInvalidate) {
       io.cpu.execute.haltIt setWhen(full)
@@ -643,7 +645,19 @@ class DataCache(val p : DataCacheConfig) extends Component{
       }
     }
 
-    val wayHits = earlyWaysHits generate ways.map(way => (io.cpu.memory.mmuBus.rsp.physicalAddress(tagRange) === way.tagsReadRsp.address && way.tagsReadRsp.valid))
+    val wayHits = earlyWaysHits generate Bits(wayCount bits)
+    val indirectTlbHitGen = (earlyWaysHits && !directTlbHit) generate new Area {
+      wayHits := B(ways.map(way => (io.cpu.memory.mmuBus.rsp.physicalAddress(tagRange) === way.tagsReadRsp.address && way.tagsReadRsp.valid)))
+    }
+    val directTlbHitGen = (earlyWaysHits && directTlbHit) generate new Area {
+      val wayTlbHits = for (way <- ways) yield for (tlb <- io.cpu.memory.mmuBus.rsp.ways) yield {
+        way.tagsReadRsp.address === tlb.physical(tagRange) && tlb.sel
+      }
+      val translatedHits = B(wayTlbHits.map(_.orR))
+      val bypassHits = B(ways.map(_.tagsReadRsp.address === io.cpu.memory.address(tagRange)))
+      wayHits := (io.cpu.memory.mmuBus.rsp.bypassTranslation ? bypassHits | translatedHits) & B(ways.map(_.tagsReadRsp.valid))
+    }
+
     val dataMux = earlyDataMux generate MuxOH(wayHits, ways.map(_.dataReadRsp))
     val wayInvalidate = stagePipe(stage0. wayInvalidate)
     val dataColisions = if(mergeExecuteMemory){

src/main/scala/vexriscv/ip/InstructionCache.scala

@@ -104,7 +104,7 @@ trait InstructionCacheCommons{
   val cacheMiss, error,  mmuRefilling, mmuException, isUser : Bool
 }
 
-case class InstructionCacheCpuFetch(p : InstructionCacheConfig) extends Bundle with IMasterSlave with InstructionCacheCommons {
+case class InstructionCacheCpuFetch(p : InstructionCacheConfig, tlbWayCount : Int) extends Bundle with IMasterSlave with InstructionCacheCommons {
   val isValid = Bool()
   val isStuck = Bool()
   val isRemoved = Bool()
@@ -112,7 +112,7 @@ case class InstructionCacheCpuFetch(p : InstructionCacheConfig) extends Bundle w
   val data = Bits(p.cpuDataWidth bits)
   val dataBypassValid = p.bypassGen generate Bool()
   val dataBypass = p.bypassGen generate Bits(p.cpuDataWidth bits)
-  val mmuBus  = MemoryTranslatorBus()
+  val mmuBus  = MemoryTranslatorBus(tlbWayCount)
   val physicalAddress = UInt(p.addressWidth bits)
   val cacheMiss, error, mmuRefilling, mmuException, isUser  = ifGen(!p.twoCycleCache)(Bool)
   val haltIt  = Bool() //Used to wait on the MMU rsp busy
@@ -141,9 +141,9 @@ case class InstructionCacheCpuDecode(p : InstructionCacheConfig) extends Bundle
   }
 }
 
-case class InstructionCacheCpuBus(p : InstructionCacheConfig) extends Bundle with IMasterSlave{
+case class InstructionCacheCpuBus(p : InstructionCacheConfig, tlbWayCount : Int) extends Bundle with IMasterSlave{
   val prefetch = InstructionCacheCpuPrefetch(p)
-  val fetch = InstructionCacheCpuFetch(p)
+  val fetch = InstructionCacheCpuFetch(p, tlbWayCount)
   val decode = InstructionCacheCpuDecode(p)
   val fill = Flow(UInt(p.addressWidth bits))
 
@@ -277,11 +277,11 @@ case class InstructionCacheFlushBus() extends Bundle with IMasterSlave{
   }
 }
 
-class InstructionCache(p : InstructionCacheConfig) extends Component{
+class InstructionCache(p : InstructionCacheConfig, tlbWayCount : Int) extends Component{
   import p._
   val io = new Bundle{
     val flush = in Bool()
-    val cpu = slave(InstructionCacheCpuBus(p))
+    val cpu = slave(InstructionCacheCpuBus(p, tlbWayCount))
     val mem = master(InstructionCacheMemBus(p))
   }
 

src/main/scala/vexriscv/plugin/DBusCachedPlugin.scala

@@ -171,9 +171,12 @@ class DBusCachedPlugin(val config : DataCacheConfig,
     import pipeline.config._
 
 
-    val cache = new DataCache(this.config.copy(
+    val cache = new DataCache(
+      this.config.copy(
         mergeExecuteMemory = writeBack == null
-    ))
+      ),
+      tlbWayCount = mmuBus.rsp.wayCount
+    )
 
     //Interconnect the plugin dBus with the cache dBus with some optional pipelining
     def optionPipe[T](cond : Boolean, on : T)(f : T => T) : T = if(cond) f(on) else on

src/main/scala/vexriscv/plugin/DBusSimplePlugin.scala

@@ -298,7 +298,6 @@ class DBusSimplePlugin(catchAddressMisaligned : Boolean = false,
   object MEMORY_ADDRESS_LOW extends Stageable(UInt(2 bits))
   object ALIGNEMENT_FAULT extends Stageable(Bool)
   object MMU_FAULT extends Stageable(Bool)
-  object MMU_RSP extends Stageable(MemoryTranslatorRsp())
   object MEMORY_ATOMIC extends Stageable(Bool)
   object ATOMIC_HIT extends Stageable(Bool)
   object MEMORY_STORE extends Stageable(Bool)
@@ -393,6 +392,8 @@ class DBusSimplePlugin(catchAddressMisaligned : Boolean = false,
     import pipeline._
     import pipeline.config._
 
+    object MMU_RSP extends Stageable(MemoryTranslatorRsp(mmuBus.rsp.wayCount))
+
     dBus = master(DBusSimpleBus()).setName("dBus")
 
 

src/main/scala/vexriscv/plugin/IBusCachedPlugin.scala

@@ -124,7 +124,7 @@ class IBusCachedPlugin(resetVector : BigInt = 0x80000000l,
     import pipeline.config._
 
     pipeline plug new FetchArea(pipeline) {
-      val cache = new InstructionCache(IBusCachedPlugin.this.config.copy(bypassGen = tightlyGen))
+      val cache = new InstructionCache(IBusCachedPlugin.this.config.copy(bypassGen = tightlyGen), mmuBus.rsp.wayCount)
       iBus = master(new InstructionCacheMemBus(IBusCachedPlugin.this.config)).setName("iBus")
       iBus <> cache.io.mem
       iBus.cmd.address.allowOverride := cache.io.mem.cmd.address

src/main/scala/vexriscv/plugin/MemoryTranslatorPlugin.scala

@@ -22,8 +22,8 @@ class MemoryTranslatorPlugin(tlbSize : Int,
   val portsInfo = ArrayBuffer[MemoryTranslatorPort]()
 
   override def newTranslationPort(priority : Int,args : Any): MemoryTranslatorBus = {
-//    val exceptionBus = pipeline.service(classOf[ExceptionService]).newExceptionPort(stage)
+    val config = args.asInstanceOf[MemoryTranslatorPortConfig]
-    val port = MemoryTranslatorPort(MemoryTranslatorBus(),priority,args.asInstanceOf[MemoryTranslatorPortConfig]/*,exceptionBus*/)
+    val port = MemoryTranslatorPort(MemoryTranslatorBus(0),priority, config/*,exceptionBus*/)
     portsInfo += port
     port.bus
   }

src/main/scala/vexriscv/plugin/MmuPlugin.scala

@@ -47,7 +47,8 @@ class MmuPlugin(ioRange : UInt => Bool,
   val portsInfo = ArrayBuffer[MmuPort]()
 
   override def newTranslationPort(priority : Int,args : Any): MemoryTranslatorBus = {
-    val port = MmuPort(MemoryTranslatorBus(),priority,args.asInstanceOf[MmuPortConfig], portsInfo.length)
+    val config = args.asInstanceOf[MmuPortConfig]
+    val port = MmuPort(MemoryTranslatorBus(config.portTlbSize),priority, config, portsInfo.length)
     portsInfo += port
     port.bus
   }
@@ -71,7 +72,7 @@ class MmuPlugin(ioRange : UInt => Bool,
     val csrService = pipeline.service(classOf[CsrInterface])
 
     //Sorted by priority
-    val sortedPortsInfo = portsInfo.sortWith((a,b) => a.priority > b.priority)
+    val sortedPortsInfo = portsInfo.sortBy(_.priority)
 
     case class CacheLine() extends Bundle {
       val valid, exception, superPage = Bool
@@ -137,6 +138,12 @@ class MmuPlugin(ioRange : UInt => Bool,
         }
         port.bus.rsp.isIoAccess := ioRange(port.bus.rsp.physicalAddress)
 
+        port.bus.rsp.bypassTranslation := !requireMmuLockup
+        for(wayId <- 0 until port.args.portTlbSize){
+          port.bus.rsp.ways(wayId).sel := cacheHits(wayId)
+          port.bus.rsp.ways(wayId).physical := cache(wayId).physicalAddress(1) @@ (cache(wayId).superPage ? port.bus.cmd.virtualAddress(21 downto 12) | cache(wayId).physicalAddress(0)) @@ port.bus.cmd.virtualAddress(11 downto 0)
+        }
+
         // Avoid keeping any invalid line in the cache after an exception.
         // https://github.com/riscv/riscv-linux/blob/8fe28cb58bcb235034b64cbbb7550a8a43fd88be/arch/riscv/include/asm/pgtable.h#L276
         when(service(classOf[IContextSwitching]).isContextSwitching) {
@@ -154,21 +161,23 @@ class MmuPlugin(ioRange : UInt => Bool,
         }
         val state = RegInit(State.IDLE)
         val vpn = Reg(Vec(UInt(10 bits), UInt(10 bits)))
-        val portId = Reg(UInt(log2Up(portsInfo.length) bits))
+        val portSortedOh = Reg(Bits(portsInfo.length bits))
         case class PTE() extends Bundle {
           val V, R, W ,X, U, G, A, D = Bool()
           val RSW = Bits(2 bits)
           val PPN0 = UInt(10 bits)
           val PPN1 = UInt(12 bits)
         }
+
+        val dBusRspStaged = dBusAccess.rsp.stage()
         val dBusRsp = new Area{
           val pte = PTE()
-          pte.assignFromBits(dBusAccess.rsp.data)
+          pte.assignFromBits(dBusRspStaged.data)
-          val exception = !pte.V || (!pte.R && pte.W) || dBusAccess.rsp.error
+          val exception = !pte.V || (!pte.R && pte.W) || dBusRspStaged.error
           val leaf = pte.R || pte.X
         }
 
-        val pteBuffer = RegNextWhen(dBusRsp.pte, dBusAccess.rsp.valid && !dBusAccess.rsp.redo)
+        val pteBuffer = RegNextWhen(dBusRsp.pte, dBusRspStaged.valid && !dBusRspStaged.redo)
 
         dBusAccess.cmd.valid := False
         dBusAccess.cmd.write := False
@@ -176,16 +185,25 @@ class MmuPlugin(ioRange : UInt => Bool,
         dBusAccess.cmd.address.assignDontCare()
         dBusAccess.cmd.data.assignDontCare()
         dBusAccess.cmd.writeMask.assignDontCare()
+
+        val refills = OHMasking.last(B(sortedPortsInfo.map(port => port.bus.cmd.isValid && port.bus.rsp.refilling)))
         switch(state){
           is(State.IDLE){
-            for(port <- portsInfo.sortBy(_.priority)){
+            when(refills.orR){
-              when(port.bus.cmd.isValid && port.bus.rsp.refilling){
+              portSortedOh := refills
-                vpn(1) := port.bus.cmd.virtualAddress(31 downto 22)
-                vpn(0) := port.bus.cmd.virtualAddress(21 downto 12)
-                portId := port.id
               state := State.L1_CMD
+              val address = MuxOH(refills, sortedPortsInfo.map(_.bus.cmd.virtualAddress))
+              vpn(1) := address(31 downto 22)
+              vpn(0) := address(21 downto 12)
             }
-            }
+//            for(port <- portsInfo.sortBy(_.priority)){
+//              when(port.bus.cmd.isValid && port.bus.rsp.refilling){
+//                vpn(1) := port.bus.cmd.virtualAddress(31 downto 22)
+//                vpn(0) := port.bus.cmd.virtualAddress(21 downto 12)
+//                portId := port.id
+//                state := State.L1_CMD
+//              }
+//            }
           }
           is(State.L1_CMD){
             dBusAccess.cmd.valid := True
@@ -195,12 +213,12 @@ class MmuPlugin(ioRange : UInt => Bool,
             }
           }
           is(State.L1_RSP){
-            when(dBusAccess.rsp.valid){
+            when(dBusRspStaged.valid){
               state := State.L0_CMD
               when(dBusRsp.leaf || dBusRsp.exception){
                 state := State.IDLE
               }
-              when(dBusAccess.rsp.redo){
+              when(dBusRspStaged.redo){
                 state := State.L1_CMD
               }
             }
@@ -213,22 +231,22 @@ class MmuPlugin(ioRange : UInt => Bool,
             }
           }
           is(State.L0_RSP){
-            when(dBusAccess.rsp.valid) {
+            when(dBusRspStaged.valid) {
               state := State.IDLE
-              when(dBusAccess.rsp.redo){
+              when(dBusRspStaged.redo){
                 state := State.L0_CMD
               }
             }
           }
         }
 
-        for(port <- ports) {
+        for((port, id) <- sortedPortsInfo.zipWithIndex) {
-          port.handle.bus.busy := state =/= State.IDLE && portId === port.id
+          port.bus.busy := state =/= State.IDLE && portSortedOh(id)
         }
 
-        when(dBusAccess.rsp.valid && !dBusAccess.rsp.redo && (dBusRsp.leaf || dBusRsp.exception)){
+        when(dBusRspStaged.valid && !dBusRspStaged.redo && (dBusRsp.leaf || dBusRsp.exception)){
-          for(port <- ports){
+          for((port, id) <- ports.zipWithIndex) {
-            when(portId === port.id) {
+            when(portSortedOh(id)) {
               port.entryToReplace.increment()
               for ((line, lineId) <- port.cache.zipWithIndex) {
                 when(port.entryToReplace === lineId){

src/main/scala/vexriscv/plugin/StaticMemoryTranslatorPlugin.scala

@@ -11,8 +11,7 @@ class StaticMemoryTranslatorPlugin(ioRange : UInt => Bool) extends Plugin[VexRis
   val portsInfo = ArrayBuffer[StaticMemoryTranslatorPort]()
 
   override def newTranslationPort(priority : Int,args : Any): MemoryTranslatorBus = {
-//    val exceptionBus = pipeline.service(classOf[ExceptionService]).newExceptionPort(stage)
+    val port = StaticMemoryTranslatorPort(MemoryTranslatorBus(0),priority)
-    val port = StaticMemoryTranslatorPort(MemoryTranslatorBus(),priority)
     portsInfo += port
     port.bus
   }

src/test/scala/vexriscv/TestIndividualFeatures.scala

@@ -436,13 +436,14 @@ class DBusDimension extends VexRiscvDimension("DBus") {
       val dBusRspSlavePipe = r.nextBoolean() || withSmp
       val relaxedMemoryTranslationRegister = r.nextBoolean()
       val earlyWaysHits = r.nextBoolean() && !noWriteBack
+      val directTlbHit = r.nextBoolean() && mmuConfig.isInstanceOf[MmuPortConfig]
       val dBusCmdMasterPipe, dBusCmdSlavePipe = false //As it create test bench issues
 
       do{
         cacheSize = 512 << r.nextInt(5)
         wayCount = 1 << r.nextInt(3)
       }while(cacheSize/wayCount < 512 || (catchAll && cacheSize/wayCount > 4096))
-      new VexRiscvPosition(s"Cached${memDataWidth}d" + "S" + cacheSize + "W" + wayCount + "BPL" + bytePerLine + (if(dBusCmdMasterPipe) "Cmp " else "") + (if(dBusCmdSlavePipe) "Csp " else "") + (if(dBusRspSlavePipe) "Rsp " else "") + (if(relaxedMemoryTranslationRegister) "Rmtr " else "") + (if(earlyWaysHits) "Ewh " else "") + (if(withAmo) "Amo " else "") + (if(withSmp) "Smp " else "")) {
+      new VexRiscvPosition(s"Cached${memDataWidth}d" + "S" + cacheSize + "W" + wayCount + "BPL" + bytePerLine + (if(dBusCmdMasterPipe) "Cmp " else "") + (if(dBusCmdSlavePipe) "Csp " else "") + (if(dBusRspSlavePipe) "Rsp " else "") + (if(relaxedMemoryTranslationRegister) "Rmtr " else "") + (if(earlyWaysHits) "Ewh " else "") + (if(withAmo) "Amo " else "") + (if(withSmp) "Smp " else "") + (if(directTlbHit) "Dtlb " else "")) {
         override def testParam = s"DBUS=CACHED DBUS_DATA_WIDTH=$memDataWidth " + (if(withLrSc) "LRSC=yes " else "")  + (if(withAmo) "AMO=yes " else "")  + (if(withSmp) "DBUS_EXCLUSIVE=yes DBUS_INVALIDATE=yes " else "")
 
         override def applyOn(config: VexRiscvConfig): Unit = {
@@ -461,7 +462,8 @@ class DBusDimension extends VexRiscvDimension("DBus") {
               withAmo = withAmo,
               earlyWaysHits = earlyWaysHits,
               withExclusive = withSmp,
-              withInvalidate = withSmp
+              withInvalidate = withSmp,
+              directTlbHit = directTlbHit
             ),
             dBusCmdMasterPipe = dBusCmdMasterPipe,
             dBusCmdSlavePipe = dBusCmdSlavePipe,