Update SMP fence (final)

src/main/scala/vexriscv/TestsWorkspace.scala

@@ -103,7 +103,8 @@ object TestsWorkspace {
               withLrSc = true,
               withAmo = true,
               withExclusive = true,
-              withInvalidate = true
+              withInvalidate = true,
+              pendingMax = 32
               //          )
             ),
             memoryTranslatorPortConfig = MmuPortConfig(

src/main/scala/vexriscv/ip/DataCache.scala

@@ -27,10 +27,11 @@ case class DataCacheConfig(cacheSize : Int,
                            withAmo : Boolean = false,
                            withExclusive : Boolean = false,
                            withInvalidate : Boolean = false,
-                           pendingMax : Int = 64,
+                           pendingMax : Int = 32,
                            mergeExecuteMemory : Boolean = false){
   assert(!(mergeExecuteMemory && (earlyDataMux || earlyWaysHits)))
   assert(!(earlyDataMux && !earlyWaysHits))
+  assert(isPow2(pendingMax))
   def withWriteResponse = withExclusive
   def burstSize = bytePerLine*8/memDataWidth
   val burstLength = bytePerLine/(memDataWidth/8)
@@ -102,10 +103,9 @@ case class DataCacheCpuExecute(p : DataCacheConfig) extends Bundle with IMasterS
   val address = UInt(p.addressWidth bit)
   val haltIt = Bool
   val args = DataCacheCpuExecuteArgs(p)
-  val totalyConsistent = Bool()
 
   override def asMaster(): Unit = {
-    out(isValid, args, address, totalyConsistent)
+    out(isValid, args, address)
     in(haltIt)
   }
 }
@@ -120,6 +120,8 @@ case class DataCacheCpuExecuteArgs(p : DataCacheConfig) extends Bundle{
     val swap = Bool()
     val alu = Bits(3 bits)
   }
+
+  val totalyConsistent = Bool() //Only for AMO/LRSC
 }
 
 case class DataCacheCpuMemory(p : DataCacheConfig) extends Bundle with IMasterSlave{
@@ -129,16 +131,31 @@ case class DataCacheCpuMemory(p : DataCacheConfig) extends Bundle with IMasterSl
   val isWrite = Bool
   val address = UInt(p.addressWidth bit)
   val mmuBus  = MemoryTranslatorBus()
-  val fenceValid = Bool()
 
   override def asMaster(): Unit = {
-    out(isValid, isStuck, isRemoved, address, fenceValid)
+    out(isValid, isStuck, isRemoved, address)
     in(isWrite)
     slave(mmuBus)
   }
 }
 
 
+case class FenceFlags() extends Bundle {
+  val SW,SR,SO,SI,PW,PR,PO,PI = Bool()
+  val FM = Bits(4 bits)
+
+  def SL = SR || SI
+  def SS = SW || SO
+  def PL = PR || PI
+  def PS = PW || PO
+  def forceAll(): Unit ={
+    List(SW,SR,SO,SI,PW,PR,PO,PI).foreach(_ := True)
+  }
+  def clearAll(): Unit ={
+    List(SW,SR,SO,SI,PW,PR,PO,PI).foreach(_ := False)
+  }
+}
+
 case class DataCacheCpuWriteBack(p : DataCacheConfig) extends Bundle with IMasterSlave{
   val isValid = Bool()
   val isStuck = Bool()
@@ -149,13 +166,10 @@ case class DataCacheCpuWriteBack(p : DataCacheConfig) extends Bundle with IMaste
   val address = UInt(p.addressWidth bit)
   val mmuException, unalignedAccess, accessError = Bool()
   val keepMemRspData = Bool() //Used by external AMO to avoid having an internal buffer
-  val fenceValid = Bool()
+  val fence = FenceFlags()
-  val fenceFire = Bool()
-
-  //  val exceptionBus = if(p.catchSomething) Flow(ExceptionCause()) else null
 
   override def asMaster(): Unit = {
-    out(isValid,isStuck,isUser, address, fenceValid, fenceFire)
+    out(isValid,isStuck,isUser, address, fence)
     in(haltIt, data, mmuException, unalignedAccess, accessError, isWrite, keepMemRspData)
   }
 }
@@ -180,6 +194,7 @@ case class DataCacheCpuBus(p : DataCacheConfig) extends Bundle with IMasterSlave
 
 case class DataCacheMemCmd(p : DataCacheConfig) extends Bundle{
   val wr = Bool
+  val uncached = Bool
   val address = UInt(p.addressWidth bit)
   val data = Bits(p.memDataWidth bits)
   val mask = Bits(p.memDataWidth/8 bits)
@@ -532,21 +547,48 @@ class DataCache(val p : DataCacheConfig) extends Component{
   val sync = withInvalidate generate new Area{
     io.mem.sync.ready := True
 
-    val pendingSync = Reg(UInt(log2Up(pendingMax) + 1 bits)) init(0)
+    val syncContext = new Area{
-    val pendingSyncNext = pendingSync + U(io.mem.cmd.fire && io.mem.cmd.wr) - U(io.mem.sync.fire)
+      val history = Mem(Bool, pendingMax)
-    pendingSync := pendingSyncNext
+      val wPtr, rPtr = Reg(UInt(log2Up(pendingMax)+1 bits)) init(0)
+      when(io.mem.cmd.fire && io.mem.cmd.wr){
+        history.write(wPtr.resized, io.mem.cmd.uncached)
+        wPtr := wPtr + 1
+      }
 
-    val full = RegNext(pendingSync.msb)
+      when(io.mem.sync.fire){
+        rPtr := rPtr + 1
+      }
+      val uncached = history.readAsync(rPtr.resized)
+      val full = RegNext(wPtr - rPtr >= pendingMax-1)
       io.cpu.execute.haltIt setWhen(full)
+    }
 
+    def pending(inc : Bool, dec : Bool) = new Area {
+      val pendingSync = Reg(UInt(log2Up(pendingMax) + 1 bits)) init(0)
+      val pendingSyncNext = pendingSync + U(io.mem.cmd.fire && io.mem.cmd.wr && inc) - U(io.mem.sync.fire && dec)
+      pendingSync := pendingSyncNext
+    }
 
-    val incoerentSync = Reg(UInt(log2Up(pendingMax) + 1 bits)) init(0)
+    val writeCached = pending(inc = !io.mem.cmd.uncached, dec = !syncContext.uncached)
-    incoerentSync := incoerentSync - U(io.mem.sync.fire && incoerentSync =/= 0)
+    val writeUncached = pending(inc = io.mem.cmd.uncached, dec = syncContext.uncached)
-    when(io.cpu.writeBack.fenceValid){ incoerentSync := pendingSyncNext }
 
+    def track(load : Bool, uncached : Boolean) = new Area {
+      val counter = Reg(UInt(log2Up(pendingMax) + 1 bits)) init(0)
+      counter := counter - U(io.mem.sync.fire && counter =/= 0 && (if(uncached) syncContext.uncached else !syncContext.uncached))
+      when(load){ counter := (if(uncached) writeUncached.pendingSyncNext else writeCached.pendingSyncNext) }
 
-    val totalyConsistent = pendingSync === 0
+      val busy = counter =/= 0
-    val fenceConsistent = incoerentSync === 0
+    }
+
+    val w2w = track(load = io.cpu.writeBack.fence.PW && io.cpu.writeBack.fence.SW, uncached = false)
+    val w2r = track(load = io.cpu.writeBack.fence.PW && io.cpu.writeBack.fence.SR, uncached = false)
+    val w2i = track(load = io.cpu.writeBack.fence.PW && io.cpu.writeBack.fence.SI, uncached = false)
+    val w2o = track(load = io.cpu.writeBack.fence.PW && io.cpu.writeBack.fence.SO, uncached = false)
+    val o2w = track(load = io.cpu.writeBack.fence.PO && io.cpu.writeBack.fence.SW, uncached =  true)
+    val o2r = track(load = io.cpu.writeBack.fence.PO && io.cpu.writeBack.fence.SR, uncached =  true)
+    //Assume o2i and o2o are ordered by the interconnect
+
+    val notTotalyConsistent = w2w.busy || w2r.busy || w2i.busy || w2o.busy || o2w.busy || o2r.busy
   }
 
 
@@ -562,18 +604,6 @@ class DataCache(val p : DataCacheConfig) extends Component{
     val wayInvalidate = B(0, wayCount bits) //Used if invalidate enabled
 
     val isAmo = if(withAmo) io.cpu.execute.isAmo else False
-
-    //Ensure write to read consistency
-    val consistancyIssue = False
-    val consistancyCheck = (withInvalidate || withWriteResponse) generate new Area {
-      val fenceConsistent =  (if(withInvalidate) sync.fenceConsistent else pending.done) && !io.cpu.writeBack.fenceValid && (if(mergeExecuteMemory) True else !io.cpu.memory.fenceValid) //Pessimistic fence tracking
-      val totalyConsistent = (if(withInvalidate) sync.totalyConsistent else pending.done) && (if(mergeExecuteMemory) True else !(io.cpu.memory.isValid && io.cpu.memory.isWrite)) && !(io.cpu.writeBack.isValid && io.cpu.memory.isWrite)
-      when(io.cpu.execute.isValid /*&& (!io.cpu.execute.args.wr || isAmo)*/){
-        when(!fenceConsistent || io.cpu.execute.totalyConsistent && !totalyConsistent){
-          consistancyIssue := True
-        }
-      }
-    }
   }
 
   val stageA = new Area{
@@ -586,10 +616,29 @@ class DataCache(val p : DataCacheConfig) extends Component{
     io.cpu.memory.mmuBus.end := !io.cpu.memory.isStuck || io.cpu.memory.isRemoved
     io.cpu.memory.isWrite := request.wr
 
+    val isAmo = if(withAmo) request.isAmo else False
+    val isLrsc = if(withAmo) request.isLrsc else False
+    val consistancyCheck = (withInvalidate || withWriteResponse) generate new Area {
+      val hazard = False
+      val w = sync.w2w.busy || sync.o2w.busy
+      val r = stagePipe(sync.w2r.busy || sync.o2r.busy) || sync.w2r.busy || sync.o2r.busy // As it use the cache, need to check against the execute stage status too
+      val o = CombInit(sync.w2o.busy)
+      val i = CombInit(sync.w2i.busy)
+
+      val s = io.cpu.memory.mmuBus.rsp.isIoAccess ? o | w
+      val l = io.cpu.memory.mmuBus.rsp.isIoAccess ? i | r
+
+      when(isAmo? (s || l) | (request.wr ? s | l)){
+        hazard := True
+      }
+      when(request.totalyConsistent && (sync.notTotalyConsistent || io.cpu.writeBack.isValid && io.cpu.writeBack.isWrite)){
+        hazard := True
+      }
+    }
+
     val wayHits = earlyWaysHits generate ways.map(way => (io.cpu.memory.mmuBus.rsp.physicalAddress(tagRange) === way.tagsReadRsp.address && way.tagsReadRsp.valid))
     val dataMux = earlyDataMux generate MuxOH(wayHits, ways.map(_.dataReadRsp))
     val wayInvalidate = stagePipe(stage0. wayInvalidate)
-    val consistancyIssue = stagePipe(stage0.consistancyIssue)
     val dataColisions = if(mergeExecuteMemory){
       stagePipe(stage0.dataColisions)
     } else {
@@ -607,7 +656,7 @@ class DataCache(val p : DataCacheConfig) extends Component{
     val tagsReadRsp = ways.map(w => ramPipe(w.tagsReadRsp))
     val dataReadRsp = !earlyDataMux generate ways.map(w => ramPipe(w.dataReadRsp))
     val wayInvalidate = stagePipe(stageA. wayInvalidate)
-    val consistancyIssue = stagePipe(stageA.consistancyIssue)
+    val consistancyHazard = if(stageA.consistancyCheck != null) stagePipe(stageA.consistancyCheck.hazard) else False
     val dataColisions = stagePipe(stageA.dataColisions)
     val waysHitsBeforeInvalidate = if(earlyWaysHits) stagePipe(B(stageA.wayHits)) else B(tagsReadRsp.map(tag => mmuRsp.physicalAddress(tagRange) === tag.address && tag.valid).asBits())
     val waysHits = waysHitsBeforeInvalidate & ~wayInvalidate
@@ -718,6 +767,7 @@ class DataCache(val p : DataCacheConfig) extends Component{
     io.mem.cmd.wr := request.wr
     io.mem.cmd.mask := mask
     io.mem.cmd.data := requestDataBypass
+    io.mem.cmd.uncached := mmuRsp.isIoAccess
     if(withExternalLrSc) io.mem.cmd.exclusive := request.isLrsc || isAmo
 
 
@@ -830,7 +880,7 @@ class DataCache(val p : DataCacheConfig) extends Component{
     }
 
     //remove side effects on exceptions
-    when(consistancyIssue || mmuRsp.refilling || io.cpu.writeBack.accessError || io.cpu.writeBack.mmuException || io.cpu.writeBack.unalignedAccess){
+    when(consistancyHazard || mmuRsp.refilling || io.cpu.writeBack.accessError || io.cpu.writeBack.mmuException || io.cpu.writeBack.unalignedAccess){
       io.mem.cmd.valid := False
       tagsWriteCmd.valid := False
       dataWriteCmd.valid := False
@@ -839,7 +889,7 @@ class DataCache(val p : DataCacheConfig) extends Component{
       if(withInternalLrSc) lrSc.reserved := lrSc.reserved
       if(withExternalAmo) amo.external.state := LR_CMD
     }
-    io.cpu.redo setWhen(io.cpu.writeBack.isValid && (mmuRsp.refilling || consistancyIssue))
+    io.cpu.redo setWhen(io.cpu.writeBack.isValid && (mmuRsp.refilling || consistancyHazard))
 
     assert(!(io.cpu.writeBack.isValid && !io.cpu.writeBack.haltIt && io.cpu.writeBack.isStuck), "writeBack stuck by another plugin is not allowed")
   }

src/main/scala/vexriscv/plugin/DBusCachedPlugin.scala

@@ -50,8 +50,7 @@ class DBusCachedPlugin(val config : DataCacheConfig,
   object MEMORY_LRSC extends Stageable(Bool)
   object MEMORY_AMO extends Stageable(Bool)
   object MEMORY_FENCE extends Stageable(Bool)
-  object MEMORY_FENCE_FRONT extends Stageable(Bool)
+  object MEMORY_FORCE_CONSTISTENCY extends Stageable(Bool)
-  object MEMORY_FENCE_BACK extends Stageable(Bool)
   object IS_DBUS_SHARING extends Stageable(Bool())
   object MEMORY_VIRTUAL_ADDRESS extends Stageable(UInt(32 bits))
 
@@ -215,31 +214,13 @@ class DBusCachedPlugin(val config : DataCacheConfig,
         arbitration.haltItself := True
       }
 
-      case class FenceFlags() extends Bundle {
-        val SW,SR,SO,SI,PW,PR,PO,PI = Bool()
-        val FM = Bits(4 bits)
-
-        def SL = SR || SI
-        def SS = SW || SO
-        def PL = PR || PI
-        def PS = PW || PO
-      }
 
       //Manage write to read hit ordering (ensure invalidation timings)
       val fence = new Area {
-        insert(MEMORY_FENCE_FRONT) := False
+        insert(MEMORY_FORCE_CONSTISTENCY) := False
-        insert(MEMORY_FENCE_BACK) := False
-        val ff = input(INSTRUCTION)(31 downto 20).as(FenceFlags())
-        if(withWriteResponse){
-          insert(MEMORY_FENCE_BACK) setWhen(input(MEMORY_FENCE) && (ff.PS && ff.SL))
-          when(input(INSTRUCTION)(26)) { //AQ
-            if(withLrSc) insert(MEMORY_FENCE_BACK) setWhen(input(MEMORY_LRSC))
-            if(withAmo)  insert(MEMORY_FENCE_BACK) setWhen(input(MEMORY_AMO))
-          }
         when(input(INSTRUCTION)(25)) { //RL
-            if(withLrSc) insert(MEMORY_FENCE_FRONT) setWhen(input(MEMORY_LRSC))
+          if (withLrSc) insert(MEMORY_FORCE_CONSTISTENCY) setWhen (input(MEMORY_LRSC))
-            if(withAmo)  insert(MEMORY_FENCE_FRONT) setWhen(input(MEMORY_AMO))
+          if (withAmo) insert(MEMORY_FORCE_CONSTISTENCY) setWhen (input(MEMORY_AMO))
-          }
         }
       }
     }
@@ -260,7 +241,7 @@ class DBusCachedPlugin(val config : DataCacheConfig,
 
 
       cache.io.cpu.flush.valid := arbitration.isValid && input(MEMORY_MANAGMENT)
-      cache.io.cpu.execute.totalyConsistent := arbitration.isValid && input(MEMORY_FENCE_FRONT)
+      cache.io.cpu.execute.args.totalyConsistent := input(MEMORY_FORCE_CONSTISTENCY)
       arbitration.haltItself setWhen(cache.io.cpu.flush.isStall || cache.io.cpu.execute.haltIt)
 
       if(withLrSc) {
@@ -302,8 +283,6 @@ class DBusCachedPlugin(val config : DataCacheConfig,
 
       cache.io.cpu.memory.mmuBus <> mmuBus
       cache.io.cpu.memory.mmuBus.rsp.isIoAccess setWhen(pipeline(DEBUG_BYPASS_CACHE) && !cache.io.cpu.memory.isWrite)
-
-      cache.io.cpu.memory.fenceValid := arbitration.isValid && input(MEMORY_FENCE_BACK)
     }
 
     val managementStage = stages.last
@@ -314,8 +293,30 @@ class DBusCachedPlugin(val config : DataCacheConfig,
       cache.io.cpu.writeBack.isUser  := (if(privilegeService != null) privilegeService.isUser() else False)
       cache.io.cpu.writeBack.address := U(input(REGFILE_WRITE_DATA))
 
-      cache.io.cpu.writeBack.fenceValid := arbitration.isValid && input(MEMORY_FENCE_BACK)
+      val fence = if(withInvalidate) {
-      cache.io.cpu.writeBack.fenceFire  := arbitration.isFiring && input(MEMORY_FENCE_BACK)
+        cache.io.cpu.writeBack.fence := input(INSTRUCTION)(31 downto 20).as(FenceFlags())
+        val aquire = False
+        if(withWriteResponse) when(input(INSTRUCTION)(26)) { //AQ
+          if(withLrSc) when(input(MEMORY_LRSC)){
+            aquire := True
+          }
+          if(withAmo) when(input(MEMORY_AMO)){
+            aquire := True
+          }
+        }
+
+        when(aquire){
+          cache.io.cpu.writeBack.fence.forceAll()
+        }
+
+        when(!input(MEMORY_FENCE) || !arbitration.isFiring){
+          cache.io.cpu.writeBack.fence.clearAll()
+        }
+
+        when(arbitration.isValid && (input(MEMORY_FENCE) || aquire)){
+          memory.arbitration.haltByOther := True //Ensure that the fence affect the memory stage instruction by stoping it
+        }
+      }
 
       redoBranch.valid := False
       redoBranch.payload := input(PC)