diff --git a/src/main/scala/SpinalRiscv/Plugin/DBusCachedPlugin.scala b/src/main/scala/SpinalRiscv/Plugin/DBusCachedPlugin.scala index e8d02aa..78aa17c 100644 --- a/src/main/scala/SpinalRiscv/Plugin/DBusCachedPlugin.scala +++ b/src/main/scala/SpinalRiscv/Plugin/DBusCachedPlugin.scala @@ -1,29 +1,11 @@ package SpinalRiscv.Plugin -import SpinalRiscv.Riscv._ +import SpinalRiscv.ip._ import SpinalRiscv._ import spinal.core._ import spinal.lib._ -case class DataCacheConfig( cacheSize : Int, - bytePerLine : Int, - wayCount : Int, - addressWidth : Int, - cpuDataWidth : Int, - memDataWidth : Int, - catchAccessError : Boolean, - catchIllegal : Boolean, - catchUnaligned : Boolean, - catchMemoryTranslationMiss : Boolean, - clearTagsAfterReset : Boolean = true, - tagSizeShift : Int = 0){ //Used to force infering ram - def burstSize = bytePerLine*8/memDataWidth - val burstLength = bytePerLine/(memDataWidth/8) - def catchSomething = catchUnaligned || catchMemoryTranslationMiss || catchIllegal || catchAccessError -} - - class DBusCachedPlugin(config : DataCacheConfig, askMemoryTranslation : Boolean = false, memoryTranslatorPortConfig : Any = null) extends Plugin[VexRiscv]{ @@ -31,6 +13,8 @@ class DBusCachedPlugin(config : DataCacheConfig, askMemoryTranslation : Boolean var dBus : DataCacheMemBus = null var mmuBus : MemoryTranslatorBus = null var exceptionBus : Flow[ExceptionCause] = null + var privilegeService : PrivilegeService = null + object MEMORY_ENABLE extends Stageable(Bool) object MEMORY_ADDRESS_LOW extends Stageable(UInt(2 bits)) @@ -76,6 +60,9 @@ class DBusCachedPlugin(config : DataCacheConfig, askMemoryTranslation : Boolean if(catchSomething) exceptionBus = pipeline.service(classOf[ExceptionService]).newExceptionPort(pipeline.writeBack) + + if(pipeline.serviceExist(classOf[PrivilegeService])) + privilegeService = pipeline.service(classOf[PrivilegeService]) } override def build(pipeline: VexRiscv): Unit = { @@ -124,6 +111,7 @@ class DBusCachedPlugin(config : DataCacheConfig, askMemoryTranslation : Boolean cache.io.cpu.memory.mmuBus.rsp.allowExecute := True cache.io.cpu.memory.mmuBus.rsp.allowRead := True cache.io.cpu.memory.mmuBus.rsp.allowWrite := True + cache.io.cpu.memory.mmuBus.rsp.allowUser := True } } @@ -131,16 +119,21 @@ class DBusCachedPlugin(config : DataCacheConfig, askMemoryTranslation : Boolean import writeBack._ cache.io.cpu.writeBack.isValid := arbitration.isValid && input(MEMORY_ENABLE) cache.io.cpu.writeBack.isStuck := arbitration.isStuck + cache.io.cpu.writeBack.isUser := (if(privilegeService != null) privilegeService.isUser(writeBack) else False) + if(catchSomething) { exceptionBus.valid := cache.io.cpu.writeBack.mmuMiss || cache.io.cpu.writeBack.accessError || cache.io.cpu.writeBack.illegalAccess || cache.io.cpu.writeBack.unalignedAccess exceptionBus.badAddr := cache.io.cpu.writeBack.badAddr - exceptionBus.code := 13 + exceptionBus.code.assignDontCare() when(cache.io.cpu.writeBack.illegalAccess || cache.io.cpu.writeBack.accessError){ exceptionBus.code := (input(INSTRUCTION)(5) ? U(7) | U(5)).resized } when(cache.io.cpu.writeBack.unalignedAccess){ exceptionBus.code := (input(INSTRUCTION)(5) ? U(6) | U(4)).resized } + when(cache.io.cpu.writeBack.mmuMiss){ + exceptionBus.code := 13 + } } arbitration.haltIt.setWhen(cache.io.cpu.writeBack.haltIt) @@ -166,555 +159,3 @@ class DBusCachedPlugin(config : DataCacheConfig, askMemoryTranslation : Boolean } - -object Bypasser{ - - //shot readValid path - def writeFirstMemWrap[T <: Data](readValid : Bool, readLastAddress : UInt, readLastData : T,writeValid : Bool, writeAddress : UInt, writeData : T) : T = { - val writeSample = readValid || (writeValid && writeAddress === readLastAddress) - val writeValidReg = RegNextWhen(writeValid,writeSample) - val writeAddressReg = RegNextWhen(writeAddress,writeSample) - val writeDataReg = RegNextWhen(writeData,writeSample) - (writeValidReg && writeAddressReg === readLastAddress) ? writeDataReg | readLastData - } - - - //shot readValid path - def writeFirstMemWrap(readValid : Bool, readLastAddress : UInt, readLastData : Bits,writeValid : Bool, writeAddress : UInt, writeData : Bits,writeMask : Bits) : Bits = { - val writeHit = writeValid && writeAddress === readLastAddress - val writeSample = readValid || writeHit - val writeValidReg = RegNextWhen(writeValid,writeSample) - val writeAddressReg = RegNextWhen(writeAddress,writeSample) - val writeDataReg = Reg(writeData) - val writeMaskReg = Reg(Bits(widthOf(writeData)/8 bits)) - val writeDataRegBytes = writeDataReg.subdivideIn(8 bits) - val writeDataBytes = writeData.subdivideIn(8 bits) - val ret = cloneOf(readLastData) - val retBytes = ret.subdivideIn(8 bits) - val readLastDataBytes = readLastData.subdivideIn(8 bits) - val writeRegHit = writeValidReg && writeAddressReg === readLastAddress - for(b <- writeMask.range){ - when(writeHit && writeMask(b)){ - writeMaskReg(b) := True - } - when(readValid) { - writeMaskReg(b) := writeMask(b) - } - when(readValid || (writeHit && writeMask(b))){ - writeDataRegBytes(b) := writeDataBytes(b) - } - - retBytes(b) := (writeRegHit && writeMaskReg(b)) ? writeDataRegBytes(b) | readLastDataBytes(b) - } - ret - } - - //Long sample path - // def writeFirstRegWrap[T <: Data](sample : Bool, sampleAddress : UInt,lastAddress : UInt, readData : T, writeValid : Bool, writeAddress : UInt, writeData : T) : (T,T) = { - // val hit = writeValid && (sample ? sampleAddress | lastAddress) === writeAddress - // val bypass = hit ? writeData | readData - // val reg = RegNextWhen(bypass,sample || hit) - // (reg,bypass) - // } - - //Short sample path - def writeFirstRegWrap[T <: Data](sample : Bool, sampleAddress : UInt,sampleLastAddress : UInt, sampleData : T, writeValid : Bool, writeAddress : UInt, writeData : T) = { - val bypass = (!sample || (writeValid && sampleAddress === writeAddress)) ? writeData | sampleData - val regEn = sample || (writeValid && sampleLastAddress === writeAddress) - val reg = RegNextWhen(bypass,regEn) - reg - } - - def writeFirstRegWrap(sample : Bool, sampleAddress : UInt,sampleLastAddress : UInt, sampleData : Bits, writeValid : Bool, writeAddress : UInt, writeData : Bits,writeMask : Bits) = { - val byteCount = widthOf(writeMask) - val sampleWriteHit = writeValid && sampleAddress === writeAddress - val sampleLastHit = writeValid && sampleLastAddress === writeAddress - val regBytes = Vec(Bits(8 bits),byteCount) - for(b <- writeMask.range){ - val bypass = Mux(!sample || (sampleWriteHit && writeMask(b)), writeData(b*8, 8 bits), sampleData(b*8, 8 bits)) - val regEn = sample || (sampleLastHit && writeMask(b)) - regBytes(b) := RegNextWhen(bypass,regEn) - } - regBytes.asBits - } -} - -object DataCacheCpuCmdKind extends SpinalEnum{ - val MEMORY,MANAGMENT = newElement() -} - -object DataCacheCpuExecute{ - implicit def implArgs(that : DataCacheCpuExecute) = that.args -} - -case class DataCacheCpuExecute(p : DataCacheConfig) extends Bundle with IMasterSlave{ - val isValid = Bool - val isStuck = Bool -// val haltIt = Bool - val args = DataCacheCpuExecuteArgs(p) - - override def asMaster(): Unit = { - out(isValid, isStuck, args) -// in(haltIt) - } -} - -case class DataCacheCpuExecuteArgs(p : DataCacheConfig) extends Bundle{ - val kind = DataCacheCpuCmdKind() - val wr = Bool - val address = UInt(p.addressWidth bit) - val data = Bits(p.cpuDataWidth bit) - val size = UInt(2 bits) - val forceUncachedAccess = Bool - val clean, invalidate, way = Bool -// val all = Bool //Address should be zero when "all" is used -} - -case class DataCacheCpuMemory(p : DataCacheConfig) extends Bundle with IMasterSlave{ - val isValid = Bool - val isStuck = Bool - val isRemoved = Bool - val mmuBus = MemoryTranslatorBus() - - override def asMaster(): Unit = { - out(isValid, isStuck, isRemoved) - slave(mmuBus) - } -} - - -case class DataCacheCpuWriteBack(p : DataCacheConfig) extends Bundle with IMasterSlave{ - val isValid = Bool - val isStuck = Bool - val haltIt = Bool - val data = Bits(p.cpuDataWidth bit) - val mmuMiss, illegalAccess, unalignedAccess , accessError = Bool - val badAddr = UInt(32 bits) -// val exceptionBus = if(p.catchSomething) Flow(ExceptionCause()) else null - - override def asMaster(): Unit = { - out(isValid,isStuck) - in(haltIt, data, mmuMiss,illegalAccess , unalignedAccess, accessError, badAddr) - } -} - -case class DataCacheCpuBus(p : DataCacheConfig) extends Bundle with IMasterSlave{ - val execute = DataCacheCpuExecute(p) - val memory = DataCacheCpuMemory(p) - val writeBack = DataCacheCpuWriteBack(p) - - override def asMaster(): Unit = { - master(execute) - master(memory) - master(writeBack) - } -} - - -case class DataCacheMemCmd(p : DataCacheConfig) extends Bundle{ - val wr = Bool - val address = UInt(p.addressWidth bit) - val data = Bits(p.memDataWidth bits) - val mask = Bits(p.memDataWidth/8 bits) - val length = UInt(log2Up(p.burstLength+1) bit) -} -case class DataCacheMemRsp(p : DataCacheConfig) extends Bundle{ - val data = Bits(p.memDataWidth bit) - val error = Bool -} - -case class DataCacheMemBus(p : DataCacheConfig) extends Bundle with IMasterSlave{ - val cmd = Stream (DataCacheMemCmd(p)) - val rsp = Flow (DataCacheMemRsp(p)) - - override def asMaster(): Unit = { - master(cmd) - slave(rsp) - } -} - - -class DataCache(p : DataCacheConfig) extends Component{ - import p._ - import DataCacheCpuCmdKind._ - assert(wayCount == 1) - assert(cpuDataWidth == memDataWidth) - - val io = new Bundle{ - val cpu = slave(DataCacheCpuBus(p)) - val mem = master(DataCacheMemBus(p)) -// val flushDone = out Bool //It pulse at the same time than the manager.request.fire - } - val haltCpu = False - val lineWidth = bytePerLine*8 - val lineCount = cacheSize/bytePerLine - val wordWidth = Math.max(memDataWidth,cpuDataWidth) - val wordWidthLog2 = log2Up(wordWidth) - val wordPerLine = lineWidth/wordWidth - val bytePerWord = wordWidth/8 - val wayLineCount = lineCount/wayCount - val wayLineLog2 = log2Up(wayLineCount) - val wayWordCount = wayLineCount * wordPerLine - val memTransactionPerLine = p.bytePerLine / (p.memDataWidth/8) - - val tagRange = addressWidth-1 downto log2Up(wayLineCount*bytePerLine) - val lineRange = tagRange.low-1 downto log2Up(bytePerLine) - val wordRange = log2Up(bytePerLine)-1 downto log2Up(bytePerWord) - - - class LineInfo() extends Bundle{ - val used = Bool - val dirty = Bool - val address = UInt(tagRange.length bit) - } - - val tagsReadCmd = Flow(UInt(log2Up(wayLineCount) bits)) - val tagsWriteCmd = Flow(new Bundle{ -// val way = UInt(log2Up(wayCount) bits) - val address = UInt(log2Up(wayLineCount) bits) - val data = new LineInfo() - }) - - val tagsWriteLastCmd = RegNext(tagsWriteCmd) - - val dataReadCmd = Flow(UInt(log2Up(wayWordCount) bits)) - val dataWriteCmd = Flow(new Bundle{ -// val way = UInt(log2Up(wayCount) bits) - val address = UInt(log2Up(wayWordCount) bits) - val data = Bits(wordWidth bits) - val mask = Bits(wordWidth/8 bits) - }) - - - tagsReadCmd.valid := False - tagsReadCmd.payload.assignDontCare() - dataReadCmd.valid := False - dataReadCmd.payload.assignDontCare() - tagsWriteCmd.valid := False - tagsWriteCmd.payload.assignDontCare() - dataWriteCmd.valid := False - dataWriteCmd.payload.assignDontCare() - io.mem.cmd.valid := False - io.mem.cmd.payload.assignDontCare() - - - val way = new Area{ - val tags = Mem(new LineInfo(),wayLineCount) - val data = Mem(Bits(wordWidth bit),wayWordCount) - - when(tagsWriteCmd.valid){ - tags(tagsWriteCmd.address) := tagsWriteCmd.data - } - when(dataWriteCmd.valid){ - data.write( - address = dataWriteCmd.address, - data = dataWriteCmd.data, - mask = dataWriteCmd.mask - ) - } - - val tagReadRspOneAddress = RegNextWhen(tagsReadCmd.payload, tagsReadCmd.valid) - val tagReadRspOne = Bypasser.writeFirstMemWrap( - readValid = tagsReadCmd.valid, - readLastAddress = tagReadRspOneAddress, - readLastData = tags.readSync(tagsReadCmd.payload,enable = tagsReadCmd.valid), - writeValid = tagsWriteCmd.valid, - writeAddress = tagsWriteCmd.address, - writeData = tagsWriteCmd.data - ) - - val dataReadRspOneKeepAddress = False - val dataReadRspOneAddress = RegNextWhen(dataReadCmd.payload, dataReadCmd.valid && !dataReadRspOneKeepAddress) - val dataReadRspOneWithoutBypass = data.readSync(dataReadCmd.payload,enable = dataReadCmd.valid) - val dataReadRspOne = Bypasser.writeFirstMemWrap( - readValid = dataReadCmd.valid, - readLastAddress = dataReadRspOneAddress, - readLastData = dataReadRspOneWithoutBypass, - writeValid = dataWriteCmd.valid, - writeAddress = dataWriteCmd.address, - writeData = dataWriteCmd.data, - writeMask = dataWriteCmd.mask - ) - - val tagReadRspTwoEnable = !io.cpu.writeBack.isStuck - val tagReadRspTwoRegIn = (tagsWriteCmd.valid && tagsWriteCmd.address === tagReadRspOneAddress) ? tagsWriteCmd.data | tagReadRspOne - val tagReadRspTwo = RegNextWhen(tagReadRspTwoRegIn ,tagReadRspTwoEnable) - - - val dataReadRspTwoEnable = !io.cpu.writeBack.isStuck - val dataReadRspTwo = Bypasser.writeFirstRegWrap( - sample = dataReadRspTwoEnable, - sampleAddress = dataReadRspOneAddress, - sampleLastAddress = RegNextWhen(dataReadRspOneAddress, dataReadRspTwoEnable), - sampleData = dataReadRspOne, - writeValid = dataWriteCmd.valid, - writeAddress = dataWriteCmd.address, - writeData = dataWriteCmd.data, - writeMask = dataWriteCmd.mask - ) - } - - when(io.cpu.execute.isValid && !io.cpu.execute.isStuck){ - tagsReadCmd.valid := True - tagsReadCmd.payload := io.cpu.execute.address(lineRange) - - dataReadCmd.valid := True - dataReadCmd.payload := io.cpu.execute.address(lineRange.high downto wordRange.low) //TODO FMAX maybe critical path could be default - } - - - val victim = new Area{ - val requestIn = Stream(cloneable(new Bundle{ -// val way = UInt(log2Up(wayCount) bits) - val address = UInt(p.addressWidth bits) - })) - requestIn.valid := False - requestIn.payload.assignDontCare() - - val request = requestIn.stage() //TODO FMAX half pipe ? - request.ready := False - - val buffer = Mem(Bits(p.memDataWidth bits),memTransactionPerLine << tagSizeShift) // WARNING << tagSizeShift could resolve cyclone II issue, //.add(new AttributeString("ramstyle","M4K")) - - //Send line read commands to fill the buffer - val readLineCmdCounter = Reg(UInt(log2Up(memTransactionPerLine + 1) bits)) init(0) - val dataReadCmdOccure = False - val dataReadRestored = RegInit(False) - when(request.valid){ - when(!readLineCmdCounter.msb) { - readLineCmdCounter := readLineCmdCounter + 1 - //dataReadCmd := request.address(lineRange.high downto wordRange.low) Done in the manager - dataReadCmdOccure := True - dataReadCmd.valid := True - dataReadCmd.payload := request.address(lineRange) @@ readLineCmdCounter(readLineCmdCounter.high - 1 downto 0) - way.dataReadRspOneKeepAddress := True - } otherwise { - when(!dataReadRestored) { - dataReadCmd.valid := True - dataReadCmd.payload := way.dataReadRspOneAddress //Restore stage one readed value - assert(io.cpu.memory.isStuck,"Should not issue instructions when a victim line is not entirly in the victim cache",FAILURE) - } - dataReadRestored := True - - } - } - - dataReadRestored clearWhen(request.ready) - - //Fill the buffer with line read responses - val readLineRspCounter = Reg(UInt(log2Up(memTransactionPerLine + 1) bits)) init(0) - when(Delay(dataReadCmdOccure,1, init=False)){ - buffer(readLineRspCounter.resized) := way.dataReadRspOneWithoutBypass - readLineRspCounter := readLineRspCounter + 1 - } - - //Send buffer read commands - val bufferReadCounter = Reg(UInt(log2Up(memTransactionPerLine + 1) bits)) init(0) - val bufferReadStream = Stream(buffer.addressType) - bufferReadStream.valid := readLineRspCounter > bufferReadCounter - bufferReadStream.payload := bufferReadCounter.resized - when(bufferReadStream.fire){ - bufferReadCounter := bufferReadCounter + 1 - } - val bufferReaded = buffer.streamReadSync(bufferReadStream).stage - bufferReaded.ready := False - - //Send memory writes from bufffer read responses - val bufferReadedCounter = Reg(UInt(log2Up(memTransactionPerLine) bits)) init(0) - val memCmdAlreadyUsed = False - when(bufferReaded.valid) { - io.mem.cmd.valid := True - io.mem.cmd.wr := True - io.mem.cmd.address := request.address(tagRange.high downto lineRange.low) @@ U(0,lineRange.low bit) - io.mem.cmd.length := p.burstLength - io.mem.cmd.data := bufferReaded.payload - io.mem.cmd.mask := (1<<(wordWidth/8))-1 - - when(!memCmdAlreadyUsed && io.mem.cmd.ready){ - bufferReaded.ready := True - bufferReadedCounter := bufferReadedCounter + 1 - when(bufferReadedCounter === bufferReadedCounter.maxValue){ - request.ready := True - } - } - } - - - val counter = Counter(memTransactionPerLine) - when(request.ready){ - readLineCmdCounter.msb := False - readLineRspCounter.msb := False - bufferReadCounter.msb := False - } - } - - - - val stageA = new Area{ - val request = RegNextWhen(io.cpu.execute.args, !io.cpu.memory.isStuck) - io.cpu.memory.mmuBus.cmd.isValid := io.cpu.memory.isValid && request.kind === MEMORY //TODO filter request kind - io.cpu.memory.mmuBus.cmd.virtualAddress := request.address - io.cpu.memory.mmuBus.cmd.bypassTranslation := request.way - } - - val stageB = new Area { - val request = RegNextWhen(stageA.request, !io.cpu.writeBack.isStuck) - val mmuRsp = RegNextWhen(io.cpu.memory.mmuBus.rsp, !io.cpu.writeBack.isStuck) -// val waysHit = RegNextWhen(way.tagReadRspTwoRegIn.used && stageA.mmuRsp.physicalAddress(tagRange) === way.tagReadRspTwoRegIn.address,!io.cpu.writeBack.isStuck) //Manual retiming - val waysHit = way.tagReadRspTwo.used && mmuRsp.physicalAddress(tagRange) === way.tagReadRspTwo.address - - - //Loader interface - val loaderValid = False - val loaderReady = False - val loadingDone = RegNext(loaderValid && loaderReady) init(False) //one cycle pulse - - //delayedXX are used to relax logic timings in flush and evict modes - val delayedIsStuck = RegNext(io.cpu.writeBack.isStuck) - val delayedWaysHitValid = RegNext(waysHit) - - val victimNotSent = RegInit(False) clearWhen(victim.requestIn.ready) setWhen(!io.cpu.memory.isStuck) - val loadingNotDone = RegInit(False) clearWhen(loaderReady) setWhen(!io.cpu.memory.isStuck) - - val writeMask = request.size.mux ( - U(0) -> B"0001", - U(1) -> B"0011", - default -> B"1111" - ) |<< mmuRsp.physicalAddress(1 downto 0) - - io.cpu.writeBack.haltIt := io.cpu.writeBack.isValid - io.cpu.writeBack.mmuMiss := False - io.cpu.writeBack.illegalAccess := False - io.cpu.writeBack.unalignedAccess := False - io.cpu.writeBack.accessError := (if(catchAccessError) io.mem.rsp.valid && io.mem.rsp.error else False) - io.cpu.writeBack.badAddr := request.address - - //Evict the cache after reset logics - val bootEvicts = if(clearTagsAfterReset) new Area { - val valid = RegInit(True) - mmuRsp.physicalAddress init (0) - when(valid) { - tagsWriteCmd.valid := valid - tagsWriteCmd.address := mmuRsp.physicalAddress(lineRange) - tagsWriteCmd.data.used := False - when(mmuRsp.physicalAddress(lineRange) =/= lineCount - 1) { - mmuRsp.physicalAddress.getDrivingReg(lineRange) := mmuRsp.physicalAddress(lineRange) + 1 - io.cpu.writeBack.haltIt := True - } otherwise { - valid := False - } - } - } - - when(io.cpu.writeBack.isValid) { - if (catchMemoryTranslationMiss) { - io.cpu.writeBack.mmuMiss := mmuRsp.miss - } - switch(request.kind) { - is(MANAGMENT) { - when(delayedIsStuck && !mmuRsp.miss) { - when(delayedWaysHitValid || (request.way && way.tagReadRspTwo.used)) { - io.cpu.writeBack.haltIt.clearWhen(!(victim.requestIn.valid && !victim.requestIn.ready)) - victim.requestIn.valid := request.clean && way.tagReadRspTwo.dirty - tagsWriteCmd.valid := victim.requestIn.ready - } otherwise{ - io.cpu.writeBack.haltIt := False - } - } - - victim.requestIn.address := way.tagReadRspTwo.address @@ mmuRsp.physicalAddress(lineRange) @@ U((lineRange.low - 1 downto 0) -> false) - tagsWriteCmd.address := mmuRsp.physicalAddress(lineRange) - tagsWriteCmd.data.used := !request.invalidate - tagsWriteCmd.data.dirty := !request.clean - } - is(MEMORY) { - val illegal = if(catchIllegal) (request.wr && !mmuRsp.allowWrite) || (!request.wr && !mmuRsp.allowRead) else False - val unaligned = if(catchUnaligned) ((request.size === 2 && mmuRsp.physicalAddress(1 downto 0) =/= 0) || (request.size === 1 && mmuRsp.physicalAddress(0 downto 0) =/= 0)) else False - io.cpu.writeBack.illegalAccess := illegal - io.cpu.writeBack.unalignedAccess := unaligned - when((Bool(!catchMemoryTranslationMiss) || !mmuRsp.miss) && !illegal && !unaligned) { - when(request.forceUncachedAccess || mmuRsp.isIoAccess) { - val memCmdSent = RegInit(False) - when(!victim.request.valid) { - //Avoid mixing memory request while victim is pending - io.mem.cmd.wr := request.wr - io.mem.cmd.address := mmuRsp.physicalAddress(tagRange.high downto wordRange.low) @@ U(0, wordRange.low bit) - io.mem.cmd.mask := writeMask - io.mem.cmd.data := request.data - io.mem.cmd.length := 1 - - when(!memCmdSent) { - io.mem.cmd.valid := True - memCmdSent setWhen (io.mem.cmd.ready) - } - - io.cpu.writeBack.haltIt.clearWhen(memCmdSent && (io.mem.rsp.fire || request.wr)) //Cut mem.cmd.ready path but insert one cycle stall when write - } - memCmdSent clearWhen (!io.cpu.writeBack.isStuck) - } otherwise { - when(waysHit || !loadingNotDone) { - io.cpu.writeBack.haltIt := False - dataWriteCmd.valid := request.wr - dataWriteCmd.address := mmuRsp.physicalAddress(lineRange.high downto wordRange.low) - dataWriteCmd.data := request.data - dataWriteCmd.mask := writeMask - - tagsWriteCmd.valid := (!loadingNotDone) || request.wr - tagsWriteCmd.address := mmuRsp.physicalAddress(lineRange) - tagsWriteCmd.data.used := True - tagsWriteCmd.data.dirty := request.wr - tagsWriteCmd.data.address := mmuRsp.physicalAddress(tagRange) - } otherwise { - val victimRequired = way.tagReadRspTwo.used && way.tagReadRspTwo.dirty - loaderValid := loadingNotDone && !(victimNotSent && victim.request.isStall) //Additional condition used to be sure of that all previous victim are written into the RAM - victim.requestIn.valid := victimRequired && victimNotSent - victim.requestIn.address := way.tagReadRspTwo.address @@ mmuRsp.physicalAddress(lineRange) @@ U((lineRange.low - 1 downto 0) -> false) - } - } - } - } - } - } - - - assert(!(io.cpu.writeBack.isValid && !io.cpu.writeBack.haltIt && io.cpu.writeBack.isStuck), "writeBack stuck by another plugin is not allowed") - io.cpu.writeBack.data := (request.forceUncachedAccess || mmuRsp.isIoAccess) ? io.mem.rsp.data | way.dataReadRspTwo //not multi ways - } - - //The whole life of a loading task, the corresponding manager request is present - val loader = new Area{ - val valid = RegNext(stageB.loaderValid) init(False) - val baseAddress = stageB.mmuRsp.physicalAddress - - val memCmdSent = RegInit(False) - when(valid && !memCmdSent) { - io.mem.cmd.valid := True - io.mem.cmd.wr := False - io.mem.cmd.address := baseAddress(tagRange.high downto lineRange.low) @@ U(0,lineRange.low bit) - io.mem.cmd.length := p.burstLength - } - - when(valid && io.mem.cmd.ready){ - memCmdSent := True - } - - when(valid && !memCmdSent) { - victim.memCmdAlreadyUsed := True - } - - val counter = Counter(memTransactionPerLine) - when(valid && io.mem.rsp.valid){ - dataWriteCmd.valid := True - dataWriteCmd.address := baseAddress(lineRange) @@ counter - dataWriteCmd.data := io.mem.rsp.data - dataWriteCmd.mask := (1<<(wordWidth/8))-1 - counter.increment() - } - - when(counter.willOverflow){ - memCmdSent := False - valid := False - stageB.loaderReady := True - } - } -} \ No newline at end of file diff --git a/src/main/scala/SpinalRiscv/Plugin/IBusCachedPlugin.scala b/src/main/scala/SpinalRiscv/Plugin/IBusCachedPlugin.scala index 00ec6dd..af8da2c 100644 --- a/src/main/scala/SpinalRiscv/Plugin/IBusCachedPlugin.scala +++ b/src/main/scala/SpinalRiscv/Plugin/IBusCachedPlugin.scala @@ -1,29 +1,11 @@ package SpinalRiscv.Plugin import SpinalRiscv._ +import SpinalRiscv.ip._ import spinal.core._ import spinal.lib._ -case class InstructionCacheConfig( cacheSize : Int, - bytePerLine : Int, - wayCount : Int, - wrappedMemAccess : Boolean, - addressWidth : Int, - cpuDataWidth : Int, - memDataWidth : Int, - catchIllegalAccess : Boolean, - catchAccessFault : Boolean, - catchMemoryTranslationMiss : Boolean, - asyncTagMemory : Boolean, - twoStageLogic : Boolean){ - def burstSize = bytePerLine*8/memDataWidth - def catchSomething = catchAccessFault || catchMemoryTranslationMiss || catchIllegalAccess - -} - - - class IBusCachedPlugin(config : InstructionCacheConfig, askMemoryTranslation : Boolean = false, memoryTranslatorPortConfig : Any = null) extends Plugin[VexRiscv] { import config._ assert(twoStageLogic || !askMemoryTranslation) @@ -31,6 +13,7 @@ class IBusCachedPlugin(config : InstructionCacheConfig, askMemoryTranslation : B var iBus : InstructionCacheMemBus = null var mmuBus : MemoryTranslatorBus = null var decodeExceptionPort : Flow[ExceptionCause] = null + var privilegeService : PrivilegeService = null object IBUS_ACCESS_ERROR extends Stageable(Bool) @@ -44,6 +27,9 @@ class IBusCachedPlugin(config : InstructionCacheConfig, askMemoryTranslation : B if(askMemoryTranslation) mmuBus = pipeline.service(classOf[MemoryTranslator]).newTranslationPort(pipeline.fetch, memoryTranslatorPortConfig) + + if(pipeline.serviceExist(classOf[PrivilegeService])) + privilegeService = pipeline.service(classOf[PrivilegeService]) } override def build(pipeline: VexRiscv): Unit = { @@ -88,6 +74,7 @@ class IBusCachedPlugin(config : InstructionCacheConfig, askMemoryTranslation : B cache.io.cpu.decode.isValid := decode.arbitration.isValid decode.arbitration.haltIt.setWhen(cache.io.cpu.decode.haltIt) cache.io.cpu.decode.isStuck := decode.arbitration.isStuck + cache.io.cpu.decode.isUser := (if(privilegeService != null) privilegeService.isUser(writeBack) else False) cache.io.cpu.decode.address := decode.input(PC) decode.insert(INSTRUCTION) := cache.io.cpu.decode.data decode.insert(INSTRUCTION_ANTICIPATED) := cache.io.cpu.decode.dataAnticipated @@ -111,403 +98,3 @@ class IBusCachedPlugin(config : InstructionCacheConfig, askMemoryTranslation : B } } } - - - -case class InstructionCacheCpuPrefetch(p : InstructionCacheConfig) extends Bundle with IMasterSlave{ - val isValid = Bool - val isFiring = Bool - val haltIt = Bool - val address = UInt(p.addressWidth bit) - - override def asMaster(): Unit = { - out(isValid, isFiring, address) - in(haltIt) - } -} - -case class InstructionCacheCpuFetch(p : InstructionCacheConfig) extends Bundle with IMasterSlave { - val isValid = Bool - val haltIt = if(!p.twoStageLogic) Bool else null - val isStuck = Bool - val isStuckByOthers = if(!p.twoStageLogic) Bool else null - val address = UInt(p.addressWidth bit) - val data = if(!p.twoStageLogic) Bits(32 bit) else null - val error = if(!p.twoStageLogic && p.catchAccessFault) Bool else null - val mmuBus = if(p.twoStageLogic) MemoryTranslatorBus() else null - - override def asMaster(): Unit = { - out(isValid, isStuck, address) - outWithNull(isStuckByOthers) - inWithNull(error,data,haltIt) - slaveWithNull(mmuBus) - } -} - -case class InstructionCacheCpuDecode(p : InstructionCacheConfig) extends Bundle with IMasterSlave { - require(p.twoStageLogic) - val isValid = Bool - val haltIt = Bool - val isStuck = Bool - val address = UInt(p.addressWidth bit) - val data = Bits(32 bit) - val dataAnticipated = Bits(32 bits) - val error = if(p.catchAccessFault) Bool else null - val mmuMiss = if(p.catchMemoryTranslationMiss) Bool else null - val illegalAccess = if(p.catchIllegalAccess) Bool else null - - override def asMaster(): Unit = { - out(isValid, isStuck, address) - in(haltIt, data, dataAnticipated) - inWithNull(error,mmuMiss,illegalAccess) - } -} - -case class InstructionCacheCpuBus(p : InstructionCacheConfig) extends Bundle with IMasterSlave{ - val prefetch = InstructionCacheCpuPrefetch(p) - val fetch = InstructionCacheCpuFetch(p) - val decode = if(p.twoStageLogic) InstructionCacheCpuDecode(p) else null - - override def asMaster(): Unit = { - master(prefetch) - master(fetch) - if(p.twoStageLogic) master(decode) - } -} - -case class InstructionCacheMemCmd(p : InstructionCacheConfig) extends Bundle{ - val address = UInt(p.addressWidth bit) -} -case class InstructionCacheMemRsp(p : InstructionCacheConfig) extends Bundle{ - val data = Bits(32 bit) - val error = Bool -} - -case class InstructionCacheMemBus(p : InstructionCacheConfig) extends Bundle with IMasterSlave{ - val cmd = Stream (InstructionCacheMemCmd(p)) - val rsp = Flow (InstructionCacheMemRsp(p)) - - override def asMaster(): Unit = { - master(cmd) - slave(rsp) - } -} - -case class InstructionCacheFlushBus() extends Bundle with IMasterSlave{ - val cmd = Event - val rsp = Bool - - override def asMaster(): Unit = { - master(cmd) - in(rsp) - } -} - -class InstructionCache(p : InstructionCacheConfig) extends Component{ - import p._ - assert(wayCount == 1) - assert(cpuDataWidth == memDataWidth) - val io = new Bundle{ - val flush = slave(InstructionCacheFlushBus()) -// val translator = master(InstructionCacheTranslationBus(p)) - val cpu = slave(InstructionCacheCpuBus(p)) - val mem = master(InstructionCacheMemBus(p)) - } -// val haltCpu = False - val lineWidth = bytePerLine*8 - val lineCount = cacheSize/bytePerLine - val wordWidth = Math.max(memDataWidth,32) - val wordWidthLog2 = log2Up(wordWidth) - val wordPerLine = lineWidth/wordWidth - val bytePerWord = wordWidth/8 - val wayLineCount = lineCount/wayCount - val wayLineLog2 = log2Up(wayLineCount) - val wayWordCount = wayLineCount * wordPerLine - - val tagRange = addressWidth-1 downto log2Up(wayLineCount*bytePerLine) - val lineRange = tagRange.low-1 downto log2Up(bytePerLine) - val wordRange = log2Up(bytePerLine)-1 downto log2Up(bytePerWord) - val tagLineRange = tagRange.high downto lineRange.low - val lineWordRange = lineRange.high downto wordRange.low - - class LineInfo extends Bundle{ - val valid = Bool - val loading = Bool - val error = if(catchAccessFault) Bool else null - val address = UInt(tagRange.length bit) - } - - class LineInfoWithHit extends LineInfo{ - val hit = Bool - } - - def LineInfoWithHit(lineInfo : LineInfo, testTag : UInt) = { - val ret = new LineInfoWithHit() - ret.assignSomeByName(lineInfo) - ret.hit := lineInfo.valid && lineInfo.address === testTag - ret - } - - - val ways = Array.fill(wayCount)(new Area{ - val tags = Mem(new LineInfo(),wayLineCount) - val datas = Mem(Bits(wordWidth bits),wayWordCount) - }) - - - io.cpu.prefetch.haltIt := False - - val lineLoader = new Area{ - val requestIn = Stream(wrap(new Bundle{ - val addr = UInt(addressWidth bits) - })) - - - - val flushCounter = Reg(UInt(log2Up(wayLineCount) + 1 bit)) init(0) - when(!flushCounter.msb){ - io.cpu.prefetch.haltIt := True - flushCounter := flushCounter + 1 - } - when(!RegNext(flushCounter.msb)){ - io.cpu.prefetch.haltIt := True - } - val flushFromInterface = RegInit(False) - when(io.flush.cmd.valid){ - io.cpu.prefetch.haltIt := True - when(io.flush.cmd.ready){ - flushCounter := 0 - flushFromInterface := True - } - } - - io.flush.rsp := flushCounter.msb.rise && flushFromInterface - - val loadingWithErrorReg = if(catchAccessFault) RegInit(False) else null - val loadingWithError = if(catchAccessFault) Bool else null - if(catchAccessFault) { - loadingWithError := loadingWithErrorReg - loadingWithErrorReg := loadingWithError - } - - - - val request = requestIn.stage() - - - //Send memory requests - val memCmdSended = RegInit(False) setWhen(io.mem.cmd.fire) - io.mem.cmd.valid := request.valid && !memCmdSended - if(wrappedMemAccess) - io.mem.cmd.address := request.addr(tagRange.high downto wordRange.low) @@ U(0,wordRange.low bit) - else - io.mem.cmd.address := request.addr(tagRange.high downto lineRange.low) @@ U(0,lineRange.low bit) - - val wordIndex = Reg(UInt(log2Up(wordPerLine) bit)) - val loadedWordsNext = Bits(wordPerLine bit) - val loadedWords = RegNext(loadedWordsNext) - val loadedWordsReadable = RegNext(loadedWords) - loadedWordsNext := loadedWords - - val waysDatasWritePort = ways(0).datas.writePort //Not multi ways - waysDatasWritePort.valid := io.mem.rsp.valid - waysDatasWritePort.address := request.addr(lineRange) @@ wordIndex - waysDatasWritePort.data := io.mem.rsp.data - when(io.mem.rsp.valid){ - wordIndex := wordIndex + 1 - loadedWordsNext(wordIndex) := True - if(catchAccessFault) loadingWithError setWhen io.mem.rsp.error - } - - val memRspLast = loadedWordsNext === B(loadedWordsNext.range -> true) - - val readyDelay = Reg(UInt(1 bit)) - when(memRspLast){ - readyDelay := readyDelay + 1 - } - request.ready := readyDelay === 1 - - val waysTagsWritePort = ways(0).tags.writePort //not multi way - waysTagsWritePort.valid := io.mem.rsp.valid || !flushCounter.msb - waysTagsWritePort.address := Mux(flushCounter.msb,request.addr(lineRange),flushCounter(flushCounter.high-1 downto 0)) - waysTagsWritePort.data.valid := flushCounter.msb - waysTagsWritePort.data.address := request.addr(tagRange) - waysTagsWritePort.data.loading := !memRspLast - if(catchAccessFault) waysTagsWritePort.data.error := loadingWithError - - - when(requestIn.ready){ - memCmdSended := False - wordIndex := requestIn.addr(wordRange) - loadedWords := 0 - loadedWordsReadable := 0 - readyDelay := 0 - if(catchAccessFault) loadingWithErrorReg := False - } - } - - val task = if(!twoStageLogic) new Area{ - val waysHitValid = False - val waysHitError = Bool.assignDontCare() - val waysHitWord = Bits(wordWidth bit) - - val waysRead = for(way <- ways) yield new Area{ - val readAddress = Mux(io.cpu.fetch.isStuck,io.cpu.fetch.address,io.cpu.prefetch.address) //TODO FMAX - val tag = if(asyncTagMemory) - way.tags.readAsync(io.cpu.fetch.address(lineRange),writeFirst) - else - way.tags.readSync(readAddress(lineRange),readUnderWrite = readFirst) - - val data = way.datas.readSync(readAddress(lineRange.high downto wordRange.low)) - waysHitWord := data //Not applicable to multi way - when(tag.valid && tag.address === io.cpu.fetch.address(tagRange)) { - waysHitValid := True - if(catchAccessFault) waysHitError := tag.error - } - } - - - val hit = waysHitValid && !(waysRead(0).tag.loading && !(if(asyncTagMemory) lineLoader.loadedWords else RegNext(lineLoader.loadedWords))(io.cpu.fetch.address(wordRange))) - io.cpu.fetch.haltIt := io.cpu.fetch.isValid && !hit - io.cpu.fetch.data := waysHitWord - if(catchAccessFault) io.cpu.fetch.error := waysRead(0).tag.error - lineLoader.requestIn.valid := io.cpu.fetch.isValid && !hit //TODO avoid duplicated request - lineLoader.requestIn.addr := io.cpu.fetch.address - } else new Area{ - //Long readValidPath -// def writeFirstMemWrap[T <: Data](readValid : Bool, readAddress : UInt, lastAddress : UInt, readData : T,writeValid : Bool, writeAddress : UInt, writeData : T) : T = { -// val hit = writeValid && (readValid ? readAddress | lastAddress) === writeAddress -// val overrideValid = RegInit(False) clearWhen(readValid) setWhen(hit) -// val overrideValue = RegNextWhen(writeData,hit) -// overrideValid ? overrideValue | readData -// } - - //shot readValid path - def writeFirstMemWrap[T <: Data](readValid : Bool, readLastAddress : UInt, readData : T,writeValid : Bool, writeAddress : UInt, writeData : T) : T = { - val writeSample = readValid || (writeValid && writeAddress === readLastAddress) - val writeValidReg = RegNextWhen(writeValid,writeSample) - val writeAddressReg = RegNextWhen(writeAddress,writeSample) - val writeDataReg = RegNextWhen(writeData,writeSample) - (writeValidReg && writeAddressReg === readLastAddress) ? writeDataReg | readData - } - - //Long sample path -// def writeFirstRegWrap[T <: Data](sample : Bool, sampleAddress : UInt,lastAddress : UInt, readData : T, writeValid : Bool, writeAddress : UInt, writeData : T) : (T,T) = { -// val hit = writeValid && (sample ? sampleAddress | lastAddress) === writeAddress -// val bypass = hit ? writeData | readData -// val reg = RegNextWhen(bypass,sample || hit) -// (reg,bypass) -// } - - //Short sample path - def writeFirstRegWrap[T <: Data](sample : Bool, sampleAddress : UInt,sampleLastAddress : UInt, readData : T, writeValid : Bool, writeAddress : UInt, writeData : T) = { - val preWrite = (writeValid && sampleAddress === writeAddress) - val postWrite = (writeValid && sampleLastAddress === writeAddress) - val bypass = (!sample || preWrite) ? writeData | readData - val regEn = sample || postWrite - val reg = RegNextWhen(bypass,regEn) - (reg,bypass,regEn,preWrite,postWrite) - } -// def writeFirstRegWrap[T <: Data](sample : Bool, sampleAddress : UInt,sampleLastAddress : UInt, readData : T, writeValid : Bool, writeAddress : UInt, writeData : T) = { -// val bypass = (!sample || (writeValid && sampleAddress === writeAddress)) ? writeData | readData -// val regEn = sample || (writeValid && sampleLastAddress === writeAddress) -// val reg = RegNextWhen(bypass,regEn) -// (reg,bypass,regEn,False,False) -// } - require(wayCount == 1) - val memRead = new Area{ - val way = ways(0) - val tag = if(asyncTagMemory) - way.tags.readAsync(io.cpu.fetch.address(lineRange),writeFirst) - else - writeFirstMemWrap( - readValid = !io.cpu.fetch.isStuck, -// readAddress = io.cpu.prefetch.address(lineRange), - readLastAddress = io.cpu.fetch.address(lineRange), - readData = way.tags.readSync(io.cpu.prefetch.address(lineRange),enable = !io.cpu.fetch.isStuck), - writeValid = lineLoader.waysTagsWritePort.valid, - writeAddress = lineLoader.waysTagsWritePort.address, - writeData = lineLoader.waysTagsWritePort.data - ) - - val data = writeFirstMemWrap( - readValid = !io.cpu.fetch.isStuck, -// readAddress = io.cpu.prefetch.address(lineWordRange), - readLastAddress = io.cpu.fetch.address(lineWordRange), - readData = way.datas.readSync(io.cpu.prefetch.address(lineWordRange),enable = !io.cpu.fetch.isStuck), - writeValid = lineLoader.waysDatasWritePort.valid, - writeAddress = lineLoader.waysDatasWritePort.address, - writeData = lineLoader.waysDatasWritePort.data - ) - } - - - val tag = writeFirstRegWrap( - sample = !io.cpu.decode.isStuck, - sampleAddress = io.cpu.fetch.address(lineRange), - sampleLastAddress = io.cpu.decode.address(lineRange), - readData = LineInfoWithHit(memRead.tag,io.cpu.fetch.address(tagRange)), - writeValid = lineLoader.waysTagsWritePort.valid, - writeAddress = lineLoader.waysTagsWritePort.address, - writeData = LineInfoWithHit(lineLoader.waysTagsWritePort.data,io.cpu.fetch.address(tagRange)) //TODO wrong address src - )._1 - - val (data,dataRegIn,dataRegEn,dataPreWrite,dataPostWrite) = writeFirstRegWrap( - sample = !io.cpu.decode.isStuck, - sampleAddress = io.cpu.fetch.address(lineWordRange), - sampleLastAddress = io.cpu.decode.address(lineWordRange), - readData = memRead.data, - writeValid = lineLoader.waysDatasWritePort.valid, - writeAddress = lineLoader.waysDatasWritePort.address, - writeData = lineLoader.waysDatasWritePort.data - ) - - io.cpu.fetch.mmuBus.cmd.isValid := io.cpu.fetch.isValid - io.cpu.fetch.mmuBus.cmd.virtualAddress := io.cpu.fetch.address - io.cpu.fetch.mmuBus.cmd.bypassTranslation := False - val mmuRsp = RegNextWhen(io.cpu.fetch.mmuBus.rsp,!io.cpu.decode.isStuck) - - val hit = tag.valid && tag.address === mmuRsp.physicalAddress(tagRange) && !(tag.loading && !lineLoader.loadedWords(mmuRsp.physicalAddress(wordRange))) -// val hit = tag.hit && !(tag.loading && !lineLoader.loadedWords(mmuRsp.physicalAddress(wordRange))) - - io.cpu.decode.haltIt := io.cpu.decode.isValid && !hit //TODO PERF not halit it when removed, Should probably be applyed in many other places - io.cpu.decode.data := data -// io.cpu.decode.dataAnticipated := dataRegEn ? dataRegIn | data - io.cpu.decode.dataAnticipated := io.cpu.decode.isStuck ? Mux(dataPostWrite,lineLoader.waysDatasWritePort.data,data) | Mux(dataPreWrite,lineLoader.waysDatasWritePort.data,memRead.data) - if(catchAccessFault) io.cpu.decode.error := tag.error - if(catchMemoryTranslationMiss) io.cpu.decode.mmuMiss := mmuRsp.miss - if(catchIllegalAccess) io.cpu.decode.illegalAccess := !mmuRsp.miss && !mmuRsp.allowExecute - - lineLoader.requestIn.valid := io.cpu.decode.isValid && !hit && !mmuRsp.miss//TODO avoid duplicated request - lineLoader.requestIn.addr := mmuRsp.physicalAddress - } - - io.flush.cmd.ready := !(lineLoader.request.valid || io.cpu.fetch.isValid) -} - - - - - - -//object InstructionCacheMain{ -// -// def main(args: Array[String]) { -// implicit val p = InstructionCacheConfig( -// cacheSize =4096, -// bytePerLine =32, -// wayCount = 1, -// wrappedMemAccess = true, -// addressWidth = 32, -// cpuDataWidth = 32, -// memDataWidth = 32, -// catchAccessFault = true) -// // val io = new Bundle{ -// // val cpu = slave(InstructionCacheCpuBus()) -// // val mem = master(InstructionCacheMemBus()) -// // } -// -// SpinalVhdl(new InstructionCache(p)) -// } -//} -// diff --git a/src/main/scala/SpinalRiscv/Plugin/MachineCsr.scala b/src/main/scala/SpinalRiscv/Plugin/MachineCsr.scala index 2fb52f8..86f9f67 100644 --- a/src/main/scala/SpinalRiscv/Plugin/MachineCsr.scala +++ b/src/main/scala/SpinalRiscv/Plugin/MachineCsr.scala @@ -62,7 +62,7 @@ case class CsrMapping(){ -class MachineCsr(config : MachineCsrConfig) extends Plugin[VexRiscv] with ExceptionService { +class MachineCsr(config : MachineCsrConfig) extends Plugin[VexRiscv] with ExceptionService with PrivilegeService{ import config._ import CsrAccess._ @@ -144,6 +144,8 @@ class MachineCsr(config : MachineCsrConfig) extends Plugin[VexRiscv] with Except } + def isUser(stage : Stage) : Bool = False + override def build(pipeline: VexRiscv): Unit = { import pipeline._ import pipeline.config._ diff --git a/src/main/scala/SpinalRiscv/Plugin/MemoryTranslatorPlugin.scala b/src/main/scala/SpinalRiscv/Plugin/MemoryTranslatorPlugin.scala index 4ca604e..98d1916 100644 --- a/src/main/scala/SpinalRiscv/Plugin/MemoryTranslatorPlugin.scala +++ b/src/main/scala/SpinalRiscv/Plugin/MemoryTranslatorPlugin.scala @@ -47,7 +47,7 @@ class MemoryTranslatorPlugin(tlbSize : Int, val valid = Bool val virtualAddress = UInt(20 bits) val physicalAddress = UInt(20 bits) - val allowRead, allowWrite, allowExecute = Bool + val allowRead, allowWrite, allowExecute, allowUser = Bool def init = { valid init (False) @@ -107,12 +107,14 @@ class MemoryTranslatorPlugin(tlbSize : Int, port.bus.rsp.allowRead := cacheLine.allowRead port.bus.rsp.allowWrite := cacheLine.allowWrite port.bus.rsp.allowExecute := cacheLine.allowExecute + port.bus.rsp.allowUser := cacheLine.allowUser port.stage.arbitration.haltIt setWhen (port.bus.cmd.isValid && !cacheHit && !sharedMiss) } otherwise { port.bus.rsp.physicalAddress := port.bus.cmd.virtualAddress port.bus.rsp.allowRead := True port.bus.rsp.allowWrite := True port.bus.rsp.allowExecute := True + port.bus.rsp.allowUser := True } port.bus.rsp.isIoAccess := ioRange(port.bus.rsp.physicalAddress) port.bus.rsp.miss := sharedMiss @@ -132,10 +134,11 @@ class MemoryTranslatorPlugin(tlbSize : Int, val line = CacheLine() line.virtualAddress := tlbWriteBuffer line.physicalAddress := input(REG2)(19 downto 0).asUInt - line.valid := input(REG2)(31) + line.allowUser := input(REG2)(27) line.allowRead := input(REG2)(28) line.allowWrite := input(REG2)(29) line.allowExecute := input(REG2)(30) + line.valid := input(REG2)(31) core.shared.cache(input(SRC1)(log2Up(tlbSize)-1 downto 0).asUInt) := line core.ports.foreach(_.cache.foreach(_.valid := False)) //Invalidate all ports caches diff --git a/src/main/scala/SpinalRiscv/Services.scala b/src/main/scala/SpinalRiscv/Services.scala index f1115fa..f091eb9 100644 --- a/src/main/scala/SpinalRiscv/Services.scala +++ b/src/main/scala/SpinalRiscv/Services.scala @@ -22,6 +22,10 @@ trait ExceptionService{ def newExceptionPort(stage : Stage, priority : Int = 0) : Flow[ExceptionCause] } +trait PrivilegeService{ + def isUser(stage : Stage) : Bool +} + case class MemoryTranslatorCmd() extends Bundle{ val isValid = Bool val virtualAddress = UInt(32 bits) @@ -30,7 +34,7 @@ case class MemoryTranslatorCmd() extends Bundle{ case class MemoryTranslatorRsp() extends Bundle{ val physicalAddress = UInt(32 bits) val isIoAccess = Bool - val allowRead, allowWrite, allowExecute = Bool + val allowRead, allowWrite, allowExecute, allowUser = Bool val miss = Bool } diff --git a/src/main/scala/SpinalRiscv/TopLevel.scala b/src/main/scala/SpinalRiscv/TopLevel.scala index 8874df9..3d4fef1 100644 --- a/src/main/scala/SpinalRiscv/TopLevel.scala +++ b/src/main/scala/SpinalRiscv/TopLevel.scala @@ -21,7 +21,7 @@ package SpinalRiscv import SpinalRiscv.Plugin._ import spinal.core._ import spinal.lib._ - +import SpinalRiscv.ip._ object TopLevel { def main(args: Array[String]) { diff --git a/src/main/scala/SpinalRiscv/ip/DataCache.scala b/src/main/scala/SpinalRiscv/ip/DataCache.scala new file mode 100644 index 0000000..8cc7ee0 --- /dev/null +++ b/src/main/scala/SpinalRiscv/ip/DataCache.scala @@ -0,0 +1,577 @@ +package SpinalRiscv.ip + +import SpinalRiscv._ +import spinal.core._ +import spinal.lib._ + + +case class DataCacheConfig( cacheSize : Int, + bytePerLine : Int, + wayCount : Int, + addressWidth : Int, + cpuDataWidth : Int, + memDataWidth : Int, + catchAccessError : Boolean, + catchIllegal : Boolean, + catchUnaligned : Boolean, + catchMemoryTranslationMiss : Boolean, + clearTagsAfterReset : Boolean = true, + tagSizeShift : Int = 0){ //Used to force infering ram +def burstSize = bytePerLine*8/memDataWidth + val burstLength = bytePerLine/(memDataWidth/8) + def catchSomething = catchUnaligned || catchMemoryTranslationMiss || catchIllegal || catchAccessError +} + + +object Bypasser{ + + //shot readValid path + def writeFirstMemWrap[T <: Data](readValid : Bool, readLastAddress : UInt, readLastData : T,writeValid : Bool, writeAddress : UInt, writeData : T) : T = { + val writeSample = readValid || (writeValid && writeAddress === readLastAddress) + val writeValidReg = RegNextWhen(writeValid,writeSample) + val writeAddressReg = RegNextWhen(writeAddress,writeSample) + val writeDataReg = RegNextWhen(writeData,writeSample) + (writeValidReg && writeAddressReg === readLastAddress) ? writeDataReg | readLastData + } + + + //shot readValid path + def writeFirstMemWrap(readValid : Bool, readLastAddress : UInt, readLastData : Bits,writeValid : Bool, writeAddress : UInt, writeData : Bits,writeMask : Bits) : Bits = { + val writeHit = writeValid && writeAddress === readLastAddress + val writeSample = readValid || writeHit + val writeValidReg = RegNextWhen(writeValid,writeSample) + val writeAddressReg = RegNextWhen(writeAddress,writeSample) + val writeDataReg = Reg(writeData) + val writeMaskReg = Reg(Bits(widthOf(writeData)/8 bits)) + val writeDataRegBytes = writeDataReg.subdivideIn(8 bits) + val writeDataBytes = writeData.subdivideIn(8 bits) + val ret = cloneOf(readLastData) + val retBytes = ret.subdivideIn(8 bits) + val readLastDataBytes = readLastData.subdivideIn(8 bits) + val writeRegHit = writeValidReg && writeAddressReg === readLastAddress + for(b <- writeMask.range){ + when(writeHit && writeMask(b)){ + writeMaskReg(b) := True + } + when(readValid) { + writeMaskReg(b) := writeMask(b) + } + when(readValid || (writeHit && writeMask(b))){ + writeDataRegBytes(b) := writeDataBytes(b) + } + + retBytes(b) := (writeRegHit && writeMaskReg(b)) ? writeDataRegBytes(b) | readLastDataBytes(b) + } + ret + } + + //Long sample path + // def writeFirstRegWrap[T <: Data](sample : Bool, sampleAddress : UInt,lastAddress : UInt, readData : T, writeValid : Bool, writeAddress : UInt, writeData : T) : (T,T) = { + // val hit = writeValid && (sample ? sampleAddress | lastAddress) === writeAddress + // val bypass = hit ? writeData | readData + // val reg = RegNextWhen(bypass,sample || hit) + // (reg,bypass) + // } + + //Short sample path + def writeFirstRegWrap[T <: Data](sample : Bool, sampleAddress : UInt,sampleLastAddress : UInt, sampleData : T, writeValid : Bool, writeAddress : UInt, writeData : T) = { + val bypass = (!sample || (writeValid && sampleAddress === writeAddress)) ? writeData | sampleData + val regEn = sample || (writeValid && sampleLastAddress === writeAddress) + val reg = RegNextWhen(bypass,regEn) + reg + } + + def writeFirstRegWrap(sample : Bool, sampleAddress : UInt,sampleLastAddress : UInt, sampleData : Bits, writeValid : Bool, writeAddress : UInt, writeData : Bits,writeMask : Bits) = { + val byteCount = widthOf(writeMask) + val sampleWriteHit = writeValid && sampleAddress === writeAddress + val sampleLastHit = writeValid && sampleLastAddress === writeAddress + val regBytes = Vec(Bits(8 bits),byteCount) + for(b <- writeMask.range){ + val bypass = Mux(!sample || (sampleWriteHit && writeMask(b)), writeData(b*8, 8 bits), sampleData(b*8, 8 bits)) + val regEn = sample || (sampleLastHit && writeMask(b)) + regBytes(b) := RegNextWhen(bypass,regEn) + } + regBytes.asBits + } +} + +object DataCacheCpuCmdKind extends SpinalEnum{ + val MEMORY,MANAGMENT = newElement() +} + +object DataCacheCpuExecute{ + implicit def implArgs(that : DataCacheCpuExecute) = that.args +} + +case class DataCacheCpuExecute(p : DataCacheConfig) extends Bundle with IMasterSlave{ + val isValid = Bool + val isStuck = Bool + // val haltIt = Bool + val args = DataCacheCpuExecuteArgs(p) + + override def asMaster(): Unit = { + out(isValid, isStuck, args) + // in(haltIt) + } +} + +case class DataCacheCpuExecuteArgs(p : DataCacheConfig) extends Bundle{ + val kind = DataCacheCpuCmdKind() + val wr = Bool + val address = UInt(p.addressWidth bit) + val data = Bits(p.cpuDataWidth bit) + val size = UInt(2 bits) + val forceUncachedAccess = Bool + val clean, invalidate, way = Bool + // val all = Bool //Address should be zero when "all" is used +} + +case class DataCacheCpuMemory(p : DataCacheConfig) extends Bundle with IMasterSlave{ + val isValid = Bool + val isStuck = Bool + val isRemoved = Bool + val mmuBus = MemoryTranslatorBus() + + override def asMaster(): Unit = { + out(isValid, isStuck, isRemoved) + slave(mmuBus) + } +} + + +case class DataCacheCpuWriteBack(p : DataCacheConfig) extends Bundle with IMasterSlave{ + val isValid = Bool + val isStuck = Bool + val isUser = Bool + val haltIt = Bool + val data = Bits(p.cpuDataWidth bit) + val mmuMiss, illegalAccess, unalignedAccess , accessError = Bool + val badAddr = UInt(32 bits) + // val exceptionBus = if(p.catchSomething) Flow(ExceptionCause()) else null + + override def asMaster(): Unit = { + out(isValid,isStuck,isUser) + in(haltIt, data, mmuMiss,illegalAccess , unalignedAccess, accessError, badAddr) + } +} + +case class DataCacheCpuBus(p : DataCacheConfig) extends Bundle with IMasterSlave{ + val execute = DataCacheCpuExecute(p) + val memory = DataCacheCpuMemory(p) + val writeBack = DataCacheCpuWriteBack(p) + + override def asMaster(): Unit = { + master(execute) + master(memory) + master(writeBack) + } +} + + +case class DataCacheMemCmd(p : DataCacheConfig) extends Bundle{ + val wr = Bool + val address = UInt(p.addressWidth bit) + val data = Bits(p.memDataWidth bits) + val mask = Bits(p.memDataWidth/8 bits) + val length = UInt(log2Up(p.burstLength+1) bit) +} +case class DataCacheMemRsp(p : DataCacheConfig) extends Bundle{ + val data = Bits(p.memDataWidth bit) + val error = Bool +} + +case class DataCacheMemBus(p : DataCacheConfig) extends Bundle with IMasterSlave{ + val cmd = Stream (DataCacheMemCmd(p)) + val rsp = Flow (DataCacheMemRsp(p)) + + override def asMaster(): Unit = { + master(cmd) + slave(rsp) + } +} + + +class DataCache(p : DataCacheConfig) extends Component{ + import p._ + import DataCacheCpuCmdKind._ + assert(wayCount == 1) + assert(cpuDataWidth == memDataWidth) + + val io = new Bundle{ + val cpu = slave(DataCacheCpuBus(p)) + val mem = master(DataCacheMemBus(p)) + // val flushDone = out Bool //It pulse at the same time than the manager.request.fire + } + val haltCpu = False + val lineWidth = bytePerLine*8 + val lineCount = cacheSize/bytePerLine + val wordWidth = Math.max(memDataWidth,cpuDataWidth) + val wordWidthLog2 = log2Up(wordWidth) + val wordPerLine = lineWidth/wordWidth + val bytePerWord = wordWidth/8 + val wayLineCount = lineCount/wayCount + val wayLineLog2 = log2Up(wayLineCount) + val wayWordCount = wayLineCount * wordPerLine + val memTransactionPerLine = p.bytePerLine / (p.memDataWidth/8) + + val tagRange = addressWidth-1 downto log2Up(wayLineCount*bytePerLine) + val lineRange = tagRange.low-1 downto log2Up(bytePerLine) + val wordRange = log2Up(bytePerLine)-1 downto log2Up(bytePerWord) + + + class LineInfo() extends Bundle{ + val used = Bool + val dirty = Bool + val address = UInt(tagRange.length bit) + } + + val tagsReadCmd = Flow(UInt(log2Up(wayLineCount) bits)) + val tagsWriteCmd = Flow(new Bundle{ + // val way = UInt(log2Up(wayCount) bits) + val address = UInt(log2Up(wayLineCount) bits) + val data = new LineInfo() + }) + + val tagsWriteLastCmd = RegNext(tagsWriteCmd) + + val dataReadCmd = Flow(UInt(log2Up(wayWordCount) bits)) + val dataWriteCmd = Flow(new Bundle{ + // val way = UInt(log2Up(wayCount) bits) + val address = UInt(log2Up(wayWordCount) bits) + val data = Bits(wordWidth bits) + val mask = Bits(wordWidth/8 bits) + }) + + + tagsReadCmd.valid := False + tagsReadCmd.payload.assignDontCare() + dataReadCmd.valid := False + dataReadCmd.payload.assignDontCare() + tagsWriteCmd.valid := False + tagsWriteCmd.payload.assignDontCare() + dataWriteCmd.valid := False + dataWriteCmd.payload.assignDontCare() + io.mem.cmd.valid := False + io.mem.cmd.payload.assignDontCare() + + + val way = new Area{ + val tags = Mem(new LineInfo(),wayLineCount) + val data = Mem(Bits(wordWidth bit),wayWordCount) + + when(tagsWriteCmd.valid){ + tags(tagsWriteCmd.address) := tagsWriteCmd.data + } + when(dataWriteCmd.valid){ + data.write( + address = dataWriteCmd.address, + data = dataWriteCmd.data, + mask = dataWriteCmd.mask + ) + } + + val tagReadRspOneAddress = RegNextWhen(tagsReadCmd.payload, tagsReadCmd.valid) + val tagReadRspOne = Bypasser.writeFirstMemWrap( + readValid = tagsReadCmd.valid, + readLastAddress = tagReadRspOneAddress, + readLastData = tags.readSync(tagsReadCmd.payload,enable = tagsReadCmd.valid), + writeValid = tagsWriteCmd.valid, + writeAddress = tagsWriteCmd.address, + writeData = tagsWriteCmd.data + ) + + val dataReadRspOneKeepAddress = False + val dataReadRspOneAddress = RegNextWhen(dataReadCmd.payload, dataReadCmd.valid && !dataReadRspOneKeepAddress) + val dataReadRspOneWithoutBypass = data.readSync(dataReadCmd.payload,enable = dataReadCmd.valid) + val dataReadRspOne = Bypasser.writeFirstMemWrap( + readValid = dataReadCmd.valid, + readLastAddress = dataReadRspOneAddress, + readLastData = dataReadRspOneWithoutBypass, + writeValid = dataWriteCmd.valid, + writeAddress = dataWriteCmd.address, + writeData = dataWriteCmd.data, + writeMask = dataWriteCmd.mask + ) + + val tagReadRspTwoEnable = !io.cpu.writeBack.isStuck + val tagReadRspTwoRegIn = (tagsWriteCmd.valid && tagsWriteCmd.address === tagReadRspOneAddress) ? tagsWriteCmd.data | tagReadRspOne + val tagReadRspTwo = RegNextWhen(tagReadRspTwoRegIn ,tagReadRspTwoEnable) + + + val dataReadRspTwoEnable = !io.cpu.writeBack.isStuck + val dataReadRspTwo = Bypasser.writeFirstRegWrap( + sample = dataReadRspTwoEnable, + sampleAddress = dataReadRspOneAddress, + sampleLastAddress = RegNextWhen(dataReadRspOneAddress, dataReadRspTwoEnable), + sampleData = dataReadRspOne, + writeValid = dataWriteCmd.valid, + writeAddress = dataWriteCmd.address, + writeData = dataWriteCmd.data, + writeMask = dataWriteCmd.mask + ) + } + + when(io.cpu.execute.isValid && !io.cpu.execute.isStuck){ + tagsReadCmd.valid := True + tagsReadCmd.payload := io.cpu.execute.address(lineRange) + + dataReadCmd.valid := True + dataReadCmd.payload := io.cpu.execute.address(lineRange.high downto wordRange.low) //TODO FMAX maybe critical path could be default + } + + + val victim = new Area{ + val requestIn = Stream(cloneable(new Bundle{ + // val way = UInt(log2Up(wayCount) bits) + val address = UInt(p.addressWidth bits) + })) + requestIn.valid := False + requestIn.payload.assignDontCare() + + val request = requestIn.stage() //TODO FMAX half pipe ? + request.ready := False + + val buffer = Mem(Bits(p.memDataWidth bits),memTransactionPerLine << tagSizeShift) // WARNING << tagSizeShift could resolve cyclone II issue, //.add(new AttributeString("ramstyle","M4K")) + + //Send line read commands to fill the buffer + val readLineCmdCounter = Reg(UInt(log2Up(memTransactionPerLine + 1) bits)) init(0) + val dataReadCmdOccure = False + val dataReadRestored = RegInit(False) + when(request.valid){ + when(!readLineCmdCounter.msb) { + readLineCmdCounter := readLineCmdCounter + 1 + //dataReadCmd := request.address(lineRange.high downto wordRange.low) Done in the manager + dataReadCmdOccure := True + dataReadCmd.valid := True + dataReadCmd.payload := request.address(lineRange) @@ readLineCmdCounter(readLineCmdCounter.high - 1 downto 0) + way.dataReadRspOneKeepAddress := True + } otherwise { + when(!dataReadRestored) { + dataReadCmd.valid := True + dataReadCmd.payload := way.dataReadRspOneAddress //Restore stage one readed value + assert(io.cpu.memory.isStuck,"Should not issue instructions when a victim line is not entirly in the victim cache",FAILURE) + } + dataReadRestored := True + + } + } + + dataReadRestored clearWhen(request.ready) + + //Fill the buffer with line read responses + val readLineRspCounter = Reg(UInt(log2Up(memTransactionPerLine + 1) bits)) init(0) + when(Delay(dataReadCmdOccure,1, init=False)){ + buffer(readLineRspCounter.resized) := way.dataReadRspOneWithoutBypass + readLineRspCounter := readLineRspCounter + 1 + } + + //Send buffer read commands + val bufferReadCounter = Reg(UInt(log2Up(memTransactionPerLine + 1) bits)) init(0) + val bufferReadStream = Stream(buffer.addressType) + bufferReadStream.valid := readLineRspCounter > bufferReadCounter + bufferReadStream.payload := bufferReadCounter.resized + when(bufferReadStream.fire){ + bufferReadCounter := bufferReadCounter + 1 + } + val bufferReaded = buffer.streamReadSync(bufferReadStream).stage + bufferReaded.ready := False + + //Send memory writes from bufffer read responses + val bufferReadedCounter = Reg(UInt(log2Up(memTransactionPerLine) bits)) init(0) + val memCmdAlreadyUsed = False + when(bufferReaded.valid) { + io.mem.cmd.valid := True + io.mem.cmd.wr := True + io.mem.cmd.address := request.address(tagRange.high downto lineRange.low) @@ U(0,lineRange.low bit) + io.mem.cmd.length := p.burstLength + io.mem.cmd.data := bufferReaded.payload + io.mem.cmd.mask := (1<<(wordWidth/8))-1 + + when(!memCmdAlreadyUsed && io.mem.cmd.ready){ + bufferReaded.ready := True + bufferReadedCounter := bufferReadedCounter + 1 + when(bufferReadedCounter === bufferReadedCounter.maxValue){ + request.ready := True + } + } + } + + + val counter = Counter(memTransactionPerLine) + when(request.ready){ + readLineCmdCounter.msb := False + readLineRspCounter.msb := False + bufferReadCounter.msb := False + } + } + + + + val stageA = new Area{ + val request = RegNextWhen(io.cpu.execute.args, !io.cpu.memory.isStuck) + io.cpu.memory.mmuBus.cmd.isValid := io.cpu.memory.isValid && request.kind === MEMORY //TODO filter request kind + io.cpu.memory.mmuBus.cmd.virtualAddress := request.address + io.cpu.memory.mmuBus.cmd.bypassTranslation := request.way + } + + val stageB = new Area { + val request = RegNextWhen(stageA.request, !io.cpu.writeBack.isStuck) + val mmuRsp = RegNextWhen(io.cpu.memory.mmuBus.rsp, !io.cpu.writeBack.isStuck) + // val waysHit = RegNextWhen(way.tagReadRspTwoRegIn.used && stageA.mmuRsp.physicalAddress(tagRange) === way.tagReadRspTwoRegIn.address,!io.cpu.writeBack.isStuck) //Manual retiming + val waysHit = way.tagReadRspTwo.used && mmuRsp.physicalAddress(tagRange) === way.tagReadRspTwo.address + + + //Loader interface + val loaderValid = False + val loaderReady = False + val loadingDone = RegNext(loaderValid && loaderReady) init(False) //one cycle pulse + + //delayedXX are used to relax logic timings in flush and evict modes + val delayedIsStuck = RegNext(io.cpu.writeBack.isStuck) + val delayedWaysHitValid = RegNext(waysHit) + + val victimNotSent = RegInit(False) clearWhen(victim.requestIn.ready) setWhen(!io.cpu.memory.isStuck) + val loadingNotDone = RegInit(False) clearWhen(loaderReady) setWhen(!io.cpu.memory.isStuck) + + val writeMask = request.size.mux ( + U(0) -> B"0001", + U(1) -> B"0011", + default -> B"1111" + ) |<< mmuRsp.physicalAddress(1 downto 0) + + io.cpu.writeBack.haltIt := io.cpu.writeBack.isValid + io.cpu.writeBack.mmuMiss := False + io.cpu.writeBack.illegalAccess := False + io.cpu.writeBack.unalignedAccess := False + io.cpu.writeBack.accessError := (if(catchAccessError) io.mem.rsp.valid && io.mem.rsp.error else False) + io.cpu.writeBack.badAddr := request.address + + //Evict the cache after reset logics + val bootEvicts = if(clearTagsAfterReset) new Area { + val valid = RegInit(True) + mmuRsp.physicalAddress init (0) + when(valid) { + tagsWriteCmd.valid := valid + tagsWriteCmd.address := mmuRsp.physicalAddress(lineRange) + tagsWriteCmd.data.used := False + when(mmuRsp.physicalAddress(lineRange) =/= lineCount - 1) { + mmuRsp.physicalAddress.getDrivingReg(lineRange) := mmuRsp.physicalAddress(lineRange) + 1 + io.cpu.writeBack.haltIt := True + } otherwise { + valid := False + } + } + } + + when(io.cpu.writeBack.isValid) { + if (catchMemoryTranslationMiss) { + io.cpu.writeBack.mmuMiss := mmuRsp.miss + } + switch(request.kind) { + is(MANAGMENT) { + when(delayedIsStuck && !mmuRsp.miss) { + when(delayedWaysHitValid || (request.way && way.tagReadRspTwo.used)) { + io.cpu.writeBack.haltIt.clearWhen(!(victim.requestIn.valid && !victim.requestIn.ready)) + victim.requestIn.valid := request.clean && way.tagReadRspTwo.dirty + tagsWriteCmd.valid := victim.requestIn.ready + } otherwise{ + io.cpu.writeBack.haltIt := False + } + } + + victim.requestIn.address := way.tagReadRspTwo.address @@ mmuRsp.physicalAddress(lineRange) @@ U((lineRange.low - 1 downto 0) -> false) + tagsWriteCmd.address := mmuRsp.physicalAddress(lineRange) + tagsWriteCmd.data.used := !request.invalidate + tagsWriteCmd.data.dirty := !request.clean + } + is(MEMORY) { + val illegal = if(catchIllegal) (request.wr && !mmuRsp.allowWrite) || (!request.wr && !mmuRsp.allowRead) || (io.cpu.writeBack.isUser && !mmuRsp.allowUser) else False + val unaligned = if(catchUnaligned) ((request.size === 2 && mmuRsp.physicalAddress(1 downto 0) =/= 0) || (request.size === 1 && mmuRsp.physicalAddress(0 downto 0) =/= 0)) else False + io.cpu.writeBack.illegalAccess := illegal + io.cpu.writeBack.unalignedAccess := unaligned + when((Bool(!catchMemoryTranslationMiss) || !mmuRsp.miss) && !illegal && !unaligned) { + when(request.forceUncachedAccess || mmuRsp.isIoAccess) { + val memCmdSent = RegInit(False) + when(!victim.request.valid) { + //Avoid mixing memory request while victim is pending + io.mem.cmd.wr := request.wr + io.mem.cmd.address := mmuRsp.physicalAddress(tagRange.high downto wordRange.low) @@ U(0, wordRange.low bit) + io.mem.cmd.mask := writeMask + io.mem.cmd.data := request.data + io.mem.cmd.length := 1 + + when(!memCmdSent) { + io.mem.cmd.valid := True + memCmdSent setWhen (io.mem.cmd.ready) + } + + io.cpu.writeBack.haltIt.clearWhen(memCmdSent && (io.mem.rsp.fire || request.wr)) //Cut mem.cmd.ready path but insert one cycle stall when write + } + memCmdSent clearWhen (!io.cpu.writeBack.isStuck) + } otherwise { + when(waysHit || !loadingNotDone) { + io.cpu.writeBack.haltIt := False + dataWriteCmd.valid := request.wr + dataWriteCmd.address := mmuRsp.physicalAddress(lineRange.high downto wordRange.low) + dataWriteCmd.data := request.data + dataWriteCmd.mask := writeMask + + tagsWriteCmd.valid := (!loadingNotDone) || request.wr + tagsWriteCmd.address := mmuRsp.physicalAddress(lineRange) + tagsWriteCmd.data.used := True + tagsWriteCmd.data.dirty := request.wr + tagsWriteCmd.data.address := mmuRsp.physicalAddress(tagRange) + } otherwise { + val victimRequired = way.tagReadRspTwo.used && way.tagReadRspTwo.dirty + loaderValid := loadingNotDone && !(victimNotSent && victim.request.isStall) //Additional condition used to be sure of that all previous victim are written into the RAM + victim.requestIn.valid := victimRequired && victimNotSent + victim.requestIn.address := way.tagReadRspTwo.address @@ mmuRsp.physicalAddress(lineRange) @@ U((lineRange.low - 1 downto 0) -> false) + } + } + } + } + } + } + + + assert(!(io.cpu.writeBack.isValid && !io.cpu.writeBack.haltIt && io.cpu.writeBack.isStuck), "writeBack stuck by another plugin is not allowed") + io.cpu.writeBack.data := (request.forceUncachedAccess || mmuRsp.isIoAccess) ? io.mem.rsp.data | way.dataReadRspTwo //not multi ways + } + + //The whole life of a loading task, the corresponding manager request is present + val loader = new Area{ + val valid = RegNext(stageB.loaderValid) init(False) + val baseAddress = stageB.mmuRsp.physicalAddress + + val memCmdSent = RegInit(False) + when(valid && !memCmdSent) { + io.mem.cmd.valid := True + io.mem.cmd.wr := False + io.mem.cmd.address := baseAddress(tagRange.high downto lineRange.low) @@ U(0,lineRange.low bit) + io.mem.cmd.length := p.burstLength + } + + when(valid && io.mem.cmd.ready){ + memCmdSent := True + } + + when(valid && !memCmdSent) { + victim.memCmdAlreadyUsed := True + } + + val counter = Counter(memTransactionPerLine) + when(valid && io.mem.rsp.valid){ + dataWriteCmd.valid := True + dataWriteCmd.address := baseAddress(lineRange) @@ counter + dataWriteCmd.data := io.mem.rsp.data + dataWriteCmd.mask := (1<<(wordWidth/8))-1 + counter.increment() + } + + when(counter.willOverflow){ + memCmdSent := False + valid := False + stageB.loaderReady := True + } + } +} \ No newline at end of file diff --git a/src/main/scala/SpinalRiscv/ip/InstructionCache.scala b/src/main/scala/SpinalRiscv/ip/InstructionCache.scala new file mode 100644 index 0000000..dee517b --- /dev/null +++ b/src/main/scala/SpinalRiscv/ip/InstructionCache.scala @@ -0,0 +1,398 @@ +package SpinalRiscv.ip + +import SpinalRiscv._ +import spinal.core._ +import spinal.lib._ + + +case class InstructionCacheConfig( cacheSize : Int, + bytePerLine : Int, + wayCount : Int, + wrappedMemAccess : Boolean, + addressWidth : Int, + cpuDataWidth : Int, + memDataWidth : Int, + catchIllegalAccess : Boolean, + catchAccessFault : Boolean, + catchMemoryTranslationMiss : Boolean, + asyncTagMemory : Boolean, + twoStageLogic : Boolean){ + def burstSize = bytePerLine*8/memDataWidth + def catchSomething = catchAccessFault || catchMemoryTranslationMiss || catchIllegalAccess + +} + + + +case class InstructionCacheCpuPrefetch(p : InstructionCacheConfig) extends Bundle with IMasterSlave{ + val isValid = Bool + val isFiring = Bool + val haltIt = Bool + val address = UInt(p.addressWidth bit) + + override def asMaster(): Unit = { + out(isValid, isFiring, address) + in(haltIt) + } +} + +case class InstructionCacheCpuFetch(p : InstructionCacheConfig) extends Bundle with IMasterSlave { + val isValid = Bool + val haltIt = if(!p.twoStageLogic) Bool else null + val isStuck = Bool + val isStuckByOthers = if(!p.twoStageLogic) Bool else null + val address = UInt(p.addressWidth bit) + val data = if(!p.twoStageLogic) Bits(32 bit) else null + val error = if(!p.twoStageLogic && p.catchAccessFault) Bool else null + val mmuBus = if(p.twoStageLogic) MemoryTranslatorBus() else null + + override def asMaster(): Unit = { + out(isValid, isStuck, address) + outWithNull(isStuckByOthers) + inWithNull(error,data,haltIt) + slaveWithNull(mmuBus) + } +} + +case class InstructionCacheCpuDecode(p : InstructionCacheConfig) extends Bundle with IMasterSlave { + require(p.twoStageLogic) + val isValid = Bool + val haltIt = Bool + val isStuck = Bool + val isUser = Bool + val address = UInt(p.addressWidth bit) + val data = Bits(32 bit) + val dataAnticipated = Bits(32 bits) + val error = if(p.catchAccessFault) Bool else null + val mmuMiss = if(p.catchMemoryTranslationMiss) Bool else null + val illegalAccess = if(p.catchIllegalAccess) Bool else null + + override def asMaster(): Unit = { + out(isValid, isStuck, address, isUser) + in(haltIt, data, dataAnticipated) + inWithNull(error,mmuMiss,illegalAccess) + } +} + +case class InstructionCacheCpuBus(p : InstructionCacheConfig) extends Bundle with IMasterSlave{ + val prefetch = InstructionCacheCpuPrefetch(p) + val fetch = InstructionCacheCpuFetch(p) + val decode = if(p.twoStageLogic) InstructionCacheCpuDecode(p) else null + + override def asMaster(): Unit = { + master(prefetch) + master(fetch) + if(p.twoStageLogic) master(decode) + } +} + +case class InstructionCacheMemCmd(p : InstructionCacheConfig) extends Bundle{ + val address = UInt(p.addressWidth bit) +} +case class InstructionCacheMemRsp(p : InstructionCacheConfig) extends Bundle{ + val data = Bits(32 bit) + val error = Bool +} + +case class InstructionCacheMemBus(p : InstructionCacheConfig) extends Bundle with IMasterSlave{ + val cmd = Stream (InstructionCacheMemCmd(p)) + val rsp = Flow (InstructionCacheMemRsp(p)) + + override def asMaster(): Unit = { + master(cmd) + slave(rsp) + } +} + +case class InstructionCacheFlushBus() extends Bundle with IMasterSlave{ + val cmd = Event + val rsp = Bool + + override def asMaster(): Unit = { + master(cmd) + in(rsp) + } +} + +class InstructionCache(p : InstructionCacheConfig) extends Component{ + import p._ + assert(wayCount == 1) + assert(cpuDataWidth == memDataWidth) + val io = new Bundle{ + val flush = slave(InstructionCacheFlushBus()) + // val translator = master(InstructionCacheTranslationBus(p)) + val cpu = slave(InstructionCacheCpuBus(p)) + val mem = master(InstructionCacheMemBus(p)) + } + // val haltCpu = False + val lineWidth = bytePerLine*8 + val lineCount = cacheSize/bytePerLine + val wordWidth = Math.max(memDataWidth,32) + val wordWidthLog2 = log2Up(wordWidth) + val wordPerLine = lineWidth/wordWidth + val bytePerWord = wordWidth/8 + val wayLineCount = lineCount/wayCount + val wayLineLog2 = log2Up(wayLineCount) + val wayWordCount = wayLineCount * wordPerLine + + val tagRange = addressWidth-1 downto log2Up(wayLineCount*bytePerLine) + val lineRange = tagRange.low-1 downto log2Up(bytePerLine) + val wordRange = log2Up(bytePerLine)-1 downto log2Up(bytePerWord) + val tagLineRange = tagRange.high downto lineRange.low + val lineWordRange = lineRange.high downto wordRange.low + + class LineInfo extends Bundle{ + val valid = Bool + val loading = Bool + val error = if(catchAccessFault) Bool else null + val address = UInt(tagRange.length bit) + } + + class LineInfoWithHit extends LineInfo{ + val hit = Bool + } + + def LineInfoWithHit(lineInfo : LineInfo, testTag : UInt) = { + val ret = new LineInfoWithHit() + ret.assignSomeByName(lineInfo) + ret.hit := lineInfo.valid && lineInfo.address === testTag + ret + } + + + val ways = Array.fill(wayCount)(new Area{ + val tags = Mem(new LineInfo(),wayLineCount) + val datas = Mem(Bits(wordWidth bits),wayWordCount) + }) + + + io.cpu.prefetch.haltIt := False + + val lineLoader = new Area{ + val requestIn = Stream(wrap(new Bundle{ + val addr = UInt(addressWidth bits) + })) + + + + val flushCounter = Reg(UInt(log2Up(wayLineCount) + 1 bit)) init(0) + when(!flushCounter.msb){ + io.cpu.prefetch.haltIt := True + flushCounter := flushCounter + 1 + } + when(!RegNext(flushCounter.msb)){ + io.cpu.prefetch.haltIt := True + } + val flushFromInterface = RegInit(False) + when(io.flush.cmd.valid){ + io.cpu.prefetch.haltIt := True + when(io.flush.cmd.ready){ + flushCounter := 0 + flushFromInterface := True + } + } + + io.flush.rsp := flushCounter.msb.rise && flushFromInterface + + val loadingWithErrorReg = if(catchAccessFault) RegInit(False) else null + val loadingWithError = if(catchAccessFault) Bool else null + if(catchAccessFault) { + loadingWithError := loadingWithErrorReg + loadingWithErrorReg := loadingWithError + } + + + + val request = requestIn.stage() + + + //Send memory requests + val memCmdSended = RegInit(False) setWhen(io.mem.cmd.fire) + io.mem.cmd.valid := request.valid && !memCmdSended + if(wrappedMemAccess) + io.mem.cmd.address := request.addr(tagRange.high downto wordRange.low) @@ U(0,wordRange.low bit) + else + io.mem.cmd.address := request.addr(tagRange.high downto lineRange.low) @@ U(0,lineRange.low bit) + + val wordIndex = Reg(UInt(log2Up(wordPerLine) bit)) + val loadedWordsNext = Bits(wordPerLine bit) + val loadedWords = RegNext(loadedWordsNext) + val loadedWordsReadable = RegNext(loadedWords) + loadedWordsNext := loadedWords + + val waysDatasWritePort = ways(0).datas.writePort //Not multi ways + waysDatasWritePort.valid := io.mem.rsp.valid + waysDatasWritePort.address := request.addr(lineRange) @@ wordIndex + waysDatasWritePort.data := io.mem.rsp.data + when(io.mem.rsp.valid){ + wordIndex := wordIndex + 1 + loadedWordsNext(wordIndex) := True + if(catchAccessFault) loadingWithError setWhen io.mem.rsp.error + } + + val memRspLast = loadedWordsNext === B(loadedWordsNext.range -> true) + + val readyDelay = Reg(UInt(1 bit)) + when(memRspLast){ + readyDelay := readyDelay + 1 + } + request.ready := readyDelay === 1 + + val waysTagsWritePort = ways(0).tags.writePort //not multi way + waysTagsWritePort.valid := io.mem.rsp.valid || !flushCounter.msb + waysTagsWritePort.address := Mux(flushCounter.msb,request.addr(lineRange),flushCounter(flushCounter.high-1 downto 0)) + waysTagsWritePort.data.valid := flushCounter.msb + waysTagsWritePort.data.address := request.addr(tagRange) + waysTagsWritePort.data.loading := !memRspLast + if(catchAccessFault) waysTagsWritePort.data.error := loadingWithError + + + when(requestIn.ready){ + memCmdSended := False + wordIndex := requestIn.addr(wordRange) + loadedWords := 0 + loadedWordsReadable := 0 + readyDelay := 0 + if(catchAccessFault) loadingWithErrorReg := False + } + } + + val task = if(!twoStageLogic) new Area{ + val waysHitValid = False + val waysHitError = Bool.assignDontCare() + val waysHitWord = Bits(wordWidth bit) + + val waysRead = for(way <- ways) yield new Area{ + val readAddress = Mux(io.cpu.fetch.isStuck,io.cpu.fetch.address,io.cpu.prefetch.address) //TODO FMAX + val tag = if(asyncTagMemory) + way.tags.readAsync(io.cpu.fetch.address(lineRange),writeFirst) + else + way.tags.readSync(readAddress(lineRange),readUnderWrite = readFirst) + + val data = way.datas.readSync(readAddress(lineRange.high downto wordRange.low)) + waysHitWord := data //Not applicable to multi way + when(tag.valid && tag.address === io.cpu.fetch.address(tagRange)) { + waysHitValid := True + if(catchAccessFault) waysHitError := tag.error + } + } + + + val hit = waysHitValid && !(waysRead(0).tag.loading && !(if(asyncTagMemory) lineLoader.loadedWords else RegNext(lineLoader.loadedWords))(io.cpu.fetch.address(wordRange))) + io.cpu.fetch.haltIt := io.cpu.fetch.isValid && !hit + io.cpu.fetch.data := waysHitWord + if(catchAccessFault) io.cpu.fetch.error := waysRead(0).tag.error + lineLoader.requestIn.valid := io.cpu.fetch.isValid && !hit //TODO avoid duplicated request + lineLoader.requestIn.addr := io.cpu.fetch.address + } else new Area{ + //Long readValidPath + // def writeFirstMemWrap[T <: Data](readValid : Bool, readAddress : UInt, lastAddress : UInt, readData : T,writeValid : Bool, writeAddress : UInt, writeData : T) : T = { + // val hit = writeValid && (readValid ? readAddress | lastAddress) === writeAddress + // val overrideValid = RegInit(False) clearWhen(readValid) setWhen(hit) + // val overrideValue = RegNextWhen(writeData,hit) + // overrideValid ? overrideValue | readData + // } + + //shot readValid path + def writeFirstMemWrap[T <: Data](readValid : Bool, readLastAddress : UInt, readData : T,writeValid : Bool, writeAddress : UInt, writeData : T) : T = { + val writeSample = readValid || (writeValid && writeAddress === readLastAddress) + val writeValidReg = RegNextWhen(writeValid,writeSample) + val writeAddressReg = RegNextWhen(writeAddress,writeSample) + val writeDataReg = RegNextWhen(writeData,writeSample) + (writeValidReg && writeAddressReg === readLastAddress) ? writeDataReg | readData + } + + //Long sample path + // def writeFirstRegWrap[T <: Data](sample : Bool, sampleAddress : UInt,lastAddress : UInt, readData : T, writeValid : Bool, writeAddress : UInt, writeData : T) : (T,T) = { + // val hit = writeValid && (sample ? sampleAddress | lastAddress) === writeAddress + // val bypass = hit ? writeData | readData + // val reg = RegNextWhen(bypass,sample || hit) + // (reg,bypass) + // } + + //Short sample path + def writeFirstRegWrap[T <: Data](sample : Bool, sampleAddress : UInt,sampleLastAddress : UInt, readData : T, writeValid : Bool, writeAddress : UInt, writeData : T) = { + val preWrite = (writeValid && sampleAddress === writeAddress) + val postWrite = (writeValid && sampleLastAddress === writeAddress) + val bypass = (!sample || preWrite) ? writeData | readData + val regEn = sample || postWrite + val reg = RegNextWhen(bypass,regEn) + (reg,bypass,regEn,preWrite,postWrite) + } + // def writeFirstRegWrap[T <: Data](sample : Bool, sampleAddress : UInt,sampleLastAddress : UInt, readData : T, writeValid : Bool, writeAddress : UInt, writeData : T) = { + // val bypass = (!sample || (writeValid && sampleAddress === writeAddress)) ? writeData | readData + // val regEn = sample || (writeValid && sampleLastAddress === writeAddress) + // val reg = RegNextWhen(bypass,regEn) + // (reg,bypass,regEn,False,False) + // } + require(wayCount == 1) + val memRead = new Area{ + val way = ways(0) + val tag = if(asyncTagMemory) + way.tags.readAsync(io.cpu.fetch.address(lineRange),writeFirst) + else + writeFirstMemWrap( + readValid = !io.cpu.fetch.isStuck, + // readAddress = io.cpu.prefetch.address(lineRange), + readLastAddress = io.cpu.fetch.address(lineRange), + readData = way.tags.readSync(io.cpu.prefetch.address(lineRange),enable = !io.cpu.fetch.isStuck), + writeValid = lineLoader.waysTagsWritePort.valid, + writeAddress = lineLoader.waysTagsWritePort.address, + writeData = lineLoader.waysTagsWritePort.data + ) + + val data = writeFirstMemWrap( + readValid = !io.cpu.fetch.isStuck, + // readAddress = io.cpu.prefetch.address(lineWordRange), + readLastAddress = io.cpu.fetch.address(lineWordRange), + readData = way.datas.readSync(io.cpu.prefetch.address(lineWordRange),enable = !io.cpu.fetch.isStuck), + writeValid = lineLoader.waysDatasWritePort.valid, + writeAddress = lineLoader.waysDatasWritePort.address, + writeData = lineLoader.waysDatasWritePort.data + ) + } + + + val tag = writeFirstRegWrap( + sample = !io.cpu.decode.isStuck, + sampleAddress = io.cpu.fetch.address(lineRange), + sampleLastAddress = io.cpu.decode.address(lineRange), + readData = LineInfoWithHit(memRead.tag,io.cpu.fetch.address(tagRange)), + writeValid = lineLoader.waysTagsWritePort.valid, + writeAddress = lineLoader.waysTagsWritePort.address, + writeData = LineInfoWithHit(lineLoader.waysTagsWritePort.data,io.cpu.fetch.address(tagRange)) //TODO wrong address src + )._1 + + val (data,dataRegIn,dataRegEn,dataPreWrite,dataPostWrite) = writeFirstRegWrap( + sample = !io.cpu.decode.isStuck, + sampleAddress = io.cpu.fetch.address(lineWordRange), + sampleLastAddress = io.cpu.decode.address(lineWordRange), + readData = memRead.data, + writeValid = lineLoader.waysDatasWritePort.valid, + writeAddress = lineLoader.waysDatasWritePort.address, + writeData = lineLoader.waysDatasWritePort.data + ) + + io.cpu.fetch.mmuBus.cmd.isValid := io.cpu.fetch.isValid + io.cpu.fetch.mmuBus.cmd.virtualAddress := io.cpu.fetch.address + io.cpu.fetch.mmuBus.cmd.bypassTranslation := False + val mmuRsp = RegNextWhen(io.cpu.fetch.mmuBus.rsp,!io.cpu.decode.isStuck) + + val hit = tag.valid && tag.address === mmuRsp.physicalAddress(tagRange) && !(tag.loading && !lineLoader.loadedWords(mmuRsp.physicalAddress(wordRange))) + // val hit = tag.hit && !(tag.loading && !lineLoader.loadedWords(mmuRsp.physicalAddress(wordRange))) + + io.cpu.decode.haltIt := io.cpu.decode.isValid && !hit //TODO PERF not halit it when removed, Should probably be applyed in many other places + io.cpu.decode.data := data + // io.cpu.decode.dataAnticipated := dataRegEn ? dataRegIn | data + io.cpu.decode.dataAnticipated := io.cpu.decode.isStuck ? Mux(dataPostWrite,lineLoader.waysDatasWritePort.data,data) | Mux(dataPreWrite,lineLoader.waysDatasWritePort.data,memRead.data) + if(catchAccessFault) io.cpu.decode.error := tag.error + if(catchMemoryTranslationMiss) io.cpu.decode.mmuMiss := mmuRsp.miss + if(catchIllegalAccess) io.cpu.decode.illegalAccess := !mmuRsp.allowExecute || (io.cpu.decode.isUser && !mmuRsp.allowUser) + + lineLoader.requestIn.valid := io.cpu.decode.isValid && !hit && !mmuRsp.miss//TODO avoid duplicated request + lineLoader.requestIn.addr := mmuRsp.physicalAddress + } + + io.flush.cmd.ready := !(lineLoader.request.valid || io.cpu.fetch.isValid) +} +