Improve testing infrastructure with more options and better readme
https://github.com/litex-hub/linux-on-litex-vexriscv/issues/112
This commit is contained in:
parent
0c255e2404
commit
559260020b
45
README.md
45
README.md
|
@ -186,24 +186,47 @@ NOTES:
|
|||
|
||||
[![Build Status](https://travis-ci.org/SpinalHDL/VexRiscv.svg?branch=master)](https://travis-ci.org/SpinalHDL/VexRiscv)
|
||||
|
||||
To run tests (need the verilator simulator), go in the src/test/cpp/regression folder and run :
|
||||
To run tests (need the java, scala, verilator), just do :
|
||||
|
||||
```sh
|
||||
# To test the GenFull CPU
|
||||
# (Don't worry about the CSR test not passing, basicaly the GenFull isn't the truly full version of the CPU, some CSR features are disable in it)
|
||||
make clean run
|
||||
|
||||
# To test the GenSmallest CPU
|
||||
make clean run IBUS=SIMPLE DBUS=SIMPLE CSR=no MMU=no DEBUG_PLUGIN=no MUL=no DIV=no
|
||||
export VEXRISCV_REGRESSION_SEED=42
|
||||
export VEXRISCV_REGRESSION_TEST_ID=
|
||||
sbt "testOnly vexriscv.TestIndividualFeatures"
|
||||
```
|
||||
|
||||
The self-test includes:
|
||||
- ISA tests from https://github.com/riscv/riscv-tests/tree/master/isa
|
||||
This will generate random VexRiscv configuration and test them with:
|
||||
- ISA tests from https://github.com/riscv/riscv-tests/tree/master/isa and https://github.com/riscv/riscv-compliance
|
||||
- Dhrystone benchmark
|
||||
- 24 FreeRTOS tests
|
||||
- Coremark benchmark
|
||||
- Zephyr os
|
||||
- Buildroot/Linux os
|
||||
- Some handwritten tests to check the CSR, debug module and MMU plugins
|
||||
|
||||
You can enable FreeRTOS tests by adding `FREERTOS=yes` to the command line, but it will take time to run. Also, it uses THREAD_COUNT host CPU threads to run multiple regression in parallel.
|
||||
You can rerun some specific test by setting VEXRISCV_REGRESSION_TEST_ID by their id. For instance, if you want to rerun :
|
||||
- test_id_5_test_IBus_CachedS1024W1BPL32Relaxvexriscv.plugin.DYNAMIC_DBus_CachedS8192W2BPL16_MulDiv_MulDivFpga_Shift_FullLate_Branch_Late_Hazard_BypassAll_RegFile_SyncDR_Src__Csr_AllNoException_Decoder__Debug_None_DBus_NoMmu
|
||||
- test_id_9_test_IBus_Simple1S2InjStagevexriscv.plugin.STATIC_DBus_SimpleLate_MulDiv_MulDivFpgaSimple_Shift_FullEarly_Branch_Late_Hazard_Interlock_RegFile_AsyncER_Src_AddSubExecute_Csr_None_Decoder__Debug_None_DBus_NoMmu
|
||||
|
||||
then :
|
||||
|
||||
```
|
||||
export VEXRISCV_REGRESSION_TEST_ID=5,9
|
||||
```
|
||||
|
||||
Also there is a few environnement variable that you can use to modulate the random generation :
|
||||
|
||||
| Parameters | range | description |
|
||||
| ------------------------------------------- | ------------------ | ----------- |
|
||||
| VEXRISCV_REGRESSION_SEED | Int | Seed used to generate the random configurations |
|
||||
| VEXRISCV_REGRESSION_TEST_ID | \[Int\[,\Int\]\*\] | Random configuration that should be keeped and tested |
|
||||
| VEXRISCV_REGRESSION_CONFIG_COUNT | Int | Number of random configurations |
|
||||
| VEXRISCV_REGRESSION_CONFIG_RVC_RATE | 0.0-1.0 | Chance to generate a RVC config |
|
||||
| VEXRISCV_REGRESSION_CONFIG_LINUX_RATE | 0.0-1.0 | Chance to generate a linux ready config |
|
||||
| VEXRISCV_REGRESSION_CONFIG_MACHINE_OS_RATE | 0.0-1.0 | Chance to generate a machine mode OS ready config |
|
||||
| VEXRISCV_REGRESSION_LINUX_REGRESSION | yes/no | Enable the linux test |
|
||||
| VEXRISCV_REGRESSION_COREMARK | yes/no | Enable the Coremark test |
|
||||
| VEXRISCV_REGRESSION_ZEPHYR_COUNT | Int | Number of zephyr tests to run on capable configs |
|
||||
| VEXRISCV_REGRESSION_CONFIG_DEMW_RATE | 0.0-1.0 | Chance to generate a config with writeback stage |
|
||||
| VEXRISCV_REGRESSION_CONFIG_DEM_RATE | 0.0-1.0 | Chance to generate a config with memory stage |
|
||||
|
||||
## Interactive debug of the simulated CPU via GDB OpenOCD and Verilator
|
||||
It's as described to run tests, but you just have to add `DEBUG_PLUGIN_EXTERNAL=yes` in the make arguments.
|
||||
|
|
|
@ -3,7 +3,7 @@ package vexriscv
|
|||
import java.io.File
|
||||
|
||||
import org.apache.commons.io.FileUtils
|
||||
import org.scalatest.FunSuite
|
||||
import org.scalatest.{BeforeAndAfterAll, FunSuite, ParallelTestExecution}
|
||||
import spinal.core._
|
||||
import vexriscv.demo._
|
||||
import vexriscv.ip.{DataCacheConfig, InstructionCacheConfig}
|
||||
|
@ -313,7 +313,7 @@ class SrcDimension extends VexRiscvDimension("Src") {
|
|||
}
|
||||
|
||||
|
||||
class IBusDimension extends VexRiscvDimension("IBus") {
|
||||
class IBusDimension(rvcRate : Double) extends VexRiscvDimension("IBus") {
|
||||
|
||||
|
||||
override def randomPositionImpl(universes: Seq[ConfigUniverse], r: Random) = {
|
||||
|
@ -322,7 +322,7 @@ class IBusDimension extends VexRiscvDimension("IBus") {
|
|||
|
||||
if(r.nextDouble() < 0.5){
|
||||
val latency = r.nextInt(5) + 1
|
||||
val compressed = r.nextBoolean()
|
||||
val compressed = r.nextDouble() < rvcRate
|
||||
val injectorStage = r.nextBoolean() || latency == 1
|
||||
val prediction = random(r, List(NONE, STATIC, DYNAMIC, DYNAMIC_TARGET))
|
||||
val catchAll = universes.contains(VexRiscvUniverse.CATCH_ALL)
|
||||
|
@ -345,7 +345,7 @@ class IBusDimension extends VexRiscvDimension("IBus") {
|
|||
}
|
||||
} else {
|
||||
val catchAll = universes.contains(VexRiscvUniverse.CATCH_ALL)
|
||||
val compressed = r.nextBoolean()
|
||||
val compressed = r.nextDouble() < rvcRate
|
||||
val tighlyCoupled = r.nextBoolean() && !catchAll
|
||||
// val tighlyCoupled = false
|
||||
val prediction = random(r, List(NONE, STATIC, DYNAMIC, DYNAMIC_TARGET))
|
||||
|
@ -497,14 +497,14 @@ class MmuDimension extends VexRiscvDimension("DBus") {
|
|||
trait CatchAllPosition
|
||||
|
||||
|
||||
class CsrDimension(freertos : String, zephyr : String) extends VexRiscvDimension("Csr") {
|
||||
class CsrDimension(freertos : String, zephyr : String, linux : String) extends VexRiscvDimension("Csr") {
|
||||
override def randomPositionImpl(universes: Seq[ConfigUniverse], r: Random) = {
|
||||
val catchAll = universes.contains(VexRiscvUniverse.CATCH_ALL)
|
||||
val supervisor = universes.contains(VexRiscvUniverse.SUPERVISOR)
|
||||
if(supervisor){
|
||||
new VexRiscvPosition("Supervisor") with CatchAllPosition{
|
||||
override def applyOn(config: VexRiscvConfig): Unit = config.plugins += new CsrPlugin(CsrPluginConfig.linuxFull(0x80000020l))
|
||||
override def testParam = s"FREERTOS=$freertos ZEPHYR=$zephyr LINUX_REGRESSION=${sys.env.getOrElse("VEXRISCV_REGRESSION_LINUX_REGRESSION", "yes")} SUPERVISOR=yes"
|
||||
override def testParam = s"FREERTOS=$freertos ZEPHYR=$zephyr LINUX_REGRESSION=$linux SUPERVISOR=yes"
|
||||
}
|
||||
} else if(catchAll){
|
||||
new VexRiscvPosition("MachineOs") with CatchAllPosition{
|
||||
|
@ -554,10 +554,35 @@ class DecoderDimension extends VexRiscvDimension("Decoder") {
|
|||
}
|
||||
|
||||
|
||||
|
||||
//class TesterPlay extends FunSuite with ParallelTestExecution {
|
||||
// def createTest(name : String): Unit ={
|
||||
// test(name){
|
||||
// for(i <- 0 to 4) {
|
||||
// println(s"$name $i")
|
||||
// Thread.sleep(2000)
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// List("a", "b","c").foreach(createTest)
|
||||
//}
|
||||
|
||||
|
||||
class TestIndividualFeatures extends FunSuite {
|
||||
val testCount = sys.env.getOrElse("VEXRISCV_REGRESSION_CONFIG_COUNT", "100").toInt
|
||||
val seed = sys.env.getOrElse("VEXRISCV_REGRESSION_SEED", Random.nextLong().toString).toLong
|
||||
val testId : Set[Int] = sys.env.get("VEXRISCV_REGRESSION_TEST_ID") match {
|
||||
case Some(x) if x != "" => x.split(',').map(_.toInt).toSet
|
||||
case _ => (0 until testCount).toSet
|
||||
}
|
||||
val rvcRate = sys.env.getOrElse("VEXRISCV_REGRESSION_CONFIG_RVC_RATE", "0.5").toDouble
|
||||
val linuxRate = sys.env.getOrElse("VEXRISCV_REGRESSION_CONFIG_LINUX_RATE", "0.3").toDouble
|
||||
val machineOsRate = sys.env.getOrElse("VEXRISCV_REGRESSION_CONFIG_MACHINE_OS_RATE", "0.5").toDouble
|
||||
val linuxRegression = sys.env.getOrElse("VEXRISCV_REGRESSION_LINUX_REGRESSION", "yes")
|
||||
val coremarkRegression = sys.env.getOrElse("VEXRISCV_REGRESSION_COREMARK", "yes")
|
||||
val zephyrCount = sys.env.getOrElse("VEXRISCV_REGRESSION_ZEPHYR_COUNT", "4")
|
||||
val demwRate = sys.env.getOrElse("VEXRISCV_REGRESSION_CONFIG_DEMW_RATE", "0.6").toDouble
|
||||
val demRate = sys.env.getOrElse("VEXRISCV_REGRESSION_CONFIG_DEM_RATE", "0.5").toDouble
|
||||
|
||||
def doCmd(cmd: String): String = {
|
||||
val stdOut = new StringBuilder()
|
||||
class Logger extends ProcessLogger {
|
||||
|
@ -578,7 +603,7 @@ class TestIndividualFeatures extends FunSuite {
|
|||
|
||||
|
||||
val dimensions = List(
|
||||
new IBusDimension,
|
||||
new IBusDimension(rvcRate),
|
||||
new DBusDimension,
|
||||
new MulDivDimension,
|
||||
new ShiftDimension,
|
||||
|
@ -586,7 +611,7 @@ class TestIndividualFeatures extends FunSuite {
|
|||
new HazardDimension,
|
||||
new RegFileDimension,
|
||||
new SrcDimension,
|
||||
new CsrDimension(/*sys.env.getOrElse("VEXRISCV_REGRESSION_FREERTOS_COUNT", "1")*/ "0", sys.env.getOrElse("VEXRISCV_REGRESSION_ZEPHYR_COUNT", "4")), //Freertos old port software is broken
|
||||
new CsrDimension(/*sys.env.getOrElse("VEXRISCV_REGRESSION_FREERTOS_COUNT", "1")*/ "0", zephyrCount, linuxRegression), //Freertos old port software is broken
|
||||
new DecoderDimension,
|
||||
new DebugDimension,
|
||||
new MmuDimension
|
||||
|
@ -612,15 +637,15 @@ class TestIndividualFeatures extends FunSuite {
|
|||
}
|
||||
|
||||
val name = (if(noMemory) "noMemoryStage_" else "") + (if(noWriteback) "noWritebackStage_" else "") + positionsToApply.map(d => d.dimension.name + "_" + d.name).mkString("_")
|
||||
test(prefix + name + "_gen") {
|
||||
test(prefix + "gen_" + name) {
|
||||
gen
|
||||
}
|
||||
|
||||
|
||||
test(prefix + name + "_test") {
|
||||
test(prefix + "test_" + name) {
|
||||
println("START TEST " + prefix + name)
|
||||
val debug = true
|
||||
val stdCmd = (s"make clean run WITH_USER_IO=no REDO=10 TRACE=${if(debug) "yes" else "no"} TRACE_START=9999924910246l STOP_ON_ERROR=no FLOW_INFO=no STOP_ON_ERROR=no DHRYSTONE=yes COREMARK=${sys.env.getOrElse("VEXRISCV_REGRESSION_COREMARK", "yes")} THREAD_COUNT=${sys.env.getOrElse("VEXRISCV_REGRESSION_THREAD_COUNT", 1)} ") + s" SEED=${testSeed} "
|
||||
val stdCmd = (s"make clean run WITH_USER_IO=no REDO=10 TRACE=${if(debug) "yes" else "no"} TRACE_START=1000000000000l STOP_ON_ERROR=no FLOW_INFO=no STOP_ON_ERROR=no DHRYSTONE=yes COREMARK=${coremarkRegression} THREAD_COUNT=1 ") + s" SEED=${testSeed} "
|
||||
val testCmd = stdCmd + (positionsToApply).map(_.testParam).mkString(" ")
|
||||
println(testCmd)
|
||||
val str = doCmd(testCmd)
|
||||
|
@ -628,42 +653,33 @@ class TestIndividualFeatures extends FunSuite {
|
|||
}
|
||||
}
|
||||
|
||||
val testId : Option[mutable.HashSet[Int]] = None
|
||||
val seed = sys.env.getOrElse("VEXRISCV_REGRESSION_SEED", Random.nextLong().toString).toLong
|
||||
//
|
||||
// val testId = Some(mutable.HashSet(3,4,9,11,13,16,18,19,20,21))
|
||||
// val testId = Some(mutable.HashSet(11))
|
||||
// val testId = Some(mutable.HashSet(4, 11))
|
||||
// val seed = 6592877339343561798l
|
||||
|
||||
|
||||
val rand = new Random(seed)
|
||||
|
||||
test("Info"){
|
||||
println(s"MAIN_SEED=$seed")
|
||||
}
|
||||
println(s"Seed=$seed")
|
||||
for(i <- 0 until sys.env.getOrElse("VEXRISCV_REGRESSION_CONFIG_COUNT", "100").toInt){
|
||||
for(i <- 0 until testCount){
|
||||
var positions : List[VexRiscvPosition] = null
|
||||
var universe = mutable.HashSet[VexRiscvUniverse]()
|
||||
if(rand.nextDouble() < 0.5) universe += VexRiscvUniverse.EXECUTE_RF
|
||||
if(sys.env.getOrElse("VEXRISCV_REGRESSION_CONFIG_LINUX_RATE", "0.3").toDouble > rand.nextDouble()) {
|
||||
if(linuxRate > rand.nextDouble()) {
|
||||
universe += VexRiscvUniverse.CATCH_ALL
|
||||
universe += VexRiscvUniverse.MMU
|
||||
universe += VexRiscvUniverse.FORCE_MULDIV
|
||||
universe += VexRiscvUniverse.SUPERVISOR
|
||||
if(sys.env.getOrElse("VEXRISCV_REGRESSION_CONFIG_DEMW_RATE", "0.6").toDouble < rand.nextDouble()){
|
||||
if(demwRate < rand.nextDouble()){
|
||||
universe += VexRiscvUniverse.NO_WRITEBACK
|
||||
}
|
||||
} else {
|
||||
if(sys.env.getOrElse("VEXRISCV_REGRESSION_CONFIG_MACHINE_OS_RATE", "0.5").toDouble > rand.nextDouble()) {
|
||||
if(machineOsRate > rand.nextDouble()) {
|
||||
universe += VexRiscvUniverse.CATCH_ALL
|
||||
if(sys.env.getOrElse("VEXRISCV_REGRESSION_CONFIG_DEMW_RATE", "0.6").toDouble < rand.nextDouble()){
|
||||
if(demwRate < rand.nextDouble()){
|
||||
universe += VexRiscvUniverse.NO_WRITEBACK
|
||||
}
|
||||
}
|
||||
if(sys.env.getOrElse("VEXRISCV_REGRESSION_CONFIG_DEMW_RATE", "0.6").toDouble > rand.nextDouble()){
|
||||
}else if(sys.env.getOrElse("VEXRISCV_REGRESSION_CONFIG_DEM_RATE", "0.5").toDouble > rand.nextDouble()){
|
||||
if(demwRate > rand.nextDouble()){
|
||||
}else if(demRate > rand.nextDouble()){
|
||||
universe += VexRiscvUniverse.NO_WRITEBACK
|
||||
} else {
|
||||
universe += VexRiscvUniverse.NO_WRITEBACK
|
||||
|
@ -676,8 +692,8 @@ class TestIndividualFeatures extends FunSuite {
|
|||
}while(!positions.forall(_.isCompatibleWith(positions)))
|
||||
|
||||
val testSeed = rand.nextInt()
|
||||
if(testId.isEmpty || testId.get.contains(i))
|
||||
doTest(positions," random_" + i + "_", testSeed, universe)
|
||||
if(testId.contains(i))
|
||||
doTest(positions," test_id_" + i + "_", testSeed, universe)
|
||||
Hack.dCounter += 1
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue