From 559260020b4dda3c5532fc281ae302936ed6baa3 Mon Sep 17 00:00:00 2001 From: Dolu1990 Date: Sun, 1 Mar 2020 13:02:08 +0100 Subject: [PATCH 1/3] Improve testing infrastructure with more options and better readme https://github.com/litex-hub/linux-on-litex-vexriscv/issues/112 --- README.md | 45 ++++++++--- .../vexriscv/TestIndividualFeatures.scala | 76 +++++++++++-------- 2 files changed, 80 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index d0deb16..d76a040 100644 --- a/README.md +++ b/README.md @@ -186,24 +186,47 @@ NOTES: [![Build Status](https://travis-ci.org/SpinalHDL/VexRiscv.svg?branch=master)](https://travis-ci.org/SpinalHDL/VexRiscv) -To run tests (need the verilator simulator), go in the src/test/cpp/regression folder and run : +To run tests (need the java, scala, verilator), just do : ```sh -# To test the GenFull CPU -# (Don't worry about the CSR test not passing, basicaly the GenFull isn't the truly full version of the CPU, some CSR features are disable in it) -make clean run - -# To test the GenSmallest CPU -make clean run IBUS=SIMPLE DBUS=SIMPLE CSR=no MMU=no DEBUG_PLUGIN=no MUL=no DIV=no +export VEXRISCV_REGRESSION_SEED=42 +export VEXRISCV_REGRESSION_TEST_ID= +sbt "testOnly vexriscv.TestIndividualFeatures" ``` -The self-test includes: -- ISA tests from https://github.com/riscv/riscv-tests/tree/master/isa +This will generate random VexRiscv configuration and test them with: +- ISA tests from https://github.com/riscv/riscv-tests/tree/master/isa and https://github.com/riscv/riscv-compliance - Dhrystone benchmark -- 24 FreeRTOS tests +- Coremark benchmark +- Zephyr os +- Buildroot/Linux os - Some handwritten tests to check the CSR, debug module and MMU plugins -You can enable FreeRTOS tests by adding `FREERTOS=yes` to the command line, but it will take time to run. Also, it uses THREAD_COUNT host CPU threads to run multiple regression in parallel. +You can rerun some specific test by setting VEXRISCV_REGRESSION_TEST_ID by their id. For instance, if you want to rerun : +- test_id_5_test_IBus_CachedS1024W1BPL32Relaxvexriscv.plugin.DYNAMIC_DBus_CachedS8192W2BPL16_MulDiv_MulDivFpga_Shift_FullLate_Branch_Late_Hazard_BypassAll_RegFile_SyncDR_Src__Csr_AllNoException_Decoder__Debug_None_DBus_NoMmu +- test_id_9_test_IBus_Simple1S2InjStagevexriscv.plugin.STATIC_DBus_SimpleLate_MulDiv_MulDivFpgaSimple_Shift_FullEarly_Branch_Late_Hazard_Interlock_RegFile_AsyncER_Src_AddSubExecute_Csr_None_Decoder__Debug_None_DBus_NoMmu + +then : + +``` +export VEXRISCV_REGRESSION_TEST_ID=5,9 +``` + +Also there is a few environnement variable that you can use to modulate the random generation : + +| Parameters | range | description | +| ------------------------------------------- | ------------------ | ----------- | +| VEXRISCV_REGRESSION_SEED | Int | Seed used to generate the random configurations | +| VEXRISCV_REGRESSION_TEST_ID | \[Int\[,\Int\]\*\] | Random configuration that should be keeped and tested | +| VEXRISCV_REGRESSION_CONFIG_COUNT | Int | Number of random configurations | +| VEXRISCV_REGRESSION_CONFIG_RVC_RATE | 0.0-1.0 | Chance to generate a RVC config | +| VEXRISCV_REGRESSION_CONFIG_LINUX_RATE | 0.0-1.0 | Chance to generate a linux ready config | +| VEXRISCV_REGRESSION_CONFIG_MACHINE_OS_RATE | 0.0-1.0 | Chance to generate a machine mode OS ready config | +| VEXRISCV_REGRESSION_LINUX_REGRESSION | yes/no | Enable the linux test | +| VEXRISCV_REGRESSION_COREMARK | yes/no | Enable the Coremark test | +| VEXRISCV_REGRESSION_ZEPHYR_COUNT | Int | Number of zephyr tests to run on capable configs | +| VEXRISCV_REGRESSION_CONFIG_DEMW_RATE | 0.0-1.0 | Chance to generate a config with writeback stage | +| VEXRISCV_REGRESSION_CONFIG_DEM_RATE | 0.0-1.0 | Chance to generate a config with memory stage | ## Interactive debug of the simulated CPU via GDB OpenOCD and Verilator It's as described to run tests, but you just have to add `DEBUG_PLUGIN_EXTERNAL=yes` in the make arguments. diff --git a/src/test/scala/vexriscv/TestIndividualFeatures.scala b/src/test/scala/vexriscv/TestIndividualFeatures.scala index 005f335..6605cb5 100644 --- a/src/test/scala/vexriscv/TestIndividualFeatures.scala +++ b/src/test/scala/vexriscv/TestIndividualFeatures.scala @@ -3,7 +3,7 @@ package vexriscv import java.io.File import org.apache.commons.io.FileUtils -import org.scalatest.FunSuite +import org.scalatest.{BeforeAndAfterAll, FunSuite, ParallelTestExecution} import spinal.core._ import vexriscv.demo._ import vexriscv.ip.{DataCacheConfig, InstructionCacheConfig} @@ -313,7 +313,7 @@ class SrcDimension extends VexRiscvDimension("Src") { } -class IBusDimension extends VexRiscvDimension("IBus") { +class IBusDimension(rvcRate : Double) extends VexRiscvDimension("IBus") { override def randomPositionImpl(universes: Seq[ConfigUniverse], r: Random) = { @@ -322,7 +322,7 @@ class IBusDimension extends VexRiscvDimension("IBus") { if(r.nextDouble() < 0.5){ val latency = r.nextInt(5) + 1 - val compressed = r.nextBoolean() + val compressed = r.nextDouble() < rvcRate val injectorStage = r.nextBoolean() || latency == 1 val prediction = random(r, List(NONE, STATIC, DYNAMIC, DYNAMIC_TARGET)) val catchAll = universes.contains(VexRiscvUniverse.CATCH_ALL) @@ -345,7 +345,7 @@ class IBusDimension extends VexRiscvDimension("IBus") { } } else { val catchAll = universes.contains(VexRiscvUniverse.CATCH_ALL) - val compressed = r.nextBoolean() + val compressed = r.nextDouble() < rvcRate val tighlyCoupled = r.nextBoolean() && !catchAll // val tighlyCoupled = false val prediction = random(r, List(NONE, STATIC, DYNAMIC, DYNAMIC_TARGET)) @@ -497,14 +497,14 @@ class MmuDimension extends VexRiscvDimension("DBus") { trait CatchAllPosition -class CsrDimension(freertos : String, zephyr : String) extends VexRiscvDimension("Csr") { +class CsrDimension(freertos : String, zephyr : String, linux : String) extends VexRiscvDimension("Csr") { override def randomPositionImpl(universes: Seq[ConfigUniverse], r: Random) = { val catchAll = universes.contains(VexRiscvUniverse.CATCH_ALL) val supervisor = universes.contains(VexRiscvUniverse.SUPERVISOR) if(supervisor){ new VexRiscvPosition("Supervisor") with CatchAllPosition{ override def applyOn(config: VexRiscvConfig): Unit = config.plugins += new CsrPlugin(CsrPluginConfig.linuxFull(0x80000020l)) - override def testParam = s"FREERTOS=$freertos ZEPHYR=$zephyr LINUX_REGRESSION=${sys.env.getOrElse("VEXRISCV_REGRESSION_LINUX_REGRESSION", "yes")} SUPERVISOR=yes" + override def testParam = s"FREERTOS=$freertos ZEPHYR=$zephyr LINUX_REGRESSION=$linux SUPERVISOR=yes" } } else if(catchAll){ new VexRiscvPosition("MachineOs") with CatchAllPosition{ @@ -554,10 +554,35 @@ class DecoderDimension extends VexRiscvDimension("Decoder") { } - +//class TesterPlay extends FunSuite with ParallelTestExecution { +// def createTest(name : String): Unit ={ +// test(name){ +// for(i <- 0 to 4) { +// println(s"$name $i") +// Thread.sleep(2000) +// } +// } +// } +// List("a", "b","c").foreach(createTest) +//} class TestIndividualFeatures extends FunSuite { + val testCount = sys.env.getOrElse("VEXRISCV_REGRESSION_CONFIG_COUNT", "100").toInt + val seed = sys.env.getOrElse("VEXRISCV_REGRESSION_SEED", Random.nextLong().toString).toLong + val testId : Set[Int] = sys.env.get("VEXRISCV_REGRESSION_TEST_ID") match { + case Some(x) if x != "" => x.split(',').map(_.toInt).toSet + case _ => (0 until testCount).toSet + } + val rvcRate = sys.env.getOrElse("VEXRISCV_REGRESSION_CONFIG_RVC_RATE", "0.5").toDouble + val linuxRate = sys.env.getOrElse("VEXRISCV_REGRESSION_CONFIG_LINUX_RATE", "0.3").toDouble + val machineOsRate = sys.env.getOrElse("VEXRISCV_REGRESSION_CONFIG_MACHINE_OS_RATE", "0.5").toDouble + val linuxRegression = sys.env.getOrElse("VEXRISCV_REGRESSION_LINUX_REGRESSION", "yes") + val coremarkRegression = sys.env.getOrElse("VEXRISCV_REGRESSION_COREMARK", "yes") + val zephyrCount = sys.env.getOrElse("VEXRISCV_REGRESSION_ZEPHYR_COUNT", "4") + val demwRate = sys.env.getOrElse("VEXRISCV_REGRESSION_CONFIG_DEMW_RATE", "0.6").toDouble + val demRate = sys.env.getOrElse("VEXRISCV_REGRESSION_CONFIG_DEM_RATE", "0.5").toDouble + def doCmd(cmd: String): String = { val stdOut = new StringBuilder() class Logger extends ProcessLogger { @@ -578,7 +603,7 @@ class TestIndividualFeatures extends FunSuite { val dimensions = List( - new IBusDimension, + new IBusDimension(rvcRate), new DBusDimension, new MulDivDimension, new ShiftDimension, @@ -586,7 +611,7 @@ class TestIndividualFeatures extends FunSuite { new HazardDimension, new RegFileDimension, new SrcDimension, - new CsrDimension(/*sys.env.getOrElse("VEXRISCV_REGRESSION_FREERTOS_COUNT", "1")*/ "0", sys.env.getOrElse("VEXRISCV_REGRESSION_ZEPHYR_COUNT", "4")), //Freertos old port software is broken + new CsrDimension(/*sys.env.getOrElse("VEXRISCV_REGRESSION_FREERTOS_COUNT", "1")*/ "0", zephyrCount, linuxRegression), //Freertos old port software is broken new DecoderDimension, new DebugDimension, new MmuDimension @@ -612,15 +637,15 @@ class TestIndividualFeatures extends FunSuite { } val name = (if(noMemory) "noMemoryStage_" else "") + (if(noWriteback) "noWritebackStage_" else "") + positionsToApply.map(d => d.dimension.name + "_" + d.name).mkString("_") - test(prefix + name + "_gen") { + test(prefix + "gen_" + name) { gen } - test(prefix + name + "_test") { + test(prefix + "test_" + name) { println("START TEST " + prefix + name) val debug = true - val stdCmd = (s"make clean run WITH_USER_IO=no REDO=10 TRACE=${if(debug) "yes" else "no"} TRACE_START=9999924910246l STOP_ON_ERROR=no FLOW_INFO=no STOP_ON_ERROR=no DHRYSTONE=yes COREMARK=${sys.env.getOrElse("VEXRISCV_REGRESSION_COREMARK", "yes")} THREAD_COUNT=${sys.env.getOrElse("VEXRISCV_REGRESSION_THREAD_COUNT", 1)} ") + s" SEED=${testSeed} " + val stdCmd = (s"make clean run WITH_USER_IO=no REDO=10 TRACE=${if(debug) "yes" else "no"} TRACE_START=1000000000000l STOP_ON_ERROR=no FLOW_INFO=no STOP_ON_ERROR=no DHRYSTONE=yes COREMARK=${coremarkRegression} THREAD_COUNT=1 ") + s" SEED=${testSeed} " val testCmd = stdCmd + (positionsToApply).map(_.testParam).mkString(" ") println(testCmd) val str = doCmd(testCmd) @@ -628,42 +653,33 @@ class TestIndividualFeatures extends FunSuite { } } - val testId : Option[mutable.HashSet[Int]] = None - val seed = sys.env.getOrElse("VEXRISCV_REGRESSION_SEED", Random.nextLong().toString).toLong -// -// val testId = Some(mutable.HashSet(3,4,9,11,13,16,18,19,20,21)) -// val testId = Some(mutable.HashSet(11)) -// val testId = Some(mutable.HashSet(4, 11)) -// val seed = 6592877339343561798l - - val rand = new Random(seed) test("Info"){ println(s"MAIN_SEED=$seed") } println(s"Seed=$seed") - for(i <- 0 until sys.env.getOrElse("VEXRISCV_REGRESSION_CONFIG_COUNT", "100").toInt){ + for(i <- 0 until testCount){ var positions : List[VexRiscvPosition] = null var universe = mutable.HashSet[VexRiscvUniverse]() if(rand.nextDouble() < 0.5) universe += VexRiscvUniverse.EXECUTE_RF - if(sys.env.getOrElse("VEXRISCV_REGRESSION_CONFIG_LINUX_RATE", "0.3").toDouble > rand.nextDouble()) { + if(linuxRate > rand.nextDouble()) { universe += VexRiscvUniverse.CATCH_ALL universe += VexRiscvUniverse.MMU universe += VexRiscvUniverse.FORCE_MULDIV universe += VexRiscvUniverse.SUPERVISOR - if(sys.env.getOrElse("VEXRISCV_REGRESSION_CONFIG_DEMW_RATE", "0.6").toDouble < rand.nextDouble()){ + if(demwRate < rand.nextDouble()){ universe += VexRiscvUniverse.NO_WRITEBACK } } else { - if(sys.env.getOrElse("VEXRISCV_REGRESSION_CONFIG_MACHINE_OS_RATE", "0.5").toDouble > rand.nextDouble()) { + if(machineOsRate > rand.nextDouble()) { universe += VexRiscvUniverse.CATCH_ALL - if(sys.env.getOrElse("VEXRISCV_REGRESSION_CONFIG_DEMW_RATE", "0.6").toDouble < rand.nextDouble()){ + if(demwRate < rand.nextDouble()){ universe += VexRiscvUniverse.NO_WRITEBACK } } - if(sys.env.getOrElse("VEXRISCV_REGRESSION_CONFIG_DEMW_RATE", "0.6").toDouble > rand.nextDouble()){ - }else if(sys.env.getOrElse("VEXRISCV_REGRESSION_CONFIG_DEM_RATE", "0.5").toDouble > rand.nextDouble()){ + if(demwRate > rand.nextDouble()){ + }else if(demRate > rand.nextDouble()){ universe += VexRiscvUniverse.NO_WRITEBACK } else { universe += VexRiscvUniverse.NO_WRITEBACK @@ -676,8 +692,8 @@ class TestIndividualFeatures extends FunSuite { }while(!positions.forall(_.isCompatibleWith(positions))) val testSeed = rand.nextInt() - if(testId.isEmpty || testId.get.contains(i)) - doTest(positions," random_" + i + "_", testSeed, universe) + if(testId.contains(i)) + doTest(positions," test_id_" + i + "_", testSeed, universe) Hack.dCounter += 1 } } \ No newline at end of file From 02545b9beae0e1420d38f7b8001dde98f4585445 Mon Sep 17 00:00:00 2001 From: Dolu1990 Date: Sun, 1 Mar 2020 13:03:40 +0100 Subject: [PATCH 2/3] typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d76a040..76f511b 100644 --- a/README.md +++ b/README.md @@ -186,7 +186,7 @@ NOTES: [![Build Status](https://travis-ci.org/SpinalHDL/VexRiscv.svg?branch=master)](https://travis-ci.org/SpinalHDL/VexRiscv) -To run tests (need the java, scala, verilator), just do : +To run tests (need java, scala, verilator), just do : ```sh export VEXRISCV_REGRESSION_SEED=42 From ea5464ea26c4151b6218ac53ed6fc5bba4c96e77 Mon Sep 17 00:00:00 2001 From: Dolu1990 Date: Sun, 1 Mar 2020 21:40:53 +0100 Subject: [PATCH 3/3] TestIndividualFeatures is now multithreaded --- src/test/cpp/regression/main.cpp | 58 +++--- src/test/cpp/regression/makefile | 15 +- .../vexriscv/TestIndividualFeatures.scala | 166 +++++++++++++----- 3 files changed, 160 insertions(+), 79 deletions(-) diff --git a/src/test/cpp/regression/main.cpp b/src/test/cpp/regression/main.cpp index 75f06bd..d983e8e 100644 --- a/src/test/cpp/regression/main.cpp +++ b/src/test/cpp/regression/main.cpp @@ -2805,7 +2805,7 @@ public: uint32_t regFileWriteRefIndex = 0; TestA() : WorkspaceRegression("testA") { - loadHex("../../resources/hex/testA.hex"); + loadHex(string(REGRESSION_PATH) + "../../resources/hex/testA.hex"); } virtual void checks(){ @@ -2831,7 +2831,7 @@ public: TestX28(string name, uint32_t *ref, uint32_t refSize) : WorkspaceRegression(name) { this->ref = ref; this->refSize = refSize; - loadHex("../../resources/hex/" + name + ".hex"); + loadHex(string(REGRESSION_PATH) + "../../resources/hex/" + name + ".hex"); } virtual void checks(){ @@ -2851,7 +2851,7 @@ public: class RiscvTest : public WorkspaceRegression{ public: RiscvTest(string name) : WorkspaceRegression(name) { - loadHex("../../resources/hex/" + name + ".hex"); + loadHex(string(REGRESSION_PATH) + "../../resources/hex/" + name + ".hex"); bootAt(0x800000bcu); } @@ -2899,7 +2899,7 @@ public: setIStall(iStall); setDStall(dStall); withRiscvRef(); - loadHex("../../resources/hex/" + hexName + ".hex"); + loadHex(string(REGRESSION_PATH) + "../../resources/hex/" + hexName + ".hex"); this->hexName = hexName; } @@ -2942,7 +2942,7 @@ public: int out32Counter = 0; Compliance(string name) : WorkspaceRegression(name) { withRiscvRef(); - loadHex("../../resources/hex/" + name + ".elf.hex"); + loadHex(string(REGRESSION_PATH) + "../../resources/hex/" + name + ".elf.hex"); out32.open (name + ".out32"); this->name = name; } @@ -2963,7 +2963,7 @@ public: virtual void pass(){ - FILE *refFile = fopen((string("../../resources/ref/") + name + ".reference_output").c_str(), "r"); + FILE *refFile = fopen((string(REGRESSION_PATH) + string("../../resources/ref/") + name + ".reference_output").c_str(), "r"); fseek(refFile, 0, SEEK_END); uint32_t refSize = ftell(refFile); fseek(refFile, 0, SEEK_SET); @@ -3152,7 +3152,7 @@ public: DebugPluginTest() : WorkspaceRegression("DebugPluginTest") { - loadHex("../../resources/hex/debugPlugin.hex"); + loadHex(string(REGRESSION_PATH) + "../../resources/hex/debugPlugin.hex"); pthread_create(&clientThreadId, NULL, &clientThreadWrapper, this); } @@ -3837,17 +3837,17 @@ int main(int argc, char **argv, char **env) { // #endif #ifdef IBUS_CACHED - redo(REDO,WorkspaceRegression("icache").withRiscvRef()->loadHex("../raw/icache/build/icache.hex")->bootAt(0x80000000u)->run(50e3);); + redo(REDO,WorkspaceRegression("icache").withRiscvRef()->loadHex(string(REGRESSION_PATH) + "../raw/icache/build/icache.hex")->bootAt(0x80000000u)->run(50e3);); #endif #ifdef DBUS_CACHED - redo(REDO,WorkspaceRegression("dcache").loadHex("../raw/dcache/build/dcache.hex")->bootAt(0x80000000u)->run(2500e3);); + redo(REDO,WorkspaceRegression("dcache").loadHex(string(REGRESSION_PATH) + "../raw/dcache/build/dcache.hex")->bootAt(0x80000000u)->run(2500e3);); #endif #ifdef MMU - redo(REDO,WorkspaceRegression("mmu").withRiscvRef()->loadHex("../raw/mmu/build/mmu.hex")->bootAt(0x80000000u)->run(50e3);); + redo(REDO,WorkspaceRegression("mmu").withRiscvRef()->loadHex(string(REGRESSION_PATH) + "../raw/mmu/build/mmu.hex")->bootAt(0x80000000u)->run(50e3);); #endif #ifdef SUPERVISOR - redo(REDO,WorkspaceRegression("deleg").withRiscvRef()->loadHex("../raw/deleg/build/deleg.hex")->bootAt(0x80000000u)->run(50e3);); + redo(REDO,WorkspaceRegression("deleg").withRiscvRef()->loadHex(string(REGRESSION_PATH) + "../raw/deleg/build/deleg.hex")->bootAt(0x80000000u)->run(50e3);); #endif #ifdef DEBUG_PLUGIN @@ -3858,20 +3858,20 @@ int main(int argc, char **argv, char **env) { #endif #ifdef CUSTOM_SIMD_ADD - redo(REDO,WorkspaceRegression("custom_simd_add").loadHex("../custom/simd_add/build/custom_simd_add.hex")->bootAt(0x00000000u)->run(50e3);); + redo(REDO,WorkspaceRegression("custom_simd_add").loadHex(string(REGRESSION_PATH) + "../custom/simd_add/build/custom_simd_add.hex")->bootAt(0x00000000u)->run(50e3);); #endif #ifdef CUSTOM_CSR - redo(REDO,WorkspaceRegression("custom_csr").loadHex("../custom/custom_csr/build/custom_csr.hex")->bootAt(0x00000000u)->run(50e3);); + redo(REDO,WorkspaceRegression("custom_csr").loadHex(string(REGRESSION_PATH) + "../custom/custom_csr/build/custom_csr.hex")->bootAt(0x00000000u)->run(50e3);); #endif #ifdef LRSC - redo(REDO,WorkspaceRegression("lrsc").withRiscvRef()->loadHex("../raw/lrsc/build/lrsc.hex")->bootAt(0x00000000u)->run(10e3);); + redo(REDO,WorkspaceRegression("lrsc").withRiscvRef()->loadHex(string(REGRESSION_PATH) + "../raw/lrsc/build/lrsc.hex")->bootAt(0x00000000u)->run(10e3);); #endif #ifdef AMO - redo(REDO,WorkspaceRegression("amo").withRiscvRef()->loadHex("../raw/amo/build/amo.hex")->bootAt(0x00000000u)->run(10e3);); + redo(REDO,WorkspaceRegression("amo").withRiscvRef()->loadHex(string(REGRESSION_PATH) + "../raw/amo/build/amo.hex")->bootAt(0x00000000u)->run(10e3);); #endif #ifdef DHRYSTONE @@ -3910,7 +3910,7 @@ int main(int argc, char **argv, char **env) { if(withStall == -1) break; #endif WorkspaceRegression("coremark_" + rv + (withStall > 0 ? "_stall" : "_nostall")).withRiscvRef() - ->loadBin("../../resources/bin/coremark_" + rv + ".bin", 0x80000000) + ->loadBin(string(REGRESSION_PATH) + "../../resources/bin/coremark_" + rv + ".bin", 0x80000000) ->bootAt(0x80000000) ->setIStall(withStall > 0) ->setDStall(withStall > 0) @@ -3930,17 +3930,17 @@ int main(int argc, char **argv, char **env) { /*for(int redo = 0;redo < 4;redo++)*/{ for(const string &name : freeRtosTests){ - tasks.push_back([=]() { WorkspaceRegression(name + "_rv32i_O0").withRiscvRef()->loadHex("../../resources/freertos/" + name + "_rv32i_O0.hex")->bootAt(0x80000000u)->run(4e6*15);}); - tasks.push_back([=]() { WorkspaceRegression(name + "_rv32i_O3").withRiscvRef()->loadHex("../../resources/freertos/" + name + "_rv32i_O3.hex")->bootAt(0x80000000u)->run(4e6*15);}); + tasks.push_back([=]() { WorkspaceRegression(name + "_rv32i_O0").withRiscvRef()->loadHex(string(REGRESSION_PATH) + "../../resources/freertos/" + name + "_rv32i_O0.hex")->bootAt(0x80000000u)->run(4e6*15);}); + tasks.push_back([=]() { WorkspaceRegression(name + "_rv32i_O3").withRiscvRef()->loadHex(string(REGRESSION_PATH) + "../../resources/freertos/" + name + "_rv32i_O3.hex")->bootAt(0x80000000u)->run(4e6*15);}); #ifdef COMPRESSED -// tasks.push_back([=]() { WorkspaceRegression(name + "_rv32ic_O0").withRiscvRef()->loadHex("../../resources/freertos/" + name + "_rv32ic_O0.hex")->bootAt(0x80000000u)->run(5e6*15);}); - tasks.push_back([=]() { WorkspaceRegression(name + "_rv32ic_O3").withRiscvRef()->loadHex("../../resources/freertos/" + name + "_rv32ic_O3.hex")->bootAt(0x80000000u)->run(4e6*15);}); +// tasks.push_back([=]() { WorkspaceRegression(name + "_rv32ic_O0").withRiscvRef()->loadHex(string(REGRESSION_PATH) + "../../resources/freertos/" + name + "_rv32ic_O0.hex")->bootAt(0x80000000u)->run(5e6*15);}); + tasks.push_back([=]() { WorkspaceRegression(name + "_rv32ic_O3").withRiscvRef()->loadHex(string(REGRESSION_PATH) + "../../resources/freertos/" + name + "_rv32ic_O3.hex")->bootAt(0x80000000u)->run(4e6*15);}); #endif #if defined(MUL) && defined(DIV) // #ifdef COMPRESSED -// tasks.push_back([=]() { WorkspaceRegression(name + "_rv32imac_O3").withRiscvRef()->loadHex("../../resources/freertos/" + name + "_rv32imac_O3.hex")->bootAt(0x80000000u)->run(4e6*15);}); +// tasks.push_back([=]() { WorkspaceRegression(name + "_rv32imac_O3").withRiscvRef()->loadHex(string(REGRESSION_PATH) + "../../resources/freertos/" + name + "_rv32imac_O3.hex")->bootAt(0x80000000u)->run(4e6*15);}); // #else - tasks.push_back([=]() { WorkspaceRegression(name + "_rv32im_O3").withRiscvRef()->loadHex("../../resources/freertos/" + name + "_rv32im_O3.hex")->bootAt(0x80000000u)->run(4e6*15);}); + tasks.push_back([=]() { WorkspaceRegression(name + "_rv32im_O3").withRiscvRef()->loadHex(string(REGRESSION_PATH) + "../../resources/freertos/" + name + "_rv32im_O3.hex")->bootAt(0x80000000u)->run(4e6*15);}); // #endif #endif } @@ -3967,12 +3967,12 @@ int main(int argc, char **argv, char **env) { /*for(int redo = 0;redo < 4;redo++)*/{ for(const string &name : zephyrTests){ #ifdef COMPRESSED - tasks.push_back([=]() { ZephyrRegression(name + "_rv32ic").withRiscvRef()->loadHex("../../resources/VexRiscvRegressionData/sim/zephyr/" + name + "_rv32ic.hex")->bootAt(0x80000000u)->run(180e6);}); + tasks.push_back([=]() { ZephyrRegression(name + "_rv32ic").withRiscvRef()->loadHex(string(REGRESSION_PATH) + "../../resources/VexRiscvRegressionData/sim/zephyr/" + name + "_rv32ic.hex")->bootAt(0x80000000u)->run(180e6);}); #else - tasks.push_back([=]() { ZephyrRegression(name + "_rv32i").withRiscvRef()->loadHex("../../resources/VexRiscvRegressionData/sim/zephyr/" + name + "_rv32i.hex")->bootAt(0x80000000u)->run(180e6);}); + tasks.push_back([=]() { ZephyrRegression(name + "_rv32i").withRiscvRef()->loadHex(string(REGRESSION_PATH) + "../../resources/VexRiscvRegressionData/sim/zephyr/" + name + "_rv32i.hex")->bootAt(0x80000000u)->run(180e6);}); #endif #if defined(MUL) && defined(DIV) - tasks.push_back([=]() { ZephyrRegression(name + "_rv32im").withRiscvRef()->loadHex("../../resources/VexRiscvRegressionData/sim/zephyr/" + name + "_rv32im.hex")->bootAt(0x80000000u)->run(180e6);}); + tasks.push_back([=]() { ZephyrRegression(name + "_rv32im").withRiscvRef()->loadHex(string(REGRESSION_PATH) + "../../resources/VexRiscvRegressionData/sim/zephyr/" + name + "_rv32im.hex")->bootAt(0x80000000u)->run(180e6);}); #endif } } @@ -3993,10 +3993,10 @@ int main(int argc, char **argv, char **env) { LinuxRegression soc("linux"); #ifndef DEBUG_PLUGIN_EXTERNAL soc.withRiscvRef(); - soc.loadBin(EMULATOR, 0x80000000); - soc.loadBin(VMLINUX, 0xC0000000); - soc.loadBin(DTB, 0xC3000000); - soc.loadBin(RAMDISK, 0xC2000000); + soc.loadBin(string(REGRESSION_PATH) + EMULATOR, 0x80000000); + soc.loadBin(string(REGRESSION_PATH) + VMLINUX, 0xC0000000); + soc.loadBin(string(REGRESSION_PATH) + DTB, 0xC3000000); + soc.loadBin(string(REGRESSION_PATH) + RAMDISK, 0xC2000000); #endif //soc.setIStall(true); //soc.setDStall(true); diff --git a/src/test/cpp/regression/makefile b/src/test/cpp/regression/makefile index ad05707..61fe9d1 100644 --- a/src/test/cpp/regression/makefile +++ b/src/test/cpp/regression/makefile @@ -1,5 +1,6 @@ DEBUG?=no - +REGRESSION_PATH?=./ +VEXRISCV_FILE?=../../../../VexRiscv.v IBUS?=CACHED IBUS_TC?=no DBUS?=CACHED @@ -38,6 +39,7 @@ STOP_ON_ERROR?=no COREMARK=no WITH_USER_IO?=no +ADDCFLAGS += -CFLAGS -DREGRESSION_PATH='\"$(REGRESSION_PATH)/\"' ADDCFLAGS += -CFLAGS -DIBUS_${IBUS} ADDCFLAGS += -CFLAGS -DDBUS_${DBUS} ADDCFLAGS += -CFLAGS -DREDO=${REDO} @@ -107,11 +109,11 @@ endif -ifneq ($(shell grep timerInterrupt ../../../../VexRiscv.v -w),) +ifneq ($(shell grep timerInterrupt ${VEXRISCV_FILE} -w),) ADDCFLAGS += -CFLAGS -DTIMER_INTERRUPT endif -ifneq ($(shell grep externalInterrupt ../../../../VexRiscv.v -w),) +ifneq ($(shell grep externalInterrupt ${VEXRISCV_FILE} -w),) ifneq ($(EXTERNAL_INTERRUPT),no) ADDCFLAGS += -CFLAGS -DEXTERNAL_INTERRUPT endif @@ -267,10 +269,9 @@ all: clean run run: compile ./obj_dir/VVexRiscv -verilate: ../../../../VexRiscv.v - rm -f VexRiscv.v*.bin - cp ../../../../VexRiscv.v*.bin . | true - verilator -cc ../../../../VexRiscv.v -O3 -CFLAGS -std=c++11 -LDFLAGS -pthread ${ADDCFLAGS} --gdbbt ${VERILATOR_ARGS} -Wno-UNOPTFLAT -Wno-WIDTH --x-assign unique --exe main.cpp +verilate: ${VEXRISCV_FILE} + cp ${VEXRISCV_FILE}*.bin . | true + verilator -cc ${VEXRISCV_FILE} -O3 -CFLAGS -std=c++11 -LDFLAGS -pthread ${ADDCFLAGS} --gdbbt ${VERILATOR_ARGS} -Wno-UNOPTFLAT -Wno-WIDTH --x-assign unique --exe main.cpp compile: verilate make -j${THREAD_COUNT} -C obj_dir/ -f VVexRiscv.mk VVexRiscv diff --git a/src/test/scala/vexriscv/TestIndividualFeatures.scala b/src/test/scala/vexriscv/TestIndividualFeatures.scala index 6605cb5..047c719 100644 --- a/src/test/scala/vexriscv/TestIndividualFeatures.scala +++ b/src/test/scala/vexriscv/TestIndividualFeatures.scala @@ -1,9 +1,10 @@ package vexriscv -import java.io.File +import java.io.{File, OutputStream} +import java.util.concurrent.TimeUnit import org.apache.commons.io.FileUtils -import org.scalatest.{BeforeAndAfterAll, FunSuite, ParallelTestExecution} +import org.scalatest.{BeforeAndAfterAll, FunSuite, ParallelTestExecution, Tag, Transformer} import spinal.core._ import vexriscv.demo._ import vexriscv.ip.{DataCacheConfig, InstructionCacheConfig} @@ -11,6 +12,8 @@ import vexriscv.plugin._ import scala.collection.mutable import scala.collection.mutable.ArrayBuffer +import scala.concurrent.duration.Duration +import scala.concurrent.{Await, ExecutionContext, Future} import scala.sys.process._ import scala.util.Random @@ -534,6 +537,7 @@ class DebugDimension extends VexRiscvDimension("Debug") { }, new VexRiscvPosition("Enable") { override def applyOn(config: VexRiscvConfig): Unit = config.plugins += new DebugPlugin(ClockDomain.current.clone(reset = Bool().setName("debugReset"))) + override def testParam = "CONCURRENT_OS_EXECUTIONS=yes" } )) } @@ -553,21 +557,80 @@ class DecoderDimension extends VexRiscvDimension("Decoder") { } } +object PlayFuture extends App{ + implicit val ec = ExecutionContext.global + val x = for(i <- 0 until 160) yield Future { + print(s"$i ") + Thread.sleep(1000) + } -//class TesterPlay extends FunSuite with ParallelTestExecution { -// def createTest(name : String): Unit ={ -// test(name){ -// for(i <- 0 to 4) { -// println(s"$name $i") -// Thread.sleep(2000) -// } -// } -// } -// List("a", "b","c").foreach(createTest) -//} + Thread.sleep(8000) +} + +class MultithreadedFunSuite extends FunSuite { + implicit val ec = ExecutionContext.global + class Job(body : => Unit){ + val originalOutput = Console.out + val buffer = mutable.Queue[Char]() + var bufferEnabled = true + def redirector() = new OutputStream{ + override def write(i: Int): Unit = synchronized { + if(bufferEnabled) buffer += i.toChar + else originalOutput.print(i.toChar) + } + } + val future = Future{ + Console.withOut(redirector()){ + Console.withErr(redirector())(body) + } + } + + def join(): Unit = { + Thread.sleep(50) + synchronized{ + bufferEnabled = false + buffer.foreach(originalOutput.print) + } + Await.result(future, Duration.Inf) + } + } + + override protected def test(testName: String, testTags: Tag*)(testFun: => Unit) { + val job = new Job(testFun) + super.test(testName, testTags :_*)(job.join()) + } + protected def testSingleThread(testName: String, testTags: Tag*)(testFun: => Unit) { + super.test(testName, testTags :_*)(testFun) + } +} -class TestIndividualFeatures extends FunSuite { +class FunTestPara extends MultithreadedFunSuite{ + def createTest(name : String): Unit ={ + test(name){ + for(i <- 0 to 4) { + println(s"$name $i") + Thread.sleep(500) + } + } + } + (0 to 80).map(_.toString).foreach(createTest) +} + +class FunTestPlay extends FunSuite { + def createTest(name : String): Unit ={ + test(name){ + Thread.sleep(500) + for(i <- 0 to 4) { + println(s"$name $i") + Thread.sleep(500) + } + } + } + (0 to 80).map(_.toString).foreach(createTest) +} + +class TestIndividualFeatures extends MultithreadedFunSuite { val testCount = sys.env.getOrElse("VEXRISCV_REGRESSION_CONFIG_COUNT", "100").toInt val seed = sys.env.getOrElse("VEXRISCV_REGRESSION_SEED", Random.nextLong().toString).toLong val testId : Set[Int] = sys.env.get("VEXRISCV_REGRESSION_TEST_ID") match { @@ -582,24 +645,8 @@ class TestIndividualFeatures extends FunSuite { val zephyrCount = sys.env.getOrElse("VEXRISCV_REGRESSION_ZEPHYR_COUNT", "4") val demwRate = sys.env.getOrElse("VEXRISCV_REGRESSION_CONFIG_DEMW_RATE", "0.6").toDouble val demRate = sys.env.getOrElse("VEXRISCV_REGRESSION_CONFIG_DEM_RATE", "0.5").toDouble + val lock = new{} - def doCmd(cmd: String): String = { - val stdOut = new StringBuilder() - class Logger extends ProcessLogger { - override def err(s: => String): Unit = { - if (!s.startsWith("ar: creating ")) println(s) - } - - override def out(s: => String): Unit = { - println(s) - stdOut ++= s - } - - override def buffer[T](f: => T) = f - } - Process(cmd, new File("src/test/cpp/regression")).!(new Logger) - stdOut.toString() - } val dimensions = List( @@ -617,12 +664,40 @@ class TestIndividualFeatures extends FunSuite { new MmuDimension ) + var clockCounter = 0l + var startAt = System.currentTimeMillis() def doTest(positionsToApply : List[VexRiscvPosition], prefix : String = "", testSeed : Int, universes : mutable.HashSet[VexRiscvUniverse]): Unit ={ val noMemory = universes.contains(VexRiscvUniverse.NO_MEMORY) val noWriteback = universes.contains(VexRiscvUniverse.NO_WRITEBACK) - def gen = { + val name = (if(noMemory) "noMemoryStage_" else "") + (if(noWriteback) "noWritebackStage_" else "") + positionsToApply.map(d => d.dimension.name + "_" + d.name).mkString("_") + val workspace = "simWorkspace" + val project = s"$workspace/$prefix" + def doCmd(cmd: String): String = { + val stdOut = new StringBuilder() + class Logger extends ProcessLogger { + override def err(s: => String): Unit = { + if (!s.startsWith("ar: creating ")) println(s) + } + override def out(s: => String): Unit = { + println(s) + stdOut ++= s + } + override def buffer[T](f: => T) = f + } + Process(cmd, new File(project)).!(new Logger) + stdOut.toString() + } + + test(prefix + name) { + println("START TEST " + prefix + name) + + //Cleanup + FileUtils.deleteDirectory(new File(project)) + FileUtils.forceMkdir(new File(project)) + + //Generate RTL FileUtils.deleteQuietly(new File("VexRiscv.v")) - SpinalVerilog{ + SpinalConfig(targetDirectory = project).generateVerilog{ val config = VexRiscvConfig( withMemoryStage = !noMemory, withWriteBackStage = !noWriteback, @@ -634,22 +709,22 @@ class TestIndividualFeatures extends FunSuite { for (positionToApply <- positionsToApply) positionToApply.applyOn(config) new VexRiscv(config) } - } - val name = (if(noMemory) "noMemoryStage_" else "") + (if(noWriteback) "noWritebackStage_" else "") + positionsToApply.map(d => d.dimension.name + "_" + d.name).mkString("_") - test(prefix + "gen_" + name) { - gen - } + //Setup test + val files = List("main.cpp", "encoding.h" ,"makefile", "dhrystoneO3.logRef", "dhrystoneO3C.logRef","dhrystoneO3MC.logRef","dhrystoneO3M.logRef") + files.foreach(f => FileUtils.copyFileToDirectory(new File(s"src/test/cpp/regression/$f"), new File(project))) - - test(prefix + "test_" + name) { - println("START TEST " + prefix + name) + //Test RTL val debug = true - val stdCmd = (s"make clean run WITH_USER_IO=no REDO=10 TRACE=${if(debug) "yes" else "no"} TRACE_START=1000000000000l STOP_ON_ERROR=no FLOW_INFO=no STOP_ON_ERROR=no DHRYSTONE=yes COREMARK=${coremarkRegression} THREAD_COUNT=1 ") + s" SEED=${testSeed} " + val stdCmd = (s"make run REGRESSION_PATH=../../src/test/cpp/regression VEXRISCV_FILE=VexRiscv.v WITH_USER_IO=no REDO=10 TRACE=${if(debug) "yes" else "no"} TRACE_START=1000000000000l STOP_ON_ERROR=no FLOW_INFO=no STOP_ON_ERROR=no DHRYSTONE=yes COREMARK=${coremarkRegression} THREAD_COUNT=1 ") + s" SEED=${testSeed} " val testCmd = stdCmd + (positionsToApply).map(_.testParam).mkString(" ") println(testCmd) val str = doCmd(testCmd) assert(str.contains("REGRESSION SUCCESS") && !str.contains("Broken pipe")) + val pattern = "Had simulate ([0-9]+)".r + val hit = pattern.findFirstMatchIn(str) + + lock.synchronized(clockCounter += hit.get.group(1).toLong) } } @@ -693,7 +768,12 @@ class TestIndividualFeatures extends FunSuite { val testSeed = rand.nextInt() if(testId.contains(i)) - doTest(positions," test_id_" + i + "_", testSeed, universe) + doTest(positions,"test_id_" + i + "_", testSeed, universe) Hack.dCounter += 1 } + testSingleThread("report"){ + val time = (System.currentTimeMillis() - startAt)*1e-3 + val clockPerSecond = (clockCounter/time*1e-3).toLong + println(s"Duration=${(time/60).toInt}mn clocks=${(clockCounter*1e-6).toLong}M clockPerSecond=${clockPerSecond}K") + } } \ No newline at end of file