Merge pull request #131 from antmicro/jboc/benchmark
Allow testing custom access patterns
This commit is contained in:
enjoy-digital
GitHub
commit
08fd2960d0
|
|
@ -185,6 +185,60 @@ class _LiteDRAMBISTGenerator(Module):
|
|||
raise NotImplementedError
|
||||
self.comb += dma.sink.data.eq(data_gen.o)
|
||||
|
||||
@ResetInserter()
|
||||
class _LiteDRAMPatternGenerator(Module):
|
||||
def __init__(self, dram_port, init=[]):
|
||||
ashift, awidth = get_ashift_awidth(dram_port)
|
||||
self.start = Signal()
|
||||
self.done = Signal()
|
||||
self.ticks = Signal(32)
|
||||
|
||||
# # #
|
||||
|
||||
# DMA --------------------------------------------------------------------------------------
|
||||
dma = LiteDRAMDMAWriter(dram_port)
|
||||
self.submodules += dma
|
||||
|
||||
cmd_counter = Signal(dram_port.address_width, reset_less=True)
|
||||
|
||||
# Data / Address FSM -----------------------------------------------------------------------
|
||||
fsm = FSM(reset_state="IDLE")
|
||||
self.submodules += fsm
|
||||
fsm.act("IDLE",
|
||||
If(self.start,
|
||||
NextValue(cmd_counter, 0),
|
||||
NextState("RUN")
|
||||
),
|
||||
NextValue(self.ticks, 0)
|
||||
)
|
||||
fsm.act("RUN",
|
||||
dma.sink.valid.eq(1),
|
||||
If(dma.sink.ready,
|
||||
NextValue(cmd_counter, cmd_counter + 1),
|
||||
If(cmd_counter == (len(init) - 1),
|
||||
NextState("DONE")
|
||||
)
|
||||
),
|
||||
NextValue(self.ticks, self.ticks + 1)
|
||||
)
|
||||
fsm.act("DONE",
|
||||
self.done.eq(1)
|
||||
)
|
||||
|
||||
if isinstance(dram_port, LiteDRAMNativePort): # addressing in dwords
|
||||
dma_sink_addr = dma.sink.address
|
||||
elif isinstance(dram_port, LiteDRAMAXIPort): # addressing in bytes
|
||||
dma_sink_addr = dma.sink.address[ashift:]
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
addr_cases = {i: dma_sink_addr.eq(addr) for i, (addr, data) in enumerate(init)}
|
||||
data_cases = {i: dma.sink.data.eq(data) for i, (addr, data) in enumerate(init)}
|
||||
self.comb += [
|
||||
Case(cmd_counter, addr_cases),
|
||||
Case(cmd_counter, data_cases),
|
||||
]
|
||||
|
||||
# LiteDRAMBISTGenerator ----------------------------------------------------------------------------
|
||||
|
||||
class LiteDRAMBISTGenerator(Module, AutoCSR):
|
||||
|
|
@ -367,6 +421,88 @@ class _LiteDRAMBISTChecker(Module, AutoCSR):
|
|||
self.done.eq(1)
|
||||
)
|
||||
|
||||
@ResetInserter()
|
||||
class _LiteDRAMPatternChecker(Module, AutoCSR):
|
||||
def __init__(self, dram_port, init=[]):
|
||||
ashift, awidth = get_ashift_awidth(dram_port)
|
||||
self.start = Signal()
|
||||
self.done = Signal()
|
||||
self.ticks = Signal(32)
|
||||
self.errors = Signal(32)
|
||||
|
||||
# # #
|
||||
|
||||
# DMA --------------------------------------------------------------------------------------
|
||||
dma = LiteDRAMDMAReader(dram_port)
|
||||
self.submodules += dma
|
||||
|
||||
# Address FSM ------------------------------------------------------------------------------
|
||||
cmd_counter = Signal(dram_port.address_width, reset_less=True)
|
||||
|
||||
cmd_fsm = FSM(reset_state="IDLE")
|
||||
self.submodules += cmd_fsm
|
||||
cmd_fsm.act("IDLE",
|
||||
If(self.start,
|
||||
NextValue(cmd_counter, 0),
|
||||
NextState("RUN")
|
||||
)
|
||||
)
|
||||
cmd_fsm.act("RUN",
|
||||
dma.sink.valid.eq(1),
|
||||
If(dma.sink.ready,
|
||||
NextValue(cmd_counter, cmd_counter + 1),
|
||||
If(cmd_counter == (len(init) - 1),
|
||||
NextState("DONE")
|
||||
)
|
||||
)
|
||||
)
|
||||
cmd_fsm.act("DONE")
|
||||
|
||||
if isinstance(dram_port, LiteDRAMNativePort): # addressing in dwords
|
||||
dma_addr_sink = dma.sink.address
|
||||
elif isinstance(dram_port, LiteDRAMAXIPort): # addressing in bytes
|
||||
dma_addr_sink = dma.sink.address[ashift:]
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
addr_cases = {i: dma_addr_sink.eq(addr) for i, (addr, data) in enumerate(init)}
|
||||
self.comb += Case(cmd_counter, addr_cases)
|
||||
|
||||
# Data FSM ---------------------------------------------------------------------------------
|
||||
data_counter = Signal(dram_port.address_width, reset_less=True)
|
||||
|
||||
expected_data = Signal.like(dma.source.data)
|
||||
data_cases = {i: expected_data.eq(data) for i, (addr, data) in enumerate(init)}
|
||||
self.comb += Case(data_counter, data_cases)
|
||||
|
||||
data_fsm = FSM(reset_state="IDLE")
|
||||
self.submodules += data_fsm
|
||||
data_fsm.act("IDLE",
|
||||
If(self.start,
|
||||
NextValue(data_counter, 0),
|
||||
NextValue(self.errors, 0),
|
||||
NextState("RUN")
|
||||
),
|
||||
NextValue(self.ticks, 0)
|
||||
)
|
||||
|
||||
data_fsm.act("RUN",
|
||||
dma.source.ready.eq(1),
|
||||
If(dma.source.valid,
|
||||
NextValue(data_counter, data_counter + 1),
|
||||
If(dma.source.data != expected_data,
|
||||
NextValue(self.errors, self.errors + 1)
|
||||
),
|
||||
If(data_counter == (len(init) - 1),
|
||||
NextState("DONE")
|
||||
)
|
||||
),
|
||||
NextValue(self.ticks, self.ticks + 1)
|
||||
)
|
||||
data_fsm.act("DONE",
|
||||
self.done.eq(1)
|
||||
)
|
||||
|
||||
# LiteDRAMBISTChecker ------------------------------------------------------------------------------
|
||||
|
||||
class LiteDRAMBISTChecker(Module, AutoCSR):
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -3,6 +3,7 @@
|
|||
# This file is Copyright (c) 2020 Florent Kermarrec <florent@enjoy-digital.fr>
|
||||
# License: BSD
|
||||
|
||||
import csv
|
||||
import argparse
|
||||
|
||||
from migen import *
|
||||
|
|
@ -16,9 +17,8 @@ from litex.soc.integration.builder import *
|
|||
|
||||
from litex.tools.litex_sim import SimSoC
|
||||
|
||||
from litedram.frontend.bist import _LiteDRAMBISTGenerator
|
||||
from litedram.frontend.bist import _LiteDRAMBISTChecker
|
||||
|
||||
from litedram.frontend.bist import _LiteDRAMBISTGenerator, _LiteDRAMBISTChecker, \
|
||||
_LiteDRAMPatternGenerator, _LiteDRAMPatternChecker
|
||||
|
||||
# LiteDRAM Benchmark SoC ---------------------------------------------------------------------------
|
||||
|
||||
|
|
@ -29,6 +29,7 @@ class LiteDRAMBenchmarkSoC(SimSoC):
|
|||
bist_base = 0x00000000,
|
||||
bist_length = 1024,
|
||||
bist_random = False,
|
||||
pattern_init = None,
|
||||
**kwargs):
|
||||
|
||||
# SimSoC -----------------------------------------------------------------------------------
|
||||
|
|
@ -42,12 +43,34 @@ class LiteDRAMBenchmarkSoC(SimSoC):
|
|||
# make sure that we perform at least one access
|
||||
bist_length = max(bist_length, self.sdram.controller.interface.data_width // 8)
|
||||
|
||||
# BIST Generator ---------------------------------------------------------------------------
|
||||
bist_generator = _LiteDRAMBISTGenerator(self.sdram.crossbar.get_port())
|
||||
self.submodules.bist_generator = bist_generator
|
||||
# BIST Generator / Checker -----------------------------------------------------------------
|
||||
if pattern_init is None:
|
||||
bist_generator = _LiteDRAMBISTGenerator(self.sdram.crossbar.get_port())
|
||||
bist_checker = _LiteDRAMBISTChecker(self.sdram.crossbar.get_port())
|
||||
|
||||
# BIST Checker -----------------------------------------------------------------------------
|
||||
bist_checker = _LiteDRAMBISTChecker(self.sdram.crossbar.get_port())
|
||||
generator_config = [
|
||||
bist_generator.base.eq(bist_base),
|
||||
bist_generator.length.eq(bist_length),
|
||||
bist_generator.random.eq(bist_random),
|
||||
]
|
||||
checker_config = [
|
||||
bist_checker.base.eq(bist_base),
|
||||
bist_checker.length.eq(bist_length),
|
||||
bist_checker.random.eq(bist_random),
|
||||
]
|
||||
else:
|
||||
# TODO: run checker in parallel to avoid overwriting previously written data
|
||||
address_set = set()
|
||||
for addr, _ in pattern_init:
|
||||
assert addr not in address_set, \
|
||||
'Duplicate address 0x%08x in pattern_init, write will overwrite previous value!' % addr
|
||||
address_set.add(addr)
|
||||
|
||||
bist_generator = _LiteDRAMPatternGenerator(self.sdram.crossbar.get_port(), init=pattern_init)
|
||||
bist_checker = _LiteDRAMPatternChecker(self.sdram.crossbar.get_port(), init=pattern_init)
|
||||
generator_config = checker_config = []
|
||||
|
||||
self.submodules.bist_generator = bist_generator
|
||||
self.submodules.bist_checker = bist_checker
|
||||
|
||||
# Sequencer --------------------------------------------------------------------------------
|
||||
|
|
@ -68,18 +91,14 @@ class LiteDRAMBenchmarkSoC(SimSoC):
|
|||
)
|
||||
fsm.act("BIST-GENERATOR",
|
||||
bist_generator.start.eq(1),
|
||||
bist_generator.base.eq(bist_base),
|
||||
bist_generator.length.eq(bist_length),
|
||||
bist_generator.random.eq(bist_random),
|
||||
*generator_config,
|
||||
If(bist_generator.done,
|
||||
NextState("BIST-CHECKER")
|
||||
)
|
||||
)
|
||||
fsm.act("BIST-CHECKER",
|
||||
bist_checker.start.eq(1),
|
||||
bist_checker.base.eq(bist_base),
|
||||
bist_checker.length.eq(bist_length),
|
||||
bist_checker.random.eq(bist_random),
|
||||
*checker_config,
|
||||
If(bist_checker.done,
|
||||
NextState("DISPLAY")
|
||||
)
|
||||
|
|
@ -109,20 +128,28 @@ class LiteDRAMBenchmarkSoC(SimSoC):
|
|||
|
||||
# Build --------------------------------------------------------------------------------------------
|
||||
|
||||
def load_access_pattern(filename):
|
||||
with open(filename, newline='') as f:
|
||||
reader = csv.reader(f)
|
||||
pattern_init = [(int(addr, 0), int(data, 0)) for addr, data in reader]
|
||||
return pattern_init
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="LiteDRAM Benchmark SoC Simulation")
|
||||
builder_args(parser)
|
||||
soc_sdram_args(parser)
|
||||
parser.add_argument("--threads", default=1, help="Set number of threads (default=1)")
|
||||
parser.add_argument("--sdram-module", default="MT48LC16M16", help="Select SDRAM chip")
|
||||
parser.add_argument("--sdram-data-width", default=32, help="Set SDRAM chip data width")
|
||||
parser.add_argument("--trace", action="store_true", help="Enable VCD tracing")
|
||||
parser.add_argument("--trace-start", default=0, help="Cycle to start VCD tracing")
|
||||
parser.add_argument("--trace-end", default=-1, help="Cycle to end VCD tracing")
|
||||
parser.add_argument("--opt-level", default="O0", help="Compilation optimization level")
|
||||
parser.add_argument("--bist-base", default="0x00000000", help="Base address of the test (default=0)")
|
||||
parser.add_argument("--bist-length", default="1024", help="Length of the test (default=1024)")
|
||||
parser.add_argument("--bist-random", action="store_true", help="Use random data during the test")
|
||||
parser.add_argument("--threads", default=1, help="Set number of threads (default=1)")
|
||||
parser.add_argument("--sdram-module", default="MT48LC16M16", help="Select SDRAM chip")
|
||||
parser.add_argument("--sdram-data-width", default=32, help="Set SDRAM chip data width")
|
||||
parser.add_argument("--trace", action="store_true", help="Enable VCD tracing")
|
||||
parser.add_argument("--trace-start", default=0, help="Cycle to start VCD tracing")
|
||||
parser.add_argument("--trace-end", default=-1, help="Cycle to end VCD tracing")
|
||||
parser.add_argument("--opt-level", default="O0", help="Compilation optimization level")
|
||||
parser.add_argument("--bist-base", default="0x00000000", help="Base address of the test (default=0)")
|
||||
parser.add_argument("--bist-length", default="1024", help="Length of the test (default=1024)")
|
||||
parser.add_argument("--bist-random", action="store_true", help="Use random data during the test")
|
||||
parser.add_argument("--access-pattern", help="Load access pattern (address, data) from CSV (ignores --bist-*)")
|
||||
args = parser.parse_args()
|
||||
|
||||
soc_kwargs = soc_sdram_argdict(args)
|
||||
|
|
@ -138,6 +165,9 @@ def main():
|
|||
soc_kwargs["bist_length"] = int(args.bist_length, 0)
|
||||
soc_kwargs["bist_random"] = args.bist_random
|
||||
|
||||
if args.access_pattern:
|
||||
soc_kwargs["pattern_init"] = load_access_pattern(args.access_pattern)
|
||||
|
||||
# SoC ------------------------------------------------------------------------------------------
|
||||
soc = LiteDRAMBenchmarkSoC(**soc_kwargs)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,76 +1,193 @@
|
|||
{
|
||||
# sequential access
|
||||
"test_0": {
|
||||
"sdram_module": 'MT48LC16M16',
|
||||
"sdram_data_width": 32,
|
||||
"bist_length": 4096,
|
||||
"bist_random": True,
|
||||
"access_pattern": {
|
||||
"bist_length": 4096,
|
||||
"bist_random": False,
|
||||
}
|
||||
},
|
||||
"test_1": {
|
||||
"sdram_module": 'MT48LC16M16',
|
||||
"sdram_data_width": 32,
|
||||
"bist_length": 512,
|
||||
"bist_random": False,
|
||||
"access_pattern": {
|
||||
"bist_length": 512,
|
||||
"bist_random": False,
|
||||
}
|
||||
},
|
||||
"test_2": {
|
||||
"sdram_module": 'MT46V32M16',
|
||||
"sdram_data_width": 32,
|
||||
"bist_length": 512,
|
||||
"bist_random": False,
|
||||
"access_pattern": {
|
||||
"bist_length": 512,
|
||||
"bist_random": False,
|
||||
}
|
||||
},
|
||||
"test_3": {
|
||||
"sdram_module": 'MT46V32M16',
|
||||
"sdram_data_width": 32,
|
||||
"bist_length": 2048,
|
||||
"bist_random": False,
|
||||
"access_pattern": {
|
||||
"bist_length": 2048,
|
||||
"bist_random": False,
|
||||
}
|
||||
},
|
||||
"test_4": {
|
||||
"sdram_module": 'MT47H64M16',
|
||||
"sdram_data_width": 32,
|
||||
"bist_length": 1024,
|
||||
"bist_random": False,
|
||||
"access_pattern": {
|
||||
"bist_length": 1024,
|
||||
"bist_random": False,
|
||||
}
|
||||
},
|
||||
"test_5": {
|
||||
"sdram_module": 'MT47H64M16',
|
||||
"sdram_data_width": 16,
|
||||
"bist_length": 1024,
|
||||
"bist_random": False,
|
||||
"access_pattern": {
|
||||
"bist_length": 1024,
|
||||
"bist_random": False,
|
||||
}
|
||||
},
|
||||
"test_6": {
|
||||
"sdram_module": 'MT41K128M16',
|
||||
"sdram_data_width": 16,
|
||||
"bist_length": 1024,
|
||||
"bist_random": False,
|
||||
"access_pattern": {
|
||||
"bist_length": 1024,
|
||||
"bist_random": False,
|
||||
}
|
||||
},
|
||||
"test_7": {
|
||||
"sdram_module": 'MT41K128M16',
|
||||
"sdram_data_width": 32,
|
||||
"bist_length": 1024,
|
||||
"bist_random": False,
|
||||
"access_pattern": {
|
||||
"bist_length": 1024,
|
||||
"bist_random": False,
|
||||
}
|
||||
},
|
||||
|
||||
# latency
|
||||
"test_8": {
|
||||
"sdram_module": 'MT48LC16M16',
|
||||
"sdram_data_width": 32,
|
||||
"bist_length": 1,
|
||||
"bist_random": False,
|
||||
"access_pattern": {
|
||||
"bist_length": 1,
|
||||
"bist_random": False,
|
||||
}
|
||||
},
|
||||
"test_9": {
|
||||
"sdram_module": 'MT46V32M16',
|
||||
"sdram_data_width": 32,
|
||||
"bist_length": 1,
|
||||
"bist_random": False,
|
||||
"access_pattern": {
|
||||
"bist_length": 1,
|
||||
"bist_random": False,
|
||||
}
|
||||
},
|
||||
"test_10": {
|
||||
"sdram_module": 'MT47H64M16',
|
||||
"sdram_data_width": 32,
|
||||
"bist_length": 1,
|
||||
"bist_random": False,
|
||||
"access_pattern": {
|
||||
"bist_length": 1,
|
||||
"bist_random": False,
|
||||
}
|
||||
},
|
||||
"test_11": {
|
||||
"sdram_module": 'MT41K128M16',
|
||||
"sdram_data_width": 16,
|
||||
"bist_length": 1,
|
||||
"bist_random": False,
|
||||
"access_pattern": {
|
||||
"bist_length": 1,
|
||||
"bist_random": False,
|
||||
}
|
||||
},
|
||||
|
||||
# random access
|
||||
"test_12": {
|
||||
"sdram_module": 'MT48LC16M16',
|
||||
"sdram_data_width": 32,
|
||||
"access_pattern": {
|
||||
"bist_length": 1024,
|
||||
"bist_random": True,
|
||||
}
|
||||
},
|
||||
"test_13": {
|
||||
"sdram_module": 'MT46V32M16',
|
||||
"sdram_data_width": 32,
|
||||
"access_pattern": {
|
||||
"bist_length": 1024,
|
||||
"bist_random": True,
|
||||
}
|
||||
},
|
||||
"test_14": {
|
||||
"sdram_module": 'MT47H64M16',
|
||||
"sdram_data_width": 32,
|
||||
"access_pattern": {
|
||||
"bist_length": 1024,
|
||||
"bist_random": True,
|
||||
}
|
||||
},
|
||||
"test_15": {
|
||||
"sdram_module": 'MT41K128M16',
|
||||
"sdram_data_width": 16,
|
||||
"access_pattern": {
|
||||
"bist_length": 1024,
|
||||
"bist_random": True,
|
||||
}
|
||||
},
|
||||
|
||||
# custom access pattern
|
||||
"test_16": {
|
||||
"sdram_module": 'MT48LC16M16',
|
||||
"sdram_data_width": 32,
|
||||
"access_pattern": {
|
||||
"pattern_file": "access_pattern.csv"
|
||||
}
|
||||
},
|
||||
"test_17": {
|
||||
"sdram_module": 'MT48LC16M16',
|
||||
"sdram_data_width": 32,
|
||||
"access_pattern": {
|
||||
"pattern_file": "access_pattern.csv",
|
||||
}
|
||||
},
|
||||
"test_18": {
|
||||
"sdram_module": 'MT46V32M16',
|
||||
"sdram_data_width": 32,
|
||||
"access_pattern": {
|
||||
"pattern_file": "access_pattern.csv",
|
||||
}
|
||||
},
|
||||
"test_19": {
|
||||
"sdram_module": 'MT46V32M16',
|
||||
"sdram_data_width": 32,
|
||||
"access_pattern": {
|
||||
"pattern_file": "access_pattern.csv",
|
||||
}
|
||||
},
|
||||
"test_20": {
|
||||
"sdram_module": 'MT47H64M16',
|
||||
"sdram_data_width": 32,
|
||||
"access_pattern": {
|
||||
"pattern_file": "access_pattern.csv",
|
||||
}
|
||||
},
|
||||
"test_21": {
|
||||
"sdram_module": 'MT47H64M16',
|
||||
"sdram_data_width": 16,
|
||||
"access_pattern": {
|
||||
"pattern_file": "access_pattern.csv",
|
||||
}
|
||||
},
|
||||
"test_22": {
|
||||
"sdram_module": 'MT41K128M16',
|
||||
"sdram_data_width": 16,
|
||||
"access_pattern": {
|
||||
"pattern_file": "access_pattern.csv",
|
||||
}
|
||||
},
|
||||
"test_23": {
|
||||
"sdram_module": 'MT41K128M16',
|
||||
"sdram_data_width": 32,
|
||||
"access_pattern": {
|
||||
"pattern_file": "access_pattern.csv",
|
||||
}
|
||||
},
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,34 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
import random
|
||||
import argparse
|
||||
|
||||
def main():
|
||||
desc = """
|
||||
Generate random access pattern.
|
||||
Each address in range [base, base+length) will be accessed only once,
|
||||
but in random order. This ensures that no data will be overwritten.
|
||||
"""
|
||||
parser = argparse.ArgumentParser(description=desc)
|
||||
parser.add_argument('base', help='Base address')
|
||||
parser.add_argument('length', help='Number of (address, data) pairs')
|
||||
parser.add_argument('data_width', help='Width of data (used to determine max value)')
|
||||
parser.add_argument('--seed', help='Use given random seed (int)')
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.seed:
|
||||
random.seed(int(args.seed, 0))
|
||||
|
||||
base = int(args.base, 0)
|
||||
length = int(args.length, 0)
|
||||
data_width = int(args.data_width, 0)
|
||||
|
||||
address = list(range(length))
|
||||
random.shuffle(address)
|
||||
data = [random.randrange(0, 2**data_width) for _ in range(length)]
|
||||
|
||||
for a, d in zip(address, data):
|
||||
print('0x{:08x},0x{:08x}'.format(a, d))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -0,0 +1,87 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
import json
|
||||
import argparse
|
||||
import itertools
|
||||
|
||||
modules = [
|
||||
'IS42S16160',
|
||||
'IS42S16320',
|
||||
'MT48LC4M16',
|
||||
'MT48LC16M16',
|
||||
'AS4C16M16',
|
||||
'AS4C32M16',
|
||||
'AS4C32M8',
|
||||
'M12L64322A',
|
||||
'M12L16161A',
|
||||
'MT46V32M16',
|
||||
'MT46H32M16',
|
||||
'MT46H32M32',
|
||||
'MT47H128M8',
|
||||
'MT47H32M16',
|
||||
'MT47H64M16',
|
||||
'P3R1GE4JGF',
|
||||
'MT41K64M16',
|
||||
'MT41J128M16',
|
||||
'MT41K128M16',
|
||||
'MT41J256M16',
|
||||
'MT41K256M16',
|
||||
'K4B1G0446F',
|
||||
'K4B2G1646F',
|
||||
'H5TC4G63CFR',
|
||||
'IS43TR16128B',
|
||||
'MT8JTF12864',
|
||||
'MT8KTF51264',
|
||||
# 'MT18KSF1G72HZ',
|
||||
# 'AS4C256M16D3A',
|
||||
# 'MT16KTF1G64HZ',
|
||||
# 'EDY4016A',
|
||||
# 'MT40A1G8',
|
||||
# 'MT40A512M16',
|
||||
]
|
||||
data_widths = [32]
|
||||
bist_lengths = [1, 1024, 8192]
|
||||
bist_randoms = [False]
|
||||
access_patterns = ['access_pattern.csv']
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Generate configuration for all possible argument combinations.',
|
||||
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
||||
parser.add_argument('--sdram-modules', nargs='+', default=modules, help='--sdram-module options')
|
||||
parser.add_argument('--sdram-data-widths', nargs='+', default=data_widths, help='--sdram-data-width options')
|
||||
parser.add_argument('--bist-lengths', nargs='+', default=bist_lengths, help='--bist-length options')
|
||||
parser.add_argument('--bist-randoms', nargs='+', default=bist_randoms, help='--bist-random options')
|
||||
parser.add_argument('--access-patterns', nargs='+', default=access_patterns, help='--access-pattern options')
|
||||
parser.add_argument('--name-format', default='test_%d', help='Name format for i-th test')
|
||||
args = parser.parse_args()
|
||||
|
||||
bist_product = itertools.product(args.sdram_modules, args.sdram_data_widths, args.bist_lengths, args.bist_randoms)
|
||||
pattern_product = itertools.product(args.sdram_modules, args.sdram_data_widths, args.access_patterns)
|
||||
|
||||
i = 0
|
||||
configurations = {}
|
||||
for module, data_width, bist_length, bist_random in bist_product:
|
||||
configurations[args.name_format % i] = {
|
||||
'sdram_module': module,
|
||||
'sdram_data_width': data_width,
|
||||
'access_pattern': {
|
||||
'bist_length': bist_length,
|
||||
'bist_random': bist_random,
|
||||
}
|
||||
}
|
||||
i += 1
|
||||
for module, data_width, access_pattern in pattern_product:
|
||||
configurations[args.name_format % i] = {
|
||||
'sdram_module': module,
|
||||
'sdram_data_width': data_width,
|
||||
'access_pattern': {
|
||||
'pattern_file': access_pattern,
|
||||
}
|
||||
}
|
||||
i += 1
|
||||
|
||||
json_str = json.dumps(configurations, indent=4)
|
||||
print(json_str)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -12,22 +12,182 @@ import subprocess
|
|||
from collections import defaultdict, namedtuple
|
||||
|
||||
import yaml
|
||||
try:
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import matplotlib
|
||||
from matplotlib.ticker import FuncFormatter, PercentFormatter, ScalarFormatter
|
||||
_summary = True
|
||||
except ImportError as e:
|
||||
_summary = False
|
||||
print('[WARNING] Results summary not available:', e, file=sys.stderr)
|
||||
|
||||
from litedram.common import Settings
|
||||
from litedram.common import Settings as _Settings
|
||||
|
||||
from .benchmark import LiteDRAMBenchmarkSoC
|
||||
from . import benchmark
|
||||
from .benchmark import LiteDRAMBenchmarkSoC, load_access_pattern
|
||||
|
||||
|
||||
# constructs python regex named group
|
||||
def ng(name, regex):
|
||||
return r'(?P<{}>{})'.format(name, regex)
|
||||
|
||||
def center(text, width, fillc=' '):
|
||||
added = width - len(text)
|
||||
left = added // 2
|
||||
right = added - left
|
||||
return fillc * left + text + fillc * right
|
||||
|
||||
|
||||
# Benchmark configuration --------------------------------------------------------------------------
|
||||
|
||||
class Settings(_Settings):
|
||||
def as_dict(self):
|
||||
d = dict()
|
||||
for attr, value in vars(self).items():
|
||||
if attr == 'self' or attr.startswith('_'):
|
||||
continue
|
||||
if isinstance(value, Settings):
|
||||
value = value.as_dict()
|
||||
d[attr] = value
|
||||
return d
|
||||
|
||||
|
||||
class GeneratedAccess(Settings):
|
||||
def __init__(self, bist_length, bist_random):
|
||||
self.set_attributes(locals())
|
||||
|
||||
@property
|
||||
def length(self):
|
||||
return self.bist_length
|
||||
|
||||
def as_args(self):
|
||||
args = ['--bist-length=%d' % self.bist_length]
|
||||
if self.bist_random:
|
||||
args.append('--bist-random')
|
||||
return args
|
||||
|
||||
|
||||
class CustomAccess(Settings):
|
||||
def __init__(self, pattern_file):
|
||||
self.set_attributes(locals())
|
||||
|
||||
@property
|
||||
def pattern(self):
|
||||
# we have to load the file to know pattern length, cache it when requested
|
||||
if not hasattr(self, '_pattern'):
|
||||
path = self.pattern_file
|
||||
if not os.path.isabs(path):
|
||||
benchmark_dir = os.path.dirname(benchmark.__file__)
|
||||
path = os.path.join(benchmark_dir, path)
|
||||
self._pattern = load_access_pattern(path)
|
||||
return self._pattern
|
||||
|
||||
@property
|
||||
def length(self):
|
||||
return len(self.pattern)
|
||||
|
||||
def as_args(self):
|
||||
return ['--access-pattern=%s' % self.pattern_file]
|
||||
|
||||
|
||||
class BenchmarkConfiguration(Settings):
|
||||
def __init__(self, name, sdram_module, sdram_data_width, access_pattern):
|
||||
self.set_attributes(locals())
|
||||
|
||||
def as_args(self):
|
||||
args = [
|
||||
'--sdram-module=%s' % self.sdram_module,
|
||||
'--sdram-data-width=%d' % self.sdram_data_width,
|
||||
]
|
||||
args += self.access_pattern.as_args()
|
||||
return args
|
||||
|
||||
def __eq__(self, other):
|
||||
if not isinstance(other, BenchmarkConfiguration):
|
||||
return NotImplemented
|
||||
return self.as_dict() == other.as_dict()
|
||||
|
||||
@property
|
||||
def length(self):
|
||||
return self.access_pattern.length
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, d):
|
||||
access_cls = CustomAccess if 'pattern_file' in d['access_pattern'] else GeneratedAccess
|
||||
d['access_pattern'] = access_cls(**d['access_pattern'])
|
||||
return cls(**d)
|
||||
|
||||
@classmethod
|
||||
def load_yaml(cls, yaml_file):
|
||||
with open(yaml_file) as f:
|
||||
description = yaml.safe_load(f)
|
||||
configs = []
|
||||
for name, desc in description.items():
|
||||
desc['name'] = name
|
||||
configs.append(cls.from_dict(desc))
|
||||
return configs
|
||||
|
||||
def __repr__(self):
|
||||
return 'BenchmarkConfiguration(%s)' % self.as_dict()
|
||||
|
||||
@property
|
||||
def soc(self):
|
||||
if not hasattr(self, '_soc'):
|
||||
kwargs = dict(
|
||||
sdram_module=self.sdram_module,
|
||||
sdram_data_width=self.sdram_data_width,
|
||||
)
|
||||
if isinstance(self.access_pattern, GeneratedAccess):
|
||||
kwargs['bist_length'] = self.access_pattern.bist_length
|
||||
kwargs['bist_random'] = self.access_pattern.bist_random
|
||||
elif isinstance(self.access_pattern, CustomAccess):
|
||||
kwargs['pattern_init'] = self.access_pattern.pattern
|
||||
else:
|
||||
raise ValueError(self.access_pattern)
|
||||
self._soc = LiteDRAMBenchmarkSoC(**kwargs)
|
||||
return self._soc
|
||||
|
||||
# Benchmark results --------------------------------------------------------------------------------
|
||||
|
||||
# constructs python regex named group
|
||||
def ng(name, regex):
|
||||
return r'(?P<{}>{})'.format(name, regex)
|
||||
|
||||
|
||||
def _compiled_pattern(stage, var):
|
||||
pattern_fmt = r'{stage}\s+{var}:\s+{value}'
|
||||
pattern = pattern_fmt.format(
|
||||
stage=stage,
|
||||
var=var,
|
||||
value=ng('value', '[0-9]+'),
|
||||
)
|
||||
return re.compile(pattern)
|
||||
result = re.search(pattern, benchmark_output)
|
||||
|
||||
|
||||
class BenchmarkResult:
|
||||
# pre-compiled patterns for all benchmarks
|
||||
patterns = {
|
||||
'generator_ticks': _compiled_pattern('BIST-GENERATOR', 'ticks'),
|
||||
'checker_errors': _compiled_pattern('BIST-CHECKER', 'errors'),
|
||||
'checker_ticks': _compiled_pattern('BIST-CHECKER', 'ticks'),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def find(pattern, output):
|
||||
result = pattern.search(output)
|
||||
assert result is not None, \
|
||||
'Could not find pattern "%s" in output:\n%s' % (pattern, benchmark_output)
|
||||
return int(result.group('value'))
|
||||
|
||||
def __init__(self, output):
|
||||
self._output = output
|
||||
for attr, pattern in self.patterns.items():
|
||||
setattr(self, attr, self.find(pattern, output))
|
||||
|
||||
def __repr__(self):
|
||||
d = {attr: getattr(self, attr) for attr in self.patterns.keys()}
|
||||
return 'BenchmarkResult(%s)' % d
|
||||
|
||||
# Results summary ----------------------------------------------------------------------------------
|
||||
|
||||
def human_readable(value):
|
||||
binary_prefixes = ['', 'k', 'M', 'G', 'T']
|
||||
mult = 1.0
|
||||
|
|
@ -37,262 +197,230 @@ def human_readable(value):
|
|||
mult /= 1024
|
||||
return mult, prefix
|
||||
|
||||
# Benchmark configuration --------------------------------------------------------------------------
|
||||
|
||||
class BenchmarkConfiguration(Settings):
|
||||
def __init__(self, sdram_module, sdram_data_width, bist_length, bist_random):
|
||||
self.set_attributes(locals())
|
||||
self._settings = {k: v for k, v in locals().items() if k != 'self'}
|
||||
def clocks_fmt(clocks):
|
||||
return '{:d} clk'.format(int(clocks))
|
||||
|
||||
def as_args(self):
|
||||
args = []
|
||||
for attr, value in self._settings.items():
|
||||
arg_string = '--%s' % attr.replace('_', '-')
|
||||
if isinstance(value, bool):
|
||||
if value:
|
||||
args.append(arg_string)
|
||||
else:
|
||||
args.extend([arg_string, str(value)])
|
||||
return args
|
||||
|
||||
def __eq__(self, other):
|
||||
if not isinstance(other, BenchmarkConfiguration):
|
||||
return NotImplemented
|
||||
return all((getattr(self, setting) == getattr(other, setting)
|
||||
for setting in self._settings.keys()))
|
||||
def bandwidth_fmt(bw):
|
||||
mult, prefix = human_readable(bw)
|
||||
return '{:.1f} {}bps'.format(bw * mult, prefix)
|
||||
|
||||
@classmethod
|
||||
def load_yaml(cls, yaml_file):
|
||||
with open(yaml_file) as f:
|
||||
description = yaml.safe_load(f)
|
||||
configurations = {name: cls(**desc) for name, desc in description.items()}
|
||||
return configurations
|
||||
|
||||
# Benchmark results --------------------------------------------------------------------------------
|
||||
def efficiency_fmt(eff):
|
||||
return '{:.1f} %'.format(eff * 100)
|
||||
|
||||
class BenchmarkResult:
|
||||
def __init__(self, config, output):
|
||||
self.config = config
|
||||
self._output = output
|
||||
self.parse_output(output)
|
||||
# instantiate the benchmarked soc to check its configuration
|
||||
self.benchmark_soc = LiteDRAMBenchmarkSoC(**self.config._settings)
|
||||
|
||||
def cmd_count(self):
|
||||
data_width = self.benchmark_soc.sdram.controller.interface.data_width
|
||||
return self.config.bist_length / (data_width // 8)
|
||||
|
||||
def clk_period(self):
|
||||
clk_freq = self.benchmark_soc.sdrphy.module.clk_freq
|
||||
return 1 / clk_freq
|
||||
|
||||
def write_bandwidth(self):
|
||||
return (8 * self.config.bist_length) / (self.generator_ticks * self.clk_period())
|
||||
|
||||
def read_bandwidth(self):
|
||||
return (8 * self.config.bist_length) / (self.checker_ticks * self.clk_period())
|
||||
|
||||
def write_efficiency(self):
|
||||
return self.cmd_count() / self.generator_ticks
|
||||
|
||||
def read_efficiency(self):
|
||||
return self.cmd_count() / self.checker_ticks
|
||||
|
||||
def write_latency(self):
|
||||
assert self.config.bist_length == 1, 'Not a latency benchmark'
|
||||
return self.generator_ticks
|
||||
|
||||
def read_latency(self):
|
||||
assert self.config.bist_length == 1, 'Not a latency benchmark'
|
||||
return self.checker_ticks
|
||||
|
||||
def parse_output(self, output):
|
||||
bist_pattern = r'{stage}\s+{var}:\s+{value}'
|
||||
|
||||
def find(stage, var):
|
||||
pattern = bist_pattern.format(
|
||||
stage=stage,
|
||||
var=var,
|
||||
value=ng('value', '[0-9]+'),
|
||||
)
|
||||
result = re.search(pattern, output)
|
||||
assert result is not None, 'Could not find pattern in output: %s, %s' % (pattern, output)
|
||||
return int(result.group('value'))
|
||||
|
||||
self.generator_ticks = find('BIST-GENERATOR', 'ticks')
|
||||
self.checker_errors = find('BIST-CHECKER', 'errors')
|
||||
self.checker_ticks = find('BIST-CHECKER', 'ticks')
|
||||
|
||||
@classmethod
|
||||
def dump_results_json(cls, results, file):
|
||||
"""Save multiple results in a JSON file.
|
||||
|
||||
Only configurations and outpits are saved, as they can be used to reconstruct BenchmarkResult.
|
||||
"""
|
||||
# simply use config._settings as it defines the BenchmarkConfiguration
|
||||
results_raw = [(r.config._settings, r._output) for r in results]
|
||||
with open(file, 'w') as f:
|
||||
json.dump(results_raw, f)
|
||||
|
||||
@classmethod
|
||||
def load_results_json(cls, file):
|
||||
"""Load results from a JSON file."""
|
||||
with open(file, 'r') as f:
|
||||
results_raw = json.load(f)
|
||||
return [cls(BenchmarkConfiguration(**settings), output) for (settings, output) in results_raw]
|
||||
|
||||
# Results summary ----------------------------------------------------------------------------------
|
||||
|
||||
class ResultsSummary:
|
||||
# value_scaling is a function: value -> (multiplier, prefix)
|
||||
Fmt = namedtuple('MetricFormatting', ['name', 'unit', 'value_scaling'])
|
||||
metric_formats = {
|
||||
'write_bandwidth': Fmt('Write bandwidth', 'bps', lambda value: human_readable(value)),
|
||||
'read_bandwidth': Fmt('Read bandwidth', 'bps', lambda value: human_readable(value)),
|
||||
'write_efficiency': Fmt('Write efficiency', '', lambda value: (100, '%')),
|
||||
'read_efficiency': Fmt('Read efficiency', '', lambda value: (100, '%')),
|
||||
'write_latency': Fmt('Write latency', 'clk', lambda value: (1, '')),
|
||||
'read_latency': Fmt('Read latency', 'clk', lambda value: (1, '')),
|
||||
}
|
||||
def __init__(self, run_data, plots_dir='plots'):
|
||||
self.plots_dir = plots_dir
|
||||
|
||||
def __init__(self, results):
|
||||
self.results = results
|
||||
# gather results into tabular data
|
||||
column_mappings = {
|
||||
'name': lambda d: d.config.name,
|
||||
'sdram_module': lambda d: d.config.sdram_module,
|
||||
'sdram_data_width': lambda d: d.config.sdram_data_width,
|
||||
'bist_length': lambda d: getattr(d.config.access_pattern, 'bist_length', None),
|
||||
'bist_random': lambda d: getattr(d.config.access_pattern, 'bist_random', None),
|
||||
'pattern_file': lambda d: getattr(d.config.access_pattern, 'pattern_file', None),
|
||||
'length': lambda d: d.config.length,
|
||||
'generator_ticks': lambda d: d.result.generator_ticks,
|
||||
'checker_errors': lambda d: d.result.checker_errors,
|
||||
'checker_ticks': lambda d: d.result.checker_ticks,
|
||||
'ctrl_data_width': lambda d: d.config.soc.sdram.controller.interface.data_width,
|
||||
'clk_freq': lambda d: d.config.soc.sdrphy.module.clk_freq,
|
||||
}
|
||||
columns = {name: [mapping(data) for data in run_data] for name, mapping, in column_mappings.items()}
|
||||
self.df = df = pd.DataFrame(columns)
|
||||
|
||||
def by_metric(self, metric):
|
||||
"""Returns pairs of value of the given metric and the configuration used for benchmark"""
|
||||
for result in self.results:
|
||||
# omit the results that should not be used to calculate given metric
|
||||
if result.config.bist_length == 1 and metric not in ['read_latency', 'write_latency'] \
|
||||
or result.config.bist_length != 1 and metric in ['read_latency', 'write_latency']:
|
||||
continue
|
||||
value = getattr(result, metric)()
|
||||
yield value, result.config
|
||||
# replace None with NaN
|
||||
df.fillna(value=np.nan, inplace=True)
|
||||
|
||||
def print(self):
|
||||
legend = '(module, datawidth, length, random, result)'
|
||||
fmt = ' {module:15} {dwidth:2} {length:4} {random:1} {result}'
|
||||
# compute other metrics based on ticks and configuration parameters
|
||||
df['clk_period'] = 1 / df['clk_freq']
|
||||
df['write_bandwidth'] = (8 * df['length']) / (df['generator_ticks'] * df['clk_period'])
|
||||
df['read_bandwidth'] = (8 * df['length']) / (df['checker_ticks'] * df['clk_period'])
|
||||
|
||||
# store formatted lines per metric
|
||||
metric_lines = defaultdict(list)
|
||||
for metric, (_, unit, formatter) in self.metric_formats.items():
|
||||
for value, config in self.by_metric(metric):
|
||||
mult, prefix = formatter(value)
|
||||
value_fmt = '{:5.1f} {}{}' if isinstance(value * mult, float) else '{:5d} {}{}'
|
||||
result = value_fmt.format(value * mult, prefix, unit)
|
||||
line = fmt.format(module=config.sdram_module,
|
||||
dwidth=config.sdram_data_width,
|
||||
length=config.bist_length,
|
||||
random=int(config.bist_random),
|
||||
result=result)
|
||||
metric_lines[metric].append(line)
|
||||
df['cmd_count'] = df['length'] / (df['ctrl_data_width'] / 8)
|
||||
df['write_efficiency'] = df['cmd_count'] / df['generator_ticks']
|
||||
df['read_efficiency'] = df['cmd_count'] / df['checker_ticks']
|
||||
|
||||
# find length of the longest line
|
||||
max_length = max((len(l) for lines in metric_lines.values() for l in lines))
|
||||
max_length = max(max_length, len(legend) + 2)
|
||||
width = max_length + 2
|
||||
df['write_latency'] = df[df['bist_length'] == 1]['generator_ticks']
|
||||
df['read_latency'] = df[df['bist_length'] == 1]['checker_ticks']
|
||||
|
||||
# print the formatted summary
|
||||
def header(text):
|
||||
mid = center(text, width - 6, '=')
|
||||
return center(mid, width, '-')
|
||||
print(header(' Summary '))
|
||||
print(center(legend, width))
|
||||
for metric, lines in metric_lines.items():
|
||||
print(center(self.metric_formats[metric].name, width))
|
||||
for line in lines:
|
||||
print(line)
|
||||
print(header(''))
|
||||
# boolean distinction between latency benchmarks and sequence benchmarks,
|
||||
# as thier results differ significanly
|
||||
df['is_latency'] = ~pd.isna(df['write_latency'])
|
||||
assert (df['is_latency'] == ~pd.isna(df['read_latency'])).all(), \
|
||||
'write_latency and read_latency should both have a value or both be NaN'
|
||||
|
||||
def plot(self, output_dir, backend='Agg', theme='default', save_format='png', **savefig_kwargs):
|
||||
"""Create plots with benchmark results summary
|
||||
# data formatting for text summary
|
||||
self.text_formatters = {
|
||||
'write_bandwidth': bandwidth_fmt,
|
||||
'read_bandwidth': bandwidth_fmt,
|
||||
'write_efficiency': efficiency_fmt,
|
||||
'read_efficiency': efficiency_fmt,
|
||||
'write_latency': clocks_fmt,
|
||||
'read_latency': clocks_fmt,
|
||||
}
|
||||
|
||||
Default backend is Agg, which is non-GUI backed and only allows
|
||||
to save figures as files. If a GUI backed is passed, plt.show()
|
||||
will be called at the end.
|
||||
"""
|
||||
# import locally here to be able to run benchmarks without installing matplotlib
|
||||
import matplotlib
|
||||
matplotlib.use(backend)
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
from matplotlib.ticker import FuncFormatter, PercentFormatter, ScalarFormatter
|
||||
|
||||
plt.style.use(theme)
|
||||
|
||||
def bandwidth_formatter_func(value, pos):
|
||||
mult, prefix = human_readable(value)
|
||||
return '{:.1f}{}bps'.format(value * mult, prefix)
|
||||
|
||||
tick_formatters = {
|
||||
'write_bandwidth': FuncFormatter(bandwidth_formatter_func),
|
||||
'read_bandwidth': FuncFormatter(bandwidth_formatter_func),
|
||||
# data formatting for plot summary
|
||||
self.plot_xticks_formatters = {
|
||||
'write_bandwidth': FuncFormatter(lambda value, pos: bandwidth_fmt(value)),
|
||||
'read_bandwidth': FuncFormatter(lambda value, pos: bandwidth_fmt(value)),
|
||||
'write_efficiency': PercentFormatter(1.0),
|
||||
'read_efficiency': PercentFormatter(1.0),
|
||||
'write_latency': ScalarFormatter(),
|
||||
'read_latency': ScalarFormatter(),
|
||||
}
|
||||
|
||||
def config_tick_name(config):
|
||||
return '{}\n{}, {}, {}'.format(config.sdram_module, config.sdram_data_width,
|
||||
config.bist_length, int(config.bist_random))
|
||||
def print_df(self, title, df):
|
||||
# make sure all data will be shown
|
||||
with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.width', None):
|
||||
print('===> {}:'.format(title))
|
||||
print(df)
|
||||
|
||||
for metric, (name, unit, _) in self.metric_formats.items():
|
||||
fig = plt.figure()
|
||||
axis = plt.gca()
|
||||
def get_summary(self, mask=None, columns=None, column_formatting=None, sort_kwargs=None):
|
||||
# work on a copy
|
||||
df = self.df.copy()
|
||||
|
||||
values, configs = zip(*self.by_metric(metric))
|
||||
ticks = np.arange(len(configs))
|
||||
if sort_kwargs is not None:
|
||||
df = df.sort_values(**sort_kwargs)
|
||||
|
||||
axis.barh(ticks, values, align='center')
|
||||
axis.set_yticks(ticks)
|
||||
axis.set_yticklabels([config_tick_name(c) for c in configs])
|
||||
axis.invert_yaxis()
|
||||
axis.xaxis.set_major_formatter(tick_formatters[metric])
|
||||
axis.xaxis.set_tick_params(rotation=30)
|
||||
axis.grid(True)
|
||||
axis.spines['top'].set_visible(False)
|
||||
axis.spines['right'].set_visible(False)
|
||||
axis.set_axisbelow(True)
|
||||
if column_formatting is not None:
|
||||
for column, mapping in column_formatting.items():
|
||||
old = '_{}'.format(column)
|
||||
df[old] = df[column].copy()
|
||||
df[column] = df[column].map(lambda value: mapping(value) if not pd.isna(value) else value)
|
||||
|
||||
# force xmax to 100%
|
||||
if metric in ['write_efficiency', 'read_efficiency']:
|
||||
axis.set_xlim(right=1.0)
|
||||
df = df[mask] if mask is not None else df
|
||||
df = df[columns] if columns is not None else df
|
||||
|
||||
title = self.metric_formats[metric].name
|
||||
axis.set_title(title, fontsize=12)
|
||||
return df
|
||||
|
||||
plt.tight_layout()
|
||||
filename = '{}.{}'.format(metric, save_format)
|
||||
fig.savefig(os.path.join(output_dir, filename), **savefig_kwargs)
|
||||
def text_summary(self):
|
||||
for title, df in self.groupped_results():
|
||||
self.print_df(title, df)
|
||||
print()
|
||||
|
||||
def groupped_results(self, formatted=True):
|
||||
df = self.df
|
||||
|
||||
formatters = self.text_formatters if formatted else {}
|
||||
|
||||
common_columns = ['name', 'sdram_module', 'sdram_data_width']
|
||||
latency_columns = ['write_latency', 'read_latency']
|
||||
performance_columns = ['write_bandwidth', 'read_bandwidth', 'write_efficiency', 'read_efficiency']
|
||||
|
||||
yield 'Latency', self.get_summary(
|
||||
mask=df['is_latency'] == True,
|
||||
columns=common_columns + latency_columns,
|
||||
column_formatting=formatters,
|
||||
)
|
||||
# yield 'Any access pattern', self.get_summary(
|
||||
# mask=(df['is_latency'] == False),
|
||||
# columns=common_columns + performance_columns + ['length', 'bist_random', 'pattern_file'],
|
||||
# column_formatting=self.text_formatters,
|
||||
# **kwargs,
|
||||
# ),
|
||||
yield 'Custom access pattern', self.get_summary(
|
||||
mask=(df['is_latency'] == False) & (~pd.isna(df['pattern_file'])),
|
||||
columns=common_columns + performance_columns + ['length', 'pattern_file'],
|
||||
column_formatting=formatters,
|
||||
),
|
||||
yield 'Sequential access pattern', self.get_summary(
|
||||
mask=(df['is_latency'] == False) & (pd.isna(df['pattern_file'])) & (df['bist_random'] == False),
|
||||
columns=common_columns + performance_columns + ['bist_length'], # could be length
|
||||
column_formatting=formatters,
|
||||
),
|
||||
yield 'Random access pattern', self.get_summary(
|
||||
mask=(df['is_latency'] == False) & (pd.isna(df['pattern_file'])) & (df['bist_random'] == True),
|
||||
columns=common_columns + performance_columns + ['bist_length'],
|
||||
column_formatting=formatters,
|
||||
),
|
||||
|
||||
def plot_summary(self, plots_dir='plots', backend='Agg', theme='default', save_format='png', **savefig_kw):
|
||||
matplotlib.use(backend)
|
||||
import matplotlib.pyplot as plt
|
||||
plt.style.use(theme)
|
||||
|
||||
for title, df in self.groupped_results(formatted=False):
|
||||
for column in self.plot_xticks_formatters.keys():
|
||||
if column not in df.columns or df[column].empty:
|
||||
continue
|
||||
axis = self.plot_df(title, df, column)
|
||||
|
||||
# construct path
|
||||
def path_name(name):
|
||||
return name.lower().replace(' ', '_')
|
||||
|
||||
filename = '{}.{}'.format(path_name(column), save_format)
|
||||
path = os.path.join(plots_dir, path_name(title), filename)
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
|
||||
# save figure
|
||||
axis.get_figure().savefig(path, **savefig_kw)
|
||||
|
||||
if backend != 'Agg':
|
||||
plt.show()
|
||||
|
||||
def plot_df(self, title, df, column, save_format='png', save_filename=None):
|
||||
if save_filename is None:
|
||||
save_filename = os.path.join(self.plots_dir, title.lower().replace(' ', '_'))
|
||||
|
||||
axis = df.plot(kind='barh', x='name', y=column, title=title, grid=True, legend=False)
|
||||
if column in self.plot_xticks_formatters:
|
||||
axis.xaxis.set_major_formatter(self.plot_xticks_formatters[column])
|
||||
axis.xaxis.set_tick_params(rotation=15)
|
||||
axis.spines['top'].set_visible(False)
|
||||
axis.spines['right'].set_visible(False)
|
||||
axis.set_axisbelow(True)
|
||||
|
||||
# # force xmax to 100%
|
||||
# if column in ['write_efficiency', 'read_efficiency']:
|
||||
# axis.set_xlim(right=1.0)
|
||||
|
||||
return axis
|
||||
|
||||
# Run ----------------------------------------------------------------------------------------------
|
||||
|
||||
def run_benchmark(cmd_args):
|
||||
# run as separate process, because else we cannot capture all output from verilator
|
||||
benchmark_script = os.path.join(os.path.dirname(__file__), 'benchmark.py')
|
||||
command = ['python3', benchmark_script, *cmd_args]
|
||||
proc = subprocess.run(command, stdout=subprocess.PIPE)
|
||||
class RunCache(list):
|
||||
RunData = namedtuple('RunData', ['config', 'result'])
|
||||
|
||||
def dump_json(self, filename):
|
||||
json_data = [{'config': data.config.as_dict(), 'output': data.result._output} for data in self]
|
||||
with open(filename, 'w') as f:
|
||||
json.dump(json_data, f)
|
||||
|
||||
@classmethod
|
||||
def load_json(cls, filename):
|
||||
with open(filename, 'r') as f:
|
||||
json_data = json.load(f)
|
||||
loaded = []
|
||||
for data in json_data:
|
||||
config = BenchmarkConfiguration.from_dict(data['config'])
|
||||
result = BenchmarkResult(data['output'])
|
||||
loaded.append(cls.RunData(config=config, result=result))
|
||||
return loaded
|
||||
|
||||
|
||||
def run_python(script, args):
|
||||
command = ['python3', script, *args]
|
||||
proc = subprocess.run(command, stdout=subprocess.PIPE, cwd=os.path.dirname(script))
|
||||
return str(proc.stdout)
|
||||
|
||||
|
||||
def run_benchmarks(configurations):
|
||||
results = []
|
||||
for name, config in configurations.items():
|
||||
cmd_args = config.as_args()
|
||||
print('{}: {}'.format(name, ' '.join(cmd_args)))
|
||||
output = run_benchmark(cmd_args)
|
||||
# exit if checker had any read error
|
||||
result = BenchmarkResult(config, output)
|
||||
if result.checker_errors != 0:
|
||||
print('Error during benchmark "{}": checker_errors = {}'.format(
|
||||
name, result.checker_errors), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
results.append(result)
|
||||
return results
|
||||
def run_benchmark(config):
|
||||
benchmark_script = os.path.join(os.path.dirname(__file__), 'benchmark.py')
|
||||
# run as separate process, because else we cannot capture all output from verilator
|
||||
output = run_python(benchmark_script, config.as_args())
|
||||
result = BenchmarkResult(output)
|
||||
# exit if checker had any read error
|
||||
if result.checker_errors != 0:
|
||||
raise RuntimeError('Error during benchmark: checker_errors = {}, args = {}'.format(
|
||||
result.checker_errors, args
|
||||
))
|
||||
return result
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
|
|
@ -308,6 +436,7 @@ def main(argv=None):
|
|||
parser.add_argument('--plot-transparent', action='store_true', help='Use transparent background when saving plots')
|
||||
parser.add_argument('--plot-output-dir', default='plots', help='Specify where to save the plots')
|
||||
parser.add_argument('--plot-theme', default='default', help='Use different matplotlib theme')
|
||||
parser.add_argument('--ignore-failures', action='store_true', help='Ignore failuers during benchmarking, continue using successful runs only')
|
||||
parser.add_argument('--results-cache', help="""Use given JSON file as results cache. If the file exists,
|
||||
it will be loaded instead of running actual benchmarks,
|
||||
else benchmarks will be run normally, and then saved
|
||||
|
|
@ -315,43 +444,58 @@ def main(argv=None):
|
|||
to generate different summary without having to rerun benchmarks.""")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
if not args.results_cache and not _summary:
|
||||
print('Summary not available and not running with --results-cache - run would not produce any results! Aborting.',
|
||||
file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# load and filter configurations
|
||||
configurations = BenchmarkConfiguration.load_yaml(args.config)
|
||||
filters = []
|
||||
if args.regex:
|
||||
filters.append(lambda name_value: re.search(args.regex, name_value[0]))
|
||||
if args.not_regex:
|
||||
filters.append(lambda name_value: not re.search(args.not_regex, name_value[0]))
|
||||
if args.names:
|
||||
filters.append(lambda name_value: name_value[0] in args.names)
|
||||
for f in filters:
|
||||
configurations = dict(filter(f, configurations.items()))
|
||||
|
||||
cache_exists = args.results_cache and os.path.isfile(args.results_cache)
|
||||
filters = {
|
||||
'regex': lambda config: re.search(args.regex, config.name),
|
||||
'not_regex': lambda config: not re.search(args.not_regex, config.name),
|
||||
'names': lambda config: config.name in args.names,
|
||||
}
|
||||
for arg, f in filters.items():
|
||||
if getattr(args, arg):
|
||||
configurations = filter(f, configurations)
|
||||
configurations = list(configurations)
|
||||
|
||||
# load outputs from cache if it exsits
|
||||
cache_exists = args.results_cache and os.path.isfile(args.results_cache)
|
||||
if args.results_cache and cache_exists:
|
||||
cached_results = BenchmarkResult.load_results_json(args.results_cache)
|
||||
cache = RunCache.load_json(args.results_cache)
|
||||
|
||||
# take only those that match configurations
|
||||
results = [r for r in cached_results if r.config in configurations.values()]
|
||||
names_to_load = [c.name for c in configurations]
|
||||
run_data = [data for data in cache if data.config.name in names_to_load]
|
||||
else: # run all the benchmarks normally
|
||||
results = run_benchmarks(configurations)
|
||||
run_data = []
|
||||
for config in configurations:
|
||||
print(' {}: {}'.format(config.name, ' '.join(config.as_args())))
|
||||
try:
|
||||
run_data.append(RunCache.RunData(config, run_benchmark(config)))
|
||||
except:
|
||||
if not args.ignore_failures:
|
||||
raise
|
||||
|
||||
# store outputs in cache
|
||||
if args.results_cache and not cache_exists:
|
||||
BenchmarkResult.dump_results_json(results, args.results_cache)
|
||||
cache = RunCache(run_data)
|
||||
cache.dump_json(args.results_cache)
|
||||
|
||||
# display the summary
|
||||
summary = ResultsSummary(results)
|
||||
summary.print()
|
||||
if args.plot:
|
||||
if not os.path.isdir(args.plot_output_dir):
|
||||
os.makedirs(args.plot_output_dir)
|
||||
summary.plot(args.plot_output_dir,
|
||||
backend=args.plot_backend,
|
||||
theme=args.plot_theme,
|
||||
save_format=args.plot_format,
|
||||
transparent=args.plot_transparent)
|
||||
# display summary
|
||||
if _summary:
|
||||
summary = ResultsSummary(run_data)
|
||||
summary.text_summary()
|
||||
if args.plot:
|
||||
summary.plot_summary(
|
||||
plots_dir=args.plot_output_dir,
|
||||
backend=args.plot_backend,
|
||||
theme=args.plot_theme,
|
||||
save_format=args.plot_format,
|
||||
transparent=args.plot_transparent,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
Loading…
Reference in New Issue