From fcbcd4d3fea76431f65751ea52045a46faaebe40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C4=99drzej=20Boczar?= Date: Tue, 4 Feb 2020 12:34:15 +0100 Subject: [PATCH 1/7] test: add option to benchmark predefined access patterns --- litedram/frontend/bist.py | 136 ++++++++++++++++++++++++++++++++++++++ test/benchmark.py | 74 ++++++++++++++------- 2 files changed, 186 insertions(+), 24 deletions(-) diff --git a/litedram/frontend/bist.py b/litedram/frontend/bist.py index 16dd7d3..1506a97 100644 --- a/litedram/frontend/bist.py +++ b/litedram/frontend/bist.py @@ -185,6 +185,60 @@ class _LiteDRAMBISTGenerator(Module): raise NotImplementedError self.comb += dma.sink.data.eq(data_gen.o) +@ResetInserter() +class _LiteDRAMPatternGenerator(Module): + def __init__(self, dram_port, init=[]): + ashift, awidth = get_ashift_awidth(dram_port) + self.start = Signal() + self.done = Signal() + self.ticks = Signal(32) + + # # # + + # DMA -------------------------------------------------------------------------------------- + dma = LiteDRAMDMAWriter(dram_port) + self.submodules += dma + + cmd_counter = Signal(dram_port.address_width, reset_less=True) + + # Data / Address FSM ----------------------------------------------------------------------- + fsm = FSM(reset_state="IDLE") + self.submodules += fsm + fsm.act("IDLE", + If(self.start, + NextValue(cmd_counter, 0), + NextState("RUN") + ), + NextValue(self.ticks, 0) + ) + fsm.act("RUN", + dma.sink.valid.eq(1), + If(dma.sink.ready, + NextValue(cmd_counter, cmd_counter + 1), + If(cmd_counter == (len(init) - 1), + NextState("DONE") + ) + ), + NextValue(self.ticks, self.ticks + 1) + ) + fsm.act("DONE", + self.done.eq(1) + ) + + if isinstance(dram_port, LiteDRAMNativePort): # addressing in dwords + dma_sink_addr = dma.sink.address + elif isinstance(dram_port, LiteDRAMAXIPort): # addressing in bytes + dma_sink_addr = dma.sink.address[ashift:] + else: + raise NotImplementedError + + addr_cases = {i: dma_sink_addr.eq(addr) for i, (addr, data) in enumerate(init)} + data_cases = {i: dma.sink.data.eq(data) for i, (addr, data) in enumerate(init)} + self.comb += [ + Case(cmd_counter, addr_cases), + Case(cmd_counter, data_cases), + ] + # LiteDRAMBISTGenerator ---------------------------------------------------------------------------- class LiteDRAMBISTGenerator(Module, AutoCSR): @@ -367,6 +421,88 @@ class _LiteDRAMBISTChecker(Module, AutoCSR): self.done.eq(1) ) +@ResetInserter() +class _LiteDRAMPatternChecker(Module, AutoCSR): + def __init__(self, dram_port, init=[]): + ashift, awidth = get_ashift_awidth(dram_port) + self.start = Signal() + self.done = Signal() + self.ticks = Signal(32) + self.errors = Signal(32) + + # # # + + # DMA -------------------------------------------------------------------------------------- + dma = LiteDRAMDMAReader(dram_port) + self.submodules += dma + + # Address FSM ------------------------------------------------------------------------------ + cmd_counter = Signal(dram_port.address_width, reset_less=True) + + cmd_fsm = FSM(reset_state="IDLE") + self.submodules += cmd_fsm + cmd_fsm.act("IDLE", + If(self.start, + NextValue(cmd_counter, 0), + NextState("RUN") + ) + ) + cmd_fsm.act("RUN", + dma.sink.valid.eq(1), + If(dma.sink.ready, + NextValue(cmd_counter, cmd_counter + 1), + If(cmd_counter == (len(init) - 1), + NextState("DONE") + ) + ) + ) + cmd_fsm.act("DONE") + + if isinstance(dram_port, LiteDRAMNativePort): # addressing in dwords + dma_addr_sink = dma.sink.address + elif isinstance(dram_port, LiteDRAMAXIPort): # addressing in bytes + dma_addr_sink = dma.sink.address[ashift:] + else: + raise NotImplementedError + + addr_cases = {i: dma_addr_sink.eq(addr) for i, (addr, data) in enumerate(init)} + self.comb += Case(cmd_counter, addr_cases) + + # Data FSM --------------------------------------------------------------------------------- + data_counter = Signal(dram_port.address_width, reset_less=True) + + expected_data = Signal.like(dma.source.data) + data_cases = {i: expected_data.eq(data) for i, (addr, data) in enumerate(init)} + self.comb += Case(data_counter, data_cases) + + data_fsm = FSM(reset_state="IDLE") + self.submodules += data_fsm + data_fsm.act("IDLE", + If(self.start, + NextValue(data_counter, 0), + NextValue(self.errors, 0), + NextState("RUN") + ), + NextValue(self.ticks, 0) + ) + + data_fsm.act("RUN", + dma.source.ready.eq(1), + If(dma.source.valid, + NextValue(data_counter, data_counter + 1), + If(dma.source.data != expected_data, + NextValue(self.errors, self.errors + 1) + ), + If(data_counter == (len(init) - 1), + NextState("DONE") + ) + ), + NextValue(self.ticks, self.ticks + 1) + ) + data_fsm.act("DONE", + self.done.eq(1) + ) + # LiteDRAMBISTChecker ------------------------------------------------------------------------------ class LiteDRAMBISTChecker(Module, AutoCSR): diff --git a/test/benchmark.py b/test/benchmark.py index 61c55fa..9681c8a 100755 --- a/test/benchmark.py +++ b/test/benchmark.py @@ -3,6 +3,7 @@ # This file is Copyright (c) 2020 Florent Kermarrec # License: BSD +import csv import argparse from migen import * @@ -16,9 +17,8 @@ from litex.soc.integration.builder import * from litex.tools.litex_sim import SimSoC -from litedram.frontend.bist import _LiteDRAMBISTGenerator -from litedram.frontend.bist import _LiteDRAMBISTChecker - +from litedram.frontend.bist import _LiteDRAMBISTGenerator, _LiteDRAMBISTChecker, \ + _LiteDRAMPatternGenerator, _LiteDRAMPatternChecker # LiteDRAM Benchmark SoC --------------------------------------------------------------------------- @@ -29,6 +29,7 @@ class LiteDRAMBenchmarkSoC(SimSoC): bist_base = 0x00000000, bist_length = 1024, bist_random = False, + pattern_init = None, **kwargs): # SimSoC ----------------------------------------------------------------------------------- @@ -42,12 +43,34 @@ class LiteDRAMBenchmarkSoC(SimSoC): # make sure that we perform at least one access bist_length = max(bist_length, self.sdram.controller.interface.data_width // 8) - # BIST Generator --------------------------------------------------------------------------- - bist_generator = _LiteDRAMBISTGenerator(self.sdram.crossbar.get_port()) - self.submodules.bist_generator = bist_generator + # BIST Generator / Checker ----------------------------------------------------------------- + if pattern_init is None: + bist_generator = _LiteDRAMBISTGenerator(self.sdram.crossbar.get_port()) + bist_checker = _LiteDRAMBISTChecker(self.sdram.crossbar.get_port()) - # BIST Checker ----------------------------------------------------------------------------- - bist_checker = _LiteDRAMBISTChecker(self.sdram.crossbar.get_port()) + generator_config = [ + bist_generator.base.eq(bist_base), + bist_generator.length.eq(bist_length), + bist_generator.random.eq(bist_random), + ] + checker_config = [ + bist_checker.base.eq(bist_base), + bist_checker.length.eq(bist_length), + bist_checker.random.eq(bist_random), + ] + else: + # TODO: run checker in parallel to avoid overwriting previously written data + address_set = set() + for addr, _ in pattern_init: + assert addr not in address_set, \ + 'Duplicate address 0x%08x in pattern_init, write will overwrite previous value!' % addr + address_set.add(addr) + + bist_generator = _LiteDRAMPatternGenerator(self.sdram.crossbar.get_port(), init=pattern_init) + bist_checker = _LiteDRAMPatternChecker(self.sdram.crossbar.get_port(), init=pattern_init) + generator_config = checker_config = [] + + self.submodules.bist_generator = bist_generator self.submodules.bist_checker = bist_checker # Sequencer -------------------------------------------------------------------------------- @@ -68,18 +91,14 @@ class LiteDRAMBenchmarkSoC(SimSoC): ) fsm.act("BIST-GENERATOR", bist_generator.start.eq(1), - bist_generator.base.eq(bist_base), - bist_generator.length.eq(bist_length), - bist_generator.random.eq(bist_random), + *generator_config, If(bist_generator.done, NextState("BIST-CHECKER") ) ) fsm.act("BIST-CHECKER", bist_checker.start.eq(1), - bist_checker.base.eq(bist_base), - bist_checker.length.eq(bist_length), - bist_checker.random.eq(bist_random), + *checker_config, If(bist_checker.done, NextState("DISPLAY") ) @@ -113,16 +132,17 @@ def main(): parser = argparse.ArgumentParser(description="LiteDRAM Benchmark SoC Simulation") builder_args(parser) soc_sdram_args(parser) - parser.add_argument("--threads", default=1, help="Set number of threads (default=1)") - parser.add_argument("--sdram-module", default="MT48LC16M16", help="Select SDRAM chip") - parser.add_argument("--sdram-data-width", default=32, help="Set SDRAM chip data width") - parser.add_argument("--trace", action="store_true", help="Enable VCD tracing") - parser.add_argument("--trace-start", default=0, help="Cycle to start VCD tracing") - parser.add_argument("--trace-end", default=-1, help="Cycle to end VCD tracing") - parser.add_argument("--opt-level", default="O0", help="Compilation optimization level") - parser.add_argument("--bist-base", default="0x00000000", help="Base address of the test (default=0)") - parser.add_argument("--bist-length", default="1024", help="Length of the test (default=1024)") - parser.add_argument("--bist-random", action="store_true", help="Use random data during the test") + parser.add_argument("--threads", default=1, help="Set number of threads (default=1)") + parser.add_argument("--sdram-module", default="MT48LC16M16", help="Select SDRAM chip") + parser.add_argument("--sdram-data-width", default=32, help="Set SDRAM chip data width") + parser.add_argument("--trace", action="store_true", help="Enable VCD tracing") + parser.add_argument("--trace-start", default=0, help="Cycle to start VCD tracing") + parser.add_argument("--trace-end", default=-1, help="Cycle to end VCD tracing") + parser.add_argument("--opt-level", default="O0", help="Compilation optimization level") + parser.add_argument("--bist-base", default="0x00000000", help="Base address of the test (default=0)") + parser.add_argument("--bist-length", default="1024", help="Length of the test (default=1024)") + parser.add_argument("--bist-random", action="store_true", help="Use random data during the test") + parser.add_argument("--access-pattern", help="Load access pattern (address, data) from CSV (ignores --bist-*)") args = parser.parse_args() soc_kwargs = soc_sdram_argdict(args) @@ -138,6 +158,12 @@ def main(): soc_kwargs["bist_length"] = int(args.bist_length, 0) soc_kwargs["bist_random"] = args.bist_random + if args.access_pattern: + with open(args.access_pattern, newline='') as f: + reader = csv.reader(f) + pattern_init = [(int(addr, 0), int(data, 0)) for addr, data in reader] + soc_kwargs["pattern_init"] = pattern_init + # SoC ------------------------------------------------------------------------------------------ soc = LiteDRAMBenchmarkSoC(**soc_kwargs) From fcd3d4ff6c58b2c9dd8b0d4d2cc038286ddfc081 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C4=99drzej=20Boczar?= Date: Tue, 4 Feb 2020 13:45:08 +0100 Subject: [PATCH 2/7] test: helper scripts for generating benchmark configurations/access patterns --- test/gen_access_pattern.py | 30 +++++++++++++++++ test/gen_config.py | 66 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100755 test/gen_access_pattern.py create mode 100755 test/gen_config.py diff --git a/test/gen_access_pattern.py b/test/gen_access_pattern.py new file mode 100755 index 0000000..f6cd200 --- /dev/null +++ b/test/gen_access_pattern.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python + +import random +import argparse + +desc = """ +Generate random access pattern. +Each address in range [base, base+length) will be accessed only once, +but in random order. This ensures that no data will be overwritten. +""" +parser = argparse.ArgumentParser(description=desc) +parser.add_argument('base', help='Base address') +parser.add_argument('length', help='Number of (address, data) pairs') +parser.add_argument('data_width', help='Width of data (used to determine max value)') +parser.add_argument('--seed', help='Use given random seed (int)') +args = parser.parse_args() + +if args.seed: + random.seed(int(args.seed, 0)) + +base = int(args.base, 0) +length = int(args.length, 0) +data_width = int(args.data_width, 0) + +address = list(range(length)) +random.shuffle(address) +data = [random.randrange(0, 2**data_width) for _ in range(length)] + +for a, d in zip(address, data): + print('0x{:08x},0x{:08x}'.format(a, d)) diff --git a/test/gen_config.py b/test/gen_config.py new file mode 100755 index 0000000..a0b9188 --- /dev/null +++ b/test/gen_config.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python + +import json +import argparse +import itertools + +modules = [ + 'IS42S16160', + 'IS42S16320', + 'MT48LC4M16', + 'MT48LC16M16', + 'AS4C16M16', + 'AS4C32M16', + 'AS4C32M8', + 'M12L64322A', + 'M12L16161A', + 'MT46V32M16', + 'MT46H32M16', + 'MT46H32M32', + 'MT47H128M8', + 'MT47H32M16', + 'MT47H64M16', + 'P3R1GE4JGF', + 'MT41K64M16', + 'MT41J128M16', + 'MT41K128M16', + 'MT41J256M16', + 'MT41K256M16', + 'K4B1G0446F', + 'K4B2G1646F', + 'H5TC4G63CFR', + 'IS43TR16128B', + 'MT8JTF12864', + 'MT8KTF51264', + # 'MT18KSF1G72HZ', + # 'AS4C256M16D3A', + # 'MT16KTF1G64HZ', + # 'EDY4016A', + # 'MT40A1G8', + # 'MT40A512M16', +] +data_widths = [32] +bist_lengths = [1, 1024, 8192] +bist_randoms = [False] + +parser = argparse.ArgumentParser(description='Generate configuration for all possible argument combinations.', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument('--sdram-modules', nargs='+', default=modules, help='--sdram-module options') +parser.add_argument('--sdram-data-widths', nargs='+', default=data_widths, help='--sdram-data-width options') +parser.add_argument('--bist-lengths', nargs='+', default=bist_lengths, help='--bist-length options') +parser.add_argument('--bist-randoms', nargs='+', default=bist_randoms, help='--bist-random options') +parser.add_argument('--name-format', default='test_%d', help='Name format for i-th test') +args = parser.parse_args() + +product = itertools.product(args.sdram_modules, args.sdram_data_widths, args.bist_lengths, args.bist_randoms) +configurations = {} +for i, (module, data_width, bist_length, bist_random) in enumerate(product): + configurations[args.name_format % i] = { + 'sdram_module': module, + 'sdram_data_width': data_width, + 'bist_length': bist_length, + 'bist_random': bist_random, + } + +json_str = json.dumps(configurations, indent=4) +print(json_str) From 7e0515c477dcbc98673006747433a8da1bfeecb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C4=99drzej=20Boczar?= Date: Tue, 4 Feb 2020 16:37:26 +0100 Subject: [PATCH 3/7] test: fix problem with helper scripts being executed by `setup.py test` --- test/gen_access_pattern.py | 46 +++++++++++++++++++++----------------- test/gen_config.py | 42 ++++++++++++++++++---------------- 2 files changed, 48 insertions(+), 40 deletions(-) diff --git a/test/gen_access_pattern.py b/test/gen_access_pattern.py index f6cd200..186dc1e 100755 --- a/test/gen_access_pattern.py +++ b/test/gen_access_pattern.py @@ -3,28 +3,32 @@ import random import argparse -desc = """ -Generate random access pattern. -Each address in range [base, base+length) will be accessed only once, -but in random order. This ensures that no data will be overwritten. -""" -parser = argparse.ArgumentParser(description=desc) -parser.add_argument('base', help='Base address') -parser.add_argument('length', help='Number of (address, data) pairs') -parser.add_argument('data_width', help='Width of data (used to determine max value)') -parser.add_argument('--seed', help='Use given random seed (int)') -args = parser.parse_args() +def main(): + desc = """ + Generate random access pattern. + Each address in range [base, base+length) will be accessed only once, + but in random order. This ensures that no data will be overwritten. + """ + parser = argparse.ArgumentParser(description=desc) + parser.add_argument('base', help='Base address') + parser.add_argument('length', help='Number of (address, data) pairs') + parser.add_argument('data_width', help='Width of data (used to determine max value)') + parser.add_argument('--seed', help='Use given random seed (int)') + args = parser.parse_args() -if args.seed: - random.seed(int(args.seed, 0)) + if args.seed: + random.seed(int(args.seed, 0)) -base = int(args.base, 0) -length = int(args.length, 0) -data_width = int(args.data_width, 0) + base = int(args.base, 0) + length = int(args.length, 0) + data_width = int(args.data_width, 0) -address = list(range(length)) -random.shuffle(address) -data = [random.randrange(0, 2**data_width) for _ in range(length)] + address = list(range(length)) + random.shuffle(address) + data = [random.randrange(0, 2**data_width) for _ in range(length)] -for a, d in zip(address, data): - print('0x{:08x},0x{:08x}'.format(a, d)) + for a, d in zip(address, data): + print('0x{:08x},0x{:08x}'.format(a, d)) + +if __name__ == "__main__": + main() diff --git a/test/gen_config.py b/test/gen_config.py index a0b9188..298e774 100755 --- a/test/gen_config.py +++ b/test/gen_config.py @@ -43,24 +43,28 @@ data_widths = [32] bist_lengths = [1, 1024, 8192] bist_randoms = [False] -parser = argparse.ArgumentParser(description='Generate configuration for all possible argument combinations.', - formatter_class=argparse.ArgumentDefaultsHelpFormatter) -parser.add_argument('--sdram-modules', nargs='+', default=modules, help='--sdram-module options') -parser.add_argument('--sdram-data-widths', nargs='+', default=data_widths, help='--sdram-data-width options') -parser.add_argument('--bist-lengths', nargs='+', default=bist_lengths, help='--bist-length options') -parser.add_argument('--bist-randoms', nargs='+', default=bist_randoms, help='--bist-random options') -parser.add_argument('--name-format', default='test_%d', help='Name format for i-th test') -args = parser.parse_args() +def main(): + parser = argparse.ArgumentParser(description='Generate configuration for all possible argument combinations.', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('--sdram-modules', nargs='+', default=modules, help='--sdram-module options') + parser.add_argument('--sdram-data-widths', nargs='+', default=data_widths, help='--sdram-data-width options') + parser.add_argument('--bist-lengths', nargs='+', default=bist_lengths, help='--bist-length options') + parser.add_argument('--bist-randoms', nargs='+', default=bist_randoms, help='--bist-random options') + parser.add_argument('--name-format', default='test_%d', help='Name format for i-th test') + args = parser.parse_args() -product = itertools.product(args.sdram_modules, args.sdram_data_widths, args.bist_lengths, args.bist_randoms) -configurations = {} -for i, (module, data_width, bist_length, bist_random) in enumerate(product): - configurations[args.name_format % i] = { - 'sdram_module': module, - 'sdram_data_width': data_width, - 'bist_length': bist_length, - 'bist_random': bist_random, - } + product = itertools.product(args.sdram_modules, args.sdram_data_widths, args.bist_lengths, args.bist_randoms) + configurations = {} + for i, (module, data_width, bist_length, bist_random) in enumerate(product): + configurations[args.name_format % i] = { + 'sdram_module': module, + 'sdram_data_width': data_width, + 'bist_length': bist_length, + 'bist_random': bist_random, + } -json_str = json.dumps(configurations, indent=4) -print(json_str) + json_str = json.dumps(configurations, indent=4) + print(json_str) + +if __name__ == "__main__": + main() From f9f86d507f945f15be35651e5a362a20c05976ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C4=99drzej=20Boczar?= Date: Wed, 5 Feb 2020 12:38:05 +0100 Subject: [PATCH 4/7] test: update benchmark configuration to account for access pattern --- test/benchmark.py | 12 +++-- test/run_benchmarks.py | 107 +++++++++++++++++++++++++++++++---------- 2 files changed, 89 insertions(+), 30 deletions(-) diff --git a/test/benchmark.py b/test/benchmark.py index 9681c8a..11f3020 100755 --- a/test/benchmark.py +++ b/test/benchmark.py @@ -128,6 +128,13 @@ class LiteDRAMBenchmarkSoC(SimSoC): # Build -------------------------------------------------------------------------------------------- +def load_access_pattern(filename): + with open(filename, newline='') as f: + reader = csv.reader(f) + pattern_init = [(int(addr, 0), int(data, 0)) for addr, data in reader] + return pattern_init + + def main(): parser = argparse.ArgumentParser(description="LiteDRAM Benchmark SoC Simulation") builder_args(parser) @@ -159,10 +166,7 @@ def main(): soc_kwargs["bist_random"] = args.bist_random if args.access_pattern: - with open(args.access_pattern, newline='') as f: - reader = csv.reader(f) - pattern_init = [(int(addr, 0), int(data, 0)) for addr, data in reader] - soc_kwargs["pattern_init"] = pattern_init + soc_kwargs["pattern_init"] = load_access_pattern(args.access_pattern) # SoC ------------------------------------------------------------------------------------------ soc = LiteDRAMBenchmarkSoC(**soc_kwargs) diff --git a/test/run_benchmarks.py b/test/run_benchmarks.py index ed884ce..598e225 100755 --- a/test/run_benchmarks.py +++ b/test/run_benchmarks.py @@ -13,9 +13,10 @@ from collections import defaultdict, namedtuple import yaml -from litedram.common import Settings +from litedram.common import Settings as _Settings -from .benchmark import LiteDRAMBenchmarkSoC +from . import benchmark +from .benchmark import LiteDRAMBenchmarkSoC, load_access_pattern # constructs python regex named group @@ -39,34 +40,88 @@ def human_readable(value): # Benchmark configuration -------------------------------------------------------------------------- -class BenchmarkConfiguration(Settings): - def __init__(self, sdram_module, sdram_data_width, bist_length, bist_random): +class Settings(_Settings): + def as_dict(self): + d = dict() + for attr, value in vars(self).items(): + if attr == 'self' or attr.startswith('_'): + continue + if isinstance(value, Settings): + value = value.as_dict() + d[attr] = value + return d + + +class GeneratedAccess(Settings): + def __init__(self, bist_length, bist_random): self.set_attributes(locals()) - self._settings = {k: v for k, v in locals().items() if k != 'self'} + + @property + def length(self): + return self.bist_length def as_args(self): - args = [] - for attr, value in self._settings.items(): - arg_string = '--%s' % attr.replace('_', '-') - if isinstance(value, bool): - if value: - args.append(arg_string) - else: - args.extend([arg_string, str(value)]) + args = ['--bist-length=%d' % self.bist_length] + if self.bist_random: + args.append('--bist-random') + return args + + +class CustomAccess(Settings): + def __init__(self, pattern_file): + self.set_attributes(locals()) + + @property + def length(self): + # we have to load the file to know pattern length, cache it when requested + if not hasattr(self, '_pattern'): + path = self.pattern_file + if not os.path.isabs(path): + benchmark_dir = os.path.dirname(benchmark.__file__) + path = os.path.join(benchmark_dir, path) + self._pattern = load_access_pattern(path) + return len(self._pattern) + + def as_args(self): + return ['--access-pattern=%s' % self.pattern_file] + + +class BenchmarkConfiguration(Settings): + def __init__(self, name, sdram_module, sdram_data_width, access_pattern): + self.set_attributes(locals()) + + def as_args(self): + args = [ + '--sdram-module=%s' % self.sdram_module, + '--sdram-data-width=%d' % self.sdram_data_width, + ] + args += self.access_pattern.as_args() return args def __eq__(self, other): if not isinstance(other, BenchmarkConfiguration): return NotImplemented - return all((getattr(self, setting) == getattr(other, setting) - for setting in self._settings.keys())) + return self.as_dict() == other.as_dict() + + @property + def length(self): + return self.access_pattern.length @classmethod def load_yaml(cls, yaml_file): with open(yaml_file) as f: description = yaml.safe_load(f) - configurations = {name: cls(**desc) for name, desc in description.items()} - return configurations + configs = [] + for name, desc in description.items(): + if 'access_pattern' in desc: + access = CustomAccess(desc.pop('access_pattern')) + else: + access = GeneratedAccess(desc.pop('bist_length'), desc.pop('bist_random')) + configs.append(cls(name, **desc, access_pattern=access)) + return configs + + def __repr__(self): + return 'BenchmarkConfiguration(%s)' % self.as_dict() # Benchmark results -------------------------------------------------------------------------------- @@ -317,15 +372,15 @@ def main(argv=None): # load and filter configurations configurations = BenchmarkConfiguration.load_yaml(args.config) - filters = [] - if args.regex: - filters.append(lambda name_value: re.search(args.regex, name_value[0])) - if args.not_regex: - filters.append(lambda name_value: not re.search(args.not_regex, name_value[0])) - if args.names: - filters.append(lambda name_value: name_value[0] in args.names) - for f in filters: - configurations = dict(filter(f, configurations.items())) + filters = { + 'regex': lambda config: re.search(args.regex, config.name), + 'not_regex': lambda config: not re.search(args.not_regex, config.name), + 'names': lambda config: config.name in args.names, + } + for arg, f in filters.items(): + if getattr(args, arg): + configurations = filter(f, configurations) + configurations = list(configurations) cache_exists = args.results_cache and os.path.isfile(args.results_cache) From 1702e2ad7c40fbc1e7b3fa452037fabba3deff4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C4=99drzej=20Boczar?= Date: Wed, 5 Feb 2020 13:41:47 +0100 Subject: [PATCH 5/7] test: update summary to work for all configurations (use pandas) --- test/run_benchmarks.py | 535 ++++++++++++++++++++++++----------------- 1 file changed, 312 insertions(+), 223 deletions(-) diff --git a/test/run_benchmarks.py b/test/run_benchmarks.py index 598e225..7ee3a53 100755 --- a/test/run_benchmarks.py +++ b/test/run_benchmarks.py @@ -12,6 +12,15 @@ import subprocess from collections import defaultdict, namedtuple import yaml +try: + import numpy as np + import pandas as pd + import matplotlib + from matplotlib.ticker import FuncFormatter, PercentFormatter, ScalarFormatter + _summary = True +except ImportError as e: + _summary = False + print('[WARNING] Results summary not available:', e, file=sys.stderr) from litedram.common import Settings as _Settings @@ -19,24 +28,12 @@ from . import benchmark from .benchmark import LiteDRAMBenchmarkSoC, load_access_pattern -# constructs python regex named group -def ng(name, regex): - return r'(?P<{}>{})'.format(name, regex) - def center(text, width, fillc=' '): added = width - len(text) left = added // 2 right = added - left return fillc * left + text + fillc * right -def human_readable(value): - binary_prefixes = ['', 'k', 'M', 'G', 'T'] - mult = 1.0 - for prefix in binary_prefixes: - if value * mult < 1024: - break - mult /= 1024 - return mult, prefix # Benchmark configuration -------------------------------------------------------------------------- @@ -72,7 +69,7 @@ class CustomAccess(Settings): self.set_attributes(locals()) @property - def length(self): + def pattern(self): # we have to load the file to know pattern length, cache it when requested if not hasattr(self, '_pattern'): path = self.pattern_file @@ -80,7 +77,11 @@ class CustomAccess(Settings): benchmark_dir = os.path.dirname(benchmark.__file__) path = os.path.join(benchmark_dir, path) self._pattern = load_access_pattern(path) - return len(self._pattern) + return self._pattern + + @property + def length(self): + return len(self.pattern) def as_args(self): return ['--access-pattern=%s' % self.pattern_file] @@ -107,247 +108,319 @@ class BenchmarkConfiguration(Settings): def length(self): return self.access_pattern.length + @classmethod + def from_dict(cls, d): + access_cls = CustomAccess if 'pattern_file' in d['access_pattern'] else GeneratedAccess + d['access_pattern'] = access_cls(**d['access_pattern']) + return cls(**d) + @classmethod def load_yaml(cls, yaml_file): with open(yaml_file) as f: description = yaml.safe_load(f) configs = [] for name, desc in description.items(): - if 'access_pattern' in desc: - access = CustomAccess(desc.pop('access_pattern')) - else: - access = GeneratedAccess(desc.pop('bist_length'), desc.pop('bist_random')) - configs.append(cls(name, **desc, access_pattern=access)) + desc['name'] = name + configs.append(cls.from_dict(desc)) return configs def __repr__(self): return 'BenchmarkConfiguration(%s)' % self.as_dict() + @property + def soc(self): + if not hasattr(self, '_soc'): + kwargs = dict( + sdram_module=self.sdram_module, + sdram_data_width=self.sdram_data_width, + ) + if isinstance(self.access_pattern, GeneratedAccess): + kwargs['bist_length'] = self.access_pattern.bist_length + kwargs['bist_random'] = self.access_pattern.bist_random + elif isinstance(self.access_pattern, CustomAccess): + kwargs['pattern_init'] = self.access_pattern.pattern + else: + raise ValueError(self.access_pattern) + self._soc = LiteDRAMBenchmarkSoC(**kwargs) + return self._soc + # Benchmark results -------------------------------------------------------------------------------- +# constructs python regex named group +def ng(name, regex): + return r'(?P<{}>{})'.format(name, regex) + + +def _compiled_pattern(stage, var): + pattern_fmt = r'{stage}\s+{var}:\s+{value}' + pattern = pattern_fmt.format( + stage=stage, + var=var, + value=ng('value', '[0-9]+'), + ) + return re.compile(pattern) + result = re.search(pattern, benchmark_output) + + class BenchmarkResult: - def __init__(self, config, output): - self.config = config + # pre-compiled patterns for all benchmarks + patterns = { + 'generator_ticks': _compiled_pattern('BIST-GENERATOR', 'ticks'), + 'checker_errors': _compiled_pattern('BIST-CHECKER', 'errors'), + 'checker_ticks': _compiled_pattern('BIST-CHECKER', 'ticks'), + } + + @staticmethod + def find(pattern, output): + result = pattern.search(output) + assert result is not None, \ + 'Could not find pattern "%s" in output:\n%s' % (pattern, benchmark_output) + return int(result.group('value')) + + def __init__(self, output): self._output = output - self.parse_output(output) - # instantiate the benchmarked soc to check its configuration - self.benchmark_soc = LiteDRAMBenchmarkSoC(**self.config._settings) + for attr, pattern in self.patterns.items(): + setattr(self, attr, self.find(pattern, output)) - def cmd_count(self): - data_width = self.benchmark_soc.sdram.controller.interface.data_width - return self.config.bist_length / (data_width // 8) - - def clk_period(self): - clk_freq = self.benchmark_soc.sdrphy.module.clk_freq - return 1 / clk_freq - - def write_bandwidth(self): - return (8 * self.config.bist_length) / (self.generator_ticks * self.clk_period()) - - def read_bandwidth(self): - return (8 * self.config.bist_length) / (self.checker_ticks * self.clk_period()) - - def write_efficiency(self): - return self.cmd_count() / self.generator_ticks - - def read_efficiency(self): - return self.cmd_count() / self.checker_ticks - - def write_latency(self): - assert self.config.bist_length == 1, 'Not a latency benchmark' - return self.generator_ticks - - def read_latency(self): - assert self.config.bist_length == 1, 'Not a latency benchmark' - return self.checker_ticks - - def parse_output(self, output): - bist_pattern = r'{stage}\s+{var}:\s+{value}' - - def find(stage, var): - pattern = bist_pattern.format( - stage=stage, - var=var, - value=ng('value', '[0-9]+'), - ) - result = re.search(pattern, output) - assert result is not None, 'Could not find pattern in output: %s, %s' % (pattern, output) - return int(result.group('value')) - - self.generator_ticks = find('BIST-GENERATOR', 'ticks') - self.checker_errors = find('BIST-CHECKER', 'errors') - self.checker_ticks = find('BIST-CHECKER', 'ticks') - - @classmethod - def dump_results_json(cls, results, file): - """Save multiple results in a JSON file. - - Only configurations and outpits are saved, as they can be used to reconstruct BenchmarkResult. - """ - # simply use config._settings as it defines the BenchmarkConfiguration - results_raw = [(r.config._settings, r._output) for r in results] - with open(file, 'w') as f: - json.dump(results_raw, f) - - @classmethod - def load_results_json(cls, file): - """Load results from a JSON file.""" - with open(file, 'r') as f: - results_raw = json.load(f) - return [cls(BenchmarkConfiguration(**settings), output) for (settings, output) in results_raw] + def __repr__(self): + d = {attr: getattr(self, attr) for attr in self.patterns.keys()} + return 'BenchmarkResult(%s)' % d # Results summary ---------------------------------------------------------------------------------- +def human_readable(value): + binary_prefixes = ['', 'k', 'M', 'G', 'T'] + mult = 1.0 + for prefix in binary_prefixes: + if value * mult < 1024: + break + mult /= 1024 + return mult, prefix + + +def clocks_fmt(clocks): + return '{:d} clk'.format(int(clocks)) + + +def bandwidth_fmt(bw): + mult, prefix = human_readable(bw) + return '{:.1f} {}bps'.format(bw * mult, prefix) + + +def efficiency_fmt(eff): + return '{:.1f} %'.format(eff * 100) + + class ResultsSummary: - # value_scaling is a function: value -> (multiplier, prefix) - Fmt = namedtuple('MetricFormatting', ['name', 'unit', 'value_scaling']) - metric_formats = { - 'write_bandwidth': Fmt('Write bandwidth', 'bps', lambda value: human_readable(value)), - 'read_bandwidth': Fmt('Read bandwidth', 'bps', lambda value: human_readable(value)), - 'write_efficiency': Fmt('Write efficiency', '', lambda value: (100, '%')), - 'read_efficiency': Fmt('Read efficiency', '', lambda value: (100, '%')), - 'write_latency': Fmt('Write latency', 'clk', lambda value: (1, '')), - 'read_latency': Fmt('Read latency', 'clk', lambda value: (1, '')), - } + def __init__(self, run_data, plots_dir='plots'): + self.plots_dir = plots_dir - def __init__(self, results): - self.results = results + # gather results into tabular data + column_mappings = { + 'name': lambda d: d.config.name, + 'sdram_module': lambda d: d.config.sdram_module, + 'sdram_data_width': lambda d: d.config.sdram_data_width, + 'bist_length': lambda d: getattr(d.config.access_pattern, 'bist_length', None), + 'bist_random': lambda d: getattr(d.config.access_pattern, 'bist_random', None), + 'pattern_file': lambda d: getattr(d.config.access_pattern, 'pattern_file', None), + 'length': lambda d: d.config.length, + 'generator_ticks': lambda d: d.result.generator_ticks, + 'checker_errors': lambda d: d.result.checker_errors, + 'checker_ticks': lambda d: d.result.checker_ticks, + 'ctrl_data_width': lambda d: d.config.soc.sdram.controller.interface.data_width, + 'clk_freq': lambda d: d.config.soc.sdrphy.module.clk_freq, + } + columns = {name: [mapping(data) for data in run_data] for name, mapping, in column_mappings.items()} + self.df = df = pd.DataFrame(columns) - def by_metric(self, metric): - """Returns pairs of value of the given metric and the configuration used for benchmark""" - for result in self.results: - # omit the results that should not be used to calculate given metric - if result.config.bist_length == 1 and metric not in ['read_latency', 'write_latency'] \ - or result.config.bist_length != 1 and metric in ['read_latency', 'write_latency']: - continue - value = getattr(result, metric)() - yield value, result.config + # replace None with NaN + df.fillna(value=np.nan, inplace=True) - def print(self): - legend = '(module, datawidth, length, random, result)' - fmt = ' {module:15} {dwidth:2} {length:4} {random:1} {result}' + # compute other metrics based on ticks and configuration parameters + df['clk_period'] = 1 / df['clk_freq'] + df['write_bandwidth'] = (8 * df['length']) / (df['generator_ticks'] * df['clk_period']) + df['read_bandwidth'] = (8 * df['length']) / (df['checker_ticks'] * df['clk_period']) - # store formatted lines per metric - metric_lines = defaultdict(list) - for metric, (_, unit, formatter) in self.metric_formats.items(): - for value, config in self.by_metric(metric): - mult, prefix = formatter(value) - value_fmt = '{:5.1f} {}{}' if isinstance(value * mult, float) else '{:5d} {}{}' - result = value_fmt.format(value * mult, prefix, unit) - line = fmt.format(module=config.sdram_module, - dwidth=config.sdram_data_width, - length=config.bist_length, - random=int(config.bist_random), - result=result) - metric_lines[metric].append(line) + df['cmd_count'] = df['length'] / (df['ctrl_data_width'] / 8) + df['write_efficiency'] = df['cmd_count'] / df['generator_ticks'] + df['read_efficiency'] = df['cmd_count'] / df['checker_ticks'] - # find length of the longest line - max_length = max((len(l) for lines in metric_lines.values() for l in lines)) - max_length = max(max_length, len(legend) + 2) - width = max_length + 2 + df['write_latency'] = df[df['bist_length'] == 1]['generator_ticks'] + df['read_latency'] = df[df['bist_length'] == 1]['checker_ticks'] - # print the formatted summary - def header(text): - mid = center(text, width - 6, '=') - return center(mid, width, '-') - print(header(' Summary ')) - print(center(legend, width)) - for metric, lines in metric_lines.items(): - print(center(self.metric_formats[metric].name, width)) - for line in lines: - print(line) - print(header('')) + # boolean distinction between latency benchmarks and sequence benchmarks, + # as thier results differ significanly + df['is_latency'] = ~pd.isna(df['write_latency']) + assert (df['is_latency'] == ~pd.isna(df['read_latency'])).all(), \ + 'write_latency and read_latency should both have a value or both be NaN' - def plot(self, output_dir, backend='Agg', theme='default', save_format='png', **savefig_kwargs): - """Create plots with benchmark results summary + # data formatting for text summary + self.text_formatters = { + 'write_bandwidth': bandwidth_fmt, + 'read_bandwidth': bandwidth_fmt, + 'write_efficiency': efficiency_fmt, + 'read_efficiency': efficiency_fmt, + 'write_latency': clocks_fmt, + 'read_latency': clocks_fmt, + } - Default backend is Agg, which is non-GUI backed and only allows - to save figures as files. If a GUI backed is passed, plt.show() - will be called at the end. - """ - # import locally here to be able to run benchmarks without installing matplotlib - import matplotlib - matplotlib.use(backend) - - import matplotlib.pyplot as plt - import numpy as np - from matplotlib.ticker import FuncFormatter, PercentFormatter, ScalarFormatter - - plt.style.use(theme) - - def bandwidth_formatter_func(value, pos): - mult, prefix = human_readable(value) - return '{:.1f}{}bps'.format(value * mult, prefix) - - tick_formatters = { - 'write_bandwidth': FuncFormatter(bandwidth_formatter_func), - 'read_bandwidth': FuncFormatter(bandwidth_formatter_func), + # data formatting for plot summary + self.plot_xticks_formatters = { + 'write_bandwidth': FuncFormatter(lambda value, pos: bandwidth_fmt(value)), + 'read_bandwidth': FuncFormatter(lambda value, pos: bandwidth_fmt(value)), 'write_efficiency': PercentFormatter(1.0), 'read_efficiency': PercentFormatter(1.0), 'write_latency': ScalarFormatter(), 'read_latency': ScalarFormatter(), } - def config_tick_name(config): - return '{}\n{}, {}, {}'.format(config.sdram_module, config.sdram_data_width, - config.bist_length, int(config.bist_random)) + def print_df(self, title, df): + # make sure all data will be shown + with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.width', None): + print('===> {}:'.format(title)) + print(df) - for metric, (name, unit, _) in self.metric_formats.items(): - fig = plt.figure() - axis = plt.gca() + def get_summary(self, mask=None, columns=None, column_formatting=None, sort_kwargs=None): + # work on a copy + df = self.df.copy() - values, configs = zip(*self.by_metric(metric)) - ticks = np.arange(len(configs)) + if sort_kwargs is not None: + df = df.sort_values(**sort_kwargs) - axis.barh(ticks, values, align='center') - axis.set_yticks(ticks) - axis.set_yticklabels([config_tick_name(c) for c in configs]) - axis.invert_yaxis() - axis.xaxis.set_major_formatter(tick_formatters[metric]) - axis.xaxis.set_tick_params(rotation=30) - axis.grid(True) - axis.spines['top'].set_visible(False) - axis.spines['right'].set_visible(False) - axis.set_axisbelow(True) + if column_formatting is not None: + for column, mapping in column_formatting.items(): + old = '_{}'.format(column) + df[old] = df[column].copy() + df[column] = df[column].map(lambda value: mapping(value) if not pd.isna(value) else value) - # force xmax to 100% - if metric in ['write_efficiency', 'read_efficiency']: - axis.set_xlim(right=1.0) + df = df[mask] if mask is not None else df + df = df[columns] if columns is not None else df - title = self.metric_formats[metric].name - axis.set_title(title, fontsize=12) + return df - plt.tight_layout() - filename = '{}.{}'.format(metric, save_format) - fig.savefig(os.path.join(output_dir, filename), **savefig_kwargs) + def text_summary(self): + for title, df in self.groupped_results(): + self.print_df(title, df) + print() + + def groupped_results(self, formatted=True): + df = self.df + + formatters = self.text_formatters if formatted else {} + + common_columns = ['name', 'sdram_module', 'sdram_data_width'] + latency_columns = ['write_latency', 'read_latency'] + performance_columns = ['write_bandwidth', 'read_bandwidth', 'write_efficiency', 'read_efficiency'] + + yield 'Latency', self.get_summary( + mask=df['is_latency'] == True, + columns=common_columns + latency_columns, + column_formatting=formatters, + ) + # yield 'Any access pattern', self.get_summary( + # mask=(df['is_latency'] == False), + # columns=common_columns + performance_columns + ['length', 'bist_random', 'pattern_file'], + # column_formatting=self.text_formatters, + # **kwargs, + # ), + yield 'Custom access pattern', self.get_summary( + mask=(df['is_latency'] == False) & (~pd.isna(df['pattern_file'])), + columns=common_columns + performance_columns + ['length', 'pattern_file'], + column_formatting=formatters, + ), + yield 'Sequential access pattern', self.get_summary( + mask=(df['is_latency'] == False) & (pd.isna(df['pattern_file'])) & (df['bist_random'] == False), + columns=common_columns + performance_columns + ['bist_length'], # could be length + column_formatting=formatters, + ), + yield 'Random access pattern', self.get_summary( + mask=(df['is_latency'] == False) & (pd.isna(df['pattern_file'])) & (df['bist_random'] == True), + columns=common_columns + performance_columns + ['bist_length'], + column_formatting=formatters, + ), + + def plot_summary(self, plots_dir='plots', backend='Agg', theme='default', save_format='png', **savefig_kw): + matplotlib.use(backend) + import matplotlib.pyplot as plt + plt.style.use(theme) + + for title, df in self.groupped_results(formatted=False): + for column in self.plot_xticks_formatters.keys(): + if column not in df.columns or df[column].empty: + continue + axis = self.plot_df(title, df, column) + + # construct path + def path_name(name): + return name.lower().replace(' ', '_') + + filename = '{}.{}'.format(path_name(column), save_format) + path = os.path.join(plots_dir, path_name(title), filename) + os.makedirs(os.path.dirname(path), exist_ok=True) + + # save figure + axis.get_figure().savefig(path, **savefig_kw) if backend != 'Agg': plt.show() + def plot_df(self, title, df, column, save_format='png', save_filename=None): + if save_filename is None: + save_filename = os.path.join(self.plots_dir, title.lower().replace(' ', '_')) + + axis = df.plot(kind='barh', x='name', y=column, title=title, grid=True, legend=False) + if column in self.plot_xticks_formatters: + axis.xaxis.set_major_formatter(self.plot_xticks_formatters[column]) + axis.xaxis.set_tick_params(rotation=15) + axis.spines['top'].set_visible(False) + axis.spines['right'].set_visible(False) + axis.set_axisbelow(True) + + # # force xmax to 100% + # if column in ['write_efficiency', 'read_efficiency']: + # axis.set_xlim(right=1.0) + + return axis + # Run ---------------------------------------------------------------------------------------------- -def run_benchmark(cmd_args): - # run as separate process, because else we cannot capture all output from verilator - benchmark_script = os.path.join(os.path.dirname(__file__), 'benchmark.py') - command = ['python3', benchmark_script, *cmd_args] - proc = subprocess.run(command, stdout=subprocess.PIPE) +class RunCache(list): + RunData = namedtuple('RunData', ['config', 'result']) + + def dump_json(self, filename): + json_data = [{'config': data.config.as_dict(), 'output': data.result._output} for data in self] + with open(filename, 'w') as f: + json.dump(json_data, f) + + @classmethod + def load_json(cls, filename): + with open(filename, 'r') as f: + json_data = json.load(f) + loaded = [] + for data in json_data: + config = BenchmarkConfiguration.from_dict(data['config']) + result = BenchmarkResult(data['output']) + loaded.append(cls.RunData(config=config, result=result)) + return loaded + + +def run_python(script, args): + command = ['python3', script, *args] + proc = subprocess.run(command, stdout=subprocess.PIPE, cwd=os.path.dirname(script)) return str(proc.stdout) -def run_benchmarks(configurations): - results = [] - for name, config in configurations.items(): - cmd_args = config.as_args() - print('{}: {}'.format(name, ' '.join(cmd_args))) - output = run_benchmark(cmd_args) - # exit if checker had any read error - result = BenchmarkResult(config, output) - if result.checker_errors != 0: - print('Error during benchmark "{}": checker_errors = {}'.format( - name, result.checker_errors), file=sys.stderr) - sys.exit(1) - results.append(result) - return results +def run_benchmark(config): + benchmark_script = os.path.join(os.path.dirname(__file__), 'benchmark.py') + # run as separate process, because else we cannot capture all output from verilator + output = run_python(benchmark_script, config.as_args()) + result = BenchmarkResult(output) + # exit if checker had any read error + if result.checker_errors != 0: + raise RuntimeError('Error during benchmark: checker_errors = {}, args = {}'.format( + result.checker_errors, args + )) + return result def main(argv=None): @@ -363,6 +436,7 @@ def main(argv=None): parser.add_argument('--plot-transparent', action='store_true', help='Use transparent background when saving plots') parser.add_argument('--plot-output-dir', default='plots', help='Specify where to save the plots') parser.add_argument('--plot-theme', default='default', help='Use different matplotlib theme') + parser.add_argument('--ignore-failures', action='store_true', help='Ignore failuers during benchmarking, continue using successful runs only') parser.add_argument('--results-cache', help="""Use given JSON file as results cache. If the file exists, it will be loaded instead of running actual benchmarks, else benchmarks will be run normally, and then saved @@ -370,6 +444,11 @@ def main(argv=None): to generate different summary without having to rerun benchmarks.""") args = parser.parse_args(argv) + if not args.results_cache and not _summary: + print('Summary not available and not running with --results-cache - run would not produce any results! Aborting.', + file=sys.stderr) + sys.exit(1) + # load and filter configurations configurations = BenchmarkConfiguration.load_yaml(args.config) filters = { @@ -382,31 +461,41 @@ def main(argv=None): configurations = filter(f, configurations) configurations = list(configurations) - cache_exists = args.results_cache and os.path.isfile(args.results_cache) - # load outputs from cache if it exsits + cache_exists = args.results_cache and os.path.isfile(args.results_cache) if args.results_cache and cache_exists: - cached_results = BenchmarkResult.load_results_json(args.results_cache) + cache = RunCache.load_json(args.results_cache) + # take only those that match configurations - results = [r for r in cached_results if r.config in configurations.values()] + names_to_load = [c.name for c in configurations] + run_data = [data for data in cache if data.config.name in names_to_load] else: # run all the benchmarks normally - results = run_benchmarks(configurations) + run_data = [] + for config in configurations: + print(' {}: {}'.format(config.name, ' '.join(config.as_args()))) + try: + run_data.append(RunCache.RunData(config, run_benchmark(config))) + except: + if not args.ignore_failures: + raise # store outputs in cache if args.results_cache and not cache_exists: - BenchmarkResult.dump_results_json(results, args.results_cache) + cache = RunCache(run_data) + cache.dump_json(args.results_cache) - # display the summary - summary = ResultsSummary(results) - summary.print() - if args.plot: - if not os.path.isdir(args.plot_output_dir): - os.makedirs(args.plot_output_dir) - summary.plot(args.plot_output_dir, - backend=args.plot_backend, - theme=args.plot_theme, - save_format=args.plot_format, - transparent=args.plot_transparent) + # display summary + if _summary: + summary = ResultsSummary(run_data) + summary.text_summary() + if args.plot: + summary.plot_summary( + plots_dir=args.plot_output_dir, + backend=args.plot_backend, + theme=args.plot_theme, + save_format=args.plot_format, + transparent=args.plot_transparent, + ) if __name__ == "__main__": From 8ba3cced603a3f24f93bbb3155be56b17af0991f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C4=99drzej=20Boczar?= Date: Wed, 5 Feb 2020 18:46:44 +0100 Subject: [PATCH 6/7] test: add new benchmark configuratiosns to example configuration file --- test/access_pattern.csv | 1024 +++++++++++++++++++++++++++++++++++++++ test/benchmarks.yml | 165 ++++++- 2 files changed, 1165 insertions(+), 24 deletions(-) create mode 100644 test/access_pattern.csv diff --git a/test/access_pattern.csv b/test/access_pattern.csv new file mode 100644 index 0000000..12ace39 --- /dev/null +++ b/test/access_pattern.csv @@ -0,0 +1,1024 @@ +0x0000012e,0x61d2c8d5 +0x000000d9,0x8ec8f8f6 +0x00000135,0xf9f6145f +0x0000033c,0xe653e338 +0x000002ff,0xbb609ae9 +0x0000037a,0xb42e5587 +0x00000228,0xce227592 +0x00000317,0xbd257969 +0x0000036e,0x7fe31c88 +0x0000011f,0xda6ba38d +0x000002c7,0x141f40ad +0x000000ab,0xa7c88a54 +0x000003f1,0xa48ddec4 +0x00000123,0xd800a7fe +0x00000098,0x45a96b1d +0x0000003b,0x4a75a15d +0x0000004f,0x90e04e1a +0x00000361,0x9af6660b +0x000000fc,0xa4830b72 +0x00000033,0xb063ccb3 +0x000000c1,0xf60e53ac +0x000002d2,0x47d9b283 +0x0000013f,0xe19843f7 +0x0000031d,0x30920818 +0x000001c3,0x2a830514 +0x000002c6,0xee6a0e7e +0x00000104,0x6ab2c91c +0x00000100,0x2dc79361 +0x000001e7,0xe9bf7c25 +0x00000149,0x29adfad6 +0x00000109,0x4f6c5d4a +0x00000285,0x9ec316ff +0x000001c4,0xf7768be4 +0x00000082,0x7478d35c +0x000001a9,0x1832672d +0x000001e5,0x12449eee +0x000002f3,0x90a205e0 +0x00000217,0x266d0830 +0x0000039a,0xe6a207ac +0x00000265,0x8fd00e38 +0x0000005e,0xc71ce3cc +0x00000116,0x03808ba0 +0x0000025d,0x9db362a9 +0x000002d8,0x04ab3a95 +0x0000018b,0xd8ccace2 +0x0000007d,0x9443ab73 +0x00000187,0x7933b972 +0x0000015e,0xcf6daccc +0x000002bc,0xbb9aa01c +0x00000274,0x790bce70 +0x00000227,0xa2e56a6d +0x0000002a,0xf2a9f0c7 +0x00000173,0x0cd5e422 +0x00000164,0x96c0513f +0x000001fb,0x5d766d17 +0x000001a0,0x55ea28bb +0x000002bd,0x38d65dfa +0x00000142,0x7212fb34 +0x000000b5,0x3b5fa506 +0x00000354,0x6a89900a +0x000001ac,0x2ce6be3f +0x00000342,0xb956396b +0x00000281,0x118b84ad +0x00000300,0x6e091a3c +0x0000029f,0x854141b2 +0x000000c6,0x3aafa04b +0x00000107,0x907c2cb4 +0x0000000a,0xc965666f +0x0000024c,0x2792b3a0 +0x00000203,0x1d19c1e1 +0x00000332,0x9f8ff8ce +0x00000092,0x86c77fe4 +0x00000101,0x5d59d5f3 +0x00000025,0x5dedd914 +0x00000258,0x9c886d9e +0x00000243,0x90a6e12a +0x00000247,0x75f3803e +0x000001ad,0x68843b6d +0x00000071,0x63b4de04 +0x0000026e,0x622eab11 +0x0000034b,0xb42696c1 +0x00000195,0x7685880a +0x000002ce,0x008d2d65 +0x000001c5,0xc064168f +0x00000019,0xf93530a6 +0x000001b3,0xec1b06db +0x000002f9,0x7ce508d3 +0x00000078,0xae78c8eb +0x00000134,0x64c63926 +0x00000255,0xd3646539 +0x00000010,0xa158c47d +0x000000f1,0xf18d4c56 +0x0000001d,0x3a693620 +0x000000e5,0x76b47c71 +0x0000001e,0xa741d479 +0x0000017f,0x84f156b2 +0x000000cb,0x8cc414d8 +0x000001a4,0xcd6cd47f +0x000003e4,0x37307894 +0x000001a1,0x041c0b6e +0x00000236,0x67741810 +0x000000b1,0x573222c0 +0x00000248,0x68e8f7f4 +0x00000316,0x16265757 +0x000001ed,0x85df14f1 +0x0000028e,0x54ec8e2f +0x000002c8,0xfae5e756 +0x000003a4,0x2cd32729 +0x0000035b,0x8ce13b5a +0x0000022d,0x943ce80f +0x00000345,0x278ff629 +0x0000008d,0x94a9c2ec +0x00000111,0xba4e5642 +0x00000011,0x22b3909c +0x000000f7,0x55338938 +0x00000186,0x9218fd6c +0x000000e0,0x3c48a497 +0x0000033a,0xda233663 +0x00000048,0x58855816 +0x0000029d,0x4df9feb1 +0x00000382,0x9f0f2502 +0x00000132,0xddaed3fe +0x000001a7,0xfde6bdba +0x00000128,0x448fce9a +0x0000037e,0x42aaaa1e +0x000001a8,0x4ea6f4df +0x000000ec,0x912aeb2c +0x00000056,0xde7d8aeb +0x000003e6,0x15f855a1 +0x0000001c,0xe0225e1e +0x00000080,0x3686bfb7 +0x000000da,0x7724b050 +0x000001b2,0xfec50f9a +0x000003c8,0x68d72fa8 +0x00000225,0x65df6eb6 +0x00000049,0x5b181430 +0x0000027e,0x35ec0cd3 +0x00000348,0xc1f5138a +0x0000036a,0x47b36f4d +0x00000178,0x073f863c +0x000001fc,0xf6ccbf5f +0x000003b3,0x3729f5ba +0x00000356,0x7f6d4988 +0x00000378,0x31d541ce +0x000001bf,0xe26033b2 +0x000002f4,0x4d009701 +0x0000012a,0x383e7b20 +0x000002f6,0x4ecb429b +0x000002ea,0xac934081 +0x000001d7,0xdf607028 +0x0000005b,0x3d48a4d7 +0x000000dc,0x9b2eed8e +0x00000089,0xe8170872 +0x0000007c,0x64439e56 +0x0000000e,0x3bb95a3c +0x00000239,0x36e6a900 +0x000001d8,0xc4c42852 +0x0000002d,0xa3a1a282 +0x0000000f,0x4f3c81e2 +0x000002c2,0x5d291767 +0x000002fb,0x2e48dcfa +0x00000119,0xc07a0c3c +0x000002e8,0x1cce5cbd +0x00000333,0x3a3a7e63 +0x00000110,0x4b501b3a +0x000001af,0xb1287fe3 +0x000001db,0xee4a7258 +0x0000018c,0xe9276e99 +0x0000013c,0x5bf99b5f +0x0000010b,0x2b1a09ff +0x0000031b,0xd9f78b82 +0x000003d9,0x505c7a02 +0x0000000d,0x9c568cbf +0x00000330,0x6f043b79 +0x000001f1,0x994dda6a +0x0000000b,0xd09629fa +0x00000136,0x41991be6 +0x0000004d,0x5e6a2d20 +0x00000198,0x0e0474d7 +0x000000ac,0x951f892d +0x00000022,0xc9ec5c7c +0x0000000c,0x95301582 +0x000002ef,0x08da98c7 +0x000003fb,0x43b4fd33 +0x000001eb,0x9e9b01ba +0x000000a3,0x325f8442 +0x00000209,0x74878d2f +0x00000159,0x4f1ec90f +0x00000172,0x6089924d +0x0000021a,0xaedaaeb8 +0x0000029a,0x7107bf13 +0x000001e1,0x584d3369 +0x00000106,0x75579cb1 +0x00000148,0x34bb1175 +0x00000398,0xb5d2e270 +0x000001ff,0x5a4cd7c1 +0x000003df,0x5ab6637c +0x0000032f,0xa588b2ab +0x0000033d,0x893d9f9c +0x00000183,0x6c833e74 +0x0000019d,0x9584ba23 +0x000000d4,0x7499251f +0x00000384,0x38c04aee +0x000002d7,0x2f780083 +0x0000003c,0x25ea371f +0x0000014d,0xa422c342 +0x000000a5,0xbb8ca7b1 +0x000000b4,0x600bfbb7 +0x000003a2,0x7536026b +0x0000019f,0xf3056983 +0x000003b5,0x14819baa +0x0000025b,0x22bfc5b4 +0x000001f4,0xf3dcc407 +0x000003d4,0xdd204ebc +0x000000e8,0x19371324 +0x0000031a,0x388f82b1 +0x000000b8,0xaa3c9d73 +0x00000370,0xdf89cab0 +0x00000251,0x149453d0 +0x00000177,0x060e6abb +0x000001d3,0x64c800bf +0x000002e0,0x197fe27c +0x000001bc,0x112c0c05 +0x00000339,0x5c1d0545 +0x0000011d,0xa9cb28e3 +0x00000020,0x9d16cec1 +0x0000037b,0xf46e9e3d +0x00000182,0x99e052fd +0x00000389,0xbd93a71e +0x000002b4,0x4bdf2842 +0x000002fc,0xa93be104 +0x00000192,0x1fd37c83 +0x0000020e,0x542b2943 +0x00000001,0x6a1469d3 +0x00000146,0xbfca75e9 +0x0000010e,0x8e00b2ee +0x00000141,0x4a5c4324 +0x00000023,0x7cfd7c93 +0x00000235,0xce9539ab +0x000002a0,0xe31e7f99 +0x00000231,0xbb3e3d23 +0x00000214,0x6eaa5dfc +0x00000319,0x9f719f0d +0x00000175,0x5ae51df8 +0x000002b0,0xa993d4c0 +0x000002da,0x9bb2a247 +0x00000053,0xd8d2dec4 +0x000003e0,0xaae00f7b +0x00000309,0x3f866f74 +0x00000276,0x4cf2e344 +0x00000055,0x4ba9a459 +0x00000216,0xee2ceffb +0x00000139,0xc9e720fb +0x00000338,0xc6204e59 +0x00000271,0xc98ed7f6 +0x000002a4,0xfd31d029 +0x00000286,0xdbcc04cb +0x0000037f,0x19bfb426 +0x00000064,0x0def53ae +0x00000335,0xbd34fdea +0x00000205,0x331768f4 +0x0000035c,0xa0e11fea +0x00000375,0x43bd9bcf +0x00000246,0xf1f27dd0 +0x00000018,0x3a703637 +0x00000388,0x4ca11d1c +0x00000169,0x7a808c4a +0x00000143,0xde360dec +0x00000088,0xf788f2f8 +0x000002c3,0x20fc77bb +0x0000018a,0x171ffa1c +0x000003bd,0xc1285057 +0x0000032b,0x47a437ab +0x000003ce,0xab34d83e +0x00000066,0x9c21cc65 +0x00000163,0x16b89209 +0x000001a5,0x39ed216e +0x000000d7,0x24e89199 +0x00000125,0xcd425b63 +0x000002e7,0xa3d10901 +0x00000219,0x16d97533 +0x00000363,0xceb7cdc8 +0x000002c9,0xb7ca8bb3 +0x000002a1,0xc9a18b20 +0x00000027,0xaf3a93d3 +0x00000343,0x724cf43c +0x0000016d,0x709cdc2b +0x000003fa,0x0bb57a27 +0x000003d0,0x50ebce7d +0x000003b4,0xcca0fcbd +0x000001d2,0xd47c15f9 +0x00000171,0x23e44721 +0x0000014b,0xfa40a59c +0x000002bf,0x2669def1 +0x000003de,0x029d6b69 +0x00000390,0xd6467cb3 +0x00000013,0x2fa1f2fe +0x00000329,0x39b5b021 +0x00000391,0x7155d9e6 +0x000000a4,0xe65f6a8b +0x0000010d,0xeb081ac3 +0x000001d9,0xc3333042 +0x0000001a,0xa7b3a7a1 +0x0000005c,0x15cc2a0b +0x000001df,0x58349e33 +0x00000081,0xbce62295 +0x00000129,0x4d34e0a5 +0x0000037d,0xae0fb82d +0x0000033f,0xbab64099 +0x000000ae,0x19cf9f0c +0x00000085,0xf65025dc +0x0000030d,0x156184eb +0x000000f3,0xd44874cb +0x000003c4,0x234a08dc +0x00000073,0x89be0b06 +0x0000008c,0x9c2f3f36 +0x0000006a,0x6dbd974e +0x000003ae,0xb1c0fac8 +0x0000036c,0x0158019e +0x0000003f,0x60d2350b +0x000000c3,0xe0bf851a +0x0000005d,0x57f4331e +0x00000160,0x2d01f727 +0x00000074,0xe7420f16 +0x0000001b,0x65193a81 +0x000003b6,0x5404408b +0x000000fe,0x4a912205 +0x0000020f,0xaf1ec7e3 +0x000003f2,0x65284d10 +0x00000038,0xc38902f9 +0x0000006b,0x93f02e10 +0x00000029,0xb73538b1 +0x000002ab,0x3832c214 +0x000001f3,0x7f04e0fe +0x000002e2,0x863662da +0x0000010f,0x6a7b7b52 +0x00000360,0x313fa92d +0x000002e1,0x29574441 +0x00000133,0x8503d7a1 +0x00000072,0xdf037127 +0x00000318,0x1f0a5242 +0x00000154,0x5d171f74 +0x00000197,0xa236ca17 +0x000000c5,0x344b386f +0x0000023b,0xbbfca537 +0x000002dc,0x8ecdbd48 +0x000000d3,0x68417535 +0x000002c4,0xa87d737e +0x00000302,0x5b4ac57d +0x000003b1,0x7208c47a +0x00000264,0xe967870e +0x00000230,0x6b8a64a4 +0x000001c0,0xa90e1999 +0x000000d0,0xfe6134c9 +0x000000e4,0x0d1e3e4f +0x000003ed,0x2765a290 +0x0000038a,0xadefbc7a +0x00000290,0x2253d279 +0x000001da,0xef8c14da +0x00000204,0xfa11139c +0x00000024,0x51773a21 +0x0000022c,0x82e40079 +0x000003e1,0x0d25dc31 +0x000002fa,0xae982f52 +0x000000aa,0xbdf45ea4 +0x0000026f,0x4441d881 +0x00000362,0x9129843f +0x000000d8,0x34175578 +0x000001f6,0x01663b92 +0x000003cc,0x341926d0 +0x00000352,0x4dc5fcc0 +0x00000036,0x1566a8eb +0x00000273,0xfdc7e15c +0x000000af,0xbaed2374 +0x00000041,0x1a0b8317 +0x000001dd,0xd18f699a +0x000000a8,0x90cb209a +0x00000257,0xbf67c32e +0x0000003d,0x0cbb59c4 +0x00000093,0xfe60eac3 +0x00000336,0xcf5be5dc +0x00000084,0x660900fc +0x00000320,0xaf5e2207 +0x0000032e,0x397069f3 +0x000001b7,0x9ed88604 +0x0000005a,0xd9e0f4a5 +0x000001cb,0xf2081a29 +0x00000270,0xe42a12a4 +0x0000027b,0x48fb577a +0x000003c3,0x239ff442 +0x0000015f,0xcef290ee +0x0000002b,0xe894ca29 +0x000002e6,0x00605a5d +0x000003d8,0x8a3309b1 +0x000003ba,0x715efaae +0x00000113,0xa0d1f6a8 +0x000001e8,0x744fd2ca +0x00000065,0x326f3881 +0x00000305,0xf508ccfc +0x00000215,0x594d1b47 +0x000002d4,0x89b4ec73 +0x000001a2,0xf12e7ff3 +0x000003ab,0x270a33a1 +0x000002d9,0x95c40cc3 +0x000001f0,0xadb2d1e6 +0x00000334,0x1f8b03c2 +0x00000359,0x0c8c22df +0x00000016,0xfdef90f8 +0x00000321,0x6d16c2b6 +0x0000012d,0x4a4a85aa +0x00000112,0x1b9612a0 +0x000000c2,0x11b7b67a +0x000003a8,0x0d2ddd6b +0x0000028c,0x9bac58e1 +0x0000001f,0x38060223 +0x00000299,0xa69b4430 +0x0000034e,0x92075ea3 +0x00000068,0x24693370 +0x00000091,0xe20db412 +0x0000029e,0x78539957 +0x00000385,0x7a646c06 +0x0000002c,0xd00f800a +0x00000188,0x0846d6bc +0x000001e0,0x1702b043 +0x000002cb,0x1f438aea +0x0000005f,0xa43309be +0x00000147,0x1609f9c1 +0x000002f7,0x7b851f8e +0x000002dd,0x6bf738a3 +0x000000ca,0x1a50ad66 +0x000001e9,0x37a9593f +0x0000006f,0x6de45853 +0x0000015c,0xeb6cb24e +0x00000030,0xce267ad7 +0x0000002e,0x4ec1453b +0x000001d1,0xfc14bec4 +0x00000017,0x876d7392 +0x000002f8,0x9ac12c44 +0x000000f6,0xf524d27d +0x0000035d,0x30a11607 +0x00000394,0xd3b26a18 +0x00000355,0xee8a455a +0x00000261,0x98c20113 +0x000000d5,0x37fbdd7f +0x00000325,0x85269178 +0x0000011e,0xf74a97ad +0x0000028d,0xb144ba63 +0x000000ef,0xec40f103 +0x000002be,0xda839587 +0x00000061,0xe116a87a +0x000002ae,0xb615d223 +0x000001ab,0x81c40cec +0x000000cd,0xe644b747 +0x0000006e,0x82bd6a8c +0x00000000,0xf000849b +0x0000032d,0xb22bc4c2 +0x00000368,0x7510c7f3 +0x00000067,0xe616e21e +0x00000358,0xb2b4e853 +0x000002a3,0x1a809df7 +0x000000a0,0xbd9649c3 +0x000003d2,0x93cb8b68 +0x00000046,0xf4aa0c1b +0x0000026a,0xb84137a0 +0x000000f4,0x5519ed9c +0x0000025e,0x27785441 +0x000000e2,0xf7c05a5c +0x00000346,0x99448f63 +0x0000008b,0xf7808cc8 +0x0000036b,0xa20fdf9e +0x000000bc,0x0d624554 +0x0000017b,0x328aa94c +0x0000023f,0xab882098 +0x0000033b,0xcb5d14ad +0x00000114,0x62fca8d3 +0x000003fc,0x80a34a21 +0x000001bd,0x2027292c +0x00000328,0x6b1ed1f1 +0x000000b6,0x3cd89d38 +0x0000027c,0xf3f04d4e +0x000003d5,0x482e2cce +0x00000054,0x5d084c63 +0x000002df,0x9694542a +0x000003b8,0x823d6559 +0x00000289,0xc3adcb32 +0x0000035f,0x88e8e6db +0x0000024f,0xc288d6fb +0x000001cc,0xf5a0c23e +0x0000015a,0x04c6ac85 +0x000000e6,0xa2c708a6 +0x000000cc,0x214ae76e +0x0000039e,0x75bf1bc8 +0x000003c9,0x3191a7eb +0x000000e9,0xec7d07db +0x00000060,0xe0710b88 +0x000002a7,0x73d0cd4e +0x000003db,0x1e017f85 +0x000000a2,0x489b1f6c +0x00000076,0x60529d31 +0x0000004b,0x93b6355e +0x00000063,0xe9691ee0 +0x000001ea,0xf1d3e628 +0x000000a6,0x3eaf45d5 +0x0000034a,0x079bc1db +0x00000003,0x2b83ee22 +0x000003cb,0xb38d5007 +0x00000315,0x005c5340 +0x000003f9,0xf61bec1d +0x000003fe,0x459a3987 +0x000001b4,0x955aa611 +0x000000c9,0x0b8502a7 +0x0000025c,0x919b4b7f +0x00000323,0x4e0b307c +0x0000039c,0x25e20b80 +0x00000035,0x15d35def +0x00000155,0xed7988b9 +0x0000007a,0x0d259437 +0x0000031c,0xc448416c +0x00000379,0x588b1ea1 +0x000000ee,0xda9033f2 +0x0000017c,0x5d8510dd +0x0000017d,0x7a845fad +0x0000009d,0x285e125f +0x000003ac,0xc3a8f4f8 +0x00000044,0x562f95f9 +0x000001c2,0xcbcbfc47 +0x000001f8,0x8bb3c481 +0x000002f1,0x5eb9554d +0x0000007e,0x3cd4d757 +0x000003c2,0xf24687c9 +0x00000208,0x22fe40f5 +0x000001c6,0xbd394a93 +0x00000207,0x9f8abb23 +0x0000003e,0x3b084161 +0x00000310,0x5dd566f4 +0x0000038e,0x93cfc737 +0x0000015d,0x248175bd +0x0000009c,0x06a757d2 +0x00000220,0x6298f764 +0x000001cd,0xb7493bd3 +0x00000340,0xcab3638c +0x0000016a,0xba6f41df +0x000000de,0xb583bd95 +0x000000b3,0x55ee3276 +0x00000105,0xe60a6ea0 +0x00000292,0xfa17da23 +0x000001c7,0xc02731ee +0x0000039b,0x314a1f3f +0x00000324,0xaa0e0330 +0x000003ef,0x5606084c +0x000000bb,0x3f139afc +0x0000027d,0x04af2287 +0x0000025f,0x1ddf8f9e +0x00000242,0x0f8a411a +0x000003c1,0xc6518d07 +0x00000303,0x465f710f +0x0000002f,0xed4d052e +0x0000024b,0x1343c957 +0x00000393,0x5c0a5fe4 +0x000002c5,0x0465e58d +0x0000009e,0x2c09e0a8 +0x00000008,0x853a3b86 +0x0000009f,0x0bb6972d +0x000002d3,0x961b173f +0x000000f0,0x9756f025 +0x00000245,0x5d446cad +0x000001b9,0xdd7862a5 +0x000002b5,0xc2d1e49b +0x00000090,0x5dcb1a93 +0x0000019b,0x17c5f622 +0x0000007f,0x2048e019 +0x000000bf,0xe575efda +0x00000050,0x29f4ff94 +0x00000326,0x5af3c8fa +0x0000019e,0xae8ad590 +0x000002ad,0x4325de4c +0x0000030c,0xd00530a4 +0x000002cc,0xf3f7fcf7 +0x0000032c,0xbc35a67f +0x0000016c,0x2ad3a928 +0x000000f8,0x4ddf8b47 +0x0000004c,0x349d1982 +0x0000039f,0x8e06d477 +0x000003ee,0x67e0cbc5 +0x0000021b,0xbbbd1879 +0x0000004a,0x2c55a027 +0x000002a2,0x2634f218 +0x000003cf,0x3d73d279 +0x000000eb,0x6e78b973 +0x0000021e,0xdd01d278 +0x00000127,0xd6fd8840 +0x00000059,0xadb7fd4a +0x000002ec,0x3ced1d8a +0x0000038f,0x1dad4ff2 +0x000002b3,0xa7b024d8 +0x000002d6,0xead8cd71 +0x00000184,0x1ee19ab9 +0x00000120,0xa362afd8 +0x00000260,0xb2d1429d +0x00000005,0x6aa3df1a +0x00000308,0x1ebb5208 +0x000000ff,0xa9df5014 +0x00000087,0x32938571 +0x00000126,0x59d446f1 +0x00000293,0xde9d5490 +0x0000029c,0xa779710c +0x000002a9,0x62c7737d +0x000003bc,0xa800328e +0x000002ca,0xd5b99fa7 +0x00000249,0x06ec5c2f +0x000003ff,0x588a9bdc +0x00000371,0x33386477 +0x000000c4,0x2feeb727 +0x000000fd,0xe950d114 +0x0000033e,0xcccfdb62 +0x000003af,0x9cbb4ec8 +0x00000045,0x4e91087f +0x000000ea,0xfa9c9aac +0x000000be,0x6af216ea +0x0000010c,0x632ae74f +0x000001e2,0xb101bded +0x0000020a,0x7a41e224 +0x00000238,0x4c4b3a6e +0x00000158,0xb7328634 +0x000001be,0xe102181b +0x000003a9,0x5c529dd4 +0x000003a7,0xec370158 +0x00000108,0x889d6ebe +0x0000025a,0x95906629 +0x00000350,0xcf1cd4e7 +0x000001f5,0x7c295b29 +0x000003bb,0xeae747ee +0x00000349,0x82ae2057 +0x000003b9,0x7a249f88 +0x000001b0,0x2563614b +0x00000174,0x325549b4 +0x000000e3,0x6e51ae06 +0x00000006,0xb845331b +0x000003b0,0xbf77e74d +0x00000322,0x444d330a +0x00000191,0x83d91767 +0x000000ad,0x477af9ba +0x0000021f,0xc071f857 +0x0000008f,0x3e8a5d1a +0x0000029b,0x9b0431fd +0x000002a8,0xcebb2ae6 +0x00000165,0x1a2c3cd4 +0x00000157,0x1629e3fe +0x000003d1,0x9dc7627c +0x00000121,0xe1ff1567 +0x0000007b,0x51610f6c +0x000000d1,0x7d5918b4 +0x000002b8,0x74025419 +0x00000395,0x6001080f +0x000000b7,0xbdfa2c1a +0x000001b1,0xe0b9b238 +0x0000038d,0xae0f8b86 +0x00000291,0x7d585c4d +0x0000018f,0x7381b5ac +0x000002ba,0xc3081096 +0x000000ce,0x7f15786e +0x0000022e,0x424bc3fc +0x0000006c,0xee2cc7a7 +0x000001a6,0xb3b219a0 +0x0000034d,0x9a9e73fe +0x00000118,0x63d7a149 +0x0000026d,0x1d6d3ea4 +0x00000162,0x44b271c9 +0x00000241,0xd6da9f44 +0x0000030b,0x3f8a78d2 +0x000001ba,0xd2c0fe52 +0x000000bd,0x723c8910 +0x00000234,0xbf48bec5 +0x000001b8,0xa7874edb +0x00000226,0xdd6aa284 +0x00000144,0xe7d2458c +0x00000012,0xd6d8c04a +0x0000017a,0xf6ae9915 +0x00000014,0xb00b0422 +0x000002f0,0x88041d25 +0x00000069,0x61278220 +0x0000030f,0xf2c3811d +0x00000037,0x9d861d63 +0x000002d5,0x6c37dd6c +0x000000c7,0xc80f3a17 +0x000001d4,0xdf893fdc +0x000002ed,0xc5ec640d +0x000002d1,0xa1bdec12 +0x000001ee,0x4e254439 +0x00000311,0x178b04fc +0x0000019a,0x8da6f6b4 +0x000003e8,0x82797f9b +0x00000152,0x9b9c0438 +0x0000014c,0x909b71e7 +0x000002b9,0x57eefc02 +0x00000279,0x21f30dd9 +0x0000013b,0x282ecf47 +0x0000012b,0xb8240d6c +0x00000272,0x56ce2bbb +0x00000040,0xba39ad57 +0x0000026c,0x35592672 +0x000003e3,0x4eb09c06 +0x000003f3,0x341149c8 +0x000001e3,0xe89b8254 +0x000002c0,0x2dd5663b +0x0000009a,0x9e6c6b56 +0x00000021,0x12fd7034 +0x00000387,0xa027ea96 +0x000001ce,0x8af8e5c7 +0x000003f0,0x4d341384 +0x00000267,0xd2f19763 +0x0000018d,0x765671c5 +0x000000a1,0xaf382a64 +0x00000221,0x7fd9a647 +0x000002fe,0x7d1b99ca +0x00000277,0x4db2b052 +0x000003dc,0xd5a05d52 +0x00000058,0x4ccbf2d4 +0x000000c0,0x33761998 +0x00000254,0xa34acbad +0x000001d5,0xe2064af6 +0x0000021d,0x33c319ae +0x00000083,0x8af6070c +0x00000062,0x2c5c3595 +0x000001b5,0x53c1a11a +0x00000365,0xb7641db3 +0x000000fa,0x8f168750 +0x000003d7,0x62700567 +0x0000035e,0x30cf6a3c +0x00000297,0xaece2cae +0x0000030e,0x4a431c09 +0x000001f2,0xb088d216 +0x0000023a,0x96ce06c1 +0x00000176,0x8a9abb34 +0x00000115,0x311a4837 +0x0000031e,0x5e85164b +0x00000047,0x00a0eeb5 +0x000001a3,0xf84eca18 +0x000000f2,0x2cbee27a +0x0000014e,0x2f191aee +0x000000b9,0x3b12a538 +0x0000026b,0x472f6ac7 +0x00000233,0xa4337bf2 +0x00000052,0xf6959222 +0x000002b7,0xfe1c9ccf +0x0000024a,0x4e6efdf3 +0x00000218,0x5496f22a +0x0000024d,0xcfdaf597 +0x00000009,0x17453936 +0x0000032a,0x8c570977 +0x00000313,0xa5a96add +0x00000179,0xfb73e3f9 +0x00000180,0x31c27b51 +0x000002fd,0x2d15a0cd +0x0000012f,0xc475fc25 +0x000002a5,0xdf36df3c +0x000003be,0xbccd34d7 +0x00000140,0x3f11ac6c +0x000002af,0x4f8f60de +0x00000200,0x6b3ed957 +0x00000372,0x3134dec0 +0x000002bb,0x4677e498 +0x000000a9,0xc657ff78 +0x00000351,0xdaa8f98a +0x00000170,0x865000f4 +0x000001c1,0x0daad104 +0x000003f6,0x54381f7c +0x00000282,0xae0de4fd +0x0000039d,0x930925cc +0x0000020b,0xc04f5b79 +0x00000007,0xe60b3af3 +0x0000027f,0xcb230d4d +0x0000034f,0x6e94cf18 +0x00000026,0x979b5acb +0x000002b2,0xd69292e1 +0x00000137,0x298e7abb +0x000001dc,0xaab2730d +0x0000020d,0x50b57ca0 +0x00000298,0x5f9276c5 +0x0000022a,0x63391f81 +0x00000314,0x00f61207 +0x0000030a,0x1d94bc11 +0x0000017e,0xece45f2a +0x00000196,0x3376ea4f +0x00000212,0x394c781e +0x0000015b,0xaa45c8e4 +0x00000262,0xee189cde +0x000000b2,0x0cf81cad +0x000003e5,0x6a469268 +0x00000122,0x0fe0bf0f +0x000003ca,0x299b8fa4 +0x0000035a,0x18a3db69 +0x00000266,0x0387ddf8 +0x000001c8,0x76d2e831 +0x000001d0,0xd3132dbd +0x00000206,0xfa3a0f73 +0x00000373,0x837f807f +0x00000015,0x8f88a8ab +0x00000386,0x3c2ee566 +0x000003f7,0x56d51358 +0x00000223,0xe644ada5 +0x000002f5,0x8f20bddd +0x0000011b,0xa72467ed +0x000001ef,0x0eeb3073 +0x000000d2,0xfceccbcc +0x000002f2,0x722f9ca7 +0x000002ac,0x134a4fbb +0x00000301,0x3cc98027 +0x0000014f,0xfbec5bf5 +0x000003d3,0x628b4ab9 +0x000001f7,0xf8e6291c +0x000002d0,0x326e58d8 +0x000001ae,0x78256d64 +0x00000057,0x403e8c61 +0x0000008a,0x1d7338bc +0x0000013e,0x1e11634f +0x000002de,0xaf0f3eb1 +0x00000199,0xae5c4f27 +0x00000102,0xb0bac110 +0x000003c6,0x485052b8 +0x0000016e,0x5a38a789 +0x00000252,0x54542916 +0x00000002,0x39b180bd +0x00000250,0x230f1e18 +0x000002db,0x1da31dad +0x0000018e,0xbf914fb7 +0x000003c0,0x63f13c95 +0x00000369,0xbc276edd +0x00000288,0xf2e5c78c +0x00000211,0x8be09b81 +0x00000193,0xa9dd3901 +0x000003d6,0x564698ff +0x000000dd,0x654586cd +0x0000013d,0x17f8cdf8 +0x000003dd,0xf55532d7 +0x000000e1,0xd21d0301 +0x00000344,0x8e5c90b7 +0x00000347,0xce2ea106 +0x000002b6,0xe1456d48 +0x00000096,0x1690a90b +0x000002e4,0xbfcf1ee5 +0x00000284,0x9bbd41e7 +0x00000376,0x0a1b239d +0x00000194,0xb8c0425c +0x000001cf,0x5a5f67b8 +0x0000019c,0x8c1365bf +0x00000269,0x7ccc3095 +0x000001fe,0xeff19023 +0x00000028,0x4c6c96e1 +0x00000075,0x6031fafc +0x000000e7,0x6d066ff7 +0x00000213,0x83a12826 +0x000003a1,0xdff42be1 +0x000000d6,0x74721815 +0x00000094,0x5e9436ba +0x000003ec,0x0f91f2dd +0x00000167,0x2f5d9a5c +0x000001bb,0x41b606be +0x00000222,0x455c3f18 +0x0000038c,0x41938755 +0x00000034,0xf3e676b0 +0x00000031,0xb3b105f0 +0x00000043,0xaebbd49c +0x00000079,0x32e9f285 +0x00000130,0x90514309 +0x00000253,0xda947617 +0x000003c7,0x3a454ada +0x00000131,0x6d9bb5fa +0x00000278,0x5c094e3b +0x00000256,0x19969979 +0x00000357,0x8f966e1a +0x00000185,0xba65d2f8 +0x00000263,0x19417509 +0x000000db,0x1eaadeab +0x00000232,0xef46402c +0x00000240,0x215c8ec5 +0x000003b7,0x1d7489dd +0x00000086,0xa9d49edc +0x000000f5,0xb85554dc +0x00000099,0x5d964de7 +0x000003fd,0x6722db64 +0x00000312,0x829a6b53 +0x000001fd,0xf5abdab3 +0x000002c1,0xe5662f67 +0x000003c5,0x2a5f11dd +0x000002ee,0x69769d62 +0x000001d6,0x76155587 +0x0000016f,0xf474b542 +0x000003b2,0x0e03d080 +0x0000006d,0xcce03f59 +0x00000244,0xae332051 +0x00000366,0xa5d17a7b +0x0000022b,0x0bb8f348 +0x00000150,0xd9d81433 +0x000000f9,0x06f2b1ca +0x000000ed,0x10f3cbb5 +0x0000028a,0xb92f3a08 +0x0000038b,0x2e7839fc +0x0000036d,0x8a55891d +0x00000306,0x004434c7 +0x0000027a,0x2c2fe04e +0x0000010a,0x23317858 +0x000003cd,0xb2e47f17 +0x000003a0,0xed86f8e0 +0x0000013a,0xf295d26e +0x00000042,0xbd71be15 +0x0000016b,0xf52fcd29 +0x000003a3,0x80b123f1 +0x000003f4,0xc7b11df1 +0x000003ad,0x93a20006 +0x000003eb,0x52d781ac +0x000002b1,0x6a5e69a3 +0x0000011a,0x0f63315c +0x00000117,0x33659391 +0x00000051,0x1a05f763 +0x0000009b,0xde4ef3eb +0x00000161,0xcd7d1638 +0x000001ec,0xee053da6 +0x000001ca,0x42044d3f +0x000001de,0xcd1d0123 +0x0000028f,0x51f7b994 +0x000003bf,0x108f008e +0x00000032,0x1ec7d1cb +0x00000124,0xab39af81 +0x00000189,0x1678f062 +0x000002cd,0x510dc040 +0x00000331,0x4d33628a +0x00000229,0x944e008a +0x00000341,0x5a7a5372 +0x00000166,0xa92e5b7d +0x00000337,0x174610f0 +0x000002e3,0x38695e89 +0x0000023e,0x96675159 +0x000003da,0x64ab9558 +0x00000097,0x3f86cd0f +0x000000fb,0x107b81b4 +0x000002eb,0x7fd9a144 +0x00000039,0x30a2b470 +0x00000145,0x5f730f7d +0x0000011c,0x966cf066 +0x00000275,0x21d87efc +0x00000202,0xa470e81e +0x00000380,0x7f1c9cfe +0x000002a6,0x83ef8a4a +0x00000392,0x7f080fa5 +0x00000304,0x24a98eb9 +0x000003f8,0xd1a6e7cb +0x00000224,0xc8497258 +0x000003a6,0x6b304020 +0x00000201,0x30733eea +0x0000004e,0xe5996b9a +0x00000383,0x11421776 +0x000000cf,0x35b5d61d +0x00000367,0x630ff9ae +0x000001aa,0xbee7db59 +0x0000021c,0x341d6960 +0x00000283,0x89c0976d +0x00000070,0x599deb7b +0x000003a5,0xb79a547e +0x000003aa,0x990a9aeb +0x00000399,0xd867d08c +0x000001b6,0x9c822c9e +0x00000153,0x05f7124a +0x00000296,0x1014c48d +0x000002e5,0xc948f761 +0x00000190,0x74483f9b +0x00000077,0x805f10a1 +0x00000280,0xb6fdd0fe +0x000000c8,0xb7147ac4 +0x000003e9,0x39daa1ed +0x00000259,0xc3d82fe5 +0x000002e9,0x1781b102 +0x000002aa,0xfb0674fd +0x00000396,0xa2f79ac4 +0x00000353,0x796131a3 +0x00000294,0x3a4f253c +0x00000268,0x8819efeb +0x00000364,0xbbac1595 +0x000003f5,0x6468a5f3 +0x00000103,0xd39a7cf1 +0x00000004,0x2d311b72 +0x000002cf,0xbeda8a15 +0x0000031f,0x92221439 +0x00000374,0xb790d8a9 +0x0000020c,0x45db5760 +0x000003e2,0x6332d00f +0x0000028b,0xcfb7b189 +0x000001c9,0x6380385f +0x0000037c,0x443714c5 +0x00000397,0x3a84c720 +0x00000377,0x6cd3807c +0x000000a7,0xeed1718a +0x00000168,0xb2d6b00f +0x000000ba,0xd4143f6b +0x000001e6,0x70a5cba8 +0x0000003a,0x6db46c23 +0x00000138,0x8b1cbc57 +0x00000327,0x1417447b +0x0000023d,0xf824f9fe +0x000000b0,0xa4afe544 +0x0000024e,0xcdeefb90 +0x00000156,0x9116659d +0x00000095,0xbec1d9ff +0x000003ea,0x418440ad +0x000001e4,0x60b154d8 +0x00000381,0x05bee376 +0x0000023c,0x56313370 +0x000000df,0x2271ed24 +0x00000237,0x5e79fb1a +0x0000034c,0x00a28d23 +0x00000307,0x9a60280a +0x00000287,0x4aebb908 +0x0000036f,0x15b250b7 +0x000003e7,0xc03cd972 +0x0000022f,0x07b8b4f6 +0x00000295,0xba38ebb6 +0x00000210,0xba34a72a +0x000001f9,0x3f3d8c6d +0x000001fa,0xeec12a22 +0x00000151,0xcf258683 +0x0000012c,0x52c63dee +0x00000181,0x7a1b33cb +0x0000014a,0x87b6f8b2 +0x0000008e,0x18c0f3a6 diff --git a/test/benchmarks.yml b/test/benchmarks.yml index 5d926a1..f58b240 100644 --- a/test/benchmarks.yml +++ b/test/benchmarks.yml @@ -1,76 +1,193 @@ { + # sequential access "test_0": { "sdram_module": 'MT48LC16M16', "sdram_data_width": 32, - "bist_length": 4096, - "bist_random": True, + "access_pattern": { + "bist_length": 4096, + "bist_random": False, + } }, "test_1": { "sdram_module": 'MT48LC16M16', "sdram_data_width": 32, - "bist_length": 512, - "bist_random": False, + "access_pattern": { + "bist_length": 512, + "bist_random": False, + } }, "test_2": { "sdram_module": 'MT46V32M16', "sdram_data_width": 32, - "bist_length": 512, - "bist_random": False, + "access_pattern": { + "bist_length": 512, + "bist_random": False, + } }, "test_3": { "sdram_module": 'MT46V32M16', "sdram_data_width": 32, - "bist_length": 2048, - "bist_random": False, + "access_pattern": { + "bist_length": 2048, + "bist_random": False, + } }, "test_4": { "sdram_module": 'MT47H64M16', "sdram_data_width": 32, - "bist_length": 1024, - "bist_random": False, + "access_pattern": { + "bist_length": 1024, + "bist_random": False, + } }, "test_5": { "sdram_module": 'MT47H64M16', "sdram_data_width": 16, - "bist_length": 1024, - "bist_random": False, + "access_pattern": { + "bist_length": 1024, + "bist_random": False, + } }, "test_6": { "sdram_module": 'MT41K128M16', "sdram_data_width": 16, - "bist_length": 1024, - "bist_random": False, + "access_pattern": { + "bist_length": 1024, + "bist_random": False, + } }, "test_7": { "sdram_module": 'MT41K128M16', "sdram_data_width": 32, - "bist_length": 1024, - "bist_random": False, + "access_pattern": { + "bist_length": 1024, + "bist_random": False, + } }, # latency "test_8": { "sdram_module": 'MT48LC16M16', "sdram_data_width": 32, - "bist_length": 1, - "bist_random": False, + "access_pattern": { + "bist_length": 1, + "bist_random": False, + } }, "test_9": { "sdram_module": 'MT46V32M16', "sdram_data_width": 32, - "bist_length": 1, - "bist_random": False, + "access_pattern": { + "bist_length": 1, + "bist_random": False, + } }, "test_10": { "sdram_module": 'MT47H64M16', "sdram_data_width": 32, - "bist_length": 1, - "bist_random": False, + "access_pattern": { + "bist_length": 1, + "bist_random": False, + } }, "test_11": { "sdram_module": 'MT41K128M16', "sdram_data_width": 16, - "bist_length": 1, - "bist_random": False, + "access_pattern": { + "bist_length": 1, + "bist_random": False, + } + }, + + # random access + "test_12": { + "sdram_module": 'MT48LC16M16', + "sdram_data_width": 32, + "access_pattern": { + "bist_length": 1024, + "bist_random": True, + } + }, + "test_13": { + "sdram_module": 'MT46V32M16', + "sdram_data_width": 32, + "access_pattern": { + "bist_length": 1024, + "bist_random": True, + } + }, + "test_14": { + "sdram_module": 'MT47H64M16', + "sdram_data_width": 32, + "access_pattern": { + "bist_length": 1024, + "bist_random": True, + } + }, + "test_15": { + "sdram_module": 'MT41K128M16', + "sdram_data_width": 16, + "access_pattern": { + "bist_length": 1024, + "bist_random": True, + } + }, + + # custom access pattern + "test_16": { + "sdram_module": 'MT48LC16M16', + "sdram_data_width": 32, + "access_pattern": { + "pattern_file": "access_pattern.csv" + } + }, + "test_17": { + "sdram_module": 'MT48LC16M16', + "sdram_data_width": 32, + "access_pattern": { + "pattern_file": "access_pattern.csv", + } + }, + "test_18": { + "sdram_module": 'MT46V32M16', + "sdram_data_width": 32, + "access_pattern": { + "pattern_file": "access_pattern.csv", + } + }, + "test_19": { + "sdram_module": 'MT46V32M16', + "sdram_data_width": 32, + "access_pattern": { + "pattern_file": "access_pattern.csv", + } + }, + "test_20": { + "sdram_module": 'MT47H64M16', + "sdram_data_width": 32, + "access_pattern": { + "pattern_file": "access_pattern.csv", + } + }, + "test_21": { + "sdram_module": 'MT47H64M16', + "sdram_data_width": 16, + "access_pattern": { + "pattern_file": "access_pattern.csv", + } + }, + "test_22": { + "sdram_module": 'MT41K128M16', + "sdram_data_width": 16, + "access_pattern": { + "pattern_file": "access_pattern.csv", + } + }, + "test_23": { + "sdram_module": 'MT41K128M16', + "sdram_data_width": 32, + "access_pattern": { + "pattern_file": "access_pattern.csv", + } }, } From 62a5473ecdf0d9327fa9315bdb1ef72f40a02cbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C4=99drzej=20Boczar?= Date: Thu, 6 Feb 2020 10:49:44 +0100 Subject: [PATCH 7/7] test: update script for generating benchmark configurations --- test/gen_config.py | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/test/gen_config.py b/test/gen_config.py index 298e774..9bb51cf 100755 --- a/test/gen_config.py +++ b/test/gen_config.py @@ -42,26 +42,43 @@ modules = [ data_widths = [32] bist_lengths = [1, 1024, 8192] bist_randoms = [False] +access_patterns = ['access_pattern.csv'] def main(): parser = argparse.ArgumentParser(description='Generate configuration for all possible argument combinations.', formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('--sdram-modules', nargs='+', default=modules, help='--sdram-module options') - parser.add_argument('--sdram-data-widths', nargs='+', default=data_widths, help='--sdram-data-width options') - parser.add_argument('--bist-lengths', nargs='+', default=bist_lengths, help='--bist-length options') - parser.add_argument('--bist-randoms', nargs='+', default=bist_randoms, help='--bist-random options') - parser.add_argument('--name-format', default='test_%d', help='Name format for i-th test') + parser.add_argument('--sdram-modules', nargs='+', default=modules, help='--sdram-module options') + parser.add_argument('--sdram-data-widths', nargs='+', default=data_widths, help='--sdram-data-width options') + parser.add_argument('--bist-lengths', nargs='+', default=bist_lengths, help='--bist-length options') + parser.add_argument('--bist-randoms', nargs='+', default=bist_randoms, help='--bist-random options') + parser.add_argument('--access-patterns', nargs='+', default=access_patterns, help='--access-pattern options') + parser.add_argument('--name-format', default='test_%d', help='Name format for i-th test') args = parser.parse_args() - product = itertools.product(args.sdram_modules, args.sdram_data_widths, args.bist_lengths, args.bist_randoms) + bist_product = itertools.product(args.sdram_modules, args.sdram_data_widths, args.bist_lengths, args.bist_randoms) + pattern_product = itertools.product(args.sdram_modules, args.sdram_data_widths, args.access_patterns) + + i = 0 configurations = {} - for i, (module, data_width, bist_length, bist_random) in enumerate(product): + for module, data_width, bist_length, bist_random in bist_product: configurations[args.name_format % i] = { 'sdram_module': module, 'sdram_data_width': data_width, - 'bist_length': bist_length, - 'bist_random': bist_random, + 'access_pattern': { + 'bist_length': bist_length, + 'bist_random': bist_random, + } } + i += 1 + for module, data_width, access_pattern in pattern_product: + configurations[args.name_format % i] = { + 'sdram_module': module, + 'sdram_data_width': data_width, + 'access_pattern': { + 'pattern_file': access_pattern, + } + } + i += 1 json_str = json.dumps(configurations, indent=4) print(json_str)