From a584923f1cd106a81a0df62b4b3eb51ac5634fd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C4=99drzej=20Boczar?= Date: Mon, 3 Feb 2020 12:13:16 +0100 Subject: [PATCH] test: use JSON instead of pickle for storing benchmarks cache --- test/run_benchmarks.py | 88 +++++++++++++++++++++++++----------------- 1 file changed, 52 insertions(+), 36 deletions(-) diff --git a/test/run_benchmarks.py b/test/run_benchmarks.py index 2652634..3d5178c 100755 --- a/test/run_benchmarks.py +++ b/test/run_benchmarks.py @@ -6,6 +6,7 @@ import os import re import sys +import json import argparse import subprocess from collections import defaultdict, namedtuple @@ -72,27 +73,11 @@ class BenchmarkConfiguration(Settings): class BenchmarkResult: def __init__(self, config, output): self.config = config + self._output = output self.parse_output(output) # instantiate the benchmarked soc to check its configuration self.benchmark_soc = LiteDRAMBenchmarkSoC(**self.config._settings) - def parse_output(self, output): - bist_pattern = r'{stage}\s+{var}:\s+{value}' - - def find(stage, var): - pattern = bist_pattern.format( - stage=stage, - var=var, - value=ng('value', '[0-9]+'), - ) - result = re.search(pattern, output) - assert result is not None, 'Could not find pattern in output: %s, %s' % (pattern, output) - return int(result.group('value')) - - self.generator_ticks = find('BIST-GENERATOR', 'ticks') - self.checker_errors = find('BIST-CHECKER', 'errors') - self.checker_ticks = find('BIST-CHECKER', 'ticks') - def cmd_count(self): data_width = self.benchmark_soc.sdram.controller.interface.data_width return self.config.bist_length / (data_width // 8) @@ -113,6 +98,41 @@ class BenchmarkResult: def read_efficiency(self): return self.cmd_count() / self.checker_ticks + def parse_output(self, output): + bist_pattern = r'{stage}\s+{var}:\s+{value}' + + def find(stage, var): + pattern = bist_pattern.format( + stage=stage, + var=var, + value=ng('value', '[0-9]+'), + ) + result = re.search(pattern, output) + assert result is not None, 'Could not find pattern in output: %s, %s' % (pattern, output) + return int(result.group('value')) + + self.generator_ticks = find('BIST-GENERATOR', 'ticks') + self.checker_errors = find('BIST-CHECKER', 'errors') + self.checker_ticks = find('BIST-CHECKER', 'ticks') + + @classmethod + def dump_results_json(cls, results, file): + """Save multiple results in a JSON file. + + Only configurations and outpits are saved, as they can be used to reconstruct BenchmarkResult. + """ + # simply use config._settings as it defines the BenchmarkConfiguration + results_raw = [(r.config._settings, r._output) for r in results] + with open(file, 'w') as f: + json.dump(results_raw, f) + + @classmethod + def load_results_json(cls, file): + """Load results from a JSON file.""" + with open(file, 'r') as f: + results_raw = json.load(f) + return [cls(BenchmarkConfiguration(**settings), output) for (settings, output) in results_raw] + # Results summary ---------------------------------------------------------------------------------- class ResultsSummary: @@ -243,22 +263,19 @@ def run_benchmark(cmd_args): def run_benchmarks(configurations): - benchmarks = [] + results = [] for name, config in configurations.items(): cmd_args = config.as_args() print('{}: {}'.format(name, ' '.join(cmd_args))) output = run_benchmark(cmd_args) - - # return raw outputs, not BenchmarkResult so that we can store them in a file - benchmarks.append((config, output)) - # exit if checker had any read error result = BenchmarkResult(config, output) if result.checker_errors != 0: print('Error during benchmark "{}": checker_errors = {}'.format( name, result.checker_errors), file=sys.stderr) sys.exit(1) - return benchmarks + results.append(result) + return results def main(argv=None): @@ -274,7 +291,11 @@ def main(argv=None): parser.add_argument('--plot-transparent', action='store_true', help='Use transparent background when saving plots') parser.add_argument('--plot-output-dir', default='plots', help='Specify where to save the plots') parser.add_argument('--plot-theme', default='default', help='Use different matplotlib theme') - parser.add_argument('--output-cache', help='Cache benchmark outputs to given file if it exists, else load them from the file without running benchmarks. This allows to run the script multiple times to produce different outputs from the same run') + parser.add_argument('--results-cache', help="""Use given JSON file as results cache. If the file exists, + it will be loaded instead of running actual benchmarks, + else benchmarks will be run normally, and then saved + to the given file. This allows to easily rerun the script + to generate different summary without having to rerun benchmarks.""") args = parser.parse_args(argv) # load and filter configurations @@ -289,26 +310,21 @@ def main(argv=None): for f in filters: configurations = dict(filter(f, configurations.items())) - cache_exists = args.output_cache and os.path.isfile(args.output_cache) + cache_exists = args.results_cache and os.path.isfile(args.results_cache) # load outputs from cache if it exsits - if args.output_cache and cache_exists: - import pickle - with open(args.output_cache, 'rb') as f: - cached_benchmarks = pickle.load(f) + if args.results_cache and cache_exists: + cached_results = BenchmarkResult.load_results_json(args.results_cache) # take only those that match configurations - benchmarks = [(c, o) for c, o in cached_benchmarks if c in configurations.values()] + results = [r for r in cached_results if r.config in configurations.values()] else: # run all the benchmarks normally - benchmarks = run_benchmarks(configurations) + results = run_benchmarks(configurations) # store outputs in cache - if args.output_cache and not cache_exists: - import pickle - with open(args.output_cache, 'wb') as f: - pickle.dump(benchmarks, f, pickle.HIGHEST_PROTOCOL) + if args.results_cache and not cache_exists: + BenchmarkResult.dump_results_json(results, args.results_cache) # display the summary - results = [BenchmarkResult(config, output) for config, output in benchmarks] summary = ResultsSummary(results) summary.print() if args.plot: