From 1a517a308d019379a3c40347d424fc0e1feb06af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C4=99drzej=20Boczar?= <jboczar@antmicro.com>
Date: Fri, 31 Jan 2020 14:16:39 +0100
Subject: [PATCH] test: cache benchmark results to be able to produce multiple
 summaries

---
 test/run_benchmarks.py | 43 +++++++++++++++++++++++++++++++++++-------
 1 file changed, 36 insertions(+), 7 deletions(-)

diff --git a/test/run_benchmarks.py b/test/run_benchmarks.py
index 81afe84..bfc6d04 100755
--- a/test/run_benchmarks.py
+++ b/test/run_benchmarks.py
@@ -40,7 +40,7 @@ def human_readable(value):
 class BenchmarkConfiguration(Settings):
     def __init__(self, sdram_module, sdram_data_width, bist_length, bist_random):
         self.set_attributes(locals())
-        self._settings = {k: v for k, v in locals().items() if v != self}
+        self._settings = {k: v for k, v in locals().items() if k != 'self'}
 
     def as_args(self):
         args = []
@@ -53,6 +53,12 @@ class BenchmarkConfiguration(Settings):
                 args.extend([arg_string, str(value)])
         return args
 
+    def __eq__(self, other):
+        if not isinstance(other, BenchmarkConfiguration):
+            return NotImplemented
+        return all((getattr(self, setting) == getattr(other, setting)
+                    for setting in self._settings.keys()))
+
     @classmethod
     def load_yaml(cls, yaml_file):
         with open(yaml_file) as f:
@@ -235,6 +241,16 @@ def run_benchmark(cmd_args):
     return proc.stdout
 
 
+def run_benchmarks(configurations):
+    benchmarks = []
+    for name, config in configurations.items():
+        cmd_args = config.as_args()
+        print('{}: {}'.format(name, ' '.join(cmd_args)))
+        output = run_benchmark(cmd_args)
+        benchmarks.append((config, output))
+    return benchmarks
+
+
 def main(argv=None):
     parser = argparse.ArgumentParser(
         description='Run LiteDRAM benchmarks and collect the results.')
@@ -248,6 +264,7 @@ def main(argv=None):
     parser.add_argument('--plot-transparent', action='store_true', help='Use transparent background when saving plots')
     parser.add_argument('--plot-output-dir',  default='plots',     help='Specify where to save the plots')
     parser.add_argument('--plot-theme',       default='default',   help='Use different matplotlib theme')
+    parser.add_argument('--output-cache',                          help='Cache benchmark outputs to given file if it exists, else load them from the file without running benchmarks. This allows to run the script multiple times to produce different outputs from the same run')
     args = parser.parse_args(argv)
 
     # load and filter configurations
@@ -262,14 +279,26 @@ def main(argv=None):
     for f in filters:
         configurations = dict(filter(f, configurations.items()))
 
-    # run the benchmarks
-    results = []
-    for name, config in configurations.items():
-        cmd_args = config.as_args()
-        print('{}: {}'.format(name, ' '.join(cmd_args)))
-        results.append(BenchmarkResult(config, run_benchmark(cmd_args)))
+    cache_exists = args.output_cache and os.path.isfile(args.output_cache)
+
+    # load outputs from cache if it exsits
+    if args.output_cache and cache_exists:
+        import pickle
+        with open(args.output_cache, 'rb') as f:
+            cached_benchmarks = pickle.load(f)
+        # take only those that match configurations
+        benchmarks = [(c, o) for c, o in cached_benchmarks if c in configurations.values()]
+    else:  # run all the benchmarks normally
+        benchmarks = run_benchmarks(configurations)
+
+    # store outputs in cache
+    if args.output_cache and not cache_exists:
+        import pickle
+        with open(args.output_cache, 'wb') as f:
+            pickle.dump(benchmarks, f, pickle.HIGHEST_PROTOCOL)
 
     # display the summary
+    results = [BenchmarkResult(config, output) for config, output in benchmarks]
     summary = ResultsSummary(results)
     summary.print()
     if args.plot: