diff --git a/test/run_benchmarks.py b/test/run_benchmarks.py index 3c4eaae..ad01ee5 100755 --- a/test/run_benchmarks.py +++ b/test/run_benchmarks.py @@ -230,9 +230,12 @@ class ResultsSummary: def __init__(self, run_data, plots_dir='plots'): self.plots_dir = plots_dir - # filter out failures - self.failed_configs = [data.config for data in run_data if data.result is None] - run_data = [data for data in run_data if data.result is not None] + # because .sdram_controller_data_width may fail for unimplemented modules + def except_none(func): + try: + return func() + except: + return None # gather results into tabular data column_mappings = { @@ -246,14 +249,14 @@ class ResultsSummary: 'bist_random': lambda d: getattr(d.config.access_pattern, 'bist_random', None), 'pattern_file': lambda d: getattr(d.config.access_pattern, 'pattern_file', None), 'length': lambda d: d.config.length, - 'generator_ticks': lambda d: d.result.generator_ticks, - 'checker_errors': lambda d: d.result.checker_errors, - 'checker_ticks': lambda d: d.result.checker_ticks, - 'ctrl_data_width': lambda d: d.config.sdram_controller_data_width, + 'generator_ticks': lambda d: getattr(d.result, 'generator_ticks', None), # None means benchmark failure + 'checker_errors': lambda d: getattr(d.result, 'checker_errors', None), + 'checker_ticks': lambda d: getattr(d.result, 'checker_ticks', None), + 'ctrl_data_width': lambda d: except_none(lambda: d.config.sdram_controller_data_width), 'clk_freq': lambda d: d.config.sdram_clk_freq, } columns = {name: [mapping(data) for data in run_data] for name, mapping, in column_mappings.items()} - self.df = df = pd.DataFrame(columns) + self._df = df = pd.DataFrame(columns) # replace None with NaN df.fillna(value=np.nan, inplace=True) @@ -300,6 +303,16 @@ class ResultsSummary: 'read_latency': ScalarFormatter(), } + def df(self, ok=True, failures=False): + is_failure = lambda df: pd.isna(df['generator_ticks']) | pd.isna(df['checker_ticks']) | pd.isna(df['checker_errors']) + df = self._df + if not ok: # remove ok + is_ok = ~is_failure(df) + df = df[~is_ok] + if not failures: # remove failures + df = df[~is_failure(df)] + return df + def header(self, text): return '===> {}'.format(text) @@ -309,9 +322,9 @@ class ResultsSummary: print(self.header(title + ':')) print(df) - def get_summary(self, mask=None, columns=None, column_formatting=None, sort_kwargs=None): + def get_summary(self, df, mask=None, columns=None, column_formatting=None, sort_kwargs=None): # work on a copy - df = self.df.copy() + df = df.copy() if sort_kwargs is not None: df = df.sort_values(**sort_kwargs) @@ -360,7 +373,7 @@ class ResultsSummary: )) def groupped_results(self, formatters=None): - df = self.df + df = self.df() if formatters is None: formatters = self.text_formatters @@ -368,27 +381,32 @@ class ResultsSummary: common_columns = ['name', 'sdram_module', 'sdram_data_width', 'bist_alternating', 'num_generators', 'num_checkers'] latency_columns = ['write_latency', 'read_latency'] performance_columns = ['write_bandwidth', 'read_bandwidth', 'write_efficiency', 'read_efficiency'] + failure_columns = ['bist_length', 'bist_random', 'pattern_file', 'length', 'generator_ticks', 'checker_errors', 'checker_ticks'] - yield 'Latency', self.get_summary( + yield 'Latency', self.get_summary(df, mask=df['is_latency'] == True, columns=common_columns + latency_columns, column_formatting=formatters, ) - yield 'Custom access pattern', self.get_summary( + yield 'Custom access pattern', self.get_summary(df, mask=(df['is_latency'] == False) & (~pd.isna(df['pattern_file'])), columns=common_columns + ['length', 'pattern_file'] + performance_columns, column_formatting=formatters, ), - yield 'Sequential access pattern', self.get_summary( + yield 'Sequential access pattern', self.get_summary(df, mask=(df['is_latency'] == False) & (pd.isna(df['pattern_file'])) & (df['bist_random'] == False), columns=common_columns + ['bist_length'] + performance_columns, # could be length column_formatting=formatters, ), - yield 'Random access pattern', self.get_summary( + yield 'Random access pattern', self.get_summary(df, mask=(df['is_latency'] == False) & (pd.isna(df['pattern_file'])) & (df['bist_random'] == True), columns=common_columns + ['bist_length'] + performance_columns, column_formatting=formatters, ), + yield 'Failures', self.get_summary(self.df(ok=False, failures=True), + columns=common_columns + failure_columns, + column_formatting=None, + ), def plot_summary(self, plots_dir='plots', backend='Agg', theme='default', save_format='png', **savefig_kw): matplotlib.use(backend) @@ -441,14 +459,6 @@ class ResultsSummary: return axis - def failures_summary(self): - if len(self.failed_configs) > 0: - print(self.header('Failures:')) - for config in self.failed_configs: - print(' {}: {}'.format(config.name, config.as_args())) - else: - print(self.header('All benchmarks ok.')) - # Run ---------------------------------------------------------------------------------------------- class RunCache(list): @@ -618,7 +628,6 @@ def main(argv=None): if _summary: summary = ResultsSummary(run_data) summary.text_summary() - summary.failures_summary() if args.html: summary.html_summary(args.html_output_dir) if args.plot: