diff --git a/test/run_benchmarks.py b/test/run_benchmarks.py index 5fb5ce5..3c4eaae 100755 --- a/test/run_benchmarks.py +++ b/test/run_benchmarks.py @@ -12,6 +12,7 @@ import re import sys import json import argparse +import datetime import subprocess from collections import defaultdict, namedtuple @@ -211,6 +212,20 @@ def efficiency_fmt(eff): return '{:.1f} %'.format(eff * 100) +def get_git_file_path(filename): + cmd = ['git', 'ls-files', '--full-name', filename] + proc = subprocess.run(cmd, stdout=subprocess.PIPE, cwd=os.path.dirname(__file__)) + return proc.stdout.decode().strip() if proc.returncode == 0 else '' + + +def get_git_revision_hash(short=False): + short = ['--short'] if short else [] + # cmd = ['git', 'rev-parse', *short, 'HEAD'] + cmd = ['git', 'rev-parse', *short, 'origin/master'] + proc = subprocess.run(cmd, stdout=subprocess.PIPE, cwd=os.path.dirname(__file__)) + return proc.stdout.decode().strip() if proc.returncode == 0 else '' + + class ResultsSummary: def __init__(self, run_data, plots_dir='plots'): self.plots_dir = plots_dir @@ -317,10 +332,38 @@ class ResultsSummary: self.print_df(title, df) print() - def groupped_results(self, formatted=True): + def html_summary(self, output_dir): + import jinja2 + + tables = {} + names = {} + for title, df in self.groupped_results(): + table_id = title.lower().replace(' ', '_') + + tables[table_id] = df.to_html(table_id=table_id, border=0) + names[table_id] = title + + template_dir = os.path.join(os.path.dirname(__file__), 'summary') + env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_dir)) + template = env.get_template('summary.html.jinja2') + + os.makedirs(output_dir, exist_ok=True) + with open(os.path.join(output_dir, 'summary.html'), 'w') as f: + f.write(template.render( + title='LiteDRAM benchmarks summary', + tables=tables, + names=names, + script_path=get_git_file_path(__file__), + revision=get_git_revision_hash(), + revision_short=get_git_revision_hash(short=True), + generation_date=datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + )) + + def groupped_results(self, formatters=None): df = self.df - formatters = self.text_formatters if formatted else {} + if formatters is None: + formatters = self.text_formatters common_columns = ['name', 'sdram_module', 'sdram_data_width', 'bist_alternating', 'num_generators', 'num_checkers'] latency_columns = ['write_latency', 'read_latency'] @@ -333,17 +376,17 @@ class ResultsSummary: ) yield 'Custom access pattern', self.get_summary( mask=(df['is_latency'] == False) & (~pd.isna(df['pattern_file'])), - columns=common_columns + performance_columns + ['length', 'pattern_file'], + columns=common_columns + ['length', 'pattern_file'] + performance_columns, column_formatting=formatters, ), yield 'Sequential access pattern', self.get_summary( mask=(df['is_latency'] == False) & (pd.isna(df['pattern_file'])) & (df['bist_random'] == False), - columns=common_columns + performance_columns + ['bist_length'], # could be length + columns=common_columns + ['bist_length'] + performance_columns, # could be length column_formatting=formatters, ), yield 'Random access pattern', self.get_summary( mask=(df['is_latency'] == False) & (pd.isna(df['pattern_file'])) & (df['bist_random'] == True), - columns=common_columns + performance_columns + ['bist_length'], + columns=common_columns + ['bist_length'] + performance_columns, column_formatting=formatters, ), @@ -352,7 +395,7 @@ class ResultsSummary: import matplotlib.pyplot as plt plt.style.use(theme) - for title, df in self.groupped_results(formatted=False): + for title, df in self.groupped_results(formatters={}): for column in self.plot_xticks_formatters.keys(): if column not in df.columns or df[column].empty: continue @@ -520,6 +563,8 @@ def main(argv=None): parser.add_argument('--names', nargs='*', help='Limit benchmarks to given names') parser.add_argument('--regex', help='Limit benchmarks to names matching the regex') parser.add_argument('--not-regex', help='Limit benchmarks to names not matching the regex') + parser.add_argument('--html', action='store_true', help='Generate HTML summary') + parser.add_argument('--html-output-dir', default='html', help='Output directory for generated HTML') parser.add_argument('--plot', action='store_true', help='Generate plots with results summary') parser.add_argument('--plot-format', default='png', help='Specify plots file format (default=png)') parser.add_argument('--plot-backend', default='Agg', help='Optionally specify matplotlib GUI backend') @@ -574,6 +619,8 @@ def main(argv=None): summary = ResultsSummary(run_data) summary.text_summary() summary.failures_summary() + if args.html: + summary.html_summary(args.html_output_dir) if args.plot: summary.plot_summary( plots_dir=args.plot_output_dir, diff --git a/test/summary/summary.css b/test/summary/summary.css new file mode 100644 index 0000000..400225f --- /dev/null +++ b/test/summary/summary.css @@ -0,0 +1,100 @@ +body { + font-family: 'Roboto', sans-serif; +} + +footer { + text-align: center; + font-size: 10px; + padding: 20px; +} + +.dataTables_filter { + margin: 15px 50px 10px 50px; +} + +.dataTables_filter input { + width: 400px; +} + +.table-select { + width: 100%; + margin: 0 auto; +} + +.table-select ul { + list-style-type: none; + margin: 0; + padding: 0; + overflow: hidden; +} + +.table-select li { + float: left; +} + +.table-select li a { + display: block; + padding: 10px 0px; + margin: 0px 20px; + text-align: center; + text-decoration: none; + font-size: 18px; + color:inherit; + border-bottom: 1px solid; + border-color: #ccc; + transition: 0.2s; +} + +.table-select li a:hover { + border-color: #111; +} + +/* did not work, .focus() couldn't turn it on */ +/* .table-select li a:focus { */ +/* border-color: #222; */ +/* } */ +.table-select-active { + border-color: #111 !important; +} + +.tables-wrapper { + width: 100%; + margin: auto; +} + +.loading { + z-index: 999; + position: absolute; + top: 50%; + left: 50%; + margin-right: -50%; + transform: translate(-50%, -50%); +} + +/* Loading animation */ +.lds-dual-ring { + display: inline-block; + width: 80px; + height: 80px; +} +.lds-dual-ring:after { + content: " "; + display: block; + width: 64px; + height: 64px; + margin: 8px; + border-radius: 50%; + border: 6px solid #fff; + border-color: #aaa transparent #aaa transparent; + animation: lds-dual-ring 1.2s linear infinite; +} +@keyframes lds-dual-ring { + 0% { + transform: rotate(0deg); + } + 100% { + transform: rotate(360deg); + } +} + +/* vim: set ts=2 sw=2: */ diff --git a/test/summary/summary.html.jinja2 b/test/summary/summary.html.jinja2 new file mode 100644 index 0000000..4227fee --- /dev/null +++ b/test/summary/summary.html.jinja2 @@ -0,0 +1,162 @@ + + +
+ + +