test: add generation of html benchmarks summary
This commit is contained in:
Jędrzej Boczar
parent
9083822a74
commit
bba49f2df8
|
|
@ -12,6 +12,7 @@ import re
|
|||
import sys
|
||||
import json
|
||||
import argparse
|
||||
import datetime
|
||||
import subprocess
|
||||
from collections import defaultdict, namedtuple
|
||||
|
||||
|
|
@ -211,6 +212,20 @@ def efficiency_fmt(eff):
|
|||
return '{:.1f} %'.format(eff * 100)
|
||||
|
||||
|
||||
def get_git_file_path(filename):
|
||||
cmd = ['git', 'ls-files', '--full-name', filename]
|
||||
proc = subprocess.run(cmd, stdout=subprocess.PIPE, cwd=os.path.dirname(__file__))
|
||||
return proc.stdout.decode().strip() if proc.returncode == 0 else ''
|
||||
|
||||
|
||||
def get_git_revision_hash(short=False):
|
||||
short = ['--short'] if short else []
|
||||
# cmd = ['git', 'rev-parse', *short, 'HEAD']
|
||||
cmd = ['git', 'rev-parse', *short, 'origin/master']
|
||||
proc = subprocess.run(cmd, stdout=subprocess.PIPE, cwd=os.path.dirname(__file__))
|
||||
return proc.stdout.decode().strip() if proc.returncode == 0 else ''
|
||||
|
||||
|
||||
class ResultsSummary:
|
||||
def __init__(self, run_data, plots_dir='plots'):
|
||||
self.plots_dir = plots_dir
|
||||
|
|
@ -317,10 +332,38 @@ class ResultsSummary:
|
|||
self.print_df(title, df)
|
||||
print()
|
||||
|
||||
def groupped_results(self, formatted=True):
|
||||
def html_summary(self, output_dir):
|
||||
import jinja2
|
||||
|
||||
tables = {}
|
||||
names = {}
|
||||
for title, df in self.groupped_results():
|
||||
table_id = title.lower().replace(' ', '_')
|
||||
|
||||
tables[table_id] = df.to_html(table_id=table_id, border=0)
|
||||
names[table_id] = title
|
||||
|
||||
template_dir = os.path.join(os.path.dirname(__file__), 'summary')
|
||||
env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_dir))
|
||||
template = env.get_template('summary.html.jinja2')
|
||||
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
with open(os.path.join(output_dir, 'summary.html'), 'w') as f:
|
||||
f.write(template.render(
|
||||
title='LiteDRAM benchmarks summary',
|
||||
tables=tables,
|
||||
names=names,
|
||||
script_path=get_git_file_path(__file__),
|
||||
revision=get_git_revision_hash(),
|
||||
revision_short=get_git_revision_hash(short=True),
|
||||
generation_date=datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||
))
|
||||
|
||||
def groupped_results(self, formatters=None):
|
||||
df = self.df
|
||||
|
||||
formatters = self.text_formatters if formatted else {}
|
||||
if formatters is None:
|
||||
formatters = self.text_formatters
|
||||
|
||||
common_columns = ['name', 'sdram_module', 'sdram_data_width', 'bist_alternating', 'num_generators', 'num_checkers']
|
||||
latency_columns = ['write_latency', 'read_latency']
|
||||
|
|
@ -333,17 +376,17 @@ class ResultsSummary:
|
|||
)
|
||||
yield 'Custom access pattern', self.get_summary(
|
||||
mask=(df['is_latency'] == False) & (~pd.isna(df['pattern_file'])),
|
||||
columns=common_columns + performance_columns + ['length', 'pattern_file'],
|
||||
columns=common_columns + ['length', 'pattern_file'] + performance_columns,
|
||||
column_formatting=formatters,
|
||||
),
|
||||
yield 'Sequential access pattern', self.get_summary(
|
||||
mask=(df['is_latency'] == False) & (pd.isna(df['pattern_file'])) & (df['bist_random'] == False),
|
||||
columns=common_columns + performance_columns + ['bist_length'], # could be length
|
||||
columns=common_columns + ['bist_length'] + performance_columns, # could be length
|
||||
column_formatting=formatters,
|
||||
),
|
||||
yield 'Random access pattern', self.get_summary(
|
||||
mask=(df['is_latency'] == False) & (pd.isna(df['pattern_file'])) & (df['bist_random'] == True),
|
||||
columns=common_columns + performance_columns + ['bist_length'],
|
||||
columns=common_columns + ['bist_length'] + performance_columns,
|
||||
column_formatting=formatters,
|
||||
),
|
||||
|
||||
|
|
@ -352,7 +395,7 @@ class ResultsSummary:
|
|||
import matplotlib.pyplot as plt
|
||||
plt.style.use(theme)
|
||||
|
||||
for title, df in self.groupped_results(formatted=False):
|
||||
for title, df in self.groupped_results(formatters={}):
|
||||
for column in self.plot_xticks_formatters.keys():
|
||||
if column not in df.columns or df[column].empty:
|
||||
continue
|
||||
|
|
@ -520,6 +563,8 @@ def main(argv=None):
|
|||
parser.add_argument('--names', nargs='*', help='Limit benchmarks to given names')
|
||||
parser.add_argument('--regex', help='Limit benchmarks to names matching the regex')
|
||||
parser.add_argument('--not-regex', help='Limit benchmarks to names not matching the regex')
|
||||
parser.add_argument('--html', action='store_true', help='Generate HTML summary')
|
||||
parser.add_argument('--html-output-dir', default='html', help='Output directory for generated HTML')
|
||||
parser.add_argument('--plot', action='store_true', help='Generate plots with results summary')
|
||||
parser.add_argument('--plot-format', default='png', help='Specify plots file format (default=png)')
|
||||
parser.add_argument('--plot-backend', default='Agg', help='Optionally specify matplotlib GUI backend')
|
||||
|
|
@ -574,6 +619,8 @@ def main(argv=None):
|
|||
summary = ResultsSummary(run_data)
|
||||
summary.text_summary()
|
||||
summary.failures_summary()
|
||||
if args.html:
|
||||
summary.html_summary(args.html_output_dir)
|
||||
if args.plot:
|
||||
summary.plot_summary(
|
||||
plots_dir=args.plot_output_dir,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,100 @@
|
|||
body {
|
||||
font-family: 'Roboto', sans-serif;
|
||||
}
|
||||
|
||||
footer {
|
||||
text-align: center;
|
||||
font-size: 10px;
|
||||
padding: 20px;
|
||||
}
|
||||
|
||||
.dataTables_filter {
|
||||
margin: 15px 50px 10px 50px;
|
||||
}
|
||||
|
||||
.dataTables_filter input {
|
||||
width: 400px;
|
||||
}
|
||||
|
||||
.table-select {
|
||||
width: 100%;
|
||||
margin: 0 auto;
|
||||
}
|
||||
|
||||
.table-select ul {
|
||||
list-style-type: none;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.table-select li {
|
||||
float: left;
|
||||
}
|
||||
|
||||
.table-select li a {
|
||||
display: block;
|
||||
padding: 10px 0px;
|
||||
margin: 0px 20px;
|
||||
text-align: center;
|
||||
text-decoration: none;
|
||||
font-size: 18px;
|
||||
color:inherit;
|
||||
border-bottom: 1px solid;
|
||||
border-color: #ccc;
|
||||
transition: 0.2s;
|
||||
}
|
||||
|
||||
.table-select li a:hover {
|
||||
border-color: #111;
|
||||
}
|
||||
|
||||
/* did not work, .focus() couldn't turn it on */
|
||||
/* .table-select li a:focus { */
|
||||
/* border-color: #222; */
|
||||
/* } */
|
||||
.table-select-active {
|
||||
border-color: #111 !important;
|
||||
}
|
||||
|
||||
.tables-wrapper {
|
||||
width: 100%;
|
||||
margin: auto;
|
||||
}
|
||||
|
||||
.loading {
|
||||
z-index: 999;
|
||||
position: absolute;
|
||||
top: 50%;
|
||||
left: 50%;
|
||||
margin-right: -50%;
|
||||
transform: translate(-50%, -50%);
|
||||
}
|
||||
|
||||
/* Loading animation */
|
||||
.lds-dual-ring {
|
||||
display: inline-block;
|
||||
width: 80px;
|
||||
height: 80px;
|
||||
}
|
||||
.lds-dual-ring:after {
|
||||
content: " ";
|
||||
display: block;
|
||||
width: 64px;
|
||||
height: 64px;
|
||||
margin: 8px;
|
||||
border-radius: 50%;
|
||||
border: 6px solid #fff;
|
||||
border-color: #aaa transparent #aaa transparent;
|
||||
animation: lds-dual-ring 1.2s linear infinite;
|
||||
}
|
||||
@keyframes lds-dual-ring {
|
||||
0% {
|
||||
transform: rotate(0deg);
|
||||
}
|
||||
100% {
|
||||
transform: rotate(360deg);
|
||||
}
|
||||
}
|
||||
|
||||
/* vim: set ts=2 sw=2: */
|
||||
|
|
@ -0,0 +1,162 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>{{ title }}</title>
|
||||
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.4.1/jquery.min.js"></script>
|
||||
|
||||
<script type="text/javascript" charset="utf8" src="https://cdn.datatables.net/1.10.19/js/jquery.dataTables.min.js"></script>
|
||||
<link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/1.10.19/css/jquery.dataTables.css">
|
||||
<script type="text/javascript" charset="utf8" src="https://cdn.datatables.net/fixedheader/3.1.5/js/dataTables.fixedHeader.min.js"></script>
|
||||
<link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/fixedheader/3.1.5/css/fixedHeader.dataTables.min.css">
|
||||
|
||||
<style type="text/css" media="all">
|
||||
{% include 'summary.css' %}
|
||||
{# Calculate size of table selection elements so that they take up whole space #}
|
||||
.table-select li {
|
||||
width: calc(100% / {{ tables.keys() | length }});
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
{# Loading symbol that gets hidden after initialisation of all tables #}
|
||||
<div class="loading lds-dual-ring"></div>
|
||||
|
||||
{# Bar for selecting the current data table #}
|
||||
<div class="table-select">
|
||||
<ul>
|
||||
{% for id, name in names.items() %}
|
||||
<li id='{{ id }}-button'><a href="#">{{ name }}</a></li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
{# <hr/> #}
|
||||
</div>
|
||||
|
||||
{# Display of the current data table #}
|
||||
<div class="tables-wrapper">
|
||||
{% for id, table in tables.items() %}
|
||||
{# Hide tables not to show them before all DataTables are loaded #}
|
||||
<div id="{{ id }}-div" style="display: none;">
|
||||
{{ table }}
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</body>
|
||||
|
||||
<footer id="footer">
|
||||
<a href="https://github.com/enjoy-digital/litedram">LiteDRAM</a> is a part of <a href="https://github.com/enjoy-digital/litex">Litex</a>.
|
||||
<br>
|
||||
Generated using
|
||||
<a href="https://github.com/enjoy-digital/litedram/blob/{{ revision }}/{{ script_path }}">{{ script_path }}</a>,
|
||||
revision
|
||||
<a href="https://github.com/enjoy-digital/litedram/commit/{{ revision }}">{{ revision_short }}</a>,
|
||||
{{ generation_date }}.
|
||||
</footer>
|
||||
|
||||
{# Script last, so that for large tables we get some content on the page before loading tables #}
|
||||
<script>
|
||||
{# Ids of the data tables #}
|
||||
table_ids = [
|
||||
{% for id in tables.keys() %}
|
||||
'{{ id }}',
|
||||
{% endfor %}
|
||||
];
|
||||
|
||||
{# Show table with given id and hide all the others #}
|
||||
show_table = function(id) {
|
||||
if (!table_ids.includes(id)) {
|
||||
console.log('Error: show_table(' + id + ')');
|
||||
return;
|
||||
}
|
||||
for (var table_div of $('.tables-wrapper').children()) {
|
||||
if (table_div.id) {
|
||||
var table_div = $('#' + table_div.id)
|
||||
if (table_div.attr('id') == id + '-div') {
|
||||
table_div.show();
|
||||
} else {
|
||||
table_div.hide();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sort human-readable values assuming format "123 Kb", only first letter of unit is used
|
||||
jQuery.fn.dataTable.ext.type.order['file-size-pre'] = function(data) {
|
||||
var matches = data.match(/^(\d+(?:\.\d+)?)\s*([a-z]+)/i);
|
||||
var multipliers = {
|
||||
k: Math.pow(2, 10),
|
||||
m: Math.pow(2, 20),
|
||||
g: Math.pow(2, 30),
|
||||
t: Math.pow(2, 40),
|
||||
};
|
||||
|
||||
console.log(matches);
|
||||
if (matches) {
|
||||
var float = parseFloat(matches[1]);
|
||||
var prefix = matches[2].toLowerCase()[0];
|
||||
var multiplier = multipliers[prefix];
|
||||
if (multiplier) {
|
||||
float = float * multiplier;
|
||||
console.log(matches, float, multiplier);
|
||||
}
|
||||
return float;
|
||||
} else {
|
||||
return -1;
|
||||
};
|
||||
};
|
||||
|
||||
{# Initialization after DOM has been loaded #}
|
||||
$(document).ready(function() {
|
||||
// generate data tables
|
||||
for (var id of table_ids) {
|
||||
// add human readable class to all bandwidth columns
|
||||
var columns = $('#' + id + ' > thead > tr > th').filter(function(index) {
|
||||
return $(this).text().toLowerCase().includes('bandwidth');
|
||||
});
|
||||
console.log(columns)
|
||||
columns.addClass('human-readable-data');
|
||||
|
||||
// construct data table
|
||||
table = $('#' + id);
|
||||
table.DataTable({
|
||||
paging: false,
|
||||
fixedHeader: true,
|
||||
columnDefs: [
|
||||
{ type: 'file-size', targets: [ 'human-readable-data' ] },
|
||||
{ className: 'dt-body-right', targets: [ '_all' ] },
|
||||
{ className: 'dt-head-center', targets: [ '_all' ] },
|
||||
]
|
||||
});
|
||||
table.addClass("stripe");
|
||||
table.addClass("hover");
|
||||
table.addClass("order-column");
|
||||
table.addClass("cell-border");
|
||||
table.addClass("row-border");
|
||||
}
|
||||
|
||||
// add click handlers that change the table being shown
|
||||
for (var id of table_ids) {
|
||||
var ahref = $('#' + id + '-button a');
|
||||
// use nested closure so that we avoid the situation
|
||||
// where all click handlers end up with the last id
|
||||
ahref.click(function(table_id) {
|
||||
return function() {
|
||||
// get rid of this class after first click
|
||||
$('.table-select a').removeClass('table-select-active');
|
||||
$(this).addClass('table-select-active');
|
||||
show_table(table_id);
|
||||
}
|
||||
}(id))
|
||||
}
|
||||
|
||||
// show the first one
|
||||
$('#' + table_ids[0] + '-button a:first').click();
|
||||
|
||||
// hide all elements of class loading
|
||||
$('.loading').hide();
|
||||
});
|
||||
</script>
|
||||
</html>
|
||||
{# vim: set ts=2 sts=2 sw=2 et: #}
|
||||
Loading…
Reference in New Issue