test: add generation of html benchmarks summary

test/run_benchmarks.py

@@ -12,6 +12,7 @@ import re
 import sys
 import json
 import argparse
+import datetime
 import subprocess
 from collections import defaultdict, namedtuple
 
@@ -211,6 +212,20 @@ def efficiency_fmt(eff):
     return '{:.1f} %'.format(eff * 100)
 
 
+def get_git_file_path(filename):
+    cmd = ['git', 'ls-files', '--full-name', filename]
+    proc = subprocess.run(cmd, stdout=subprocess.PIPE, cwd=os.path.dirname(__file__))
+    return proc.stdout.decode().strip() if proc.returncode == 0 else ''
+
+
+def get_git_revision_hash(short=False):
+    short = ['--short'] if short else []
+    #  cmd = ['git', 'rev-parse', *short, 'HEAD']
+    cmd = ['git', 'rev-parse', *short, 'origin/master']
+    proc = subprocess.run(cmd, stdout=subprocess.PIPE, cwd=os.path.dirname(__file__))
+    return proc.stdout.decode().strip() if proc.returncode == 0 else ''
+
+
 class ResultsSummary:
     def __init__(self, run_data, plots_dir='plots'):
         self.plots_dir = plots_dir
@@ -317,10 +332,38 @@ class ResultsSummary:
             self.print_df(title, df)
             print()
 
-    def groupped_results(self, formatted=True):
+    def html_summary(self, output_dir):
+        import jinja2
+
+        tables = {}
+        names = {}
+        for title, df in self.groupped_results():
+            table_id = title.lower().replace(' ', '_')
+
+            tables[table_id] = df.to_html(table_id=table_id, border=0)
+            names[table_id] = title
+
+        template_dir = os.path.join(os.path.dirname(__file__), 'summary')
+        env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_dir))
+        template = env.get_template('summary.html.jinja2')
+
+        os.makedirs(output_dir, exist_ok=True)
+        with open(os.path.join(output_dir, 'summary.html'), 'w') as f:
+            f.write(template.render(
+                title='LiteDRAM benchmarks summary',
+                tables=tables,
+                names=names,
+                script_path=get_git_file_path(__file__),
+                revision=get_git_revision_hash(),
+                revision_short=get_git_revision_hash(short=True),
+                generation_date=datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+            ))
+
+    def groupped_results(self, formatters=None):
         df = self.df
 
-        formatters = self.text_formatters if formatted else {}
+        if formatters is None:
+            formatters = self.text_formatters
 
         common_columns = ['name', 'sdram_module', 'sdram_data_width', 'bist_alternating', 'num_generators', 'num_checkers']
         latency_columns = ['write_latency', 'read_latency']
@@ -333,17 +376,17 @@ class ResultsSummary:
         )
         yield 'Custom access pattern', self.get_summary(
             mask=(df['is_latency'] == False) & (~pd.isna(df['pattern_file'])),
-            columns=common_columns + performance_columns + ['length', 'pattern_file'],
+            columns=common_columns + ['length', 'pattern_file'] + performance_columns,
             column_formatting=formatters,
         ),
         yield 'Sequential access pattern', self.get_summary(
             mask=(df['is_latency'] == False) & (pd.isna(df['pattern_file'])) & (df['bist_random'] == False),
-            columns=common_columns + performance_columns + ['bist_length'], # could be length
+            columns=common_columns + ['bist_length'] + performance_columns, # could be length
             column_formatting=formatters,
         ),
         yield 'Random access pattern', self.get_summary(
             mask=(df['is_latency'] == False) & (pd.isna(df['pattern_file'])) & (df['bist_random'] == True),
-            columns=common_columns + performance_columns + ['bist_length'],
+            columns=common_columns + ['bist_length'] + performance_columns,
             column_formatting=formatters,
         ),
 
@@ -352,7 +395,7 @@ class ResultsSummary:
         import matplotlib.pyplot as plt
         plt.style.use(theme)
 
-        for title, df in self.groupped_results(formatted=False):
+        for title, df in self.groupped_results(formatters={}):
             for column in self.plot_xticks_formatters.keys():
                 if column not in df.columns or df[column].empty:
                     continue
@@ -520,6 +563,8 @@ def main(argv=None):
     parser.add_argument('--names',            nargs='*',           help='Limit benchmarks to given names')
     parser.add_argument('--regex',                                 help='Limit benchmarks to names matching the regex')
     parser.add_argument('--not-regex',                             help='Limit benchmarks to names not matching the regex')
+    parser.add_argument('--html',             action='store_true', help='Generate HTML summary')
+    parser.add_argument('--html-output-dir',  default='html',      help='Output directory for generated HTML')
     parser.add_argument('--plot',             action='store_true', help='Generate plots with results summary')
     parser.add_argument('--plot-format',      default='png',       help='Specify plots file format (default=png)')
     parser.add_argument('--plot-backend',     default='Agg',       help='Optionally specify matplotlib GUI backend')
@@ -574,6 +619,8 @@ def main(argv=None):
         summary = ResultsSummary(run_data)
         summary.text_summary()
         summary.failures_summary()
+        if args.html:
+            summary.html_summary(args.html_output_dir)
         if args.plot:
             summary.plot_summary(
                 plots_dir=args.plot_output_dir,

test/summary/summary.css

@@ -0,0 +1,100 @@
+body {
+  font-family: 'Roboto', sans-serif;
+}
+
+footer {
+  text-align: center;
+  font-size: 10px;
+  padding: 20px;
+}
+
+.dataTables_filter {
+  margin: 15px 50px 10px 50px;
+}
+
+.dataTables_filter input {
+  width: 400px;
+}
+
+.table-select {
+  width: 100%;
+  margin: 0 auto;
+}
+
+.table-select ul {
+  list-style-type: none;
+  margin: 0;
+  padding: 0;
+  overflow: hidden;
+}
+
+.table-select li {
+  float: left;
+}
+
+.table-select li a {
+  display: block;
+  padding: 10px 0px;
+  margin: 0px 20px;
+  text-align: center;
+  text-decoration: none;
+  font-size: 18px;
+  color:inherit;
+  border-bottom: 1px solid;
+  border-color: #ccc;
+  transition: 0.2s;
+}
+
+.table-select li a:hover {
+  border-color: #111;
+}
+
+/* did not work, .focus() couldn't turn it on */
+/* .table-select li a:focus { */
+/*   border-color: #222; */
+/* } */
+.table-select-active {
+  border-color: #111 !important;
+}
+
+.tables-wrapper {
+  width: 100%;
+  margin: auto;
+}
+
+.loading {
+  z-index: 999;
+  position: absolute;
+  top: 50%;
+  left: 50%;
+  margin-right: -50%;
+  transform: translate(-50%, -50%);
+}
+
+/* Loading animation */
+.lds-dual-ring {
+  display: inline-block;
+  width: 80px;
+  height: 80px;
+}
+.lds-dual-ring:after {
+  content: " ";
+  display: block;
+  width: 64px;
+  height: 64px;
+  margin: 8px;
+  border-radius: 50%;
+  border: 6px solid #fff;
+  border-color: #aaa transparent #aaa transparent;
+  animation: lds-dual-ring 1.2s linear infinite;
+}
+@keyframes lds-dual-ring {
+  0% {
+    transform: rotate(0deg);
+  }
+  100% {
+    transform: rotate(360deg);
+  }
+}
+
+/* vim: set ts=2 sw=2: */

test/summary/summary.html.jinja2

@@ -0,0 +1,162 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <title>{{ title }}</title>
+
+    <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.4.1/jquery.min.js"></script>
+
+    <script type="text/javascript" charset="utf8" src="https://cdn.datatables.net/1.10.19/js/jquery.dataTables.min.js"></script>
+    <link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/1.10.19/css/jquery.dataTables.css">
+    <script type="text/javascript" charset="utf8" src="https://cdn.datatables.net/fixedheader/3.1.5/js/dataTables.fixedHeader.min.js"></script>
+    <link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/fixedheader/3.1.5/css/fixedHeader.dataTables.min.css">
+
+    <style type="text/css" media="all">
+{% include 'summary.css' %}
+{# Calculate size of table selection elements so that they take up whole space #}
+.table-select li {
+  width: calc(100% / {{ tables.keys() | length }});
+}
+    </style>
+  </head>
+  <body>
+    {# Loading symbol that gets hidden after initialisation of all tables #}
+    <div class="loading lds-dual-ring"></div>
+
+    {# Bar for selecting the current data table #}
+    <div class="table-select">
+      <ul>
+        {% for id, name in names.items() %}
+          <li id='{{ id }}-button'><a href="#">{{ name }}</a></li>
+        {% endfor %}
+      </ul>
+      {# <hr/> #}
+    </div>
+
+    {# Display of the current data table #}
+    <div class="tables-wrapper">
+      {% for id, table in tables.items() %}
+        {# Hide tables not to show them before all DataTables are loaded #}
+        <div id="{{ id }}-div" style="display: none;">
+          {{ table }}
+        </div>
+      {% endfor %}
+    </div>
+  </body>
+
+  <footer id="footer">
+    <a href="https://github.com/enjoy-digital/litedram">LiteDRAM</a> is a part of <a href="https://github.com/enjoy-digital/litex">Litex</a>.
+    <br>
+    Generated using
+    <a href="https://github.com/enjoy-digital/litedram/blob/{{ revision }}/{{ script_path }}">{{ script_path }}</a>,
+    revision
+    <a href="https://github.com/enjoy-digital/litedram/commit/{{ revision }}">{{ revision_short }}</a>,
+    {{ generation_date }}.
+  </footer>
+
+  {# Script last, so that for large tables we get some content on the page before loading tables #}
+  <script>
+    {# Ids of the data tables #}
+    table_ids = [
+      {% for id in tables.keys() %}
+        '{{ id }}',
+    {% endfor %}
+    ];
+
+    {# Show table with given id and hide all the others #}
+    show_table = function(id) {
+      if (!table_ids.includes(id)) {
+        console.log('Error: show_table(' + id + ')');
+        return;
+      }
+      for (var table_div of $('.tables-wrapper').children()) {
+        if (table_div.id) {
+          var table_div = $('#' + table_div.id)
+          if (table_div.attr('id') == id + '-div') {
+            table_div.show();
+          } else {
+            table_div.hide();
+          }
+        }
+      }
+    }
+
+    // sort human-readable values assuming format "123 Kb", only first letter of unit is used
+    jQuery.fn.dataTable.ext.type.order['file-size-pre'] = function(data) {
+      var matches = data.match(/^(\d+(?:\.\d+)?)\s*([a-z]+)/i);
+      var multipliers = {
+        k: Math.pow(2, 10),
+        m: Math.pow(2, 20),
+        g: Math.pow(2, 30),
+        t: Math.pow(2, 40),
+      };
+
+      console.log(matches);
+      if (matches) {
+        var float = parseFloat(matches[1]);
+        var prefix = matches[2].toLowerCase()[0];
+        var multiplier = multipliers[prefix];
+        if (multiplier) {
+          float = float * multiplier;
+          console.log(matches, float, multiplier);
+        }
+        return float;
+      } else {
+        return -1;
+      };
+    };
+
+    {# Initialization after DOM has been loaded #}
+    $(document).ready(function() {
+      // generate data tables
+      for (var id of table_ids) {
+        // add human readable class to all bandwidth columns
+        var columns = $('#' + id + ' > thead > tr > th').filter(function(index) {
+          return $(this).text().toLowerCase().includes('bandwidth');
+        });
+        console.log(columns)
+        columns.addClass('human-readable-data');
+
+        // construct data table
+        table = $('#' + id);
+        table.DataTable({
+          paging: false,
+          fixedHeader: true,
+          columnDefs: [
+            { type: 'file-size', targets: [ 'human-readable-data' ] },
+            { className: 'dt-body-right', targets: [ '_all' ] },
+            { className: 'dt-head-center', targets: [ '_all' ] },
+          ]
+        });
+        table.addClass("stripe");
+        table.addClass("hover");
+        table.addClass("order-column");
+        table.addClass("cell-border");
+        table.addClass("row-border");
+      }
+
+      // add click handlers that change the table being shown
+      for (var id of table_ids) {
+        var ahref = $('#' + id + '-button a');
+        // use nested closure so that we avoid the situation
+        // where all click handlers end up with the last id
+        ahref.click(function(table_id) {
+          return function() {
+            // get rid of this class after first click
+            $('.table-select a').removeClass('table-select-active');
+            $(this).addClass('table-select-active');
+            show_table(table_id);
+          }
+        }(id))
+      }
+
+      // show the first one
+      $('#' + table_ids[0] + '-button a:first').click();
+
+      // hide all elements of class loading
+      $('.loading').hide();
+    });
+  </script>
+</html>
+{# vim: set ts=2 sts=2 sw=2 et: #}