litedram/test/run_benchmarks.py

#!/usr/bin/env python3

# This file is Copyright (c) 2020 Antmicro <www.antmicro.com>
# License: BSD

# Limitations/TODO
# - add configurable sdram_clk_freq - using hardcoded value now
# - sdram_controller_data_width - try to expose the value from litex_sim to avoid duplicated code

import os
import re
import sys
import json
import argparse
import datetime
import subprocess
from collections import defaultdict, namedtuple

import yaml
try:
    import numpy as np
    import pandas as pd
    import matplotlib
    from matplotlib.ticker import FuncFormatter, PercentFormatter, ScalarFormatter
    _summary = True
except ImportError as e:
    _summary = False
    print('[WARNING] Results summary not available:', e, file=sys.stderr)

from litex.tools.litex_sim import get_sdram_phy_settings, sdram_module_nphases
from litedram import modules as litedram_modules
from litedram.common import Settings as _Settings

from test import benchmark

# Benchmark configuration --------------------------------------------------------------------------

class Settings(_Settings):
    def as_dict(self):
        d = dict()
        for attr, value in vars(self).items():
            if attr == 'self' or attr.startswith('_'):
                continue
            if isinstance(value, Settings):
                value = value.as_dict()
            d[attr] = value
        return d


class GeneratedAccess(Settings):
    def __init__(self, bist_length, bist_random):
        self.set_attributes(locals())

    @property
    def length(self):
        return self.bist_length

    def as_args(self):
        args = ['--bist-length=%d' % self.bist_length]
        if self.bist_random:
            args.append('--bist-random')
        return args


class CustomAccess(Settings):
    def __init__(self, pattern_file):
        self.set_attributes(locals())

    @property
    def pattern(self):
        # we have to load the file to know pattern length, cache it when requested
        if not hasattr(self, '_pattern'):
            path = self.pattern_file
            if not os.path.isabs(path):
                benchmark_dir = os.path.dirname(benchmark.__file__)
                path = os.path.join(benchmark_dir, path)
            self._pattern = benchmark.load_access_pattern(path)
        return self._pattern

    @property
    def length(self):
        return len(self.pattern)

    def as_args(self):
        return ['--access-pattern=%s' % self.pattern_file]


class BenchmarkConfiguration(Settings):
    def __init__(self, name, sdram_module, sdram_data_width, bist_alternating,
                 num_generators, num_checkers, access_pattern):
        self.set_attributes(locals())

    def as_args(self):
        args = [
            '--sdram-module=%s' % self.sdram_module,
            '--sdram-data-width=%d' % self.sdram_data_width,
            '--num-generators=%d' % self.num_generators,
            '--num-checkers=%d' % self.num_checkers,
        ]
        if self.bist_alternating:
            args.append('--bist-alternating')
        args += self.access_pattern.as_args()
        return args

    def __eq__(self, other):
        if not isinstance(other, BenchmarkConfiguration):
            return NotImplemented
        return self.as_dict() == other.as_dict()

    @property
    def length(self):
        return self.access_pattern.length

    @classmethod
    def from_dict(cls, d):
        access_cls = CustomAccess if 'pattern_file' in d['access_pattern'] else GeneratedAccess
        d['access_pattern'] = access_cls(**d['access_pattern'])
        return cls(**d)

    @classmethod
    def load_yaml(cls, yaml_file):
        with open(yaml_file) as f:
            description = yaml.safe_load(f)
        configs = []
        for name, desc in description.items():
            desc['name'] = name
            configs.append(cls.from_dict(desc))
        return configs

    def __repr__(self):
        return 'BenchmarkConfiguration(%s)' % self.as_dict()

    @property
    def sdram_clk_freq(self):
        return 100e6  # FIXME: value of 100MHz is hardcoded in litex_sim

    @property
    def sdram_memtype(self):
        # use values from module class (no need to instantiate it)
        sdram_module_cls = getattr(litedram_modules, self.sdram_module)
        return sdram_module_cls.memtype

    @property
    def sdram_controller_data_width(self):
        nphases = sdram_module_nphases[self.sdram_memtype]
        dfi_databits = self.sdram_data_width * (1 if self.sdram_memtype == 'SDR' else 2)
        return dfi_databits * nphases

# Benchmark results --------------------------------------------------------------------------------

# constructs python regex named group
def ng(name, regex):
    return r'(?P<{}>{})'.format(name, regex)


def _compiled_pattern(stage, var):
    pattern_fmt = r'{stage}\s+{var}:\s+{value}'
    pattern = pattern_fmt.format(
        stage=stage,
        var=var,
        value=ng('value', '[0-9]+'),
    )
    return re.compile(pattern)
    result = re.search(pattern, benchmark_output)


class BenchmarkResult:
    # pre-compiled patterns for all benchmarks
    patterns = {
        'generator_ticks': _compiled_pattern('BIST-GENERATOR', 'ticks'),
        'checker_errors': _compiled_pattern('BIST-CHECKER', 'errors'),
        'checker_ticks': _compiled_pattern('BIST-CHECKER', 'ticks'),
    }

    @staticmethod
    def find(pattern, output):
        result = pattern.search(output)
        assert result is not None, \
            'Could not find pattern "%s" in output' % (pattern)
        return int(result.group('value'))

    def __init__(self, output):
        self._output = output
        for attr, pattern in self.patterns.items():
            setattr(self, attr, self.find(pattern, output))

    def __repr__(self):
        d = {attr: getattr(self, attr) for attr in self.patterns.keys()}
        return 'BenchmarkResult(%s)' % d

# Results summary ----------------------------------------------------------------------------------

def human_readable(value):
    binary_prefixes = ['', 'k', 'M', 'G', 'T']
    mult = 1.0
    for prefix in binary_prefixes:
        if value * mult < 1024:
            break
        mult /= 1024
    return mult, prefix


def clocks_fmt(clocks):
    return '{:d} clk'.format(int(clocks))


def bandwidth_fmt(bw):
    mult, prefix = human_readable(bw)
    return '{:.1f} {}bps'.format(bw * mult, prefix)


def efficiency_fmt(eff):
    return '{:.1f} %'.format(eff * 100)


def get_git_file_path(filename):
    cmd = ['git', 'ls-files', '--full-name', filename]
    proc = subprocess.run(cmd, stdout=subprocess.PIPE, cwd=os.path.dirname(__file__))
    return proc.stdout.decode().strip() if proc.returncode == 0 else ''


def get_git_revision_hash(short=False):
    short = ['--short'] if short else []
    cmd = ['git', 'rev-parse', *short, 'HEAD']
    proc = subprocess.run(cmd, stdout=subprocess.PIPE, cwd=os.path.dirname(__file__))
    return proc.stdout.decode().strip() if proc.returncode == 0 else ''


class ResultsSummary:
    def __init__(self, run_data, plots_dir='plots'):
        self.plots_dir = plots_dir

        # because .sdram_controller_data_width may fail for unimplemented modules
        def except_none(func):
            try:
                return func()
            except:
                return None

        # gather results into tabular data
        column_mappings = {
            'name':             lambda d: d.config.name,
            'sdram_module':     lambda d: d.config.sdram_module,
            'sdram_data_width': lambda d: d.config.sdram_data_width,
            'bist_alternating': lambda d: d.config.bist_alternating,
            'num_generators':   lambda d: d.config.num_generators,
            'num_checkers':     lambda d: d.config.num_checkers,
            'bist_length':      lambda d: getattr(d.config.access_pattern, 'bist_length', None),
            'bist_random':      lambda d: getattr(d.config.access_pattern, 'bist_random', None),
            'pattern_file':     lambda d: getattr(d.config.access_pattern, 'pattern_file', None),
            'length':           lambda d: d.config.length,
            'generator_ticks':  lambda d: getattr(d.result, 'generator_ticks', None),  # None means benchmark failure
            'checker_errors':   lambda d: getattr(d.result, 'checker_errors', None),
            'checker_ticks':    lambda d: getattr(d.result, 'checker_ticks', None),
            'ctrl_data_width':  lambda d: except_none(lambda: d.config.sdram_controller_data_width),
            'sdram_memtype':    lambda d: except_none(lambda: d.config.sdram_memtype),
            'clk_freq':         lambda d: d.config.sdram_clk_freq,
        }
        columns = {name: [mapping(data) for data in run_data] for name, mapping, in column_mappings.items()}
        self._df = df = pd.DataFrame(columns)

        # replace None with NaN
        df.fillna(value=np.nan, inplace=True)

        # compute other metrics based on ticks and configuration parameters
        df['clk_period'] = 1 / df['clk_freq']
        # bandwidth is the number of bits per time
        # in case with N generators/checkers we actually process N times more data
        df['write_bandwidth'] = (8 * df['length'] * df['num_generators']) / (df['generator_ticks'] * df['clk_period'])
        df['read_bandwidth']  = (8 * df['length'] * df['num_checkers']) / (df['checker_ticks'] * df['clk_period'])

        # efficiency calculated as number of write/read commands to number of cycles spent on writing/reading (ticks)
        # for multiple generators/checkers multiply by their number
        df['cmd_count'] = df['length'] / (df['ctrl_data_width'] / 8)
        df['write_efficiency'] = df['cmd_count'] * df['num_generators'] / df['generator_ticks']
        df['read_efficiency'] = df['cmd_count'] * df['num_checkers'] / df['checker_ticks']

        df['write_latency'] = df[df['bist_length'] == 1]['generator_ticks']
        df['read_latency'] = df[df['bist_length'] == 1]['checker_ticks']

        # boolean distinction between latency benchmarks and sequence benchmarks,
        # as thier results differ significanly
        df['is_latency'] = ~pd.isna(df['write_latency'])
        assert (df['is_latency'] == ~pd.isna(df['read_latency'])).all(), \
            'write_latency and read_latency should both have a value or both be NaN'

        # data formatting for text summary
        self.text_formatters = {
            'write_bandwidth':  bandwidth_fmt,
            'read_bandwidth':   bandwidth_fmt,
            'write_efficiency': efficiency_fmt,
            'read_efficiency':  efficiency_fmt,
            'write_latency':    clocks_fmt,
            'read_latency':     clocks_fmt,
        }

        # data formatting for plot summary
        self.plot_xticks_formatters = {
            'write_bandwidth':  FuncFormatter(lambda value, pos: bandwidth_fmt(value)),
            'read_bandwidth':   FuncFormatter(lambda value, pos: bandwidth_fmt(value)),
            'write_efficiency': PercentFormatter(1.0),
            'read_efficiency':  PercentFormatter(1.0),
            'write_latency':    ScalarFormatter(),
            'read_latency':     ScalarFormatter(),
        }

    def df(self, ok=True, failures=False):
        is_failure = lambda df: pd.isna(df['generator_ticks']) | pd.isna(df['checker_ticks']) | pd.isna(df['checker_errors'])
        df = self._df
        if not ok:  # remove ok
            is_ok = ~is_failure(df)
            df = df[~is_ok]
        if not failures:  # remove failures
            df = df[~is_failure(df)]
        return df

    def header(self, text):
        return '===> {}'.format(text)

    def print_df(self, title, df):
        # make sure all data will be shown
        with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.width', None):
            print(self.header(title + ':'))
            print(df)

    def get_summary(self, df, mask=None, columns=None, column_formatting=None, sort_kwargs=None):
        # work on a copy
        df = df.copy()

        if sort_kwargs is not None:
            df = df.sort_values(**sort_kwargs)

        if column_formatting is not None:
            for column, mapping in column_formatting.items():
                old = '_{}'.format(column)
                df[old] = df[column].copy()
                df[column] = df[column].map(lambda value: mapping(value) if not pd.isna(value) else value)

        df = df[mask] if mask is not None else df
        df = df[columns] if columns is not None else df

        return df

    def text_summary(self):
        for title, df in self.groupped_results():
            self.print_df(title, df)
            print()

    def html_summary(self, output_dir):
        import jinja2

        tables = {}
        names = {}
        for title, df in self.groupped_results():
            table_id = title.lower().replace(' ', '_')

            tables[table_id] = df.to_html(table_id=table_id, border=0)
            names[table_id] = title

        template_dir = os.path.join(os.path.dirname(__file__), 'summary')
        env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_dir))
        template = env.get_template('summary.html.jinja2')

        os.makedirs(output_dir, exist_ok=True)
        with open(os.path.join(output_dir, 'summary.html'), 'w') as f:
            f.write(template.render(
                title='LiteDRAM benchmarks summary',
                tables=tables,
                names=names,
                script_path=get_git_file_path(__file__),
                revision=get_git_revision_hash(),
                revision_short=get_git_revision_hash(short=True),
                generation_date=datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
            ))

    def groupped_results(self, formatters=None):
        df = self.df()

        if formatters is None:
            formatters = self.text_formatters

        common_columns = [
            'name', 'sdram_module', 'sdram_memtype', 'sdram_data_width',
            'bist_alternating', 'num_generators', 'num_checkers'
        ]
        latency_columns = ['write_latency', 'read_latency']
        performance_columns = [
            'write_bandwidth', 'read_bandwidth', 'write_efficiency', 'read_efficiency'
        ]
        failure_columns = [
            'bist_length', 'bist_random', 'pattern_file', 'length',
            'generator_ticks', 'checker_errors', 'checker_ticks'
        ]

        yield 'Latency', self.get_summary(df,
            mask=df['is_latency'] == True,
            columns=common_columns + latency_columns,
            column_formatting=formatters,
        )
        yield 'Custom access pattern', self.get_summary(df,
            mask=(df['is_latency'] == False) & (~pd.isna(df['pattern_file'])),
            columns=common_columns + ['length', 'pattern_file'] + performance_columns,
            column_formatting=formatters,
        ),
        yield 'Sequential access pattern', self.get_summary(df,
            mask=(df['is_latency'] == False) & (pd.isna(df['pattern_file'])) & (df['bist_random'] == False),
            columns=common_columns + ['bist_length'] + performance_columns, # could be length
            column_formatting=formatters,
        ),
        yield 'Random access pattern', self.get_summary(df,
            mask=(df['is_latency'] == False) & (pd.isna(df['pattern_file'])) & (df['bist_random'] == True),
            columns=common_columns + ['bist_length'] + performance_columns,
            column_formatting=formatters,
        ),
        yield 'Failures', self.get_summary(self.df(ok=False, failures=True),
            columns=common_columns + failure_columns,
            column_formatting=None,
        ),

    def plot_summary(self, plots_dir='plots', backend='Agg', theme='default', save_format='png', **savefig_kw):
        matplotlib.use(backend)
        import matplotlib.pyplot as plt
        plt.style.use(theme)

        for title, df in self.groupped_results(formatters={}):
            for column in self.plot_xticks_formatters.keys():
                if column not in df.columns or df[column].empty:
                    continue
                axis = self.plot_df(title, df, column)

                # construct path
                def path_name(name):
                    return name.lower().replace(' ', '_')

                filename = '{}.{}'.format(path_name(column), save_format)
                path = os.path.join(plots_dir, path_name(title), filename)
                os.makedirs(os.path.dirname(path), exist_ok=True)

                # save figure
                axis.get_figure().savefig(path, **savefig_kw)

        if backend != 'Agg':
            plt.show()

    def plot_df(self, title, df, column, fig_width=6.4, fig_min_height=2.2, save_format='png', save_filename=None):
        if save_filename is None:
            save_filename = os.path.join(self.plots_dir, title.lower().replace(' ', '_'))

        axis = df.plot(kind='barh', x='name', y=column, title=title, grid=True, legend=False)
        fig = axis.get_figure()

        if column in self.plot_xticks_formatters:
            axis.xaxis.set_major_formatter(self.plot_xticks_formatters[column])
            axis.xaxis.set_tick_params(rotation=15)
        axis.spines['top'].set_visible(False)
        axis.spines['right'].set_visible(False)
        axis.set_axisbelow(True)
        axis.set_ylabel('')  # no need for label as we have only one series

        # for large number of rows, the bar labels start overlapping
        # use fixed ratio between number of rows and height of figure
        n_ok = 16
        new_height = (fig_width / n_ok) * len(df)
        fig.set_size_inches(fig_width, max(fig_min_height, new_height))

        # remove empty spaces
        fig.tight_layout()

        return axis

# Run ----------------------------------------------------------------------------------------------

class RunCache(list):
    RunData = namedtuple('RunData', ['config', 'result'])

    def dump_json(self, filename):
        json_data = [{'config': data.config.as_dict(), 'output': getattr(data.result, '_output', None) } for data in self]
        with open(filename, 'w') as f:
            json.dump(json_data, f)

    @classmethod
    def load_json(cls, filename):
        with open(filename, 'r') as f:
            json_data = json.load(f)
        loaded = []
        for data in json_data:
            config = BenchmarkConfiguration.from_dict(data['config'])
            result = BenchmarkResult(data['output']) if data['output'] is not None else None
            loaded.append(cls.RunData(config=config, result=result))
        return loaded


def run_python(script, args, **kwargs):
    command = ['python3', script, *args]
    proc = subprocess.run(command, stdout=subprocess.PIPE, cwd=os.path.dirname(script), **kwargs)
    return str(proc.stdout)


BenchmarkArgs = namedtuple('BenchmarkArgs', ['config', 'output_dir', 'ignore_failures', 'timeout'])


def run_single_benchmark(fargs):
    # run as separate process, because else we cannot capture all output from verilator
    print('  {}: {}'.format(fargs.config.name, ' '.join(fargs.config.as_args())))
    try:
        args = fargs.config.as_args() + ['--output-dir', fargs.output_dir, '--log-level', 'warning']
        output = run_python(benchmark.__file__, args, timeout=fargs.timeout)
        result = BenchmarkResult(output)
        # exit if checker had any read error
        if result.checker_errors != 0:
            raise RuntimeError('Error during benchmark: checker_errors = {}, args = {}'.format(
                result.checker_errors, fargs.config.as_args()
            ))
    except Exception as e:
        if fargs.ignore_failures:
            print('  {}: ERROR: {}'.format(fargs.config.name, e))
            return None
        else:
            raise
    print('  {}: ok'.format(fargs.config.name))
    return result


InQueueItem = namedtuple('InQueueItem', ['index', 'config'])
OutQueueItem = namedtuple('OutQueueItem', ['index', 'result'])


def run_parallel(configurations, output_base_dir, njobs, ignore_failures, timeout):
    from multiprocessing import Process, Queue
    import queue

    def worker(in_queue, out_queue, out_dir):
        while True:
            in_item = in_queue.get()
            if in_item is None:
                return
            fargs = BenchmarkArgs(in_item.config, out_dir, ignore_failures, timeout)
            result = run_single_benchmark(fargs)
            out_queue.put(OutQueueItem(in_item.index, result))

    if njobs == 0:
        njobs = os.cpu_count()
    print('Using {:d} parallel jobs'.format(njobs))

    # use one directory per worker, as running each benchmark in separate directory
    # takes too much disk space (~2GB per 100 benchmarks)
    dir_pool = [os.path.join(output_base_dir, 'worker_%02d' % i) for i in range(njobs)]

    in_queue, out_queue = Queue(), Queue()
    workers = [Process(target=worker, args=(in_queue, out_queue, dir)) for dir in dir_pool]
    for w in workers:
        w.start()

    # put all benchmark configurations with index to retrieve them in order
    for i, config in enumerate(configurations):
        in_queue.put(InQueueItem(i, config))

    # send "finish signal" for each worker
    for _ in workers:
        in_queue.put(None)

    # retrieve results in proper order
    out_items = [out_queue.get() for _ in configurations]
    results = [out.result for out in sorted(out_items, key=lambda o: o.index)]

    for p in workers:
        p.join()

    return results


def run_benchmarks(configurations, output_base_dir, njobs, ignore_failures, timeout):
    print('Running {:d} benchmarks ...'.format(len(configurations)))
    if njobs == 1:
        results = [run_single_benchmark(BenchmarkArgs(config, output_base_dir, ignore_failures, timeout))
                   for config in configurations]
    else:
        results = run_parallel(configurations, output_base_dir, njobs, ignore_failures, timeout)
    run_data = [RunCache.RunData(config, result) for config, result in zip(configurations, results)]
    return run_data


def main(argv=None):
    parser = argparse.ArgumentParser(
        description='Run LiteDRAM benchmarks and collect the results.')
    parser.add_argument("config",                                  help="YAML config file")
    parser.add_argument('--names',            nargs='*',           help='Limit benchmarks to given names')
    parser.add_argument('--regex',                                 help='Limit benchmarks to names matching the regex')
    parser.add_argument('--not-regex',                             help='Limit benchmarks to names not matching the regex')
    parser.add_argument('--html',             action='store_true', help='Generate HTML summary')
    parser.add_argument('--html-output-dir',  default='html',      help='Output directory for generated HTML')
    parser.add_argument('--plot',             action='store_true', help='Generate plots with results summary')
    parser.add_argument('--plot-format',      default='png',       help='Specify plots file format (default=png)')
    parser.add_argument('--plot-backend',     default='Agg',       help='Optionally specify matplotlib GUI backend')
    parser.add_argument('--plot-transparent', action='store_true', help='Use transparent background when saving plots')
    parser.add_argument('--plot-output-dir',  default='plots',     help='Specify where to save the plots')
    parser.add_argument('--plot-theme',       default='default',   help='Use different matplotlib theme')
    parser.add_argument('--fail-fast',        action='store_true', help='Exit on any benchmark error, do not continue')
    parser.add_argument('--output-dir',       default='build',     help='Directory to store benchmark build output')
    parser.add_argument('--njobs',            default=0, type=int, help='Use N parallel jobs to run benchmarks (default=0, which uses CPU count)')
    parser.add_argument('--heartbeat',        default=0, type=int, help='Print heartbeat message with given interval (default=0 => never)')
    parser.add_argument('--timeout',          default=None,        help='Set timeout for a single benchmark')
    parser.add_argument('--results-cache',                         help="""Use given JSON file as results cache. If the file exists,
                                                                           it will be loaded instead of running actual benchmarks,
                                                                           else benchmarks will be run normally, and then saved
                                                                           to the given file. This allows to easily rerun the script
                                                                           to generate different summary without having to rerun benchmarks.""")
    args = parser.parse_args(argv)

    if not args.results_cache and not _summary:
        print('Summary not available and not running with --results-cache - run would not produce any results! Aborting.',
              file=sys.stderr)
        sys.exit(1)

    # load and filter configurations
    configurations = BenchmarkConfiguration.load_yaml(args.config)
    filters = {
        'regex':     lambda config: re.search(args.regex, config.name),
        'not_regex': lambda config: not re.search(args.not_regex, config.name),
        'names':     lambda config: config.name in args.names,
    }
    for arg, f in filters.items():
        if getattr(args, arg):
            configurations = filter(f, configurations)
    configurations = list(configurations)

    # load outputs from cache if it exsits
    cache_exists = args.results_cache and os.path.isfile(args.results_cache)
    if args.results_cache and cache_exists:
        cache = RunCache.load_json(args.results_cache)

        # take only those that match configurations
        names_to_load = [c.name for c in configurations]
        run_data = [data for  data in cache if data.config.name in names_to_load]
    else:  # run all the benchmarks normally
        if args.heartbeat:
            heartbeat_cmd = ['/bin/sh', '-c', 'while true; do sleep %d; echo Heartbeat...; done' % args.heartbeat]
            heartbeat = subprocess.Popen(heartbeat_cmd)
        if args.timeout is not None:
            args.timeout = int(args.timeout)
        run_data = run_benchmarks(configurations, args.output_dir, args.njobs, not args.fail_fast, args.timeout)
        if args.heartbeat:
            heartbeat.kill()

    # store outputs in cache
    if args.results_cache and not cache_exists:
        cache = RunCache(run_data)
        cache.dump_json(args.results_cache)

    # display summary
    if _summary:
        summary = ResultsSummary(run_data)
        summary.text_summary()
        if args.html:
            summary.html_summary(args.html_output_dir)
        if args.plot:
            summary.plot_summary(
                plots_dir=args.plot_output_dir,
                backend=args.plot_backend,
                theme=args.plot_theme,
                save_format=args.plot_format,
                transparent=args.plot_transparent,
            )

    # exit with error when there is no single benchmark that succeeded
    succeeded = sum(1 if d.result is not None else 0 for d in run_data)
    if succeeded == 0:
        sys.exit(1)

if __name__ == "__main__":
    main()