From 811c73254b6bac0df780fd29cd265b2dcbdccc2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C4=99drzej=20Boczar?= Date: Fri, 31 Jan 2020 15:16:37 +0100 Subject: [PATCH] test: benchmark script exits with error on any checker error --- test/run_benchmarks.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/run_benchmarks.py b/test/run_benchmarks.py index bfc6d04..da8a6cd 100755 --- a/test/run_benchmarks.py +++ b/test/run_benchmarks.py @@ -5,6 +5,7 @@ import os import re +import sys import argparse import subprocess from collections import defaultdict, namedtuple @@ -247,7 +248,16 @@ def run_benchmarks(configurations): cmd_args = config.as_args() print('{}: {}'.format(name, ' '.join(cmd_args))) output = run_benchmark(cmd_args) + + # return raw outputs, not BenchmarkResult so that we can store them in a file benchmarks.append((config, output)) + + # exit if checker had any read error + result = BenchmarkResult(config, output) + if result.checker_errors != 0: + print('Error during benchmark "{}": checker_errors = {}'.format( + name, result.checker_errors), file=sys.stderr) + sys.exit(1) return benchmarks