197 lines · plain
1#!/usr/bin/env python32 3import argparse4import functools5import pathlib6import re7import statistics8import sys9import tempfile10 11import numpy12import pandas13import plotly.express14import tabulate15 16def parse_lnt(lines, aggregate=statistics.median):17 """18 Parse lines in LNT format and return a list of dictionnaries of the form:19 20 [21 {22 'benchmark': <benchmark1>,23 <metric1>: float,24 <metric2>: float,25 ...26 },27 {28 'benchmark': <benchmark2>,29 <metric1>: float,30 <metric2>: float,31 ...32 },33 ...34 ]35 36 If a metric has multiple values associated to it, they are aggregated into a single37 value using the provided aggregation function.38 """39 results = {}40 for line in lines:41 line = line.strip()42 if not line:43 continue44 45 (identifier, value) = line.split(' ')46 (benchmark, metric) = identifier.split('.')47 if benchmark not in results:48 results[benchmark] = {'benchmark': benchmark}49 50 entry = results[benchmark]51 if metric not in entry:52 entry[metric] = []53 entry[metric].append(float(value))54 55 for (bm, entry) in results.items():56 for metric in entry:57 if isinstance(entry[metric], list):58 entry[metric] = aggregate(entry[metric])59 60 return list(results.values())61 62def plain_text_comparison(data, metric, baseline_name=None, candidate_name=None):63 """64 Create a tabulated comparison of the baseline and the candidate for the given metric.65 """66 data = data.replace(numpy.nan, None) # avoid NaNs in tabulate output67 headers = ['Benchmark', baseline_name, candidate_name, 'Difference', '% Difference']68 fmt = (None, '.2f', '.2f', '.2f', '.2%')69 table = data[['benchmark', f'{metric}_0', f'{metric}_1', 'difference', 'percent']]70 71 # Compute the geomean and report on their difference72 geomean_0 = statistics.geometric_mean(data[f'{metric}_0'].dropna())73 geomean_1 = statistics.geometric_mean(data[f'{metric}_1'].dropna())74 geomean_row = ['Geomean', geomean_0, geomean_1, (geomean_1 - geomean_0), (geomean_1 - geomean_0) / geomean_0]75 table.loc[table.index.max() + 1] = geomean_row76 77 return tabulate.tabulate(table.set_index('benchmark'), headers=headers, floatfmt=fmt, numalign='right')78 79def create_chart(data, metric, subtitle=None, series_names=None):80 """81 Create a bar chart comparing the given metric across the provided series.82 """83 data = data.rename(columns={f'{metric}_{i}': series_names[i] for i in range(len(series_names))})84 title = ' vs '.join(series_names)85 figure = plotly.express.bar(data, title=title, subtitle=subtitle, x='benchmark', y=series_names, barmode='group')86 figure.update_layout(xaxis_title='', yaxis_title='', legend_title='')87 return figure88 89def main(argv):90 parser = argparse.ArgumentParser(91 prog='compare-benchmarks',92 description='Compare the results of multiple sets of benchmarks in LNT format.',93 epilog='This script depends on the modules listed in `libcxx/utils/requirements.txt`.')94 parser.add_argument('files', type=argparse.FileType('r'), nargs='+',95 help='Path to LNT format files containing the benchmark results to compare. In the text format, '96 'exactly two files must be compared.')97 parser.add_argument('--output', '-o', type=pathlib.Path, required=False,98 help='Path of a file where to output the resulting comparison. If the output format is `text`, '99 'default to stdout. If the output format is `chart`, default to a temporary file which is '100 'opened automatically once generated, but not removed after creation.')101 parser.add_argument('--metric', type=str, default='execution_time',102 help='The metric to compare. LNT data may contain multiple metrics (e.g. code size, execution time, etc) -- '103 'this option allows selecting which metric is being analyzed. The default is `execution_time`.')104 parser.add_argument('--filter', type=str, required=False,105 help='An optional regular expression used to filter the benchmarks included in the comparison. '106 'Only benchmarks whose names match the regular expression will be included.')107 parser.add_argument('--sort', type=str, required=False, default='benchmark',108 choices=['benchmark', 'baseline', 'candidate', 'percent_diff'],109 help='Optional sorting criteria for displaying results. By default, results are displayed in '110 'alphabetical order of the benchmark. Supported sorting criteria are: '111 '`benchmark` (sort using the alphabetical name of the benchmark), '112 '`baseline` (sort using the absolute number of the baseline run), '113 '`candidate` (sort using the absolute number of the candidate run), '114 'and `percent_diff` (sort using the percent difference between the baseline and the candidate). '115 'Note that when more than two input files are compared, the only valid sorting order is `benchmark`.')116 parser.add_argument('--format', type=str, choices=['text', 'chart'], default='text',117 help='Select the output format. `text` generates a plain-text comparison in tabular form, and `chart` '118 'generates a self-contained HTML graph that can be opened in a browser. The default is `text`.')119 parser.add_argument('--open', action='store_true',120 help='Whether to automatically open the generated HTML file when finished. This option only makes sense '121 'when the output format is `chart`.')122 parser.add_argument('--series-names', type=str, required=False,123 help='Optional comma-delimited list of names to use for the various series. By default, we use '124 'Baseline and Candidate for two input files, and CandidateN for subsequent inputs.')125 parser.add_argument('--subtitle', type=str, required=False,126 help='Optional subtitle to use for the chart. This can be used to help identify the contents of the chart. '127 'This option cannot be used with the plain text output.')128 args = parser.parse_args(argv)129 130 # Validate arguments (the values admissible for various arguments depend on other131 # arguments, the number of inputs, etc)132 if args.format == 'text':133 if len(args.files) != 2:134 parser.error('--format=text requires exactly two input files to compare')135 if args.subtitle is not None:136 parser.error('Passing --subtitle makes no sense with --format=text')137 if args.open:138 parser.error('Passing --open makes no sense with --format=text')139 140 if len(args.files) != 2 and args.sort != 'benchmark':141 parser.error('Using any sort order other than `benchmark` requires exactly two input files.')142 143 if args.series_names is None:144 args.series_names = ['Baseline']145 if len(args.files) == 2:146 args.series_names += ['Candidate']147 elif len(args.files) > 2:148 args.series_names.extend(f'Candidate{n}' for n in range(1, len(args.files)))149 else:150 args.series_names = args.series_names.split(',')151 if len(args.series_names) != len(args.files):152 parser.error(f'Passed incorrect number of series names: got {len(args.series_names)} series names but {len(args.files)} inputs to compare')153 154 # Parse the raw LNT data and store each input in a dataframe155 lnt_inputs = [parse_lnt(file.readlines()) for file in args.files]156 inputs = [pandas.DataFrame(lnt).rename(columns={args.metric: f'{args.metric}_{i}'}) for (i, lnt) in enumerate(lnt_inputs)]157 158 # Join the inputs into a single dataframe159 data = functools.reduce(lambda a, b: a.merge(b, how='outer', on='benchmark'), inputs)160 161 # If we have exactly two data sets, compute additional info in new columns162 if len(lnt_inputs) == 2:163 data['difference'] = data[f'{args.metric}_1'] - data[f'{args.metric}_0']164 data['percent'] = data['difference'] / data[f'{args.metric}_0']165 166 if args.filter is not None:167 keeplist = [b for b in data['benchmark'] if re.search(args.filter, b) is not None]168 data = data[data['benchmark'].isin(keeplist)]169 170 # Sort the data by the appropriate criteria171 if args.sort == 'benchmark':172 data = data.sort_values(by='benchmark')173 elif args.sort == 'baseline':174 data = data.sort_values(by=f'{args.metric}_0')175 elif args.sort == 'candidate':176 data = data.sort_values(by=f'{args.metric}_1')177 elif args.sort == 'percent_diff':178 data = data.sort_values(by=f'percent')179 180 if args.format == 'chart':181 figure = create_chart(data, args.metric, subtitle=args.subtitle, series_names=args.series_names)182 do_open = args.output is None or args.open183 output = args.output or tempfile.NamedTemporaryFile(suffix='.html').name184 plotly.io.write_html(figure, file=output, auto_open=do_open)185 else:186 diff = plain_text_comparison(data, args.metric, baseline_name=args.series_names[0],187 candidate_name=args.series_names[1])188 diff += '\n'189 if args.output is not None:190 with open(args.output, 'w') as out:191 out.write(diff)192 else:193 sys.stdout.write(diff)194 195if __name__ == '__main__':196 main(sys.argv[1:])197