268 lines · plain
1#!/usr/bin/env python32 3import argparse4import datetime5import functools6import os7import pathlib8import re9import statistics10import subprocess11import sys12import tempfile13 14import git15import pandas16import plotly17import plotly.express18import tqdm19 20@functools.total_ordering21class Commit:22 """23 This class represents a commit inside a given Git repository.24 """25 26 def __init__(self, git_repo, sha):27 self._git_repo = git_repo28 self._sha = sha29 30 def __eq__(self, other):31 """32 Return whether two commits refer to the same commit.33 34 This doesn't take into account the content of the Git tree at those commits, only the35 'identity' of the commits themselves.36 """37 return self.fullrev == other.fullrev38 39 def __lt__(self, other):40 """41 Return whether a commit is an ancestor of another commit in the Git repository.42 """43 # Is self._sha an ancestor of other._sha?44 res = subprocess.run(['git', '-C', self._git_repo, 'merge-base', '--is-ancestor', self._sha, other._sha])45 if res.returncode not in (0, 1):46 raise RuntimeError(f'Error when trying to obtain the commit order for {self._sha} and {other._sha}')47 return res.returncode == 048 49 def __hash__(self):50 """51 Return the full revision for this commit.52 """53 return hash(self.fullrev)54 55 @functools.cache56 def show(self, include_diff=False):57 """58 Return the commit information equivalent to `git show` associated to this commit.59 """60 cmd = ['git', '-C', self._git_repo, 'show', self._sha]61 if not include_diff:62 cmd.append('--no-patch')63 return subprocess.check_output(cmd, text=True)64 65 @functools.cached_property66 def shortrev(self):67 """68 Return the shortened version of the given SHA.69 """70 return subprocess.check_output(['git', '-C', self._git_repo, 'rev-parse', '--short', self._sha], text=True).strip()71 72 @functools.cached_property73 def fullrev(self):74 """75 Return the full SHA associated to this commit.76 """77 return subprocess.check_output(['git', '-C', self._git_repo, 'rev-parse', self._sha], text=True).strip()78 79 @functools.cached_property80 def commit_date(self):81 """82 Return the date of the commit as a `datetime.datetime` object.83 """84 repo = git.Repo(self._git_repo)85 return datetime.datetime.fromtimestamp(repo.commit(self._sha).committed_date)86 87 def prefetch(self):88 """89 Prefetch cached properties associated to this commit object.90 91 This makes it possible to control when time is spent recovering that information from Git for92 e.g. better reporting to the user.93 """94 self.commit_date95 self.fullrev96 self.shortrev97 self.show()98 99 def __str__(self):100 return self._sha101 102def truncate_lines(string, n, marker=None):103 """104 Truncate the given string at a certain number of lines.105 106 Optionally, add a marker on the last line to identify that truncation has happened.107 """108 lines = string.splitlines()109 truncated = lines[:n]110 if marker is not None and len(lines) > len(truncated):111 truncated[-1] = marker112 assert len(truncated) <= n, "broken post-condition"113 return '\n'.join(truncated)114 115def create_plot(data, metric, trendline=None, subtitle=None):116 """117 Create a plot object showing the evolution of each benchmark throughout the given commits for118 the given metric.119 """120 data = data.sort_values(by=['revlist_order', 'benchmark'])121 revlist = pandas.unique(data['commit']) # list of all commits in chronological order122 hover_info = {c: truncate_lines(c.show(), 30, marker='...').replace('\n', '<br>') for c in revlist}123 figure = plotly.express.scatter(data, title=f"{revlist[0].shortrev} to {revlist[-1].shortrev}",124 subtitle=subtitle,125 x='revlist_order', y=metric,126 symbol='benchmark',127 color='benchmark',128 hover_name=[hover_info[c] for c in data['commit']],129 trendline=trendline)130 return figure131 132def directory_path(string):133 if os.path.isdir(string):134 return pathlib.Path(string)135 else:136 raise NotADirectoryError(string)137 138def parse_lnt(lines, aggregate=statistics.median):139 """140 Parse lines in LNT format and return a list of dictionnaries of the form:141 142 [143 {144 'benchmark': <benchmark1>,145 <metric1>: float,146 <metric2>: float,147 ...148 },149 {150 'benchmark': <benchmark2>,151 <metric1>: float,152 <metric2>: float,153 ...154 },155 ...156 ]157 158 If a metric has multiple values associated to it, they are aggregated into a single159 value using the provided aggregation function.160 """161 results = {}162 for line in lines:163 line = line.strip()164 if not line:165 continue166 167 (identifier, value) = line.split(' ')168 (benchmark, metric) = identifier.split('.')169 if benchmark not in results:170 results[benchmark] = {'benchmark': benchmark}171 172 entry = results[benchmark]173 if metric not in entry:174 entry[metric] = []175 entry[metric].append(float(value))176 177 for (bm, entry) in results.items():178 for metric in entry:179 if isinstance(entry[metric], list):180 entry[metric] = aggregate(entry[metric])181 182 return list(results.values())183 184def sorted_revlist(git_repo, commits):185 """186 Return the list of commits sorted by their chronological order (from oldest to newest) in the187 provided Git repository. Items earlier in the list are older than items later in the list.188 """189 revlist_cmd = ['git', '-C', git_repo, 'rev-list', '--no-walk'] + list(commits)190 revlist = subprocess.check_output(revlist_cmd, text=True).strip().splitlines()191 return list(reversed(revlist))192 193def main(argv):194 parser = argparse.ArgumentParser(195 prog='visualize-historical',196 description='Visualize historical data in LNT format. This program generates a HTML file that embeds an '197 'interactive plot with the provided data. The HTML file can then be opened in a browser to '198 'visualize the data as a chart.',199 epilog='This script depends on the modules listed in `libcxx/utils/requirements.txt`.')200 parser.add_argument('directory', type=directory_path,201 help='Path to a valid directory containing benchmark data in LNT format, each file being named <commit>.lnt. '202 'This is also the format generated by the `benchmark-historical` utility.')203 parser.add_argument('--output', '-o', type=pathlib.Path, required=False,204 help='Optional path where to output the resulting HTML file. If it already exists, it is overwritten. '205 'Defaults to a temporary file which is opened automatically once generated, but not removed after '206 'creation.')207 parser.add_argument('--metric', type=str, default='execution_time',208 help='The metric to compare. LNT data may contain multiple metrics (e.g. code size, execution time, etc) -- '209 'this option allows selecting which metric is being visualized. The default is "execution_time".')210 parser.add_argument('--filter', type=str, required=False,211 help='An optional regular expression used to filter the benchmarks included in the chart. '212 'Only benchmarks whose names match the regular expression will be included. '213 'Since the chart is interactive, it generally makes most sense to include all the benchmarks '214 'and to then filter them in the browser, but in some cases producing a chart with a reduced '215 'number of data series is useful.')216 parser.add_argument('--subtitle', type=str, required=False,217 help='Optional subtitle for the chart. This can be used to help identify the contents of the chart.')218 parser.add_argument('--git-repo', type=directory_path, default=pathlib.Path(os.getcwd()),219 help='Path to the git repository to use for ordering commits in time. '220 'By default, the current working directory is used.')221 parser.add_argument('--open', action='store_true',222 help='Whether to automatically open the generated HTML file when finished. If no output file is provided, '223 'the resulting benchmark is opened automatically by default.')224 parser.add_argument('--trendline', type=str, required=False, default=None, choices=('ols', 'lowess', 'expanding'),225 help='Optional trendline to add on each series in the chart. See the documentation in '226 'https://plotly.com/python-api-reference/generated/plotly.express.trendline_functions.html '227 'details on each option.')228 args = parser.parse_args(argv)229 230 # Extract benchmark data from the directory.231 data = {}232 files = [f for f in args.directory.glob('*.lnt')]233 for file in tqdm.tqdm(files, desc='Parsing LNT files'):234 rows = parse_lnt(file.read_text().splitlines())235 (commit, _) = os.path.splitext(os.path.basename(file))236 commit = Commit(args.git_repo, commit)237 data[commit] = rows238 239 # Obtain commit information which is then cached throughout the program. Do this240 # eagerly so we can provide a progress bar.241 for commit in tqdm.tqdm(data.keys(), desc='Prefetching Git information'):242 commit.prefetch()243 244 # Create a dataframe from the raw data and add some columns to it:245 # - 'commit' represents the Commit object associated to the results in that row246 # - `revlist_order` represents the order of the commit within the Git repository.247 # - `date` represents the commit date248 revlist = sorted_revlist(args.git_repo, [c.fullrev for c in data.keys()])249 data = pandas.DataFrame([row | {'commit': c} for (c, rows) in data.items() for row in rows])250 data = data.join(pandas.DataFrame([{'revlist_order': revlist.index(c.fullrev)} for c in data['commit']]))251 data = data.join(pandas.DataFrame([{'date': c.commit_date} for c in data['commit']]))252 253 # Filter the benchmarks if needed.254 if args.filter is not None:255 keeplist = [b for b in data['benchmark'] if re.search(args.filter, b) is not None]256 data = data[data['benchmark'].isin(keeplist)]257 if len(data) == 0:258 raise RuntimeError(f'Filter "{args.filter}" resulted in empty data set -- nothing to plot')259 260 # Plot the data for all the required benchmarks.261 figure = create_plot(data, args.metric, trendline=args.trendline, subtitle=args.subtitle)262 do_open = args.output is None or args.open263 output = args.output if args.output is not None else tempfile.NamedTemporaryFile(suffix='.html').name264 plotly.io.write_html(figure, file=output, auto_open=do_open)265 266if __name__ == '__main__':267 main(sys.argv[1:])268