472 lines · python
1#!/usr/bin/env python2 3from __future__ import print_function4 5import argparse6import errno7import functools8import html9import io10from multiprocessing import cpu_count11import os.path12import re13import shutil14import sys15 16from pygments import highlight17from pygments.lexers.c_cpp import CppLexer18from pygments.formatters import HtmlFormatter19 20import optpmap21import optrecord22 23 24desc = """Generate HTML output to visualize optimization records from the YAML files25generated with -fsave-optimization-record and -fdiagnostics-show-hotness.26 27The tools requires PyYAML and Pygments Python packages."""28 29 30# This allows passing the global context to the child processes.31class Context:32 def __init__(self, caller_loc=dict()):33 # Map function names to their source location for function where inlining happened34 self.caller_loc = caller_loc35 36 37context = Context()38 39 40def suppress(remark):41 if remark.Name == "sil.Specialized":42 return remark.getArgDict()["Function"][0].startswith('"Swift.')43 elif remark.Name == "sil.Inlined":44 return remark.getArgDict()["Callee"][0].startswith(45 ('"Swift.', '"specialized Swift.')46 )47 return False48 49 50class SourceFileRenderer:51 def __init__(self, source_dir, output_dir, filename, no_highlight):52 self.filename = filename53 existing_filename = None54 if os.path.exists(filename):55 existing_filename = filename56 else:57 fn = os.path.join(source_dir, filename)58 if os.path.exists(fn):59 existing_filename = fn60 61 self.no_highlight = no_highlight62 self.stream = io.open(63 os.path.join(output_dir, optrecord.html_file_name(filename)),64 "w",65 encoding="utf-8",66 )67 if existing_filename:68 self.source_stream = io.open(existing_filename, encoding="utf-8")69 else:70 self.source_stream = None71 print(72 """73<html>74<h1>Unable to locate file {}</h1>75</html>76 """.format(77 filename78 ),79 file=self.stream,80 )81 82 self.html_formatter = HtmlFormatter(encoding="utf-8")83 self.cpp_lexer = CppLexer(stripnl=False)84 85 def render_source_lines(self, stream, line_remarks):86 file_text = stream.read()87 88 if self.no_highlight:89 html_highlighted = file_text90 else:91 html_highlighted = highlight(file_text, self.cpp_lexer, self.html_formatter)92 93 # Note that the API is different between Python 2 and 3. On94 # Python 3, pygments.highlight() returns a bytes object, so we95 # have to decode. On Python 2, the output is str but since we96 # support unicode characters and the output streams is unicode we97 # decode too.98 html_highlighted = html_highlighted.decode("utf-8")99 100 # Take off the header and footer, these must be101 # reapplied line-wise, within the page structure102 html_highlighted = html_highlighted.replace(103 '<div class="highlight"><pre>', ""104 )105 html_highlighted = html_highlighted.replace("</pre></div>", "")106 107 for (linenum, html_line) in enumerate(html_highlighted.split("\n"), start=1):108 print(109 """110<tr>111<td><a name=\"L{linenum}\">{linenum}</a></td>112<td></td>113<td></td>114<td><div class="highlight"><pre>{html_line}</pre></div></td>115</tr>""".format(116 **locals()117 ),118 file=self.stream,119 )120 121 for remark in line_remarks.get(linenum, []):122 if not suppress(remark):123 self.render_inline_remarks(remark, html_line)124 125 def render_inline_remarks(self, r, line):126 inlining_context = r.DemangledFunctionName127 dl = context.caller_loc.get(r.Function)128 if dl:129 dl_dict = dict(list(dl))130 link = optrecord.make_link(dl_dict["File"], dl_dict["Line"] - 2)131 inlining_context = "<a href={link}>{r.DemangledFunctionName}</a>".format(132 **locals()133 )134 135 # Column is the number of characters *including* tabs, keep those and136 # replace everything else with spaces.137 indent = line[: max(r.Column, 1) - 1]138 indent = re.sub("\S", " ", indent)139 140 # Create expanded message and link if we have a multiline message.141 lines = r.message.split("\n")142 if len(lines) > 1:143 expand_link = '<a style="text-decoration: none;" href="" onclick="toggleExpandedMessage(this); return false;">+</a>'144 message = lines[0]145 expand_message = """146<div class="full-info" style="display:none;">147 <div class="col-left"><pre style="display:inline">{}</pre></div>148 <div class="expanded col-left"><pre>{}</pre></div>149</div>""".format(150 indent, "\n".join(lines[1:])151 )152 else:153 expand_link = ""154 expand_message = ""155 message = r.message156 print(157 """158<tr>159<td></td>160<td>{r.RelativeHotness}</td>161<td class=\"column-entry-{r.color}\">{r.PassWithDiffPrefix}</td>162<td><pre style="display:inline">{indent}</pre><span class=\"column-entry-yellow\">{expand_link} {message} </span>{expand_message}</td>163<td class=\"column-entry-yellow\">{inlining_context}</td>164</tr>""".format(165 **locals()166 ),167 file=self.stream,168 )169 170 def render(self, line_remarks):171 if not self.source_stream:172 return173 174 print(175 """176<html>177<title>{}</title>178<meta charset="utf-8" />179<head>180<link rel='stylesheet' type='text/css' href='style.css'>181<script type="text/javascript">182/* Simple helper to show/hide the expanded message of a remark. */183function toggleExpandedMessage(e) {{184 var FullTextElems = e.parentElement.parentElement.getElementsByClassName("full-info");185 if (!FullTextElems || FullTextElems.length < 1) {{186 return false;187 }}188 var FullText = FullTextElems[0];189 if (FullText.style.display == 'none') {{190 e.innerHTML = '-';191 FullText.style.display = 'block';192 }} else {{193 e.innerHTML = '+';194 FullText.style.display = 'none';195 }}196}}197</script>198</head>199<body>200<div class="centered">201<table class="source">202<thead>203<tr>204<th style="width: 2%">Line</td>205<th style="width: 3%">Hotness</td>206<th style="width: 10%">Optimization</td>207<th style="width: 70%">Source</td>208<th style="width: 15%">Inline Context</td>209</tr>210</thead>211<tbody>""".format(212 os.path.basename(self.filename)213 ),214 file=self.stream,215 )216 self.render_source_lines(self.source_stream, line_remarks)217 218 print(219 """220</tbody>221</table>222</body>223</html>""",224 file=self.stream,225 )226 227 228class IndexRenderer:229 def __init__(230 self, output_dir, should_display_hotness, max_hottest_remarks_on_index231 ):232 self.stream = io.open(233 os.path.join(output_dir, "index.html"), "w", encoding="utf-8"234 )235 self.should_display_hotness = should_display_hotness236 self.max_hottest_remarks_on_index = max_hottest_remarks_on_index237 238 def render_entry(self, r, odd):239 escaped_name = html.escape(r.DemangledFunctionName)240 print(241 """242<tr>243<td class=\"column-entry-{odd}\"><a href={r.Link}>{r.DebugLocString}</a></td>244<td class=\"column-entry-{odd}\">{r.RelativeHotness}</td>245<td class=\"column-entry-{odd}\">{escaped_name}</td>246<td class=\"column-entry-{r.color}\">{r.PassWithDiffPrefix}</td>247</tr>""".format(248 **locals()249 ),250 file=self.stream,251 )252 253 def render(self, all_remarks):254 print(255 """256<html>257<meta charset="utf-8" />258<head>259<link rel='stylesheet' type='text/css' href='style.css'>260</head>261<body>262<div class="centered">263<table>264<tr>265<td>Source Location</td>266<td>Hotness</td>267<td>Function</td>268<td>Pass</td>269</tr>""",270 file=self.stream,271 )272 273 max_entries = None274 if self.should_display_hotness:275 max_entries = self.max_hottest_remarks_on_index276 277 for i, remark in enumerate(all_remarks[:max_entries]):278 if not suppress(remark):279 self.render_entry(remark, i % 2)280 print(281 """282</table>283</body>284</html>""",285 file=self.stream,286 )287 288 289def _render_file(source_dir, output_dir, ctx, no_highlight, entry, filter_):290 global context291 context = ctx292 filename, remarks = entry293 SourceFileRenderer(source_dir, output_dir, filename, no_highlight).render(remarks)294 295 296def map_remarks(all_remarks):297 # Set up a map between function names and their source location for298 # function where inlining happened299 for remark in optrecord.itervalues(all_remarks):300 if (301 isinstance(remark, optrecord.Passed)302 and remark.Pass == "inline"303 and remark.Name == "Inlined"304 ):305 for arg in remark.Args:306 arg_dict = dict(list(arg))307 caller = arg_dict.get("Caller")308 if caller:309 try:310 context.caller_loc[caller] = arg_dict["DebugLoc"]311 except KeyError:312 pass313 314 315def generate_report(316 all_remarks,317 file_remarks,318 source_dir,319 output_dir,320 no_highlight,321 should_display_hotness,322 max_hottest_remarks_on_index,323 num_jobs,324 should_print_progress,325):326 try:327 os.makedirs(output_dir)328 except OSError as e:329 if e.errno == errno.EEXIST and os.path.isdir(output_dir):330 pass331 else:332 raise333 334 if should_print_progress:335 print("Rendering index page...")336 if should_display_hotness:337 sorted_remarks = sorted(338 optrecord.itervalues(all_remarks),339 key=lambda r: (340 r.Hotness,341 r.File,342 r.Line,343 r.Column,344 r.PassWithDiffPrefix,345 r.yaml_tag,346 r.Function,347 ),348 reverse=True,349 )350 else:351 sorted_remarks = sorted(352 optrecord.itervalues(all_remarks),353 key=lambda r: (354 r.File,355 r.Line,356 r.Column,357 r.PassWithDiffPrefix,358 r.yaml_tag,359 r.Function,360 ),361 )362 IndexRenderer(363 output_dir, should_display_hotness, max_hottest_remarks_on_index364 ).render(sorted_remarks)365 366 shutil.copy(367 os.path.join(os.path.dirname(os.path.realpath(__file__)), "style.css"),368 output_dir,369 )370 371 _render_file_bound = functools.partial(372 _render_file, source_dir, output_dir, context, no_highlight373 )374 if should_print_progress:375 print("Rendering HTML files...")376 optpmap.pmap(377 _render_file_bound, file_remarks.items(), num_jobs, should_print_progress378 )379 380 381def main():382 parser = argparse.ArgumentParser(description=desc)383 parser.add_argument(384 "yaml_dirs_or_files",385 nargs="+",386 help="List of optimization record files or directories searched "387 "for optimization record files.",388 )389 parser.add_argument(390 "--output-dir",391 "-o",392 default="html",393 help="Path to a directory where generated HTML files will be output. "394 "If the directory does not already exist, it will be created. "395 '"%(default)s" by default.',396 )397 parser.add_argument(398 "--jobs",399 "-j",400 default=None,401 type=int,402 help="Max job count (defaults to %(default)s, the current CPU count)",403 )404 parser.add_argument("--source-dir", "-s", default="", help="set source directory")405 parser.add_argument(406 "--no-progress-indicator",407 "-n",408 action="store_true",409 default=False,410 help="Do not display any indicator of how many YAML files were read "411 "or rendered into HTML.",412 )413 parser.add_argument(414 "--max-hottest-remarks-on-index",415 default=1000,416 type=int,417 help="Maximum number of the hottest remarks to appear on the index page",418 )419 parser.add_argument(420 "--no-highlight",421 action="store_true",422 default=False,423 help="Do not use a syntax highlighter when rendering the source code",424 )425 parser.add_argument(426 "--demangler",427 help="Set the demangler to be used (defaults to %s)"428 % optrecord.Remark.default_demangler,429 )430 431 parser.add_argument(432 "--filter",433 default="",434 help="Only display remarks from passes matching filter expression",435 )436 437 # Do not make this a global variable. Values needed to be propagated through438 # to individual classes and functions to be portable with multiprocessing across439 # Windows and non-Windows.440 args = parser.parse_args()441 442 print_progress = not args.no_progress_indicator443 if args.demangler:444 optrecord.Remark.set_demangler(args.demangler)445 446 files = optrecord.find_opt_files(*args.yaml_dirs_or_files)447 if not files:448 parser.error("No *.opt.yaml files found")449 sys.exit(1)450 451 all_remarks, file_remarks, should_display_hotness = optrecord.gather_results(452 files, args.jobs, print_progress, args.filter453 )454 455 map_remarks(all_remarks)456 457 generate_report(458 all_remarks,459 file_remarks,460 args.source_dir,461 args.output_dir,462 args.no_highlight,463 should_display_hotness,464 args.max_hottest_remarks_on_index,465 args.jobs,466 print_progress,467 )468 469 470if __name__ == "__main__":471 main()472