352 lines · python
1#!/usr/bin/env python2 3from __future__ import print_function4 5import io6import yaml7 8# Try to use the C parser.9try:10 from yaml import CLoader as Loader11except ImportError:12 print("For faster parsing, you may want to install libYAML for PyYAML")13 from yaml import Loader14 15import html16from collections import defaultdict17import fnmatch18import functools19from multiprocessing import Lock20import os, os.path21import subprocess22from sys import intern23import re24 25import optpmap26 27 28def itervalues(d):29 return iter(d.values())30 31 32def iteritems(d):33 return iter(d.items())34 35 36def html_file_name(filename):37 return filename.replace("/", "_").replace("#", "_") + ".html"38 39 40def make_link(File, Line):41 return '"{}#L{}"'.format(html_file_name(File), Line)42 43 44class Remark(yaml.YAMLObject):45 # Work-around for http://pyyaml.org/ticket/154.46 yaml_loader = Loader47 48 default_demangler = "c++filt -n"49 demangler_proc = None50 demangler_lock = Lock()51 52 @classmethod53 def set_demangler(cls, demangler):54 cls.demangler_proc = subprocess.Popen(55 demangler.split(), stdin=subprocess.PIPE, stdout=subprocess.PIPE56 )57 58 @classmethod59 def demangle(cls, name):60 with cls.demangler_lock:61 if not cls.demangler_proc:62 cls.set_demangler(cls.default_demangler)63 cls.demangler_proc.stdin.write((name + "\n").encode("utf-8"))64 cls.demangler_proc.stdin.flush()65 return cls.demangler_proc.stdout.readline().rstrip().decode("utf-8")66 67 # Intern all strings since we have lot of duplication across filenames,68 # remark text.69 #70 # Change Args from a list of dicts to a tuple of tuples. This saves71 # memory in two ways. One, a small tuple is significantly smaller than a72 # small dict. Two, using tuple instead of list allows Args to be directly73 # used as part of the key (in Python only immutable types are hashable).74 def _reduce_memory(self):75 self.Pass = intern(self.Pass)76 self.Name = intern(self.Name)77 try:78 # Can't intern unicode strings.79 self.Function = intern(self.Function)80 except:81 pass82 83 def _reduce_memory_dict(old_dict):84 new_dict = dict()85 for (k, v) in iteritems(old_dict):86 if type(k) is str:87 k = intern(k)88 89 if type(v) is str:90 v = intern(v)91 elif type(v) is dict:92 # This handles [{'Caller': ..., 'DebugLoc': { 'File': ... }}]93 v = _reduce_memory_dict(v)94 new_dict[k] = v95 return tuple(new_dict.items())96 97 self.Args = tuple([_reduce_memory_dict(arg_dict) for arg_dict in self.Args])98 99 # The inverse operation of the dictonary-related memory optimization in100 # _reduce_memory_dict. E.g.101 # (('DebugLoc', (('File', ...) ... ))) -> [{'DebugLoc': {'File': ...} ....}]102 def recover_yaml_structure(self):103 def tuple_to_dict(t):104 d = dict()105 for (k, v) in t:106 if type(v) is tuple:107 v = tuple_to_dict(v)108 d[k] = v109 return d110 111 self.Args = [tuple_to_dict(arg_tuple) for arg_tuple in self.Args]112 113 def canonicalize(self):114 if not hasattr(self, "Hotness"):115 self.Hotness = 0116 if not hasattr(self, "Args"):117 self.Args = []118 self._reduce_memory()119 120 @property121 def File(self):122 return self.DebugLoc["File"]123 124 @property125 def Line(self):126 return int(self.DebugLoc["Line"])127 128 @property129 def Column(self):130 return self.DebugLoc["Column"]131 132 @property133 def DebugLocString(self):134 return "{}:{}:{}".format(self.File, self.Line, self.Column)135 136 @property137 def DemangledFunctionName(self):138 return self.demangle(self.Function)139 140 @property141 def Link(self):142 return make_link(self.File, self.Line)143 144 def getArgString(self, mapping):145 mapping = dict(list(mapping))146 dl = mapping.get("DebugLoc")147 if dl:148 del mapping["DebugLoc"]149 150 assert len(mapping) == 1151 (key, value) = list(mapping.items())[0]152 153 if key == "Caller" or key == "Callee" or key == "DirectCallee":154 value = html.escape(self.demangle(value))155 156 if dl and key != "Caller":157 dl_dict = dict(list(dl))158 return "<a href={}>{}</a>".format(159 make_link(dl_dict["File"], dl_dict["Line"]), value160 )161 else:162 return value163 164 # Return a cached dictionary for the arguments. The key for each entry is165 # the argument key (e.g. 'Callee' for inlining remarks. The value is a166 # list containing the value (e.g. for 'Callee' the function) and167 # optionally a DebugLoc.168 def getArgDict(self):169 if hasattr(self, "ArgDict"):170 return self.ArgDict171 self.ArgDict = {}172 for arg in self.Args:173 if len(arg) == 2:174 if arg[0][0] == "DebugLoc":175 dbgidx = 0176 else:177 assert arg[1][0] == "DebugLoc"178 dbgidx = 1179 180 key = arg[1 - dbgidx][0]181 entry = (arg[1 - dbgidx][1], arg[dbgidx][1])182 else:183 arg = arg[0]184 key = arg[0]185 entry = (arg[1],)186 187 self.ArgDict[key] = entry188 return self.ArgDict189 190 def getDiffPrefix(self):191 if hasattr(self, "Added"):192 if self.Added:193 return "+"194 else:195 return "-"196 return ""197 198 @property199 def PassWithDiffPrefix(self):200 return self.getDiffPrefix() + self.Pass201 202 @property203 def message(self):204 # Args is a list of mappings (dictionaries)205 values = [self.getArgString(mapping) for mapping in self.Args]206 return "".join(values)207 208 @property209 def RelativeHotness(self):210 if self.max_hotness:211 return "{0:.2f}%".format(self.Hotness * 100.0 / self.max_hotness)212 else:213 return ""214 215 @property216 def key(self):217 return (218 self.__class__,219 self.PassWithDiffPrefix,220 self.Name,221 self.File,222 self.Line,223 self.Column,224 self.Function,225 self.Args,226 )227 228 def __hash__(self):229 return hash(self.key)230 231 def __eq__(self, other):232 return self.key == other.key233 234 def __repr__(self):235 return str(self.key)236 237 238class Analysis(Remark):239 yaml_tag = "!Analysis"240 241 @property242 def color(self):243 return "white"244 245 246class AnalysisFPCommute(Analysis):247 yaml_tag = "!AnalysisFPCommute"248 249 250class AnalysisAliasing(Analysis):251 yaml_tag = "!AnalysisAliasing"252 253 254class Passed(Remark):255 yaml_tag = "!Passed"256 257 @property258 def color(self):259 return "green"260 261 262class Missed(Remark):263 yaml_tag = "!Missed"264 265 @property266 def color(self):267 return "red"268 269 270class Failure(Missed):271 yaml_tag = "!Failure"272 273 274def get_remarks(input_file, filter_=None):275 max_hotness = 0276 all_remarks = dict()277 file_remarks = defaultdict(functools.partial(defaultdict, list))278 279 with io.open(input_file, encoding="utf-8") as f:280 docs = yaml.load_all(f, Loader=Loader)281 282 filter_e = None283 if filter_:284 filter_e = re.compile(filter_)285 for remark in docs:286 remark.canonicalize()287 # Avoid remarks withoug debug location or if they are duplicated288 if not hasattr(remark, "DebugLoc") or remark.key in all_remarks:289 continue290 291 if filter_e and not filter_e.search(remark.Pass):292 continue293 294 all_remarks[remark.key] = remark295 296 file_remarks[remark.File][remark.Line].append(remark)297 298 # If we're reading a back a diff yaml file, max_hotness is already299 # captured which may actually be less than the max hotness found300 # in the file.301 if hasattr(remark, "max_hotness"):302 max_hotness = remark.max_hotness303 max_hotness = max(max_hotness, remark.Hotness)304 305 return max_hotness, all_remarks, file_remarks306 307 308def gather_results(filenames, num_jobs, should_print_progress, filter_=None):309 if should_print_progress:310 print("Reading YAML files...")311 remarks = optpmap.pmap(312 get_remarks, filenames, num_jobs, should_print_progress, filter_313 )314 max_hotness = max(entry[0] for entry in remarks)315 316 def merge_file_remarks(file_remarks_job, all_remarks, merged):317 for filename, d in iteritems(file_remarks_job):318 for line, remarks in iteritems(d):319 for remark in remarks:320 # Bring max_hotness into the remarks so that321 # RelativeHotness does not depend on an external global.322 remark.max_hotness = max_hotness323 if remark.key not in all_remarks:324 merged[filename][line].append(remark)325 326 all_remarks = dict()327 file_remarks = defaultdict(functools.partial(defaultdict, list))328 for _, all_remarks_job, file_remarks_job in remarks:329 merge_file_remarks(file_remarks_job, all_remarks, file_remarks)330 all_remarks.update(all_remarks_job)331 332 return all_remarks, file_remarks, max_hotness != 0333 334 335def find_opt_files(*dirs_or_files):336 all = []337 for dir_or_file in dirs_or_files:338 if os.path.isfile(dir_or_file):339 all.append(dir_or_file)340 else:341 for dir, subdirs, files in os.walk(dir_or_file):342 # Exclude mounted directories and symlinks (os.walk default).343 subdirs[:] = [344 d for d in subdirs if not os.path.ismount(os.path.join(dir, d))345 ]346 for file in files:347 if fnmatch.fnmatch(file, "*.opt.yaml*") or fnmatch.fnmatch(348 file, "*.opt.ld.yaml*"349 ):350 all.append(os.path.join(dir, file))351 return all352