1190 lines · python
1#!/usr/bin/env python2# ===- lib/asan/scripts/asan_symbolize.py -----------------------------------===#3#4# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.5# See https://llvm.org/LICENSE.txt for license information.6# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception7#8# ===------------------------------------------------------------------------===#9"""10Example of use:11 asan_symbolize.py -c "$HOME/opt/cross/bin/arm-linux-gnueabi-" -s "$HOME/SymbolFiles" < asan.log12 13PLUGINS14 15This script provides a way for external plug-ins to hook into the behaviour of16various parts of this script (see `--plugins`). This is useful for situations17where it is necessary to handle site-specific quirks (e.g. binaries with debug18symbols only accessible via a remote service) without having to modify the19script itself.20 21"""22import argparse23import bisect24import errno25import getopt26import logging27import os28import re29import shutil30import subprocess31import sys32 33symbolizers = {}34demangle = False35binutils_prefix = None36fix_filename_patterns = None37logfile = sys.stdin38allow_system_symbolizer = True39force_system_symbolizer = False40 41# FIXME: merge the code that calls fix_filename().42def fix_filename(file_name):43 if fix_filename_patterns:44 for path_to_cut in fix_filename_patterns:45 file_name = re.sub(".*" + path_to_cut, "", file_name)46 file_name = re.sub(".*asan_[a-z_]*.(cc|cpp):[0-9]*", "_asan_rtl_", file_name)47 file_name = re.sub(".*crtstuff.c:0", "???:0", file_name)48 return file_name49 50 51def is_valid_arch(s):52 return s in [53 "i386",54 "x86_64",55 "x86_64h",56 "arm",57 "armv6",58 "armv7",59 "armv7s",60 "armv7k",61 "arm64",62 "arm64e",63 "powerpc",64 "powerpc64",65 "powerpc64le",66 "s390x",67 "s390",68 "riscv64",69 "loongarch64",70 ]71 72 73def guess_arch(addr):74 # Guess which arch we're running. 10 = len('0x') + 8 hex digits.75 if len(addr) > 10:76 return "x86_64"77 else:78 return "i386"79 80 81class Symbolizer(object):82 def __init__(self):83 pass84 85 def symbolize(self, addr, binary, offset):86 """Symbolize the given address (pair of binary and offset).87 88 Overriden in subclasses.89 Args:90 addr: virtual address of an instruction.91 binary: path to executable/shared object containing this instruction.92 offset: instruction offset in the @binary.93 Returns:94 list of strings (one string for each inlined frame) describing95 the code locations for this instruction (that is, function name, file96 name, line and column numbers).97 """98 return None99 100 101class LLVMSymbolizer(Symbolizer):102 def __init__(self, symbolizer_path, default_arch, system, dsym_hints=[]):103 super(LLVMSymbolizer, self).__init__()104 self.symbolizer_path = symbolizer_path105 self.default_arch = default_arch106 self.system = system107 self.dsym_hints = dsym_hints108 self.pipe = self.open_llvm_symbolizer()109 110 def open_llvm_symbolizer(self):111 cmd = [112 self.symbolizer_path,113 ("--demangle" if demangle else "--no-demangle"),114 "--functions=linkage",115 "--inlines",116 "--default-arch=%s" % self.default_arch,117 ]118 if self.system == "Darwin":119 for hint in self.dsym_hints:120 cmd.append("--dsym-hint=%s" % hint)121 logging.debug(" ".join(cmd))122 try:123 result = subprocess.Popen(124 cmd,125 stdin=subprocess.PIPE,126 stdout=subprocess.PIPE,127 bufsize=0,128 universal_newlines=True,129 )130 except OSError:131 result = None132 return result133 134 def symbolize(self, addr, binary, offset):135 """Overrides Symbolizer.symbolize."""136 if not self.pipe:137 return None138 result = []139 try:140 symbolizer_input = '"%s" %s' % (binary, offset)141 logging.debug(symbolizer_input)142 self.pipe.stdin.write("%s\n" % symbolizer_input)143 while True:144 function_name = self.pipe.stdout.readline().rstrip()145 if not function_name:146 break147 file_name = self.pipe.stdout.readline().rstrip()148 file_name = fix_filename(file_name)149 if not function_name.startswith("??") or not file_name.startswith("??"):150 # Append only non-trivial frames.151 result.append("%s in %s %s" % (addr, function_name, file_name))152 except Exception:153 result = []154 if not result:155 result = None156 return result157 158 159def LLVMSymbolizerFactory(system, default_arch, dsym_hints=[]):160 symbolizer_path = os.getenv("LLVM_SYMBOLIZER_PATH")161 if not symbolizer_path:162 symbolizer_path = os.getenv("ASAN_SYMBOLIZER_PATH")163 if not symbolizer_path:164 # Assume llvm-symbolizer is in PATH.165 symbolizer_path = "llvm-symbolizer"166 return LLVMSymbolizer(symbolizer_path, default_arch, system, dsym_hints)167 168 169class Addr2LineSymbolizer(Symbolizer):170 def __init__(self, binary):171 super(Addr2LineSymbolizer, self).__init__()172 self.binary = binary173 self.pipe = self.open_addr2line()174 self.output_terminator = -1175 176 def open_addr2line(self):177 addr2line_tool = "addr2line"178 if binutils_prefix:179 addr2line_tool = binutils_prefix + addr2line_tool180 logging.debug("addr2line binary is %s" % shutil.which(addr2line_tool))181 cmd = [addr2line_tool, "-fi"]182 if demangle:183 cmd += ["--demangle"]184 cmd += ["-e", self.binary]185 logging.debug(" ".join(cmd))186 return subprocess.Popen(187 cmd,188 stdin=subprocess.PIPE,189 stdout=subprocess.PIPE,190 bufsize=0,191 universal_newlines=True,192 )193 194 def symbolize(self, addr, binary, offset):195 """Overrides Symbolizer.symbolize."""196 if self.binary != binary:197 return None198 lines = []199 try:200 self.pipe.stdin.write("%s\n" % offset)201 self.pipe.stdin.write("%s\n" % self.output_terminator)202 is_first_frame = True203 while True:204 function_name = self.pipe.stdout.readline().rstrip()205 logging.debug("read function_name='%s' from addr2line" % function_name)206 # If llvm-symbolizer is installed as addr2line, older versions of207 # llvm-symbolizer will print -1 when presented with -1 and not print208 # a second line. In that case we will block for ever trying to read the209 # file name. This also happens for non-existent files, in which case GNU210 # addr2line exits immediate, but llvm-symbolizer does not (see211 # https://llvm.org/PR42754).212 if function_name == "-1":213 logging.debug("got function '-1' -> no more input")214 break215 file_name = self.pipe.stdout.readline().rstrip()216 logging.debug("read file_name='%s' from addr2line" % file_name)217 if is_first_frame:218 is_first_frame = False219 elif function_name == "??":220 assert file_name == "??:0", file_name221 logging.debug("got function '??' -> no more input")222 break223 elif not function_name:224 assert not file_name, file_name225 logging.debug("got empty function name -> no more input")226 break227 if not function_name and not file_name:228 logging.debug(229 "got empty function and file name -> unknown function"230 )231 function_name = "??"232 file_name = "??:0"233 lines.append((function_name, file_name))234 except IOError as e:235 # EPIPE happens if addr2line exits early (which some implementations do236 # if an invalid file is passed).237 if e.errno == errno.EPIPE:238 logging.debug(239 f"addr2line exited early (broken pipe) returncode={self.pipe.poll()}"240 )241 else:242 logging.debug(243 "unexpected I/O exception communicating with addr2line", exc_info=e244 )245 lines.append(("??", "??:0"))246 except Exception as e:247 logging.debug(248 "got unknown exception communicating with addr2line", exc_info=e249 )250 lines.append(("??", "??:0"))251 return [252 "%s in %s %s" % (addr, function, fix_filename(file))253 for (function, file) in lines254 ]255 256 257class UnbufferedLineConverter(object):258 """259 Wrap a child process that responds to each line of input with one line of260 output. Uses pty to trick the child into providing unbuffered output.261 """262 263 def __init__(self, args, close_stderr=False):264 # Local imports so that the script can start on Windows.265 import pty266 import termios267 268 pid, fd = pty.fork()269 if pid == 0:270 # We're the child. Transfer control to command.271 if close_stderr:272 dev_null = os.open("/dev/null", 0)273 os.dup2(dev_null, 2)274 os.execvp(args[0], args)275 else:276 # Disable echoing.277 attr = termios.tcgetattr(fd)278 attr[3] = attr[3] & ~termios.ECHO279 termios.tcsetattr(fd, termios.TCSANOW, attr)280 # Set up a file()-like interface to the child process281 self.r = os.fdopen(fd, "r", 1)282 self.w = os.fdopen(os.dup(fd), "w", 1)283 284 def convert(self, line):285 self.w.write(line + "\n")286 return self.readline()287 288 def readline(self):289 return self.r.readline().rstrip()290 291 292class DarwinSymbolizer(Symbolizer):293 def __init__(self, addr, binary, arch):294 super(DarwinSymbolizer, self).__init__()295 self.binary = binary296 self.arch = arch297 self.open_atos()298 299 def open_atos(self):300 logging.debug("atos -o %s -arch %s", self.binary, self.arch)301 cmdline = ["atos", "-o", self.binary, "-arch", self.arch]302 self.atos = UnbufferedLineConverter(cmdline, close_stderr=True)303 304 def symbolize(self, addr, binary, offset):305 """Overrides Symbolizer.symbolize."""306 if self.binary != binary:307 return None308 if not os.path.exists(binary):309 # If the binary doesn't exist atos will exit which will lead to IOError310 # exceptions being raised later on so just don't try to symbolize.311 return ["{} ({}:{}+{})".format(addr, binary, self.arch, offset)]312 atos_line = self.atos.convert("0x%x" % int(offset, 16))313 while "got symbolicator for" in atos_line:314 atos_line = self.atos.readline()315 # A well-formed atos response looks like this:316 # foo(type1, type2) (in object.name) (filename.cc:80)317 # NOTE:318 # * For C functions atos omits parentheses and argument types.319 # * For C++ functions the function name (i.e., `foo` above) may contain320 # templates which may contain parentheses.321 match = re.match(r"^(.*) \(in (.*)\) \((.*:\d*)\)$", atos_line)322 logging.debug("atos_line: %s", atos_line)323 if match:324 function_name = match.group(1)325 file_name = fix_filename(match.group(3))326 return ["%s in %s %s" % (addr, function_name, file_name)]327 else:328 return ["%s in %s" % (addr, atos_line)]329 330 331# Chain several symbolizers so that if one symbolizer fails, we fall back332# to the next symbolizer in chain.333class ChainSymbolizer(Symbolizer):334 def __init__(self, symbolizer_list):335 super(ChainSymbolizer, self).__init__()336 self.symbolizer_list = symbolizer_list337 338 def symbolize(self, addr, binary, offset):339 """Overrides Symbolizer.symbolize."""340 for symbolizer in self.symbolizer_list:341 if symbolizer:342 result = symbolizer.symbolize(addr, binary, offset)343 if result:344 return result345 return None346 347 def append_symbolizer(self, symbolizer):348 self.symbolizer_list.append(symbolizer)349 350 351def BreakpadSymbolizerFactory(binary):352 suffix = os.getenv("BREAKPAD_SUFFIX")353 if suffix:354 filename = binary + suffix355 if os.access(filename, os.F_OK):356 return BreakpadSymbolizer(filename)357 return None358 359 360def SystemSymbolizerFactory(system, addr, binary, arch):361 if system == "Darwin":362 return DarwinSymbolizer(addr, binary, arch)363 elif system in ["Linux", "FreeBSD", "NetBSD", "SunOS"]:364 return Addr2LineSymbolizer(binary)365 366 367class BreakpadSymbolizer(Symbolizer):368 def __init__(self, filename):369 super(BreakpadSymbolizer, self).__init__()370 self.filename = filename371 lines = file(filename).readlines()372 self.files = []373 self.symbols = {}374 self.address_list = []375 self.addresses = {}376 # MODULE mac x86_64 A7001116478B33F18FF9BEDE9F615F190 t377 fragments = lines[0].rstrip().split()378 self.arch = fragments[2]379 self.debug_id = fragments[3]380 self.binary = " ".join(fragments[4:])381 self.parse_lines(lines[1:])382 383 def parse_lines(self, lines):384 cur_function_addr = ""385 for line in lines:386 fragments = line.split()387 if fragments[0] == "FILE":388 assert int(fragments[1]) == len(self.files)389 self.files.append(" ".join(fragments[2:]))390 elif fragments[0] == "PUBLIC":391 self.symbols[int(fragments[1], 16)] = " ".join(fragments[3:])392 elif fragments[0] in ["CFI", "STACK"]:393 pass394 elif fragments[0] == "FUNC":395 cur_function_addr = int(fragments[1], 16)396 if not cur_function_addr in self.symbols.keys():397 self.symbols[cur_function_addr] = " ".join(fragments[4:])398 else:399 # Line starting with an address.400 addr = int(fragments[0], 16)401 self.address_list.append(addr)402 # Tuple of symbol address, size, line, file number.403 self.addresses[addr] = (404 cur_function_addr,405 int(fragments[1], 16),406 int(fragments[2]),407 int(fragments[3]),408 )409 self.address_list.sort()410 411 def get_sym_file_line(self, addr):412 key = None413 if addr in self.addresses.keys():414 key = addr415 else:416 index = bisect.bisect_left(self.address_list, addr)417 if index == 0:418 return None419 else:420 key = self.address_list[index - 1]421 sym_id, size, line_no, file_no = self.addresses[key]422 symbol = self.symbols[sym_id]423 filename = self.files[file_no]424 if addr < key + size:425 return symbol, filename, line_no426 else:427 return None428 429 def symbolize(self, addr, binary, offset):430 if self.binary != binary:431 return None432 res = self.get_sym_file_line(int(offset, 16))433 if res:434 function_name, file_name, line_no = res435 result = ["%s in %s %s:%d" % (addr, function_name, file_name, line_no)]436 print(result)437 return result438 else:439 return None440 441 442class SymbolizationLoop(object):443 def __init__(self, plugin_proxy=None, dsym_hint_producer=None):444 self.plugin_proxy = plugin_proxy445 if sys.platform == "win32":446 # ASan on Windows uses dbghelp.dll to symbolize in-process, which works447 # even in sandboxed processes. Nothing needs to be done here.448 self.process_line = self.process_line_echo449 else:450 # Used by clients who may want to supply a different binary name.451 # E.g. in Chrome several binaries may share a single .dSYM.452 self.dsym_hint_producer = dsym_hint_producer453 self.system = os.uname()[0]454 if self.system not in [455 "Linux",456 "Darwin",457 "FreeBSD",458 "NetBSD",459 "SunOS",460 "AIX",461 ]:462 raise Exception("Unknown system")463 self.llvm_symbolizers = {}464 self.last_llvm_symbolizer = None465 self.dsym_hints = set([])466 self.frame_no = 0467 self.process_line = self.process_line_posix468 self.using_module_map = plugin_proxy.has_plugin(ModuleMapPlugIn.get_name())469 470 def symbolize_address(self, addr, binary, offset, arch):471 # On non-Darwin (i.e. on platforms without .dSYM debug info) always use472 # a single symbolizer binary.473 # On Darwin, if the dsym hint producer is present:474 # 1. check whether we've seen this binary already; if so,475 # use |llvm_symbolizers[binary]|, which has already loaded the debug476 # info for this binary (might not be the case for477 # |last_llvm_symbolizer|);478 # 2. otherwise check if we've seen all the hints for this binary already;479 # if so, reuse |last_llvm_symbolizer| which has the full set of hints;480 # 3. otherwise create a new symbolizer and pass all currently known481 # .dSYM hints to it.482 result = None483 if not force_system_symbolizer:484 if not binary in self.llvm_symbolizers:485 use_new_symbolizer = True486 if self.system == "Darwin" and self.dsym_hint_producer:487 dsym_hints_for_binary = set(self.dsym_hint_producer(binary))488 use_new_symbolizer = bool(dsym_hints_for_binary - self.dsym_hints)489 self.dsym_hints |= dsym_hints_for_binary490 if self.last_llvm_symbolizer and not use_new_symbolizer:491 self.llvm_symbolizers[binary] = self.last_llvm_symbolizer492 else:493 self.last_llvm_symbolizer = LLVMSymbolizerFactory(494 self.system, arch, self.dsym_hints495 )496 self.llvm_symbolizers[binary] = self.last_llvm_symbolizer497 # Use the chain of symbolizers:498 # Breakpad symbolizer -> LLVM symbolizer -> addr2line/atos499 # (fall back to next symbolizer if the previous one fails).500 if not binary in symbolizers:501 symbolizers[binary] = ChainSymbolizer(502 [BreakpadSymbolizerFactory(binary), self.llvm_symbolizers[binary]]503 )504 result = symbolizers[binary].symbolize(addr, binary, offset)505 else:506 symbolizers[binary] = ChainSymbolizer([])507 if result is None:508 if not allow_system_symbolizer:509 raise Exception("Failed to launch or use llvm-symbolizer.")510 # Initialize system symbolizer only if other symbolizers failed.511 symbolizers[binary].append_symbolizer(512 SystemSymbolizerFactory(self.system, addr, binary, arch)513 )514 result = symbolizers[binary].symbolize(addr, binary, offset)515 # The system symbolizer must produce some result.516 assert result517 return result518 519 def get_symbolized_lines(self, symbolized_lines):520 if not symbolized_lines:521 # If it is an unparsable frame, but contains a frame counter and address522 # replace the frame counter so the stack is still consistent.523 unknown_stack_frame_format = r"^( *#([0-9]+) +)(0x[0-9a-f]+) +.*"524 match = re.match(unknown_stack_frame_format, self.current_line)525 if match:526 rewritten_line = (527 self.current_line[: match.start(2)]528 + str(self.frame_no)529 + self.current_line[match.end(2) :]530 )531 self.frame_no += 1532 return [rewritten_line]533 # Not a frame line so don't increment the frame counter.534 return [self.current_line]535 result = []536 for symbolized_frame in symbolized_lines:537 result.append(538 " #%s %s" % (str(self.frame_no), symbolized_frame.rstrip())539 )540 self.frame_no += 1541 return result542 543 def process_logfile(self):544 self.frame_no = 0545 for line in logfile:546 processed = self.process_line(line)547 print("\n".join(processed))548 549 def process_line_echo(self, line):550 return [line.rstrip()]551 552 def process_line_posix(self, line):553 self.current_line = line.rstrip()554 # Unsymbolicated:555 # #0 0x7f6e35cf2e45 (/blah/foo.so+0x11fe45)556 # Partially symbolicated:557 # #0 0x7f6e35cf2e45 in foo (foo.so+0x11fe45)558 # NOTE: We have to very liberal with symbol559 # names in the regex because it could be an560 # Objective-C or C++ demangled name.561 stack_trace_line_format = (562 r"^( *#([0-9]+) *)(0x[0-9a-f]+) *(?:in *.+)? *\((.*)\+(0x[0-9a-f]+)\)"563 )564 match = re.match(stack_trace_line_format, line)565 if not match:566 logging.debug('Line "{}" does not match regex'.format(line))567 return self.get_symbolized_lines(None)568 logging.debug(line)569 _, frameno_str, addr, binary, offset = match.groups()570 571 if not self.using_module_map and not os.path.isabs(binary):572 # Do not try to symbolicate if the binary is just the module file name573 # and a module map is unavailable.574 # FIXME(dliew): This is currently necessary for reports on Darwin that are575 # partially symbolicated by `atos`.576 return self.get_symbolized_lines(None)577 arch = ""578 # Arch can be embedded in the filename, e.g.: "libabc.dylib:x86_64h"579 colon_pos = binary.rfind(":")580 if colon_pos != -1:581 maybe_arch = binary[colon_pos + 1 :]582 if is_valid_arch(maybe_arch):583 arch = maybe_arch584 binary = binary[0:colon_pos]585 if arch == "":586 arch = guess_arch(addr)587 if frameno_str == "0":588 # Assume that frame #0 is the first frame of new stack trace.589 self.frame_no = 0590 original_binary = binary591 binary = self.plugin_proxy.filter_binary_path(binary)592 if binary is None:593 # The binary filter has told us this binary can't be symbolized.594 logging.debug('Skipping symbolication of binary "%s"', original_binary)595 return self.get_symbolized_lines(None)596 symbolized_line = self.symbolize_address(addr, binary, offset, arch)597 if not symbolized_line:598 if original_binary != binary:599 symbolized_line = self.symbolize_address(600 addr, original_binary, offset, arch601 )602 return self.get_symbolized_lines(symbolized_line)603 604 605class AsanSymbolizerPlugInProxy(object):606 """607 Serves several purposes:608 - Manages the lifetime of plugins (must be used a `with` statement).609 - Provides interface for calling into plugins from within this script.610 """611 612 def __init__(self):613 self._plugins = []614 self._plugin_names = set()615 616 def _load_plugin_from_file_impl_py_gt_2(self, file_path, globals_space):617 with open(file_path, "r") as f:618 exec(f.read(), globals_space, None)619 620 def load_plugin_from_file(self, file_path):621 logging.info('Loading plugins from "{}"'.format(file_path))622 globals_space = dict(globals())623 # Provide function to register plugins624 def register_plugin(plugin):625 logging.info("Registering plugin %s", plugin.get_name())626 self.add_plugin(plugin)627 628 globals_space["register_plugin"] = register_plugin629 if sys.version_info.major < 3:630 execfile(file_path, globals_space, None)631 else:632 # Indirection here is to avoid a bug in older Python 2 versions:633 # `SyntaxError: unqualified exec is not allowed in function ...`634 self._load_plugin_from_file_impl_py_gt_2(file_path, globals_space)635 636 def add_plugin(self, plugin):637 assert isinstance(plugin, AsanSymbolizerPlugIn)638 self._plugins.append(plugin)639 self._plugin_names.add(plugin.get_name())640 plugin._receive_proxy(self)641 642 def remove_plugin(self, plugin):643 assert isinstance(plugin, AsanSymbolizerPlugIn)644 self._plugins.remove(plugin)645 self._plugin_names.remove(plugin.get_name())646 logging.debug("Removing plugin %s", plugin.get_name())647 plugin.destroy()648 649 def has_plugin(self, name):650 """651 Returns true iff the plugin name is currently652 being managed by AsanSymbolizerPlugInProxy.653 """654 return name in self._plugin_names655 656 def register_cmdline_args(self, parser):657 plugins = list(self._plugins)658 for plugin in plugins:659 plugin.register_cmdline_args(parser)660 661 def process_cmdline_args(self, pargs):662 # Use copy so we can remove items as we iterate.663 plugins = list(self._plugins)664 for plugin in plugins:665 keep = plugin.process_cmdline_args(pargs)666 assert isinstance(keep, bool)667 if not keep:668 self.remove_plugin(plugin)669 670 def __enter__(self):671 return self672 673 def __exit__(self, exc_type, exc_val, exc_tb):674 for plugin in self._plugins:675 plugin.destroy()676 # Don't suppress raised exceptions677 return False678 679 def _filter_single_value(self, function_name, input_value):680 """681 Helper for filter style plugin functions.682 """683 new_value = input_value684 for plugin in self._plugins:685 result = getattr(plugin, function_name)(new_value)686 if result is None:687 return None688 new_value = result689 return new_value690 691 def filter_binary_path(self, binary_path):692 """693 Consult available plugins to filter the path to a binary694 to make it suitable for symbolication.695 696 Returns `None` if symbolication should not be attempted for this697 binary.698 """699 return self._filter_single_value("filter_binary_path", binary_path)700 701 def filter_module_desc(self, module_desc):702 """703 Consult available plugins to determine the module704 description suitable for symbolication.705 706 Returns `None` if symbolication should not be attempted for this module.707 """708 assert isinstance(module_desc, ModuleDesc)709 return self._filter_single_value("filter_module_desc", module_desc)710 711 712class AsanSymbolizerPlugIn(object):713 """714 This is the interface the `asan_symbolize.py` code uses to talk715 to plugins.716 """717 718 @classmethod719 def get_name(cls):720 """721 Returns the name of the plugin.722 """723 return cls.__name__724 725 def _receive_proxy(self, proxy):726 assert isinstance(proxy, AsanSymbolizerPlugInProxy)727 self.proxy = proxy728 729 def register_cmdline_args(self, parser):730 """731 Hook for registering command line arguments to be732 consumed in `process_cmdline_args()`.733 734 `parser` - Instance of `argparse.ArgumentParser`.735 """736 pass737 738 def process_cmdline_args(self, pargs):739 """740 Hook for handling parsed arguments. Implementations741 should not modify `pargs`.742 743 `pargs` - Instance of `argparse.Namespace` containing744 parsed command line arguments.745 746 Return `True` if plug-in should be used, otherwise747 return `False`.748 """749 return True750 751 def destroy(self):752 """753 Hook called when a plugin is about to be destroyed.754 Implementations should free any allocated resources here.755 """756 pass757 758 # Symbolization hooks759 def filter_binary_path(self, binary_path):760 """761 Given a binary path return a binary path suitable for symbolication.762 763 Implementations should return `None` if symbolication of this binary764 should be skipped.765 """766 return binary_path767 768 def filter_module_desc(self, module_desc):769 """770 Given a ModuleDesc object (`module_desc`) return771 a ModuleDesc suitable for symbolication.772 773 Implementations should return `None` if symbolication of this binary774 should be skipped.775 """776 return module_desc777 778 779class ModuleDesc(object):780 def __init__(self, name, arch, start_addr, end_addr, module_path, uuid):781 self.name = name782 self.arch = arch783 self.start_addr = start_addr784 self.end_addr = end_addr785 # Module path from an ASan report.786 self.module_path = module_path787 # Module for performing symbolization, by default same as above.788 self.module_path_for_symbolization = module_path789 self.uuid = uuid790 assert self.is_valid()791 792 def __str__(self):793 assert self.is_valid()794 return "{name} {arch} {start_addr:#016x}-{end_addr:#016x} {module_path} {uuid}".format(795 name=self.name,796 arch=self.arch,797 start_addr=self.start_addr,798 end_addr=self.end_addr,799 module_path=self.module_path800 if self.module_path == self.module_path_for_symbolization801 else "{} ({})".format(self.module_path_for_symbolization, self.module_path),802 uuid=self.uuid,803 )804 805 def is_valid(self):806 if not isinstance(self.name, str):807 return False808 if not isinstance(self.arch, str):809 return False810 if not isinstance(self.start_addr, int):811 return False812 if self.start_addr < 0:813 return False814 if not isinstance(self.end_addr, int):815 return False816 if self.end_addr <= self.start_addr:817 return False818 if not isinstance(self.module_path, str):819 return False820 if not os.path.isabs(self.module_path):821 return False822 if not isinstance(self.module_path_for_symbolization, str):823 return False824 if not os.path.isabs(self.module_path_for_symbolization):825 return False826 if not isinstance(self.uuid, str):827 return False828 return True829 830 831class GetUUIDFromBinaryException(Exception):832 def __init__(self, msg):833 super(GetUUIDFromBinaryException, self).__init__(msg)834 835 836_get_uuid_from_binary_cache = dict()837 838 839def get_uuid_from_binary(path_to_binary, arch=None):840 cache_key = (path_to_binary, arch)841 cached_value = _get_uuid_from_binary_cache.get(cache_key)842 if cached_value:843 return cached_value844 if not os.path.exists(path_to_binary):845 raise GetUUIDFromBinaryException(846 'Binary "{}" does not exist'.format(path_to_binary)847 )848 cmd = ["/usr/bin/otool", "-l"]849 if arch:850 cmd.extend(["-arch", arch])851 cmd.append(path_to_binary)852 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)853 # Look for this output:854 # cmd LC_UUID855 # cmdsize 24856 # uuid 4CA778FE-5BF9-3C45-AE59-7DF01B2BE83F857 if isinstance(output, str):858 output_str = output859 else:860 assert isinstance(output, bytes)861 output_str = output.decode()862 assert isinstance(output_str, str)863 lines = output_str.split("\n")864 uuid = None865 for index, line in enumerate(lines):866 stripped_line = line.strip()867 if not stripped_line.startswith("cmd LC_UUID"):868 continue869 uuid_line = lines[index + 2].strip()870 if not uuid_line.startswith("uuid"):871 raise GetUUIDFromBinaryException('Malformed output: "{}"'.format(uuid_line))872 split_uuid_line = uuid_line.split()873 uuid = split_uuid_line[1]874 break875 if uuid is None:876 logging.error("Failed to retrieve UUID from binary {}".format(path_to_binary))877 logging.error("otool output was:\n{}".format(output_str))878 raise GetUUIDFromBinaryException(879 'Failed to retrieve UUID from binary "{}"'.format(path_to_binary)880 )881 else:882 # Update cache883 _get_uuid_from_binary_cache[cache_key] = uuid884 return uuid885 886 887class ModuleMap(object):888 def __init__(self):889 self._module_name_to_description_map = dict()890 891 def add_module(self, desc):892 assert isinstance(desc, ModuleDesc)893 assert desc.name not in self._module_name_to_description_map894 self._module_name_to_description_map[desc.name] = desc895 896 def find_module_by_name(self, name):897 return self._module_name_to_description_map.get(name, None)898 899 def __str__(self):900 s = "{} modules:\n".format(self.num_modules)901 for module_desc in sorted(902 self._module_name_to_description_map.values(), key=lambda v: v.start_addr903 ):904 s += str(module_desc) + "\n"905 return s906 907 @property908 def num_modules(self):909 return len(self._module_name_to_description_map)910 911 @property912 def modules(self):913 return set(self._module_name_to_description_map.values())914 915 def get_module_path_for_symbolication(self, module_name, proxy, validate_uuid):916 module_desc = self.find_module_by_name(module_name)917 if module_desc is None:918 return None919 # Allow a plug-in to change the module description to make it920 # suitable for symbolication or avoid symbolication altogether.921 module_desc = proxy.filter_module_desc(module_desc)922 if module_desc is None:923 return None924 if validate_uuid:925 logging.debug(926 "Validating UUID of {}".format(927 module_desc.module_path_for_symbolization928 )929 )930 try:931 uuid = get_uuid_from_binary(932 module_desc.module_path_for_symbolization, arch=module_desc.arch933 )934 if uuid != module_desc.uuid:935 logging.warning(936 "Detected UUID mismatch {} != {}".format(uuid, module_desc.uuid)937 )938 # UUIDs don't match. Tell client to not symbolize this.939 return None940 except GetUUIDFromBinaryException as e:941 logging.error("Failed to get binary from UUID: %s", str(e))942 return None943 else:944 logging.warning(945 "Skipping validation of UUID of {}".format(946 module_desc.module_path_for_symbolization947 )948 )949 return module_desc.module_path_for_symbolization950 951 @staticmethod952 def parse_from_file(module_map_path):953 if not os.path.exists(module_map_path):954 raise Exception('module map "{}" does not exist'.format(module_map_path))955 with open(module_map_path, "r") as f:956 mm = None957 # E.g.958 # 0x2db4000-0x102ddc000 /path/to (arm64) <0D6BBDE0-FF90-3680-899D-8E6F9528E04C>959 hex_regex = lambda name: r"0x(?P<" + name + r">[0-9a-f]+)"960 module_path_regex = r"(?P<path>.+)"961 arch_regex = r"\((?P<arch>.+)\)"962 uuid_regex = r"<(?P<uuid>[0-9A-Z-]+)>"963 line_regex = r"^{}-{}\s+{}\s+{}\s+{}".format(964 hex_regex("start_addr"),965 hex_regex("end_addr"),966 module_path_regex,967 arch_regex,968 uuid_regex,969 )970 matcher = re.compile(line_regex)971 line_num = 0972 line = "dummy"973 while line != "":974 line = f.readline()975 line_num += 1976 if mm is None:977 if line.startswith("Process module map:"):978 mm = ModuleMap()979 continue980 if line.startswith("End of module map"):981 break982 m_obj = matcher.match(line)983 if not m_obj:984 raise Exception(985 'Failed to parse line {} "{}"'.format(line_num, line)986 )987 arch = m_obj.group("arch")988 start_addr = int(m_obj.group("start_addr"), base=16)989 end_addr = int(m_obj.group("end_addr"), base=16)990 module_path = m_obj.group("path")991 uuid = m_obj.group("uuid")992 module_desc = ModuleDesc(993 name=os.path.basename(module_path),994 arch=arch,995 start_addr=start_addr,996 end_addr=end_addr,997 module_path=module_path,998 uuid=uuid,999 )1000 mm.add_module(module_desc)1001 if mm is not None:1002 logging.debug(1003 'Loaded Module map from "{}":\n{}'.format(f.name, str(mm))1004 )1005 return mm1006 1007 1008class SysRootFilterPlugIn(AsanSymbolizerPlugIn):1009 """1010 Simple plug-in to add sys root prefix to all binary paths1011 used for symbolication.1012 """1013 1014 def __init__(self):1015 self.sysroot_path = ""1016 1017 def register_cmdline_args(self, parser):1018 parser.add_argument(1019 "-s",1020 dest="sys_root",1021 metavar="SYSROOT",1022 help="set path to sysroot for sanitized binaries",1023 )1024 1025 def process_cmdline_args(self, pargs):1026 if pargs.sys_root is None:1027 # Not being used so remove ourselves.1028 return False1029 self.sysroot_path = pargs.sys_root1030 return True1031 1032 def filter_binary_path(self, path):1033 return self.sysroot_path + path1034 1035 1036class ModuleMapPlugIn(AsanSymbolizerPlugIn):1037 def __init__(self):1038 self._module_map = None1039 self._uuid_validation = True1040 1041 def register_cmdline_args(self, parser):1042 parser.add_argument(1043 "--module-map",1044 help="Path to text file containing module map"1045 "output. See print_module_map ASan option.",1046 )1047 parser.add_argument(1048 "--skip-uuid-validation",1049 default=False,1050 action="store_true",1051 help="Skips validating UUID of modules using otool.",1052 )1053 1054 def process_cmdline_args(self, pargs):1055 if not pargs.module_map:1056 return False1057 self._module_map = ModuleMap.parse_from_file(args.module_map)1058 if self._module_map is None:1059 msg = "Failed to find module map"1060 logging.error(msg)1061 raise Exception(msg)1062 self._uuid_validation = not pargs.skip_uuid_validation1063 return True1064 1065 def filter_binary_path(self, binary_path):1066 if os.path.isabs(binary_path):1067 # This is a binary path so transform into1068 # a module name1069 module_name = os.path.basename(binary_path)1070 else:1071 module_name = binary_path1072 return self._module_map.get_module_path_for_symbolication(1073 module_name, self.proxy, self._uuid_validation1074 )1075 1076 1077def add_logging_args(parser):1078 parser.add_argument(1079 "--log-dest",1080 default=None,1081 help="Destination path for script logging (default stderr).",1082 )1083 parser.add_argument(1084 "--log-level",1085 choices=["debug", "info", "warning", "error", "critical"],1086 default="info",1087 help="Log level for script (default: %(default)s).",1088 )1089 1090 1091def setup_logging():1092 # Set up a parser just for parsing the logging arguments.1093 # This is necessary because logging should be configured before we1094 # perform the main argument parsing.1095 parser = argparse.ArgumentParser(add_help=False)1096 add_logging_args(parser)1097 pargs, unparsed_args = parser.parse_known_args()1098 1099 log_level = getattr(logging, pargs.log_level.upper())1100 if log_level == logging.DEBUG:1101 log_format = (1102 "%(levelname)s: [%(funcName)s() %(filename)s:%(lineno)d] %(message)s"1103 )1104 else:1105 log_format = "%(levelname)s: %(message)s"1106 basic_config = {"level": log_level, "format": log_format}1107 log_dest = pargs.log_dest1108 if log_dest:1109 basic_config["filename"] = log_dest1110 logging.basicConfig(**basic_config)1111 logging.debug(1112 'Logging level set to "{}" and directing output to "{}"'.format(1113 pargs.log_level, "stderr" if log_dest is None else log_dest1114 )1115 )1116 return unparsed_args1117 1118 1119def add_load_plugin_args(parser):1120 parser.add_argument("-p", "--plugins", help="Load plug-in", nargs="+", default=[])1121 1122 1123def setup_plugins(plugin_proxy, args):1124 parser = argparse.ArgumentParser(add_help=False)1125 add_load_plugin_args(parser)1126 pargs, unparsed_args = parser.parse_known_args()1127 for plugin_path in pargs.plugins:1128 plugin_proxy.load_plugin_from_file(plugin_path)1129 # Add built-in plugins.1130 plugin_proxy.add_plugin(ModuleMapPlugIn())1131 plugin_proxy.add_plugin(SysRootFilterPlugIn())1132 return unparsed_args1133 1134 1135if __name__ == "__main__":1136 remaining_args = setup_logging()1137 with AsanSymbolizerPlugInProxy() as plugin_proxy:1138 remaining_args = setup_plugins(plugin_proxy, remaining_args)1139 parser = argparse.ArgumentParser(1140 formatter_class=argparse.RawDescriptionHelpFormatter,1141 description="ASan symbolization script",1142 epilog=__doc__,1143 )1144 parser.add_argument(1145 "path_to_cut",1146 nargs="*",1147 help="pattern to be cut from the result file path ",1148 )1149 parser.add_argument(1150 "-d", "--demangle", action="store_true", help="demangle function names"1151 )1152 parser.add_argument(1153 "-c", metavar="CROSS_COMPILE", help="set prefix for binutils"1154 )1155 parser.add_argument(1156 "-l",1157 "--logfile",1158 default=sys.stdin,1159 type=argparse.FileType("r"),1160 help="set log file name to parse, default is stdin",1161 )1162 parser.add_argument(1163 "--force-system-symbolizer",1164 action="store_true",1165 help="don't use llvm-symbolizer",1166 )1167 # Add logging arguments so that `--help` shows them.1168 add_logging_args(parser)1169 # Add load plugin arguments so that `--help` shows them.1170 add_load_plugin_args(parser)1171 plugin_proxy.register_cmdline_args(parser)1172 args = parser.parse_args(remaining_args)1173 plugin_proxy.process_cmdline_args(args)1174 if args.path_to_cut:1175 fix_filename_patterns = args.path_to_cut1176 if args.demangle:1177 demangle = True1178 if args.c:1179 binutils_prefix = args.c1180 if args.logfile:1181 logfile = args.logfile1182 else:1183 logfile = sys.stdin1184 if args.force_system_symbolizer:1185 force_system_symbolizer = True1186 if force_system_symbolizer:1187 assert allow_system_symbolizer1188 loop = SymbolizationLoop(plugin_proxy)1189 loop.process_logfile()1190