527 lines · python
1#!/usr/bin/env python2 3"""A tool for extracting a list of symbols to export4 5When exporting symbols from a dll or exe we either need to mark the symbols in6the source code as __declspec(dllexport) or supply a list of symbols to the7linker. This program automates the latter by inspecting the symbol tables of a8list of link inputs and deciding which of those symbols need to be exported.9 10We can't just export all the defined symbols, as there's a limit of 6553511exported symbols and in clang we go way over that, particularly in a debug12build. Therefore a large part of the work is pruning symbols either which can't13be imported, or which we think are things that have definitions in public header14files (i.e. template instantiations) and we would get defined in the thing15importing these symbols anyway.16"""17 18from __future__ import print_function19import sys20import re21import os22import subprocess23import multiprocessing24import argparse25import platform26 27# Define a function which extracts a list of pairs of (symbols, is_def) from a28# library using llvm-nm becuase it can work both with regular and bitcode files.29# We use subprocess.Popen and yield a symbol at a time instead of using30# subprocess.check_output and returning a list as, especially on Windows, waiting31# for the entire output to be ready can take a significant amount of time.32def nm_get_symbols(tool, lib):33 # '-P' means the output is in portable format,34 # '-g' means we only get global symbols,35 # '-Xany' enforce handling both 32- and 64-bit objects on AIX,36 # '--no-demangle' ensure that C++ symbol names are not demangled; note37 # that llvm-nm do not demangle by default, but the system nm on AIX does38 # that, so the behavior may change in the future,39 # '-p' do not waste time sorting the symbols.40 cmd = [tool, "-P", "-g", "-Xany", "--no-demangle", "-p"]41 process = subprocess.Popen(42 cmd + [lib],43 bufsize=1,44 stdout=subprocess.PIPE,45 stdin=subprocess.PIPE,46 universal_newlines=True,47 )48 process.stdin.close()49 for line in process.stdout:50 # Look for external symbols that are defined in some section51 # The POSIX format is:52 # name type value size53 # The -P flag displays the size field for symbols only when applicable,54 # so the last field is optional. There's no space after the value field,55 # but \s+ match newline also, so \s+\S* will match the optional size field.56 match = re.match(r"^(\S+)\s+[BDGRSTuVW]\s+\S+\s+\S*$", line)57 if match:58 yield (match.group(1), True)59 # Look for undefined symbols, which have type U and may or may not60 # (depending on which nm is being used) have value and size.61 match = re.match(r"^(\S+)\s+U\s+(\S+\s+\S*)?$", line)62 if match:63 yield (match.group(1), False)64 process.wait()65 66 67# Define a function which determines if the target is 32-bit Windows (as that's68# where calling convention name decoration happens).69def readobj_is_32bit_windows(tool, lib):70 output = subprocess.check_output(71 [tool, "--file-header", lib], universal_newlines=True72 )73 for line in output.splitlines():74 match = re.match(r"Format: (\S+)", line)75 if match:76 return match.group(1) == "COFF-i386"77 return False78 79 80# MSVC mangles names to ?<identifier_mangling>@<type_mangling>. By examining the81# identifier/type mangling we can decide which symbols could possibly be82# required and which we can discard.83def should_keep_microsoft_symbol(symbol, calling_convention_decoration):84 # Keep unmangled (i.e. extern "C") names85 if not "?" in symbol:86 if calling_convention_decoration:87 # Remove calling convention decoration from names88 match = re.match(r"[_@]([^@]+)", symbol)89 if match:90 symbol = match.group(1)91 # Discard floating point/SIMD constants.92 if symbol.startswith(("__xmm@", "__ymm@", "__real@")):93 return None94 return symbol95 # Deleting destructors start with ?_G or ?_E and can be discarded because96 # link.exe gives you a warning telling you they can't be exported if you97 # don't98 elif symbol.startswith("??_G") or symbol.startswith("??_E"):99 return None100 # Delete template instantiations. These start with ?$ and can be discarded101 # because they will be instantiated in the importing translation unit if102 # needed.103 elif symbol.startswith("??$"):104 return None105 # Delete lambda object constructors and operator() functions. These start106 # with ??R<lambda_ or ??0<lambda_ and can be discarded because lambdas are107 # usually local to a function.108 elif symbol.startswith("??R<lambda_") or symbol.startswith("??0<lambda_"):109 return None110 # An anonymous namespace is mangled as ?A(maybe hex number)@. Any symbol111 # that mentions an anonymous namespace can be discarded, as the anonymous112 # namespace doesn't exist outside of that translation unit.113 elif re.search(r"\?A(0x\w+)?@", symbol):114 return None115 # Skip X86GenMnemonicTables functions, they are not exposed from llvm/include/.116 elif re.match(r"\?is[A-Z0-9]*@X86@llvm", symbol):117 return None118 # Keep Registry<T>::Head and Registry<T>::Tail static members for plugin support.119 # Pattern matches: ?Head@?$Registry@<template_args>@llvm@@ or ?Tail@?$Registry@...120 elif (121 "?$Registry@" in symbol122 and "@llvm@@" in symbol123 and (symbol.startswith("?Head@") or symbol.startswith("?Tail@"))124 ):125 return symbol126 # Keep mangled llvm:: and clang:: function symbols. How we detect these is a127 # bit of a mess and imprecise, but that avoids having to completely demangle128 # the symbol name. The outermost namespace is at the end of the identifier129 # mangling, and the identifier mangling is followed by the type mangling, so130 # we look for (llvm|clang)@@ followed by something that looks like a131 # function type mangling. To spot a function type we use (this is derived132 # from clang/lib/AST/MicrosoftMangle.cpp):133 # <function-type> ::= <function-class> <this-cvr-qualifiers>134 # <calling-convention> <return-type>135 # <argument-list> <throw-spec>136 # <function-class> ::= [A-Z]137 # <this-cvr-qualifiers> ::= [A-Z0-9_]*138 # <calling-convention> ::= [A-JQ]139 # <return-type> ::= .+140 # <argument-list> ::= X (void)141 # ::= .+@ (list of types)142 # ::= .*Z (list of types, varargs)143 # <throw-spec> ::= exceptions are not allowed144 elif re.search(r"@(llvm|clang)@@[A-Z][A-Z0-9_]*[A-JQ].+(X|.+@|.*Z)$", symbol):145 # Remove llvm::<Class>::dump and clang::<Class>::dump methods because146 # they are used for debugging only.147 if symbol.startswith("?dump@"):148 return None149 return symbol150 # Keep mangled global variables and static class members in llvm:: namespace.151 # These have a type mangling that looks like (this is derived from152 # clang/lib/AST/MicrosoftMangle.cpp):153 # <type-encoding> ::= <storage-class> <variable-type>154 # <storage-class> ::= 0 # private static member155 # ::= 1 # protected static member156 # ::= 2 # public static member157 # ::= 3 # global158 # ::= 4 # static local159 # <variable-type> ::= <type> <cvr-qualifiers>160 # ::= <type> <pointee-cvr-qualifiers> # pointers, references161 elif re.search(r"@llvm@@[0-3].*$", symbol):162 return symbol163 return None164 165 166# Itanium manglings are of the form _Z<identifier_mangling><type_mangling>. We167# demangle the identifier mangling to identify symbols that can be safely168# discarded.169def should_keep_itanium_symbol(symbol, calling_convention_decoration):170 # Start by removing any calling convention decoration (which we expect to171 # see on all symbols, even mangled C++ symbols)172 if calling_convention_decoration and symbol.startswith("_"):173 symbol = symbol[1:]174 # Keep unmangled names175 if not symbol.startswith("_") and not symbol.startswith("."):176 return symbol177 # Discard manglings that aren't nested names178 match = re.match(r"\.?_Z(T[VTIS])?(N.+)", symbol)179 if not match:180 return None181 # Demangle the name. If the name is too complex then we don't need to keep182 # it, but it the demangling fails then keep the symbol just in case.183 try:184 names, _ = parse_itanium_nested_name(match.group(2))185 except TooComplexName:186 return None187 if not names:188 return symbol189 # Keep llvm:: and clang:: names190 elif names[0][0] == "4llvm" or names[0][0] == "5clang":191 return symbol192 # Discard everything else193 else:194 return None195 196 197# Certain kinds of complex manglings we assume cannot be part of a public198# interface, and we handle them by raising an exception.199class TooComplexName(Exception):200 pass201 202 203# Parse an itanium mangled name from the start of a string and return a204# (name, rest of string) pair.205def parse_itanium_name(arg):206 # Check for a normal name207 match = re.match(r"(\d+)(.+)", arg)208 if match:209 n = int(match.group(1))210 name = match.group(1) + match.group(2)[:n]211 rest = match.group(2)[n:]212 return name, rest213 # Check for constructor/destructor names214 match = re.match(r"([CD][123])(.+)", arg)215 if match:216 return match.group(1), match.group(2)217 # Assume that a sequence of characters that doesn't end a nesting is an218 # operator (this is very imprecise, but appears to be good enough)219 match = re.match(r"([^E]+)(.+)", arg)220 if match:221 return match.group(1), match.group(2)222 # Anything else: we can't handle it223 return None, arg224 225 226# Parse an itanium mangled template argument list from the start of a string227# and throw it away, returning the rest of the string.228def skip_itanium_template(arg):229 # A template argument list starts with I230 assert arg.startswith("I"), arg231 tmp = arg[1:]232 while tmp:233 # Check for names234 match = re.match(r"(\d+)(.+)", tmp)235 if match:236 n = int(match.group(1))237 tmp = match.group(2)[n:]238 continue239 # Check for substitutions240 match = re.match(r"S[A-Z0-9]*_(.+)", tmp)241 if match:242 tmp = match.group(1)243 # Start of a template244 elif tmp.startswith("I"):245 tmp = skip_itanium_template(tmp)246 # Start of a nested name247 elif tmp.startswith("N"):248 _, tmp = parse_itanium_nested_name(tmp)249 # Start of an expression: assume that it's too complicated250 elif tmp.startswith("L") or tmp.startswith("X"):251 raise TooComplexName252 # End of the template253 elif tmp.startswith("E"):254 return tmp[1:]255 # Something else: probably a type, skip it256 else:257 tmp = tmp[1:]258 return None259 260 261# Parse an itanium mangled nested name and transform it into a list of pairs of262# (name, is_template), returning (list, rest of string).263def parse_itanium_nested_name(arg):264 # A nested name starts with N265 assert arg.startswith("N"), arg266 ret = []267 268 # Skip past the N, and possibly a substitution269 match = re.match(r"NS[A-Z0-9]*_(.+)", arg)270 if match:271 tmp = match.group(1)272 else:273 tmp = arg[1:]274 275 # Skip past CV-qualifiers and ref qualifiers276 match = re.match(r"[rVKRO]*(.+)", tmp)277 if match:278 tmp = match.group(1)279 280 # Repeatedly parse names from the string until we reach the end of the281 # nested name282 while tmp:283 # An E ends the nested name284 if tmp.startswith("E"):285 return ret, tmp[1:]286 # Parse a name287 name_part, tmp = parse_itanium_name(tmp)288 if not name_part:289 # If we failed then we don't know how to demangle this290 return None, None291 is_template = False292 # If this name is a template record that, then skip the template293 # arguments294 if tmp.startswith("I"):295 tmp = skip_itanium_template(tmp)296 is_template = True297 # Add the name to the list298 ret.append((name_part, is_template))299 300 # If we get here then something went wrong301 return None, None302 303 304# Parse a microsoft mangled symbol and return a list of pairs of305# (name, is_template). This is very rudimentary and does just enough306# in order to determine if the first or second component is a template.307def parse_microsoft_mangling(arg):308 # If the name doesn't start with ? this isn't a mangled name309 if not arg.startswith("?"):310 return [(arg, False)]311 arg = arg[1:]312 components = []313 while len(arg) > 0:314 # If we see an empty component we've reached the end315 if arg.startswith("@"):316 return components317 # Check for a simple name318 match = re.match(r"(\w+)@(.+)", arg)319 if match:320 components.append((match.group(1), False))321 arg = match.group(2)322 continue323 # Check for a special function name324 match = re.match(r"(\?_?\w)(.+)", arg)325 if match:326 components.append((match.group(1), False))327 arg = match.group(2)328 continue329 # Check for a template name330 match = re.match(r"\?\$(\w+)@[^@]+@(.+)", arg)331 if match:332 components.append((match.group(1), True))333 arg = match.group(2)334 continue335 # Some other kind of name that we can't handle336 components.append((arg, False))337 return components338 return components339 340 341def extract_symbols(arg):342 llvm_nm_path, should_keep_symbol, calling_convention_decoration, lib = arg343 symbol_defs = dict()344 symbol_refs = set()345 for (symbol, is_def) in nm_get_symbols(llvm_nm_path, lib):346 symbol = should_keep_symbol(symbol, calling_convention_decoration)347 if symbol:348 if is_def:349 symbol_defs[symbol] = 1 + symbol_defs.setdefault(symbol, 0)350 else:351 symbol_refs.add(symbol)352 return (symbol_defs, symbol_refs)353 354 355def get_template_name(sym, mangling):356 # Parse the mangling into a list of (name, is_template)357 try:358 if mangling == "microsoft":359 names = parse_microsoft_mangling(sym)360 else:361 match = re.match(r"\.?_Z(T[VTIS])?(N.+)", sym)362 if match:363 names, _ = parse_itanium_nested_name(match.group(2))364 else:365 names = None366 except TooComplexName:367 return None368 369 if not names:370 return None371 372 # If any component is a template then return it373 for name, is_template in names:374 if is_template:375 return name376 377 # Not a template378 return None379 380 381def parse_tool_path(parser, tool, val):382 try:383 # Close std streams as we don't want any output and we don't384 # want the process to wait for something on stdin.385 p = subprocess.Popen(386 [val],387 stdout=subprocess.PIPE,388 stderr=subprocess.PIPE,389 stdin=subprocess.PIPE,390 universal_newlines=True,391 )392 p.stdout.close()393 p.stderr.close()394 p.stdin.close()395 p.wait()396 return val397 except Exception:398 parser.error(f"Invalid path for {tool}")399 400 401if __name__ == "__main__":402 parser = argparse.ArgumentParser(403 description="Extract symbols to export from libraries"404 )405 parser.add_argument(406 "--mangling",407 choices=["itanium", "microsoft"],408 required=True,409 help="expected symbol mangling scheme",410 )411 parser.add_argument(412 "--nm",413 metavar="path",414 type=lambda x: parse_tool_path(parser, "nm", x),415 help="path to the llvm-nm executable",416 )417 parser.add_argument(418 "--readobj",419 metavar="path",420 type=lambda x: parse_tool_path(parser, "readobj", x),421 help="path to the llvm-readobj executable",422 )423 parser.add_argument(424 "libs",425 metavar="lib",426 type=str,427 nargs="+",428 help="libraries to extract symbols from",429 )430 parser.add_argument("-o", metavar="file", type=str, help="output to file")431 args = parser.parse_args()432 433 # How we determine which symbols to keep and which to discard depends on434 # the mangling scheme435 if args.mangling == "microsoft":436 should_keep_symbol = should_keep_microsoft_symbol437 else:438 should_keep_symbol = should_keep_itanium_symbol439 440 # Get the list of libraries to extract symbols from441 libs = list()442 for lib in args.libs:443 # When invoked by cmake the arguments are the cmake target names of the444 # libraries, so we need to add .lib/.a to the end and maybe lib to the445 # start to get the filename. Also allow objects.446 suffixes = [".lib", ".a", ".obj", ".o"]447 if not any([lib.endswith(s) for s in suffixes]):448 for s in suffixes:449 if os.path.exists(lib + s):450 lib = lib + s451 break452 if os.path.exists("lib" + lib + s):453 lib = "lib" + lib + s454 break455 if not any([lib.endswith(s) for s in suffixes]):456 print("Don't know what to do with argument " + lib, file=sys.stderr)457 exit(1)458 libs.append(lib)459 460 # Check if calling convention decoration is used by inspecting the first461 # library in the list462 calling_convention_decoration = readobj_is_32bit_windows(args.readobj, libs[0])463 464 # Extract symbols from libraries in parallel. This is a huge time saver when465 # doing a debug build, as there are hundreds of thousands of symbols in each466 # library.467 # FIXME: On AIX, the default pool size can be too big for a logical468 # partition's allocated memory, and can lead to an out of memory469 # IO error. We are setting the pool size to 8 to avoid such470 # errors at the moment, and will look for a graceful solution later.471 pool = multiprocessing.Pool(8) if platform.system() == "AIX" \472 else multiprocessing.Pool()473 try:474 # Only one argument can be passed to the mapping function, and we can't475 # use a lambda or local function definition as that doesn't work on476 # windows, so create a list of tuples which duplicates the arguments477 # that are the same in all calls.478 vals = [479 (args.nm, should_keep_symbol, calling_convention_decoration, x)480 for x in libs481 ]482 # Do an async map then wait for the result to make sure that483 # KeyboardInterrupt gets caught correctly (see484 # http://bugs.python.org/issue8296)485 result = pool.map_async(extract_symbols, vals)486 pool.close()487 libs_symbols = result.get(3600)488 except KeyboardInterrupt:489 # On Ctrl-C terminate everything and exit490 pool.terminate()491 pool.join()492 exit(1)493 494 # Merge everything into a single dict495 symbol_defs = dict()496 symbol_refs = set()497 for (this_lib_defs, this_lib_refs) in libs_symbols:498 for k, v in list(this_lib_defs.items()):499 symbol_defs[k] = v + symbol_defs.setdefault(k, 0)500 for sym in list(this_lib_refs):501 symbol_refs.add(sym)502 503 # Find which template instantiations are referenced at least once.504 template_instantiation_refs = set()505 for sym in list(symbol_refs):506 template = get_template_name(sym, args.mangling)507 if template:508 template_instantiation_refs.add(template)509 510 # Print symbols which both:511 # * Appear in exactly one input, as symbols defined in multiple512 # objects/libraries are assumed to have public definitions.513 # * Are not a template instantiation that isn't referenced anywhere. This514 # is because we need to export any explicitly instantiated templates,515 # and we expect those to be referenced in some object.516 if args.o:517 outfile = open(args.o, "w")518 else:519 outfile = sys.stdout520 for k, v in list(symbol_defs.items()):521 # On AIX, export function descriptors instead of function entries.522 if platform.system() == "AIX" and k.startswith("."):523 continue524 template = get_template_name(k, args.mangling)525 if v == 1 and (not template or template in template_instantiation_refs):526 print(k, file=outfile)527