489 lines · python
1#!/usr/bin/env python32"""3This script:4- Builds clang with user-defined flags5- Uses that clang to build an instrumented clang, which can be used to collect6 PGO samples7- Builds a user-defined set of sources (default: clang) to act as a8 "benchmark" to generate a PGO profile9- Builds clang once more with the PGO profile generated above10 11This is a total of four clean builds of clang (by default). This may take a12while. :)13 14This scripts duplicates https://llvm.org/docs/AdvancedBuilds.html#multi-stage-pgo15Eventually, it will be updated to instead call the cmake cache mentioned there.16"""17 18import argparse19import collections20import multiprocessing21import os22import shlex23import shutil24import subprocess25import sys26 27### User configuration28 29 30# If you want to use a different 'benchmark' than building clang, make this31# function do what you want. out_dir is the build directory for clang, so all32# of the clang binaries will live under "${out_dir}/bin/". Using clang in33# ${out_dir} will magically have the profiles go to the right place.34#35# You may assume that out_dir is a freshly-built directory that you can reach36# in to build more things, if you'd like.37def _run_benchmark(env, out_dir, include_debug_info):38 """The 'benchmark' we run to generate profile data."""39 target_dir = env.output_subdir("instrumentation_run")40 41 # `check-llvm` and `check-clang` are cheap ways to increase coverage. The42 # former lets us touch on the non-x86 backends a bit if configured, and the43 # latter gives us more C to chew on (and will send us through diagnostic44 # paths a fair amount, though the `if (stuff_is_broken) { diag() ... }`45 # branches should still heavily be weighted in the not-taken direction,46 # since we built all of LLVM/etc).47 _build_things_in(env, out_dir, what=["check-llvm", "check-clang"])48 49 # Building tblgen gets us coverage; don't skip it. (out_dir may also not50 # have them anyway, but that's less of an issue)51 cmake = _get_cmake_invocation_for_bootstrap_from(env, out_dir, skip_tablegens=False)52 53 if include_debug_info:54 cmake.add_flag("CMAKE_BUILD_TYPE", "RelWithDebInfo")55 56 _run_fresh_cmake(env, cmake, target_dir)57 58 # Just build all the things. The more data we have, the better.59 _build_things_in(env, target_dir, what=["all"])60 61 62### Script63 64 65class CmakeInvocation:66 _cflags = ["CMAKE_C_FLAGS", "CMAKE_CXX_FLAGS"]67 _ldflags = [68 "CMAKE_EXE_LINKER_FLAGS",69 "CMAKE_MODULE_LINKER_FLAGS",70 "CMAKE_SHARED_LINKER_FLAGS",71 ]72 73 def __init__(self, cmake, maker, cmake_dir):74 self._prefix = [cmake, "-G", maker, cmake_dir]75 76 # Map of str -> (list|str).77 self._flags = {}78 for flag in CmakeInvocation._cflags + CmakeInvocation._ldflags:79 self._flags[flag] = []80 81 def add_new_flag(self, key, value):82 self.add_flag(key, value, allow_overwrites=False)83 84 def add_flag(self, key, value, allow_overwrites=True):85 if key not in self._flags:86 self._flags[key] = value87 return88 89 existing_value = self._flags[key]90 if isinstance(existing_value, list):91 existing_value.append(value)92 return93 94 if not allow_overwrites:95 raise ValueError("Invalid overwrite of %s requested" % key)96 97 self._flags[key] = value98 99 def add_cflags(self, flags):100 # No, I didn't intend to append ['-', 'O', '2'] to my flags, thanks :)101 assert not isinstance(flags, str)102 for f in CmakeInvocation._cflags:103 self._flags[f].extend(flags)104 105 def add_ldflags(self, flags):106 assert not isinstance(flags, str)107 for f in CmakeInvocation._ldflags:108 self._flags[f].extend(flags)109 110 def to_args(self):111 args = self._prefix.copy()112 for key, value in sorted(self._flags.items()):113 if isinstance(value, list):114 # We preload all of the list-y values (cflags, ...). If we've115 # nothing to add, don't.116 if not value:117 continue118 value = " ".join(value)119 120 arg = "-D" + key121 if value != "":122 arg += "=" + value123 args.append(arg)124 return args125 126 127class Env:128 def __init__(self, llvm_dir, use_make, output_dir, default_cmake_args, dry_run):129 self.llvm_dir = llvm_dir130 self.use_make = use_make131 self.output_dir = output_dir132 self.default_cmake_args = default_cmake_args.copy()133 self.dry_run = dry_run134 135 def get_default_cmake_args_kv(self):136 return self.default_cmake_args.items()137 138 def get_cmake_maker(self):139 return "Ninja" if not self.use_make else "Unix Makefiles"140 141 def get_make_command(self):142 if self.use_make:143 return ["make", "-j{}".format(multiprocessing.cpu_count())]144 return ["ninja"]145 146 def output_subdir(self, name):147 return os.path.join(self.output_dir, name)148 149 def has_llvm_subproject(self, name):150 if name == "compiler-rt":151 subdir = "../compiler-rt"152 elif name == "clang":153 subdir = "../clang"154 else:155 raise ValueError("Unknown subproject: %s" % name)156 157 return os.path.isdir(os.path.join(self.llvm_dir, subdir))158 159 # Note that we don't allow capturing stdout/stderr. This works quite nicely160 # with dry_run.161 def run_command(self, cmd, cwd=None, check=False, silent_unless_error=False):162 print("Running `%s` in %s" % (cmd, shlex.quote(cwd or os.getcwd())))163 164 if self.dry_run:165 return166 167 if silent_unless_error:168 stdout, stderr = subprocess.PIPE, subprocess.STDOUT169 else:170 stdout, stderr = None, None171 172 # Don't use subprocess.run because it's >= py3.5 only, and it's not too173 # much extra effort to get what it gives us anyway.174 popen = subprocess.Popen(175 cmd, stdin=subprocess.DEVNULL, stdout=stdout, stderr=stderr, cwd=cwd176 )177 stdout, _ = popen.communicate()178 return_code = popen.wait(timeout=0)179 180 if not return_code:181 return182 183 if silent_unless_error:184 print(stdout.decode("utf-8", "ignore"))185 186 if check:187 raise subprocess.CalledProcessError(188 returncode=return_code, cmd=cmd, output=stdout, stderr=None189 )190 191 192def _get_default_cmake_invocation(env):193 inv = CmakeInvocation(194 cmake="cmake", maker=env.get_cmake_maker(), cmake_dir=env.llvm_dir195 )196 for key, value in env.get_default_cmake_args_kv():197 inv.add_new_flag(key, value)198 return inv199 200 201def _get_cmake_invocation_for_bootstrap_from(env, out_dir, skip_tablegens=True):202 clang = os.path.join(out_dir, "bin", "clang")203 cmake = _get_default_cmake_invocation(env)204 cmake.add_new_flag("CMAKE_C_COMPILER", clang)205 cmake.add_new_flag("CMAKE_CXX_COMPILER", clang + "++")206 207 # We often get no value out of building new tblgens; the previous build208 # should have them. It's still correct to build them, just slower.209 def add_tablegen(key, binary):210 path = os.path.join(out_dir, "bin", binary)211 212 # Check that this exists, since the user's allowed to specify their own213 # stage1 directory (which is generally where we'll source everything214 # from). Dry runs should hope for the best from our user, as well.215 if env.dry_run or os.path.exists(path):216 cmake.add_new_flag(key, path)217 218 if skip_tablegens:219 add_tablegen("LLVM_TABLEGEN", "llvm-tblgen")220 add_tablegen("CLANG_TABLEGEN", "clang-tblgen")221 222 return cmake223 224 225def _build_things_in(env, target_dir, what):226 cmd = env.get_make_command() + what227 env.run_command(cmd, cwd=target_dir, check=True)228 229 230def _run_fresh_cmake(env, cmake, target_dir):231 if not env.dry_run:232 try:233 shutil.rmtree(target_dir)234 except FileNotFoundError:235 pass236 237 os.makedirs(target_dir, mode=0o755)238 239 cmake_args = cmake.to_args()240 env.run_command(cmake_args, cwd=target_dir, check=True, silent_unless_error=True)241 242 243def _build_stage1_clang(env):244 target_dir = env.output_subdir("stage1")245 cmake = _get_default_cmake_invocation(env)246 _run_fresh_cmake(env, cmake, target_dir)247 _build_things_in(env, target_dir, what=["clang", "llvm-profdata", "profile"])248 return target_dir249 250 251def _generate_instrumented_clang_profile(env, stage1_dir, profile_dir, output_file):252 llvm_profdata = os.path.join(stage1_dir, "bin", "llvm-profdata")253 if env.dry_run:254 profiles = [os.path.join(profile_dir, "*.profraw")]255 else:256 profiles = [257 os.path.join(profile_dir, f)258 for f in os.listdir(profile_dir)259 if f.endswith(".profraw")260 ]261 cmd = [llvm_profdata, "merge", "-output=" + output_file] + profiles262 env.run_command(cmd, check=True)263 264 265def _build_instrumented_clang(env, stage1_dir):266 assert os.path.isabs(stage1_dir)267 268 target_dir = os.path.join(env.output_dir, "instrumented")269 cmake = _get_cmake_invocation_for_bootstrap_from(env, stage1_dir)270 cmake.add_new_flag("LLVM_BUILD_INSTRUMENTED", "IR")271 272 # libcxx's configure step messes with our link order: we'll link273 # libclang_rt.profile after libgcc, and the former requires atexit from the274 # latter. So, configure checks fail.275 #276 # Since we don't need libcxx or compiler-rt anyway, just disable them.277 cmake.add_new_flag("LLVM_BUILD_RUNTIME", "No")278 279 _run_fresh_cmake(env, cmake, target_dir)280 _build_things_in(env, target_dir, what=["clang", "lld"])281 282 profiles_dir = os.path.join(target_dir, "profiles")283 return target_dir, profiles_dir284 285 286def _build_optimized_clang(env, stage1_dir, profdata_file):287 if not env.dry_run and not os.path.exists(profdata_file):288 raise ValueError(289 "Looks like the profdata file at %s doesn't exist" % profdata_file290 )291 292 target_dir = os.path.join(env.output_dir, "optimized")293 cmake = _get_cmake_invocation_for_bootstrap_from(env, stage1_dir)294 cmake.add_new_flag("LLVM_PROFDATA_FILE", os.path.abspath(profdata_file))295 296 # We'll get complaints about hash mismatches in `main` in tools/etc. Ignore297 # it.298 cmake.add_cflags(["-Wno-backend-plugin"])299 _run_fresh_cmake(env, cmake, target_dir)300 _build_things_in(env, target_dir, what=["clang"])301 return target_dir302 303 304Args = collections.namedtuple(305 "Args",306 [307 "do_optimized_build",308 "include_debug_info",309 "profile_location",310 "stage1_dir",311 ],312)313 314 315def _parse_args():316 parser = argparse.ArgumentParser(317 description="Builds LLVM and Clang with instrumentation, collects "318 "instrumentation profiles for them, and (optionally) builds things "319 "with these PGO profiles. By default, it's assumed that you're "320 "running this from your LLVM root, and all build artifacts will be "321 "saved to $PWD/out."322 )323 parser.add_argument(324 "--cmake-extra-arg",325 action="append",326 default=[],327 help="an extra arg to pass to all cmake invocations. Note that this "328 "is interpreted as a -D argument, e.g. --cmake-extra-arg FOO=BAR will "329 "be passed as -DFOO=BAR. This may be specified multiple times.",330 )331 parser.add_argument(332 "--dry-run", action="store_true", help="print commands instead of running them"333 )334 parser.add_argument(335 "--llvm-dir",336 default=".",337 help="directory containing an LLVM checkout (default: $PWD)",338 )339 parser.add_argument(340 "--no-optimized-build",341 action="store_true",342 help="disable the final, PGO-optimized build",343 )344 parser.add_argument(345 "--out-dir", help="directory to write artifacts to (default: $llvm_dir/out)"346 )347 parser.add_argument(348 "--profile-output",349 help="where to output the profile (default is $out/pgo_profile.prof)",350 )351 parser.add_argument(352 "--stage1-dir",353 help="instead of having an initial build of everything, use the given "354 "directory. It is expected that this directory will have clang, "355 "llvm-profdata, and the appropriate libclang_rt.profile already built",356 )357 parser.add_argument(358 "--use-debug-info-in-benchmark",359 action="store_true",360 help="use a regular build instead of RelWithDebInfo in the benchmark. "361 "This increases benchmark execution time and disk space requirements, "362 "but gives more coverage over debuginfo bits in LLVM and clang.",363 )364 parser.add_argument(365 "--use-make",366 action="store_true",367 default=shutil.which("ninja") is None,368 help="use Makefiles instead of ninja",369 )370 371 args = parser.parse_args()372 373 llvm_dir = os.path.abspath(args.llvm_dir)374 if args.out_dir is None:375 output_dir = os.path.join(llvm_dir, "out")376 else:377 output_dir = os.path.abspath(args.out_dir)378 379 extra_args = {380 "CMAKE_BUILD_TYPE": "Release",381 "LLVM_ENABLE_PROJECTS": "clang;compiler-rt;lld",382 }383 for arg in args.cmake_extra_arg:384 if arg.startswith("-D"):385 arg = arg[2:]386 elif arg.startswith("-"):387 raise ValueError(388 "Unknown not- -D arg encountered; you may need "389 "to tweak the source..."390 )391 split = arg.split("=", 1)392 if len(split) == 1:393 key, val = split[0], ""394 else:395 key, val = split396 extra_args[key] = val397 398 env = Env(399 default_cmake_args=extra_args,400 dry_run=args.dry_run,401 llvm_dir=llvm_dir,402 output_dir=output_dir,403 use_make=args.use_make,404 )405 406 if args.profile_output is not None:407 profile_location = args.profile_output408 else:409 profile_location = os.path.join(env.output_dir, "pgo_profile.prof")410 411 result_args = Args(412 do_optimized_build=not args.no_optimized_build,413 include_debug_info=args.use_debug_info_in_benchmark,414 profile_location=profile_location,415 stage1_dir=args.stage1_dir,416 )417 418 return env, result_args419 420 421def _looks_like_llvm_dir(directory):422 """Arbitrary set of heuristics to determine if `directory` is an llvm dir.423 424 Errs on the side of false-positives."""425 426 contents = set(os.listdir(directory))427 expected_contents = [428 "Maintainers.md",429 "cmake",430 "docs",431 "include",432 "utils",433 ]434 435 if not all(c in contents for c in expected_contents):436 return False437 438 try:439 include_listing = os.listdir(os.path.join(directory, "include"))440 except NotADirectoryError:441 return False442 443 return "llvm" in include_listing444 445 446def _die(*args, **kwargs):447 kwargs["file"] = sys.stderr448 print(*args, **kwargs)449 sys.exit(1)450 451 452def _main():453 env, args = _parse_args()454 455 if not _looks_like_llvm_dir(env.llvm_dir):456 _die("Looks like %s isn't an LLVM directory; please see --help" % env.llvm_dir)457 if not env.has_llvm_subproject("clang"):458 _die("Need a clang checkout at tools/clang")459 if not env.has_llvm_subproject("compiler-rt"):460 _die("Need a compiler-rt checkout at projects/compiler-rt")461 462 def status(*args):463 print(*args, file=sys.stderr)464 465 if args.stage1_dir is None:466 status("*** Building stage1 clang...")467 stage1_out = _build_stage1_clang(env)468 else:469 stage1_out = args.stage1_dir470 471 status("*** Building instrumented clang...")472 instrumented_out, profile_dir = _build_instrumented_clang(env, stage1_out)473 status("*** Running profdata benchmarks...")474 _run_benchmark(env, instrumented_out, args.include_debug_info)475 status("*** Generating profile...")476 _generate_instrumented_clang_profile(477 env, stage1_out, profile_dir, args.profile_location478 )479 480 print("Final profile:", args.profile_location)481 if args.do_optimized_build:482 status("*** Building PGO-optimized binaries...")483 optimized_out = _build_optimized_clang(env, stage1_out, args.profile_location)484 print("Final build directory:", optimized_out)485 486 487if __name__ == "__main__":488 _main()489