2402 lines · cpp
1//===- Construction of pass pipelines -------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8/// \file9///10/// This file provides the implementation of the PassBuilder based on our11/// static pass registry as well as related functionality. It also provides12/// helpers to aid in analyzing, debugging, and testing passes and pass13/// pipelines.14///15//===----------------------------------------------------------------------===//16 17#include "llvm/ADT/Statistic.h"18#include "llvm/Analysis/AliasAnalysis.h"19#include "llvm/Analysis/BasicAliasAnalysis.h"20#include "llvm/Analysis/CGSCCPassManager.h"21#include "llvm/Analysis/CtxProfAnalysis.h"22#include "llvm/Analysis/GlobalsModRef.h"23#include "llvm/Analysis/InlineAdvisor.h"24#include "llvm/Analysis/ProfileSummaryInfo.h"25#include "llvm/Analysis/ScopedNoAliasAA.h"26#include "llvm/Analysis/TypeBasedAliasAnalysis.h"27#include "llvm/IR/PassManager.h"28#include "llvm/Pass.h"29#include "llvm/Passes/OptimizationLevel.h"30#include "llvm/Passes/PassBuilder.h"31#include "llvm/Support/CommandLine.h"32#include "llvm/Support/ErrorHandling.h"33#include "llvm/Support/PGOOptions.h"34#include "llvm/Support/VirtualFileSystem.h"35#include "llvm/Target/TargetMachine.h"36#include "llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h"37#include "llvm/Transforms/Coroutines/CoroAnnotationElide.h"38#include "llvm/Transforms/Coroutines/CoroCleanup.h"39#include "llvm/Transforms/Coroutines/CoroConditionalWrapper.h"40#include "llvm/Transforms/Coroutines/CoroEarly.h"41#include "llvm/Transforms/Coroutines/CoroElide.h"42#include "llvm/Transforms/Coroutines/CoroSplit.h"43#include "llvm/Transforms/HipStdPar/HipStdPar.h"44#include "llvm/Transforms/IPO/AlwaysInliner.h"45#include "llvm/Transforms/IPO/Annotation2Metadata.h"46#include "llvm/Transforms/IPO/ArgumentPromotion.h"47#include "llvm/Transforms/IPO/Attributor.h"48#include "llvm/Transforms/IPO/CalledValuePropagation.h"49#include "llvm/Transforms/IPO/ConstantMerge.h"50#include "llvm/Transforms/IPO/CrossDSOCFI.h"51#include "llvm/Transforms/IPO/DeadArgumentElimination.h"52#include "llvm/Transforms/IPO/ElimAvailExtern.h"53#include "llvm/Transforms/IPO/EmbedBitcodePass.h"54#include "llvm/Transforms/IPO/ExpandVariadics.h"55#include "llvm/Transforms/IPO/FatLTOCleanup.h"56#include "llvm/Transforms/IPO/ForceFunctionAttrs.h"57#include "llvm/Transforms/IPO/FunctionAttrs.h"58#include "llvm/Transforms/IPO/GlobalDCE.h"59#include "llvm/Transforms/IPO/GlobalOpt.h"60#include "llvm/Transforms/IPO/GlobalSplit.h"61#include "llvm/Transforms/IPO/HotColdSplitting.h"62#include "llvm/Transforms/IPO/IROutliner.h"63#include "llvm/Transforms/IPO/InferFunctionAttrs.h"64#include "llvm/Transforms/IPO/Inliner.h"65#include "llvm/Transforms/IPO/LowerTypeTests.h"66#include "llvm/Transforms/IPO/MemProfContextDisambiguation.h"67#include "llvm/Transforms/IPO/MergeFunctions.h"68#include "llvm/Transforms/IPO/ModuleInliner.h"69#include "llvm/Transforms/IPO/OpenMPOpt.h"70#include "llvm/Transforms/IPO/PartialInlining.h"71#include "llvm/Transforms/IPO/SCCP.h"72#include "llvm/Transforms/IPO/SampleProfile.h"73#include "llvm/Transforms/IPO/SampleProfileProbe.h"74#include "llvm/Transforms/IPO/WholeProgramDevirt.h"75#include "llvm/Transforms/InstCombine/InstCombine.h"76#include "llvm/Transforms/Instrumentation/CGProfile.h"77#include "llvm/Transforms/Instrumentation/ControlHeightReduction.h"78#include "llvm/Transforms/Instrumentation/InstrProfiling.h"79#include "llvm/Transforms/Instrumentation/MemProfInstrumentation.h"80#include "llvm/Transforms/Instrumentation/MemProfUse.h"81#include "llvm/Transforms/Instrumentation/PGOCtxProfFlattening.h"82#include "llvm/Transforms/Instrumentation/PGOCtxProfLowering.h"83#include "llvm/Transforms/Instrumentation/PGOForceFunctionAttrs.h"84#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"85#include "llvm/Transforms/Scalar/ADCE.h"86#include "llvm/Transforms/Scalar/AlignmentFromAssumptions.h"87#include "llvm/Transforms/Scalar/AnnotationRemarks.h"88#include "llvm/Transforms/Scalar/BDCE.h"89#include "llvm/Transforms/Scalar/CallSiteSplitting.h"90#include "llvm/Transforms/Scalar/ConstraintElimination.h"91#include "llvm/Transforms/Scalar/CorrelatedValuePropagation.h"92#include "llvm/Transforms/Scalar/DFAJumpThreading.h"93#include "llvm/Transforms/Scalar/DeadStoreElimination.h"94#include "llvm/Transforms/Scalar/DivRemPairs.h"95#include "llvm/Transforms/Scalar/DropUnnecessaryAssumes.h"96#include "llvm/Transforms/Scalar/EarlyCSE.h"97#include "llvm/Transforms/Scalar/Float2Int.h"98#include "llvm/Transforms/Scalar/GVN.h"99#include "llvm/Transforms/Scalar/IndVarSimplify.h"100#include "llvm/Transforms/Scalar/InferAlignment.h"101#include "llvm/Transforms/Scalar/InstSimplifyPass.h"102#include "llvm/Transforms/Scalar/JumpTableToSwitch.h"103#include "llvm/Transforms/Scalar/JumpThreading.h"104#include "llvm/Transforms/Scalar/LICM.h"105#include "llvm/Transforms/Scalar/LoopDeletion.h"106#include "llvm/Transforms/Scalar/LoopDistribute.h"107#include "llvm/Transforms/Scalar/LoopFlatten.h"108#include "llvm/Transforms/Scalar/LoopFuse.h"109#include "llvm/Transforms/Scalar/LoopIdiomRecognize.h"110#include "llvm/Transforms/Scalar/LoopInstSimplify.h"111#include "llvm/Transforms/Scalar/LoopInterchange.h"112#include "llvm/Transforms/Scalar/LoopLoadElimination.h"113#include "llvm/Transforms/Scalar/LoopPassManager.h"114#include "llvm/Transforms/Scalar/LoopRotation.h"115#include "llvm/Transforms/Scalar/LoopSimplifyCFG.h"116#include "llvm/Transforms/Scalar/LoopSink.h"117#include "llvm/Transforms/Scalar/LoopUnrollAndJamPass.h"118#include "llvm/Transforms/Scalar/LoopUnrollPass.h"119#include "llvm/Transforms/Scalar/LoopVersioningLICM.h"120#include "llvm/Transforms/Scalar/LowerConstantIntrinsics.h"121#include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"122#include "llvm/Transforms/Scalar/LowerMatrixIntrinsics.h"123#include "llvm/Transforms/Scalar/MemCpyOptimizer.h"124#include "llvm/Transforms/Scalar/MergedLoadStoreMotion.h"125#include "llvm/Transforms/Scalar/NewGVN.h"126#include "llvm/Transforms/Scalar/Reassociate.h"127#include "llvm/Transforms/Scalar/SCCP.h"128#include "llvm/Transforms/Scalar/SROA.h"129#include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h"130#include "llvm/Transforms/Scalar/SimplifyCFG.h"131#include "llvm/Transforms/Scalar/SpeculativeExecution.h"132#include "llvm/Transforms/Scalar/TailRecursionElimination.h"133#include "llvm/Transforms/Scalar/WarnMissedTransforms.h"134#include "llvm/Transforms/Utils/AddDiscriminators.h"135#include "llvm/Transforms/Utils/AssumeBundleBuilder.h"136#include "llvm/Transforms/Utils/CanonicalizeAliases.h"137#include "llvm/Transforms/Utils/CountVisits.h"138#include "llvm/Transforms/Utils/EntryExitInstrumenter.h"139#include "llvm/Transforms/Utils/ExtraPassManager.h"140#include "llvm/Transforms/Utils/InjectTLIMappings.h"141#include "llvm/Transforms/Utils/LibCallsShrinkWrap.h"142#include "llvm/Transforms/Utils/Mem2Reg.h"143#include "llvm/Transforms/Utils/MoveAutoInit.h"144#include "llvm/Transforms/Utils/NameAnonGlobals.h"145#include "llvm/Transforms/Utils/RelLookupTableConverter.h"146#include "llvm/Transforms/Utils/SimplifyCFGOptions.h"147#include "llvm/Transforms/Vectorize/LoopVectorize.h"148#include "llvm/Transforms/Vectorize/SLPVectorizer.h"149#include "llvm/Transforms/Vectorize/VectorCombine.h"150 151using namespace llvm;152 153namespace llvm {154 155static cl::opt<InliningAdvisorMode> UseInlineAdvisor(156 "enable-ml-inliner", cl::init(InliningAdvisorMode::Default), cl::Hidden,157 cl::desc("Enable ML policy for inliner. Currently trained for -Oz only"),158 cl::values(clEnumValN(InliningAdvisorMode::Default, "default",159 "Heuristics-based inliner version"),160 clEnumValN(InliningAdvisorMode::Development, "development",161 "Use development mode (runtime-loadable model)"),162 clEnumValN(InliningAdvisorMode::Release, "release",163 "Use release mode (AOT-compiled model)")));164 165/// Flag to enable inline deferral during PGO.166static cl::opt<bool>167 EnablePGOInlineDeferral("enable-npm-pgo-inline-deferral", cl::init(true),168 cl::Hidden,169 cl::desc("Enable inline deferral during PGO"));170 171static cl::opt<bool> EnableModuleInliner("enable-module-inliner",172 cl::init(false), cl::Hidden,173 cl::desc("Enable module inliner"));174 175static cl::opt<bool> PerformMandatoryInliningsFirst(176 "mandatory-inlining-first", cl::init(false), cl::Hidden,177 cl::desc("Perform mandatory inlinings module-wide, before performing "178 "inlining"));179 180static cl::opt<bool> EnableEagerlyInvalidateAnalyses(181 "eagerly-invalidate-analyses", cl::init(true), cl::Hidden,182 cl::desc("Eagerly invalidate more analyses in default pipelines"));183 184static cl::opt<bool> EnableMergeFunctions(185 "enable-merge-functions", cl::init(false), cl::Hidden,186 cl::desc("Enable function merging as part of the optimization pipeline"));187 188static cl::opt<bool> EnablePostPGOLoopRotation(189 "enable-post-pgo-loop-rotation", cl::init(true), cl::Hidden,190 cl::desc("Run the loop rotation transformation after PGO instrumentation"));191 192static cl::opt<bool> EnableGlobalAnalyses(193 "enable-global-analyses", cl::init(true), cl::Hidden,194 cl::desc("Enable inter-procedural analyses"));195 196static cl::opt<bool> RunPartialInlining("enable-partial-inlining",197 cl::init(false), cl::Hidden,198 cl::desc("Run Partial inlining pass"));199 200static cl::opt<bool> ExtraVectorizerPasses(201 "extra-vectorizer-passes", cl::init(false), cl::Hidden,202 cl::desc("Run cleanup optimization passes after vectorization"));203 204static cl::opt<bool> RunNewGVN("enable-newgvn", cl::init(false), cl::Hidden,205 cl::desc("Run the NewGVN pass"));206 207static cl::opt<bool>208 EnableLoopInterchange("enable-loopinterchange", cl::init(false), cl::Hidden,209 cl::desc("Enable the LoopInterchange Pass"));210 211static cl::opt<bool> EnableUnrollAndJam("enable-unroll-and-jam",212 cl::init(false), cl::Hidden,213 cl::desc("Enable Unroll And Jam Pass"));214 215static cl::opt<bool> EnableLoopFlatten("enable-loop-flatten", cl::init(false),216 cl::Hidden,217 cl::desc("Enable the LoopFlatten Pass"));218 219// Experimentally allow loop header duplication. This should allow for better220// optimization at Oz, since loop-idiom recognition can then recognize things221// like memcpy. If this ends up being useful for many targets, we should drop222// this flag and make a code generation option that can be controlled223// independent of the opt level and exposed through the frontend.224static cl::opt<bool> EnableLoopHeaderDuplication(225 "enable-loop-header-duplication", cl::init(false), cl::Hidden,226 cl::desc("Enable loop header duplication at any optimization level"));227 228static cl::opt<bool>229 EnableDFAJumpThreading("enable-dfa-jump-thread",230 cl::desc("Enable DFA jump threading"),231 cl::init(false), cl::Hidden);232 233static cl::opt<bool>234 EnableHotColdSplit("hot-cold-split",235 cl::desc("Enable hot-cold splitting pass"));236 237static cl::opt<bool> EnableIROutliner("ir-outliner", cl::init(false),238 cl::Hidden,239 cl::desc("Enable ir outliner pass"));240 241static cl::opt<bool>242 DisablePreInliner("disable-preinline", cl::init(false), cl::Hidden,243 cl::desc("Disable pre-instrumentation inliner"));244 245static cl::opt<int> PreInlineThreshold(246 "preinline-threshold", cl::Hidden, cl::init(75),247 cl::desc("Control the amount of inlining in pre-instrumentation inliner "248 "(default = 75)"));249 250static cl::opt<bool>251 EnableGVNHoist("enable-gvn-hoist",252 cl::desc("Enable the GVN hoisting pass (default = off)"));253 254static cl::opt<bool>255 EnableGVNSink("enable-gvn-sink",256 cl::desc("Enable the GVN sinking pass (default = off)"));257 258static cl::opt<bool> EnableJumpTableToSwitch(259 "enable-jump-table-to-switch",260 cl::desc("Enable JumpTableToSwitch pass (default = off)"));261 262// This option is used in simplifying testing SampleFDO optimizations for263// profile loading.264static cl::opt<bool>265 EnableCHR("enable-chr", cl::init(true), cl::Hidden,266 cl::desc("Enable control height reduction optimization (CHR)"));267 268static cl::opt<bool> FlattenedProfileUsed(269 "flattened-profile-used", cl::init(false), cl::Hidden,270 cl::desc("Indicate the sample profile being used is flattened, i.e., "271 "no inline hierarchy exists in the profile"));272 273static cl::opt<bool>274 EnableMatrix("enable-matrix", cl::init(false), cl::Hidden,275 cl::desc("Enable lowering of the matrix intrinsics"));276 277static cl::opt<bool> EnableConstraintElimination(278 "enable-constraint-elimination", cl::init(true), cl::Hidden,279 cl::desc(280 "Enable pass to eliminate conditions based on linear constraints"));281 282static cl::opt<AttributorRunOption> AttributorRun(283 "attributor-enable", cl::Hidden, cl::init(AttributorRunOption::NONE),284 cl::desc("Enable the attributor inter-procedural deduction pass"),285 cl::values(clEnumValN(AttributorRunOption::ALL, "all",286 "enable all attributor runs"),287 clEnumValN(AttributorRunOption::MODULE, "module",288 "enable module-wide attributor runs"),289 clEnumValN(AttributorRunOption::CGSCC, "cgscc",290 "enable call graph SCC attributor runs"),291 clEnumValN(AttributorRunOption::NONE, "none",292 "disable attributor runs")));293 294static cl::opt<bool> EnableSampledInstr(295 "enable-sampled-instrumentation", cl::init(false), cl::Hidden,296 cl::desc("Enable profile instrumentation sampling (default = off)"));297static cl::opt<bool> UseLoopVersioningLICM(298 "enable-loop-versioning-licm", cl::init(false), cl::Hidden,299 cl::desc("Enable the experimental Loop Versioning LICM pass"));300 301static cl::opt<std::string> InstrumentColdFuncOnlyPath(302 "instrument-cold-function-only-path", cl::init(""),303 cl::desc("File path for cold function only instrumentation(requires use "304 "with --pgo-instrument-cold-function-only)"),305 cl::Hidden);306 307extern cl::opt<std::string> UseCtxProfile;308extern cl::opt<bool> PGOInstrumentColdFunctionOnly;309 310extern cl::opt<bool> EnableMemProfContextDisambiguation;311} // namespace llvm312 313PipelineTuningOptions::PipelineTuningOptions() {314 LoopInterleaving = true;315 LoopVectorization = true;316 SLPVectorization = false;317 LoopUnrolling = true;318 LoopInterchange = EnableLoopInterchange;319 LoopFusion = false;320 ForgetAllSCEVInLoopUnroll = ForgetSCEVInLoopUnroll;321 LicmMssaOptCap = SetLicmMssaOptCap;322 LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap;323 CallGraphProfile = true;324 UnifiedLTO = false;325 MergeFunctions = EnableMergeFunctions;326 InlinerThreshold = -1;327 EagerlyInvalidateAnalyses = EnableEagerlyInvalidateAnalyses;328}329 330namespace llvm {331extern cl::opt<unsigned> MaxDevirtIterations;332} // namespace llvm333 334void PassBuilder::invokePeepholeEPCallbacks(FunctionPassManager &FPM,335 OptimizationLevel Level) {336 for (auto &C : PeepholeEPCallbacks)337 C(FPM, Level);338}339void PassBuilder::invokeLateLoopOptimizationsEPCallbacks(340 LoopPassManager &LPM, OptimizationLevel Level) {341 for (auto &C : LateLoopOptimizationsEPCallbacks)342 C(LPM, Level);343}344void PassBuilder::invokeLoopOptimizerEndEPCallbacks(LoopPassManager &LPM,345 OptimizationLevel Level) {346 for (auto &C : LoopOptimizerEndEPCallbacks)347 C(LPM, Level);348}349void PassBuilder::invokeScalarOptimizerLateEPCallbacks(350 FunctionPassManager &FPM, OptimizationLevel Level) {351 for (auto &C : ScalarOptimizerLateEPCallbacks)352 C(FPM, Level);353}354void PassBuilder::invokeCGSCCOptimizerLateEPCallbacks(CGSCCPassManager &CGPM,355 OptimizationLevel Level) {356 for (auto &C : CGSCCOptimizerLateEPCallbacks)357 C(CGPM, Level);358}359void PassBuilder::invokeVectorizerStartEPCallbacks(FunctionPassManager &FPM,360 OptimizationLevel Level) {361 for (auto &C : VectorizerStartEPCallbacks)362 C(FPM, Level);363}364void PassBuilder::invokeVectorizerEndEPCallbacks(FunctionPassManager &FPM,365 OptimizationLevel Level) {366 for (auto &C : VectorizerEndEPCallbacks)367 C(FPM, Level);368}369void PassBuilder::invokeOptimizerEarlyEPCallbacks(ModulePassManager &MPM,370 OptimizationLevel Level,371 ThinOrFullLTOPhase Phase) {372 for (auto &C : OptimizerEarlyEPCallbacks)373 C(MPM, Level, Phase);374}375void PassBuilder::invokeOptimizerLastEPCallbacks(ModulePassManager &MPM,376 OptimizationLevel Level,377 ThinOrFullLTOPhase Phase) {378 for (auto &C : OptimizerLastEPCallbacks)379 C(MPM, Level, Phase);380}381void PassBuilder::invokeFullLinkTimeOptimizationEarlyEPCallbacks(382 ModulePassManager &MPM, OptimizationLevel Level) {383 for (auto &C : FullLinkTimeOptimizationEarlyEPCallbacks)384 C(MPM, Level);385}386void PassBuilder::invokeFullLinkTimeOptimizationLastEPCallbacks(387 ModulePassManager &MPM, OptimizationLevel Level) {388 for (auto &C : FullLinkTimeOptimizationLastEPCallbacks)389 C(MPM, Level);390}391void PassBuilder::invokePipelineStartEPCallbacks(ModulePassManager &MPM,392 OptimizationLevel Level) {393 for (auto &C : PipelineStartEPCallbacks)394 C(MPM, Level);395}396void PassBuilder::invokePipelineEarlySimplificationEPCallbacks(397 ModulePassManager &MPM, OptimizationLevel Level, ThinOrFullLTOPhase Phase) {398 for (auto &C : PipelineEarlySimplificationEPCallbacks)399 C(MPM, Level, Phase);400}401 402// Helper to add AnnotationRemarksPass.403static void addAnnotationRemarksPass(ModulePassManager &MPM) {404 MPM.addPass(createModuleToFunctionPassAdaptor(AnnotationRemarksPass()));405}406 407// Helper to check if the current compilation phase is preparing for LTO408static bool isLTOPreLink(ThinOrFullLTOPhase Phase) {409 return Phase == ThinOrFullLTOPhase::ThinLTOPreLink ||410 Phase == ThinOrFullLTOPhase::FullLTOPreLink;411}412 413// Helper to check if the current compilation phase is LTO backend414static bool isLTOPostLink(ThinOrFullLTOPhase Phase) {415 return Phase == ThinOrFullLTOPhase::ThinLTOPostLink ||416 Phase == ThinOrFullLTOPhase::FullLTOPostLink;417}418 419// Helper to wrap conditionally Coro passes.420static CoroConditionalWrapper buildCoroWrapper(ThinOrFullLTOPhase Phase) {421 // TODO: Skip passes according to Phase.422 ModulePassManager CoroPM;423 CoroPM.addPass(CoroEarlyPass());424 CGSCCPassManager CGPM;425 CGPM.addPass(CoroSplitPass());426 CoroPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));427 CoroPM.addPass(CoroCleanupPass());428 CoroPM.addPass(GlobalDCEPass());429 return CoroConditionalWrapper(std::move(CoroPM));430}431 432// TODO: Investigate the cost/benefit of tail call elimination on debugging.433FunctionPassManager434PassBuilder::buildO1FunctionSimplificationPipeline(OptimizationLevel Level,435 ThinOrFullLTOPhase Phase) {436 437 FunctionPassManager FPM;438 439 if (AreStatisticsEnabled())440 FPM.addPass(CountVisitsPass());441 442 // Form SSA out of local memory accesses after breaking apart aggregates into443 // scalars.444 FPM.addPass(SROAPass(SROAOptions::ModifyCFG));445 446 // Catch trivial redundancies447 FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */));448 449 // Hoisting of scalars and load expressions.450 FPM.addPass(451 SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));452 FPM.addPass(InstCombinePass());453 454 FPM.addPass(LibCallsShrinkWrapPass());455 456 invokePeepholeEPCallbacks(FPM, Level);457 458 FPM.addPass(459 SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));460 461 // Form canonically associated expression trees, and simplify the trees using462 // basic mathematical properties. For example, this will form (nearly)463 // minimal multiplication trees.464 FPM.addPass(ReassociatePass());465 466 // Add the primary loop simplification pipeline.467 // FIXME: Currently this is split into two loop pass pipelines because we run468 // some function passes in between them. These can and should be removed469 // and/or replaced by scheduling the loop pass equivalents in the correct470 // positions. But those equivalent passes aren't powerful enough yet.471 // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still472 // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to473 // fully replace `SimplifyCFGPass`, and the closest to the other we have is474 // `LoopInstSimplify`.475 LoopPassManager LPM1, LPM2;476 477 // Simplify the loop body. We do this initially to clean up after other loop478 // passes run, either when iterating on a loop or on inner loops with479 // implications on the outer loop.480 LPM1.addPass(LoopInstSimplifyPass());481 LPM1.addPass(LoopSimplifyCFGPass());482 483 // Try to remove as much code from the loop header as possible,484 // to reduce amount of IR that will have to be duplicated. However,485 // do not perform speculative hoisting the first time as LICM486 // will destroy metadata that may not need to be destroyed if run487 // after loop rotation.488 // TODO: Investigate promotion cap for O1.489 LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,490 /*AllowSpeculation=*/false));491 492 LPM1.addPass(LoopRotatePass(/* Disable header duplication */ true,493 isLTOPreLink(Phase)));494 // TODO: Investigate promotion cap for O1.495 LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,496 /*AllowSpeculation=*/true));497 LPM1.addPass(SimpleLoopUnswitchPass());498 if (EnableLoopFlatten)499 LPM1.addPass(LoopFlattenPass());500 501 LPM2.addPass(LoopIdiomRecognizePass());502 LPM2.addPass(IndVarSimplifyPass());503 504 invokeLateLoopOptimizationsEPCallbacks(LPM2, Level);505 506 LPM2.addPass(LoopDeletionPass());507 508 // Do not enable unrolling in PreLinkThinLTO phase during sample PGO509 // because it changes IR to makes profile annotation in back compile510 // inaccurate. The normal unroller doesn't pay attention to forced full unroll511 // attributes so we need to make sure and allow the full unroll pass to pay512 // attention to it.513 if (Phase != ThinOrFullLTOPhase::ThinLTOPreLink || !PGOOpt ||514 PGOOpt->Action != PGOOptions::SampleUse)515 LPM2.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(),516 /* OnlyWhenForced= */ !PTO.LoopUnrolling,517 PTO.ForgetAllSCEVInLoopUnroll));518 519 invokeLoopOptimizerEndEPCallbacks(LPM2, Level);520 521 FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM1),522 /*UseMemorySSA=*/true));523 FPM.addPass(524 SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));525 FPM.addPass(InstCombinePass());526 // The loop passes in LPM2 (LoopFullUnrollPass) do not preserve MemorySSA.527 // *All* loop passes must preserve it, in order to be able to use it.528 FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM2),529 /*UseMemorySSA=*/false));530 531 // Delete small array after loop unroll.532 FPM.addPass(SROAPass(SROAOptions::ModifyCFG));533 534 // Specially optimize memory movement as it doesn't look like dataflow in SSA.535 FPM.addPass(MemCpyOptPass());536 537 // Sparse conditional constant propagation.538 // FIXME: It isn't clear why we do this *after* loop passes rather than539 // before...540 FPM.addPass(SCCPPass());541 542 // Delete dead bit computations (instcombine runs after to fold away the dead543 // computations, and then ADCE will run later to exploit any new DCE544 // opportunities that creates).545 FPM.addPass(BDCEPass());546 547 // Run instcombine after redundancy and dead bit elimination to exploit548 // opportunities opened up by them.549 FPM.addPass(InstCombinePass());550 invokePeepholeEPCallbacks(FPM, Level);551 552 FPM.addPass(CoroElidePass());553 554 invokeScalarOptimizerLateEPCallbacks(FPM, Level);555 556 // Finally, do an expensive DCE pass to catch all the dead code exposed by557 // the simplifications and basic cleanup after all the simplifications.558 // TODO: Investigate if this is too expensive.559 FPM.addPass(ADCEPass());560 FPM.addPass(561 SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));562 FPM.addPass(InstCombinePass());563 invokePeepholeEPCallbacks(FPM, Level);564 565 return FPM;566}567 568FunctionPassManager569PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level,570 ThinOrFullLTOPhase Phase) {571 assert(Level != OptimizationLevel::O0 && "Must request optimizations!");572 573 // The O1 pipeline has a separate pipeline creation function to simplify574 // construction readability.575 if (Level.getSpeedupLevel() == 1)576 return buildO1FunctionSimplificationPipeline(Level, Phase);577 578 FunctionPassManager FPM;579 580 if (AreStatisticsEnabled())581 FPM.addPass(CountVisitsPass());582 583 // Form SSA out of local memory accesses after breaking apart aggregates into584 // scalars.585 FPM.addPass(SROAPass(SROAOptions::ModifyCFG));586 587 // Catch trivial redundancies588 FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */));589 if (EnableKnowledgeRetention)590 FPM.addPass(AssumeSimplifyPass());591 592 // Hoisting of scalars and load expressions.593 if (EnableGVNHoist)594 FPM.addPass(GVNHoistPass());595 596 // Global value numbering based sinking.597 if (EnableGVNSink) {598 FPM.addPass(GVNSinkPass());599 FPM.addPass(600 SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));601 }602 603 // Speculative execution if the target has divergent branches; otherwise nop.604 FPM.addPass(SpeculativeExecutionPass(/* OnlyIfDivergentTarget =*/true));605 606 // Optimize based on known information about branches, and cleanup afterward.607 FPM.addPass(JumpThreadingPass());608 FPM.addPass(CorrelatedValuePropagationPass());609 610 // Jump table to switch conversion.611 if (EnableJumpTableToSwitch)612 FPM.addPass(JumpTableToSwitchPass(613 /*InLTO=*/Phase == ThinOrFullLTOPhase::ThinLTOPostLink ||614 Phase == ThinOrFullLTOPhase::FullLTOPostLink));615 616 FPM.addPass(617 SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));618 FPM.addPass(InstCombinePass());619 FPM.addPass(AggressiveInstCombinePass());620 621 if (!Level.isOptimizingForSize())622 FPM.addPass(LibCallsShrinkWrapPass());623 624 invokePeepholeEPCallbacks(FPM, Level);625 626 // For PGO use pipeline, try to optimize memory intrinsics such as memcpy627 // using the size value profile. Don't perform this when optimizing for size.628 if (PGOOpt && PGOOpt->Action == PGOOptions::IRUse &&629 !Level.isOptimizingForSize())630 FPM.addPass(PGOMemOPSizeOpt());631 632 FPM.addPass(TailCallElimPass(/*UpdateFunctionEntryCount=*/633 isInstrumentedPGOUse()));634 FPM.addPass(635 SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));636 637 // Form canonically associated expression trees, and simplify the trees using638 // basic mathematical properties. For example, this will form (nearly)639 // minimal multiplication trees.640 FPM.addPass(ReassociatePass());641 642 if (EnableConstraintElimination)643 FPM.addPass(ConstraintEliminationPass());644 645 // Add the primary loop simplification pipeline.646 // FIXME: Currently this is split into two loop pass pipelines because we run647 // some function passes in between them. These can and should be removed648 // and/or replaced by scheduling the loop pass equivalents in the correct649 // positions. But those equivalent passes aren't powerful enough yet.650 // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still651 // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to652 // fully replace `SimplifyCFGPass`, and the closest to the other we have is653 // `LoopInstSimplify`.654 LoopPassManager LPM1, LPM2;655 656 // Simplify the loop body. We do this initially to clean up after other loop657 // passes run, either when iterating on a loop or on inner loops with658 // implications on the outer loop.659 LPM1.addPass(LoopInstSimplifyPass());660 LPM1.addPass(LoopSimplifyCFGPass());661 662 // Try to remove as much code from the loop header as possible,663 // to reduce amount of IR that will have to be duplicated. However,664 // do not perform speculative hoisting the first time as LICM665 // will destroy metadata that may not need to be destroyed if run666 // after loop rotation.667 // TODO: Investigate promotion cap for O1.668 LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,669 /*AllowSpeculation=*/false));670 671 // Disable header duplication in loop rotation at -Oz.672 LPM1.addPass(LoopRotatePass(EnableLoopHeaderDuplication ||673 Level != OptimizationLevel::Oz,674 isLTOPreLink(Phase)));675 // TODO: Investigate promotion cap for O1.676 LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,677 /*AllowSpeculation=*/true));678 LPM1.addPass(679 SimpleLoopUnswitchPass(/* NonTrivial */ Level == OptimizationLevel::O3));680 if (EnableLoopFlatten)681 LPM1.addPass(LoopFlattenPass());682 683 LPM2.addPass(LoopIdiomRecognizePass());684 LPM2.addPass(IndVarSimplifyPass());685 686 {687 ExtraLoopPassManager<ShouldRunExtraSimpleLoopUnswitch> ExtraPasses;688 ExtraPasses.addPass(SimpleLoopUnswitchPass(/* NonTrivial */ Level ==689 OptimizationLevel::O3));690 LPM2.addPass(std::move(ExtraPasses));691 }692 693 invokeLateLoopOptimizationsEPCallbacks(LPM2, Level);694 695 LPM2.addPass(LoopDeletionPass());696 697 // Do not enable unrolling in PreLinkThinLTO phase during sample PGO698 // because it changes IR to makes profile annotation in back compile699 // inaccurate. The normal unroller doesn't pay attention to forced full unroll700 // attributes so we need to make sure and allow the full unroll pass to pay701 // attention to it.702 if (Phase != ThinOrFullLTOPhase::ThinLTOPreLink || !PGOOpt ||703 PGOOpt->Action != PGOOptions::SampleUse)704 LPM2.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(),705 /* OnlyWhenForced= */ !PTO.LoopUnrolling,706 PTO.ForgetAllSCEVInLoopUnroll));707 708 invokeLoopOptimizerEndEPCallbacks(LPM2, Level);709 710 FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM1),711 /*UseMemorySSA=*/true));712 FPM.addPass(713 SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));714 FPM.addPass(InstCombinePass());715 // The loop passes in LPM2 (LoopIdiomRecognizePass, IndVarSimplifyPass,716 // LoopDeletionPass and LoopFullUnrollPass) do not preserve MemorySSA.717 // *All* loop passes must preserve it, in order to be able to use it.718 FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM2),719 /*UseMemorySSA=*/false));720 721 // Delete small array after loop unroll.722 FPM.addPass(SROAPass(SROAOptions::ModifyCFG));723 724 // Try vectorization/scalarization transforms that are both improvements725 // themselves and can allow further folds with GVN and InstCombine.726 FPM.addPass(VectorCombinePass(/*TryEarlyFoldsOnly=*/true));727 728 // Eliminate redundancies.729 FPM.addPass(MergedLoadStoreMotionPass());730 if (RunNewGVN)731 FPM.addPass(NewGVNPass());732 else733 FPM.addPass(GVNPass());734 735 // Sparse conditional constant propagation.736 // FIXME: It isn't clear why we do this *after* loop passes rather than737 // before...738 FPM.addPass(SCCPPass());739 740 // Delete dead bit computations (instcombine runs after to fold away the dead741 // computations, and then ADCE will run later to exploit any new DCE742 // opportunities that creates).743 FPM.addPass(BDCEPass());744 745 // Run instcombine after redundancy and dead bit elimination to exploit746 // opportunities opened up by them.747 FPM.addPass(InstCombinePass());748 invokePeepholeEPCallbacks(FPM, Level);749 750 // Re-consider control flow based optimizations after redundancy elimination,751 // redo DCE, etc.752 if (EnableDFAJumpThreading)753 FPM.addPass(DFAJumpThreadingPass());754 755 FPM.addPass(JumpThreadingPass());756 FPM.addPass(CorrelatedValuePropagationPass());757 758 // Finally, do an expensive DCE pass to catch all the dead code exposed by759 // the simplifications and basic cleanup after all the simplifications.760 // TODO: Investigate if this is too expensive.761 FPM.addPass(ADCEPass());762 763 // Specially optimize memory movement as it doesn't look like dataflow in SSA.764 FPM.addPass(MemCpyOptPass());765 766 FPM.addPass(DSEPass());767 FPM.addPass(MoveAutoInitPass());768 769 FPM.addPass(createFunctionToLoopPassAdaptor(770 LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,771 /*AllowSpeculation=*/true),772 /*UseMemorySSA=*/true));773 774 FPM.addPass(CoroElidePass());775 776 invokeScalarOptimizerLateEPCallbacks(FPM, Level);777 778 FPM.addPass(SimplifyCFGPass(SimplifyCFGOptions()779 .convertSwitchRangeToICmp(true)780 .convertSwitchToArithmetic(true)781 .hoistCommonInsts(true)782 .sinkCommonInsts(true)));783 FPM.addPass(InstCombinePass());784 invokePeepholeEPCallbacks(FPM, Level);785 786 return FPM;787}788 789void PassBuilder::addRequiredLTOPreLinkPasses(ModulePassManager &MPM) {790 MPM.addPass(CanonicalizeAliasesPass());791 MPM.addPass(NameAnonGlobalPass());792}793 794void PassBuilder::addPreInlinerPasses(ModulePassManager &MPM,795 OptimizationLevel Level,796 ThinOrFullLTOPhase LTOPhase) {797 assert(Level != OptimizationLevel::O0 && "Not expecting O0 here!");798 if (DisablePreInliner)799 return;800 InlineParams IP;801 802 IP.DefaultThreshold = PreInlineThreshold;803 804 // FIXME: The hint threshold has the same value used by the regular inliner805 // when not optimzing for size. This should probably be lowered after806 // performance testing.807 // FIXME: this comment is cargo culted from the old pass manager, revisit).808 IP.HintThreshold = Level.isOptimizingForSize() ? PreInlineThreshold : 325;809 ModuleInlinerWrapperPass MIWP(810 IP, /* MandatoryFirst */ true,811 InlineContext{LTOPhase, InlinePass::EarlyInliner});812 CGSCCPassManager &CGPipeline = MIWP.getPM();813 814 FunctionPassManager FPM;815 FPM.addPass(SROAPass(SROAOptions::ModifyCFG));816 FPM.addPass(EarlyCSEPass()); // Catch trivial redundancies.817 FPM.addPass(SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(818 true))); // Merge & remove basic blocks.819 FPM.addPass(InstCombinePass()); // Combine silly sequences.820 invokePeepholeEPCallbacks(FPM, Level);821 822 CGPipeline.addPass(createCGSCCToFunctionPassAdaptor(823 std::move(FPM), PTO.EagerlyInvalidateAnalyses));824 825 MPM.addPass(std::move(MIWP));826 827 // Delete anything that is now dead to make sure that we don't instrument828 // dead code. Instrumentation can end up keeping dead code around and829 // dramatically increase code size.830 MPM.addPass(GlobalDCEPass());831}832 833void PassBuilder::addPostPGOLoopRotation(ModulePassManager &MPM,834 OptimizationLevel Level) {835 if (EnablePostPGOLoopRotation) {836 // Disable header duplication in loop rotation at -Oz.837 MPM.addPass(createModuleToFunctionPassAdaptor(838 createFunctionToLoopPassAdaptor(839 LoopRotatePass(EnableLoopHeaderDuplication ||840 Level != OptimizationLevel::Oz),841 /*UseMemorySSA=*/false),842 PTO.EagerlyInvalidateAnalyses));843 }844}845 846void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM,847 OptimizationLevel Level, bool RunProfileGen,848 bool IsCS, bool AtomicCounterUpdate,849 std::string ProfileFile,850 std::string ProfileRemappingFile) {851 assert(Level != OptimizationLevel::O0 && "Not expecting O0 here!");852 853 if (!RunProfileGen) {854 assert(!ProfileFile.empty() && "Profile use expecting a profile file!");855 MPM.addPass(856 PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS, FS));857 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert858 // RequireAnalysisPass for PSI before subsequent non-module passes.859 MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());860 return;861 }862 863 // Perform PGO instrumentation.864 MPM.addPass(PGOInstrumentationGen(IsCS ? PGOInstrumentationType::CSFDO865 : PGOInstrumentationType::FDO));866 867 addPostPGOLoopRotation(MPM, Level);868 // Add the profile lowering pass.869 InstrProfOptions Options;870 if (!ProfileFile.empty())871 Options.InstrProfileOutput = ProfileFile;872 // Do counter promotion at Level greater than O0.873 Options.DoCounterPromotion = true;874 Options.UseBFIInPromotion = IsCS;875 if (EnableSampledInstr) {876 Options.Sampling = true;877 // With sampling, there is little beneifit to enable counter promotion.878 // But note that sampling does work with counter promotion.879 Options.DoCounterPromotion = false;880 }881 Options.Atomic = AtomicCounterUpdate;882 MPM.addPass(InstrProfilingLoweringPass(Options, IsCS));883}884 885void PassBuilder::addPGOInstrPassesForO0(ModulePassManager &MPM,886 bool RunProfileGen, bool IsCS,887 bool AtomicCounterUpdate,888 std::string ProfileFile,889 std::string ProfileRemappingFile) {890 if (!RunProfileGen) {891 assert(!ProfileFile.empty() && "Profile use expecting a profile file!");892 MPM.addPass(893 PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS, FS));894 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert895 // RequireAnalysisPass for PSI before subsequent non-module passes.896 MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());897 return;898 }899 900 // Perform PGO instrumentation.901 MPM.addPass(PGOInstrumentationGen(IsCS ? PGOInstrumentationType::CSFDO902 : PGOInstrumentationType::FDO));903 // Add the profile lowering pass.904 InstrProfOptions Options;905 if (!ProfileFile.empty())906 Options.InstrProfileOutput = ProfileFile;907 // Do not do counter promotion at O0.908 Options.DoCounterPromotion = false;909 Options.UseBFIInPromotion = IsCS;910 Options.Atomic = AtomicCounterUpdate;911 MPM.addPass(InstrProfilingLoweringPass(Options, IsCS));912}913 914static InlineParams getInlineParamsFromOptLevel(OptimizationLevel Level) {915 return getInlineParams(Level.getSpeedupLevel(), Level.getSizeLevel());916}917 918ModuleInlinerWrapperPass919PassBuilder::buildInlinerPipeline(OptimizationLevel Level,920 ThinOrFullLTOPhase Phase) {921 InlineParams IP;922 if (PTO.InlinerThreshold == -1)923 IP = getInlineParamsFromOptLevel(Level);924 else925 IP = getInlineParams(PTO.InlinerThreshold);926 // For PreLinkThinLTO + SamplePGO or PreLinkFullLTO + SamplePGO,927 // set hot-caller threshold to 0 to disable hot928 // callsite inline (as much as possible [1]) because it makes929 // profile annotation in the backend inaccurate.930 //931 // [1] Note the cost of a function could be below zero due to erased932 // prologue / epilogue.933 if (isLTOPreLink(Phase) && PGOOpt && PGOOpt->Action == PGOOptions::SampleUse)934 IP.HotCallSiteThreshold = 0;935 936 if (PGOOpt)937 IP.EnableDeferral = EnablePGOInlineDeferral;938 939 ModuleInlinerWrapperPass MIWP(IP, PerformMandatoryInliningsFirst,940 InlineContext{Phase, InlinePass::CGSCCInliner},941 UseInlineAdvisor, MaxDevirtIterations);942 943 // Require the GlobalsAA analysis for the module so we can query it within944 // the CGSCC pipeline.945 if (EnableGlobalAnalyses) {946 MIWP.addModulePass(RequireAnalysisPass<GlobalsAA, Module>());947 // Invalidate AAManager so it can be recreated and pick up the newly948 // available GlobalsAA.949 MIWP.addModulePass(950 createModuleToFunctionPassAdaptor(InvalidateAnalysisPass<AAManager>()));951 }952 953 // Require the ProfileSummaryAnalysis for the module so we can query it within954 // the inliner pass.955 MIWP.addModulePass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());956 957 // Now begin the main postorder CGSCC pipeline.958 // FIXME: The current CGSCC pipeline has its origins in the legacy pass959 // manager and trying to emulate its precise behavior. Much of this doesn't960 // make a lot of sense and we should revisit the core CGSCC structure.961 CGSCCPassManager &MainCGPipeline = MIWP.getPM();962 963 // Note: historically, the PruneEH pass was run first to deduce nounwind and964 // generally clean up exception handling overhead. It isn't clear this is965 // valuable as the inliner doesn't currently care whether it is inlining an966 // invoke or a call.967 968 if (AttributorRun & AttributorRunOption::CGSCC)969 MainCGPipeline.addPass(AttributorCGSCCPass());970 971 // Deduce function attributes. We do another run of this after the function972 // simplification pipeline, so this only needs to run when it could affect the973 // function simplification pipeline, which is only the case with recursive974 // functions.975 MainCGPipeline.addPass(PostOrderFunctionAttrsPass(/*SkipNonRecursive*/ true));976 977 // When at O3 add argument promotion to the pass pipeline.978 // FIXME: It isn't at all clear why this should be limited to O3.979 if (Level == OptimizationLevel::O3)980 MainCGPipeline.addPass(ArgumentPromotionPass());981 982 // Try to perform OpenMP specific optimizations. This is a (quick!) no-op if983 // there are no OpenMP runtime calls present in the module.984 if (Level == OptimizationLevel::O2 || Level == OptimizationLevel::O3)985 MainCGPipeline.addPass(OpenMPOptCGSCCPass(Phase));986 987 invokeCGSCCOptimizerLateEPCallbacks(MainCGPipeline, Level);988 989 // Add the core function simplification pipeline nested inside the990 // CGSCC walk.991 MainCGPipeline.addPass(createCGSCCToFunctionPassAdaptor(992 buildFunctionSimplificationPipeline(Level, Phase),993 PTO.EagerlyInvalidateAnalyses, /*NoRerun=*/true));994 995 // Finally, deduce any function attributes based on the fully simplified996 // function.997 MainCGPipeline.addPass(PostOrderFunctionAttrsPass());998 999 // Mark that the function is fully simplified and that it shouldn't be1000 // simplified again if we somehow revisit it due to CGSCC mutations unless1001 // it's been modified since.1002 MainCGPipeline.addPass(createCGSCCToFunctionPassAdaptor(1003 RequireAnalysisPass<ShouldNotRunFunctionPassesAnalysis, Function>()));1004 1005 if (Phase != ThinOrFullLTOPhase::ThinLTOPreLink) {1006 MainCGPipeline.addPass(CoroSplitPass(Level != OptimizationLevel::O0));1007 MainCGPipeline.addPass(CoroAnnotationElidePass());1008 }1009 1010 // Make sure we don't affect potential future NoRerun CGSCC adaptors.1011 MIWP.addLateModulePass(createModuleToFunctionPassAdaptor(1012 InvalidateAnalysisPass<ShouldNotRunFunctionPassesAnalysis>()));1013 1014 return MIWP;1015}1016 1017ModulePassManager1018PassBuilder::buildModuleInlinerPipeline(OptimizationLevel Level,1019 ThinOrFullLTOPhase Phase) {1020 ModulePassManager MPM;1021 1022 InlineParams IP = getInlineParamsFromOptLevel(Level);1023 // For PreLinkThinLTO + SamplePGO or PreLinkFullLTO + SamplePGO,1024 // set hot-caller threshold to 0 to disable hot1025 // callsite inline (as much as possible [1]) because it makes1026 // profile annotation in the backend inaccurate.1027 //1028 // [1] Note the cost of a function could be below zero due to erased1029 // prologue / epilogue.1030 if (isLTOPreLink(Phase) && PGOOpt && PGOOpt->Action == PGOOptions::SampleUse)1031 IP.HotCallSiteThreshold = 0;1032 1033 if (PGOOpt)1034 IP.EnableDeferral = EnablePGOInlineDeferral;1035 1036 // The inline deferral logic is used to avoid losing some1037 // inlining chance in future. It is helpful in SCC inliner, in which1038 // inlining is processed in bottom-up order.1039 // While in module inliner, the inlining order is a priority-based order1040 // by default. The inline deferral is unnecessary there. So we disable the1041 // inline deferral logic in module inliner.1042 IP.EnableDeferral = false;1043 1044 MPM.addPass(ModuleInlinerPass(IP, UseInlineAdvisor, Phase));1045 if (!UseCtxProfile.empty() && Phase == ThinOrFullLTOPhase::ThinLTOPostLink) {1046 MPM.addPass(GlobalOptPass());1047 MPM.addPass(GlobalDCEPass());1048 MPM.addPass(PGOCtxProfFlatteningPass(/*IsPreThinlink=*/false));1049 }1050 1051 MPM.addPass(createModuleToFunctionPassAdaptor(1052 buildFunctionSimplificationPipeline(Level, Phase),1053 PTO.EagerlyInvalidateAnalyses));1054 1055 if (Phase != ThinOrFullLTOPhase::ThinLTOPreLink) {1056 MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(1057 CoroSplitPass(Level != OptimizationLevel::O0)));1058 MPM.addPass(1059 createModuleToPostOrderCGSCCPassAdaptor(CoroAnnotationElidePass()));1060 }1061 1062 return MPM;1063}1064 1065ModulePassManager1066PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,1067 ThinOrFullLTOPhase Phase) {1068 assert(Level != OptimizationLevel::O0 &&1069 "Should not be used for O0 pipeline");1070 1071 assert(Phase != ThinOrFullLTOPhase::FullLTOPostLink &&1072 "FullLTOPostLink shouldn't call buildModuleSimplificationPipeline!");1073 1074 ModulePassManager MPM;1075 1076 // Place pseudo probe instrumentation as the first pass of the pipeline to1077 // minimize the impact of optimization changes.1078 if (PGOOpt && PGOOpt->PseudoProbeForProfiling &&1079 Phase != ThinOrFullLTOPhase::ThinLTOPostLink)1080 MPM.addPass(SampleProfileProbePass(TM));1081 1082 bool HasSampleProfile = PGOOpt && (PGOOpt->Action == PGOOptions::SampleUse);1083 1084 // In ThinLTO mode, when flattened profile is used, all the available1085 // profile information will be annotated in PreLink phase so there is1086 // no need to load the profile again in PostLink.1087 bool LoadSampleProfile =1088 HasSampleProfile &&1089 !(FlattenedProfileUsed && Phase == ThinOrFullLTOPhase::ThinLTOPostLink);1090 1091 // During the ThinLTO backend phase we perform early indirect call promotion1092 // here, before globalopt. Otherwise imported available_externally functions1093 // look unreferenced and are removed. If we are going to load the sample1094 // profile then defer until later.1095 // TODO: See if we can move later and consolidate with the location where1096 // we perform ICP when we are loading a sample profile.1097 // TODO: We pass HasSampleProfile (whether there was a sample profile file1098 // passed to the compile) to the SamplePGO flag of ICP. This is used to1099 // determine whether the new direct calls are annotated with prof metadata.1100 // Ideally this should be determined from whether the IR is annotated with1101 // sample profile, and not whether the a sample profile was provided on the1102 // command line. E.g. for flattened profiles where we will not be reloading1103 // the sample profile in the ThinLTO backend, we ideally shouldn't have to1104 // provide the sample profile file.1105 if (Phase == ThinOrFullLTOPhase::ThinLTOPostLink && !LoadSampleProfile)1106 MPM.addPass(PGOIndirectCallPromotion(true /* InLTO */, HasSampleProfile));1107 1108 // Create an early function pass manager to cleanup the output of the1109 // frontend. Not necessary with LTO post link pipelines since the pre link1110 // pipeline already cleaned up the frontend output.1111 if (Phase != ThinOrFullLTOPhase::ThinLTOPostLink) {1112 // Do basic inference of function attributes from known properties of system1113 // libraries and other oracles.1114 MPM.addPass(InferFunctionAttrsPass());1115 MPM.addPass(CoroEarlyPass());1116 1117 FunctionPassManager EarlyFPM;1118 EarlyFPM.addPass(EntryExitInstrumenterPass(/*PostInlining=*/false));1119 // Lower llvm.expect to metadata before attempting transforms.1120 // Compare/branch metadata may alter the behavior of passes like1121 // SimplifyCFG.1122 EarlyFPM.addPass(LowerExpectIntrinsicPass());1123 EarlyFPM.addPass(SimplifyCFGPass());1124 EarlyFPM.addPass(SROAPass(SROAOptions::ModifyCFG));1125 EarlyFPM.addPass(EarlyCSEPass());1126 if (Level == OptimizationLevel::O3)1127 EarlyFPM.addPass(CallSiteSplittingPass());1128 MPM.addPass(createModuleToFunctionPassAdaptor(1129 std::move(EarlyFPM), PTO.EagerlyInvalidateAnalyses));1130 }1131 1132 if (LoadSampleProfile) {1133 // Annotate sample profile right after early FPM to ensure freshness of1134 // the debug info.1135 MPM.addPass(SampleProfileLoaderPass(1136 PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile, Phase, FS));1137 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert1138 // RequireAnalysisPass for PSI before subsequent non-module passes.1139 MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());1140 // Do not invoke ICP in the LTOPrelink phase as it makes it hard1141 // for the profile annotation to be accurate in the LTO backend.1142 if (!isLTOPreLink(Phase))1143 // We perform early indirect call promotion here, before globalopt.1144 // This is important for the ThinLTO backend phase because otherwise1145 // imported available_externally functions look unreferenced and are1146 // removed.1147 MPM.addPass(1148 PGOIndirectCallPromotion(true /* IsInLTO */, true /* SamplePGO */));1149 }1150 1151 // Try to perform OpenMP specific optimizations on the module. This is a1152 // (quick!) no-op if there are no OpenMP runtime calls present in the module.1153 MPM.addPass(OpenMPOptPass(Phase));1154 1155 if (AttributorRun & AttributorRunOption::MODULE)1156 MPM.addPass(AttributorPass());1157 1158 // Lower type metadata and the type.test intrinsic in the ThinLTO1159 // post link pipeline after ICP. This is to enable usage of the type1160 // tests in ICP sequences.1161 if (Phase == ThinOrFullLTOPhase::ThinLTOPostLink)1162 MPM.addPass(LowerTypeTestsPass(nullptr, nullptr,1163 lowertypetests::DropTestKind::Assume));1164 1165 invokePipelineEarlySimplificationEPCallbacks(MPM, Level, Phase);1166 1167 // Interprocedural constant propagation now that basic cleanup has occurred1168 // and prior to optimizing globals.1169 // FIXME: This position in the pipeline hasn't been carefully considered in1170 // years, it should be re-analyzed.1171 MPM.addPass(IPSCCPPass(1172 IPSCCPOptions(/*AllowFuncSpec=*/1173 Level != OptimizationLevel::Os &&1174 Level != OptimizationLevel::Oz &&1175 !isLTOPreLink(Phase))));1176 1177 // Attach metadata to indirect call sites indicating the set of functions1178 // they may target at run-time. This should follow IPSCCP.1179 MPM.addPass(CalledValuePropagationPass());1180 1181 // Optimize globals to try and fold them into constants.1182 MPM.addPass(GlobalOptPass());1183 1184 // Create a small function pass pipeline to cleanup after all the global1185 // optimizations.1186 FunctionPassManager GlobalCleanupPM;1187 // FIXME: Should this instead by a run of SROA?1188 GlobalCleanupPM.addPass(PromotePass());1189 GlobalCleanupPM.addPass(InstCombinePass());1190 invokePeepholeEPCallbacks(GlobalCleanupPM, Level);1191 GlobalCleanupPM.addPass(1192 SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));1193 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(GlobalCleanupPM),1194 PTO.EagerlyInvalidateAnalyses));1195 1196 // We already asserted this happens in non-FullLTOPostLink earlier.1197 const bool IsPreLink = Phase != ThinOrFullLTOPhase::ThinLTOPostLink;1198 // Enable contextual profiling instrumentation.1199 const bool IsCtxProfGen =1200 IsPreLink && PGOCtxProfLoweringPass::isCtxIRPGOInstrEnabled();1201 const bool IsPGOPreLink = !IsCtxProfGen && PGOOpt && IsPreLink;1202 const bool IsPGOInstrGen =1203 IsPGOPreLink && PGOOpt->Action == PGOOptions::IRInstr;1204 const bool IsPGOInstrUse =1205 IsPGOPreLink && PGOOpt->Action == PGOOptions::IRUse;1206 const bool IsMemprofUse = IsPGOPreLink && !PGOOpt->MemoryProfile.empty();1207 // We don't want to mix pgo ctx gen and pgo gen; we also don't currently1208 // enable ctx profiling from the frontend.1209 assert(!(IsPGOInstrGen && PGOCtxProfLoweringPass::isCtxIRPGOInstrEnabled()) &&1210 "Enabling both instrumented PGO and contextual instrumentation is not "1211 "supported.");1212 const bool IsCtxProfUse =1213 !UseCtxProfile.empty() && Phase == ThinOrFullLTOPhase::ThinLTOPreLink;1214 1215 assert(1216 (InstrumentColdFuncOnlyPath.empty() || PGOInstrumentColdFunctionOnly) &&1217 "--instrument-cold-function-only-path is provided but "1218 "--pgo-instrument-cold-function-only is not enabled");1219 const bool IsColdFuncOnlyInstrGen = PGOInstrumentColdFunctionOnly &&1220 IsPGOPreLink &&1221 !InstrumentColdFuncOnlyPath.empty();1222 1223 if (IsPGOInstrGen || IsPGOInstrUse || IsMemprofUse || IsCtxProfGen ||1224 IsCtxProfUse || IsColdFuncOnlyInstrGen)1225 addPreInlinerPasses(MPM, Level, Phase);1226 1227 // Add all the requested passes for instrumentation PGO, if requested.1228 if (IsPGOInstrGen || IsPGOInstrUse) {1229 addPGOInstrPasses(MPM, Level,1230 /*RunProfileGen=*/IsPGOInstrGen,1231 /*IsCS=*/false, PGOOpt->AtomicCounterUpdate,1232 PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile);1233 } else if (IsCtxProfGen || IsCtxProfUse) {1234 MPM.addPass(PGOInstrumentationGen(PGOInstrumentationType::CTXPROF));1235 // In pre-link, we just want the instrumented IR. We use the contextual1236 // profile in the post-thinlink phase.1237 // The instrumentation will be removed in post-thinlink after IPO.1238 // FIXME(mtrofin): move AssignGUIDPass if there is agreement to use this1239 // mechanism for GUIDs.1240 MPM.addPass(AssignGUIDPass());1241 if (IsCtxProfUse) {1242 MPM.addPass(PGOCtxProfFlatteningPass(/*IsPreThinlink=*/true));1243 return MPM;1244 }1245 // Block further inlining in the instrumented ctxprof case. This avoids1246 // confusingly collecting profiles for the same GUID corresponding to1247 // different variants of the function. We could do like PGO and identify1248 // functions by a (GUID, Hash) tuple, but since the ctxprof "use" waits for1249 // thinlto to happen before performing any further optimizations, it's1250 // unnecessary to collect profiles for non-prevailing copies.1251 MPM.addPass(NoinlineNonPrevailing());1252 addPostPGOLoopRotation(MPM, Level);1253 MPM.addPass(PGOCtxProfLoweringPass());1254 } else if (IsColdFuncOnlyInstrGen) {1255 addPGOInstrPasses(MPM, Level, /* RunProfileGen */ true, /* IsCS */ false,1256 /* AtomicCounterUpdate */ false,1257 InstrumentColdFuncOnlyPath,1258 /* ProfileRemappingFile */ "");1259 }1260 1261 if (IsPGOInstrGen || IsPGOInstrUse || IsCtxProfGen)1262 MPM.addPass(PGOIndirectCallPromotion(false, false));1263 1264 if (IsPGOPreLink && PGOOpt->CSAction == PGOOptions::CSIRInstr)1265 MPM.addPass(PGOInstrumentationGenCreateVar(PGOOpt->CSProfileGenFile,1266 EnableSampledInstr));1267 1268 if (IsMemprofUse)1269 MPM.addPass(MemProfUsePass(PGOOpt->MemoryProfile, FS));1270 1271 if (PGOOpt && (PGOOpt->Action == PGOOptions::IRUse ||1272 PGOOpt->Action == PGOOptions::SampleUse))1273 MPM.addPass(PGOForceFunctionAttrsPass(PGOOpt->ColdOptType));1274 1275 MPM.addPass(AlwaysInlinerPass(/*InsertLifetimeIntrinsics=*/true));1276 1277 if (EnableModuleInliner)1278 MPM.addPass(buildModuleInlinerPipeline(Level, Phase));1279 else1280 MPM.addPass(buildInlinerPipeline(Level, Phase));1281 1282 // Remove any dead arguments exposed by cleanups, constant folding globals,1283 // and argument promotion.1284 MPM.addPass(DeadArgumentEliminationPass());1285 1286 if (Phase == ThinOrFullLTOPhase::ThinLTOPostLink)1287 MPM.addPass(SimplifyTypeTestsPass());1288 1289 if (Phase != ThinOrFullLTOPhase::ThinLTOPreLink)1290 MPM.addPass(CoroCleanupPass());1291 1292 // Optimize globals now that functions are fully simplified.1293 MPM.addPass(GlobalOptPass());1294 MPM.addPass(GlobalDCEPass());1295 1296 return MPM;1297}1298 1299/// TODO: Should LTO cause any differences to this set of passes?1300void PassBuilder::addVectorPasses(OptimizationLevel Level,1301 FunctionPassManager &FPM,1302 ThinOrFullLTOPhase LTOPhase) {1303 const bool IsFullLTO = LTOPhase == ThinOrFullLTOPhase::FullLTOPostLink;1304 1305 FPM.addPass(LoopVectorizePass(1306 LoopVectorizeOptions(!PTO.LoopInterleaving, !PTO.LoopVectorization)));1307 1308 // Drop dereferenceable assumes after vectorization, as they are no longer1309 // needed and can inhibit further optimization.1310 if (!isLTOPreLink(LTOPhase))1311 FPM.addPass(DropUnnecessaryAssumesPass(/*DropDereferenceable=*/true));1312 1313 FPM.addPass(InferAlignmentPass());1314 if (IsFullLTO) {1315 // The vectorizer may have significantly shortened a loop body; unroll1316 // again. Unroll small loops to hide loop backedge latency and saturate any1317 // parallel execution resources of an out-of-order processor. We also then1318 // need to clean up redundancies and loop invariant code.1319 // FIXME: It would be really good to use a loop-integrated instruction1320 // combiner for cleanup here so that the unrolling and LICM can be pipelined1321 // across the loop nests.1322 // We do UnrollAndJam in a separate LPM to ensure it happens before unroll1323 if (EnableUnrollAndJam && PTO.LoopUnrolling)1324 FPM.addPass(createFunctionToLoopPassAdaptor(1325 LoopUnrollAndJamPass(Level.getSpeedupLevel())));1326 FPM.addPass(LoopUnrollPass(LoopUnrollOptions(1327 Level.getSpeedupLevel(), /*OnlyWhenForced=*/!PTO.LoopUnrolling,1328 PTO.ForgetAllSCEVInLoopUnroll)));1329 FPM.addPass(WarnMissedTransformationsPass());1330 // Now that we are done with loop unrolling, be it either by LoopVectorizer,1331 // or LoopUnroll passes, some variable-offset GEP's into alloca's could have1332 // become constant-offset, thus enabling SROA and alloca promotion. Do so.1333 // NOTE: we are very late in the pipeline, and we don't have any LICM1334 // or SimplifyCFG passes scheduled after us, that would cleanup1335 // the CFG mess this may created if allowed to modify CFG, so forbid that.1336 FPM.addPass(SROAPass(SROAOptions::PreserveCFG));1337 }1338 1339 if (!IsFullLTO) {1340 // Eliminate loads by forwarding stores from the previous iteration to loads1341 // of the current iteration.1342 FPM.addPass(LoopLoadEliminationPass());1343 }1344 // Cleanup after the loop optimization passes.1345 FPM.addPass(InstCombinePass());1346 1347 if (Level.getSpeedupLevel() > 1 && ExtraVectorizerPasses) {1348 ExtraFunctionPassManager<ShouldRunExtraVectorPasses> ExtraPasses;1349 // At higher optimization levels, try to clean up any runtime overlap and1350 // alignment checks inserted by the vectorizer. We want to track correlated1351 // runtime checks for two inner loops in the same outer loop, fold any1352 // common computations, hoist loop-invariant aspects out of any outer loop,1353 // and unswitch the runtime checks if possible. Once hoisted, we may have1354 // dead (or speculatable) control flows or more combining opportunities.1355 ExtraPasses.addPass(EarlyCSEPass());1356 ExtraPasses.addPass(CorrelatedValuePropagationPass());1357 ExtraPasses.addPass(InstCombinePass());1358 LoopPassManager LPM;1359 LPM.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,1360 /*AllowSpeculation=*/true));1361 LPM.addPass(SimpleLoopUnswitchPass(/* NonTrivial */ Level ==1362 OptimizationLevel::O3));1363 ExtraPasses.addPass(1364 createFunctionToLoopPassAdaptor(std::move(LPM), /*UseMemorySSA=*/true));1365 ExtraPasses.addPass(1366 SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));1367 ExtraPasses.addPass(InstCombinePass());1368 FPM.addPass(std::move(ExtraPasses));1369 }1370 1371 // Now that we've formed fast to execute loop structures, we do further1372 // optimizations. These are run afterward as they might block doing complex1373 // analyses and transforms such as what are needed for loop vectorization.1374 1375 // Cleanup after loop vectorization, etc. Simplification passes like CVP and1376 // GVN, loop transforms, and others have already run, so it's now better to1377 // convert to more optimized IR using more aggressive simplify CFG options.1378 // The extra sinking transform can create larger basic blocks, so do this1379 // before SLP vectorization.1380 FPM.addPass(SimplifyCFGPass(SimplifyCFGOptions()1381 .forwardSwitchCondToPhi(true)1382 .convertSwitchRangeToICmp(true)1383 .convertSwitchToArithmetic(true)1384 .convertSwitchToLookupTable(true)1385 .needCanonicalLoops(false)1386 .hoistCommonInsts(true)1387 .sinkCommonInsts(true)));1388 1389 if (IsFullLTO) {1390 FPM.addPass(SCCPPass());1391 FPM.addPass(InstCombinePass());1392 FPM.addPass(BDCEPass());1393 }1394 1395 // Optimize parallel scalar instruction chains into SIMD instructions.1396 if (PTO.SLPVectorization) {1397 FPM.addPass(SLPVectorizerPass());1398 if (Level.getSpeedupLevel() > 1 && ExtraVectorizerPasses) {1399 FPM.addPass(EarlyCSEPass());1400 }1401 }1402 // Enhance/cleanup vector code.1403 FPM.addPass(VectorCombinePass());1404 1405 if (!IsFullLTO) {1406 FPM.addPass(InstCombinePass());1407 // Unroll small loops to hide loop backedge latency and saturate any1408 // parallel execution resources of an out-of-order processor. We also then1409 // need to clean up redundancies and loop invariant code.1410 // FIXME: It would be really good to use a loop-integrated instruction1411 // combiner for cleanup here so that the unrolling and LICM can be pipelined1412 // across the loop nests.1413 // We do UnrollAndJam in a separate LPM to ensure it happens before unroll1414 if (EnableUnrollAndJam && PTO.LoopUnrolling) {1415 FPM.addPass(createFunctionToLoopPassAdaptor(1416 LoopUnrollAndJamPass(Level.getSpeedupLevel())));1417 }1418 FPM.addPass(LoopUnrollPass(LoopUnrollOptions(1419 Level.getSpeedupLevel(), /*OnlyWhenForced=*/!PTO.LoopUnrolling,1420 PTO.ForgetAllSCEVInLoopUnroll)));1421 FPM.addPass(WarnMissedTransformationsPass());1422 // Now that we are done with loop unrolling, be it either by LoopVectorizer,1423 // or LoopUnroll passes, some variable-offset GEP's into alloca's could have1424 // become constant-offset, thus enabling SROA and alloca promotion. Do so.1425 // NOTE: we are very late in the pipeline, and we don't have any LICM1426 // or SimplifyCFG passes scheduled after us, that would cleanup1427 // the CFG mess this may created if allowed to modify CFG, so forbid that.1428 FPM.addPass(SROAPass(SROAOptions::PreserveCFG));1429 }1430 1431 FPM.addPass(InferAlignmentPass());1432 FPM.addPass(InstCombinePass());1433 1434 // This is needed for two reasons:1435 // 1. It works around problems that instcombine introduces, such as sinking1436 // expensive FP divides into loops containing multiplications using the1437 // divide result.1438 // 2. It helps to clean up some loop-invariant code created by the loop1439 // unroll pass when IsFullLTO=false.1440 FPM.addPass(createFunctionToLoopPassAdaptor(1441 LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,1442 /*AllowSpeculation=*/true),1443 /*UseMemorySSA=*/true));1444 1445 // Now that we've vectorized and unrolled loops, we may have more refined1446 // alignment information, try to re-derive it here.1447 FPM.addPass(AlignmentFromAssumptionsPass());1448}1449 1450ModulePassManager1451PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level,1452 ThinOrFullLTOPhase LTOPhase) {1453 const bool LTOPreLink = isLTOPreLink(LTOPhase);1454 ModulePassManager MPM;1455 1456 // Run partial inlining pass to partially inline functions that have1457 // large bodies.1458 if (RunPartialInlining)1459 MPM.addPass(PartialInlinerPass());1460 1461 // Remove avail extern fns and globals definitions since we aren't compiling1462 // an object file for later LTO. For LTO we want to preserve these so they1463 // are eligible for inlining at link-time. Note if they are unreferenced they1464 // will be removed by GlobalDCE later, so this only impacts referenced1465 // available externally globals. Eventually they will be suppressed during1466 // codegen, but eliminating here enables more opportunity for GlobalDCE as it1467 // may make globals referenced by available external functions dead and saves1468 // running remaining passes on the eliminated functions. These should be1469 // preserved during prelinking for link-time inlining decisions.1470 if (!LTOPreLink)1471 MPM.addPass(EliminateAvailableExternallyPass());1472 1473 // Do RPO function attribute inference across the module to forward-propagate1474 // attributes where applicable.1475 // FIXME: Is this really an optimization rather than a canonicalization?1476 MPM.addPass(ReversePostOrderFunctionAttrsPass());1477 1478 // Do a post inline PGO instrumentation and use pass. This is a context1479 // sensitive PGO pass. We don't want to do this in LTOPreLink phrase as1480 // cross-module inline has not been done yet. The context sensitive1481 // instrumentation is after all the inlines are done.1482 if (!LTOPreLink && PGOOpt) {1483 if (PGOOpt->CSAction == PGOOptions::CSIRInstr)1484 addPGOInstrPasses(MPM, Level, /*RunProfileGen=*/true,1485 /*IsCS=*/true, PGOOpt->AtomicCounterUpdate,1486 PGOOpt->CSProfileGenFile, PGOOpt->ProfileRemappingFile);1487 else if (PGOOpt->CSAction == PGOOptions::CSIRUse)1488 addPGOInstrPasses(MPM, Level, /*RunProfileGen=*/false,1489 /*IsCS=*/true, PGOOpt->AtomicCounterUpdate,1490 PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile);1491 }1492 1493 // Re-compute GlobalsAA here prior to function passes. This is particularly1494 // useful as the above will have inlined, DCE'ed, and function-attr1495 // propagated everything. We should at this point have a reasonably minimal1496 // and richly annotated call graph. By computing aliasing and mod/ref1497 // information for all local globals here, the late loop passes and notably1498 // the vectorizer will be able to use them to help recognize vectorizable1499 // memory operations.1500 if (EnableGlobalAnalyses)1501 MPM.addPass(RecomputeGlobalsAAPass());1502 1503 invokeOptimizerEarlyEPCallbacks(MPM, Level, LTOPhase);1504 1505 FunctionPassManager OptimizePM;1506 1507 // Only drop unnecessary assumes post-inline and post-link, as otherwise1508 // additional uses of the affected value may be introduced through inlining1509 // and CSE.1510 if (!isLTOPreLink(LTOPhase))1511 OptimizePM.addPass(DropUnnecessaryAssumesPass());1512 1513 // Scheduling LoopVersioningLICM when inlining is over, because after that1514 // we may see more accurate aliasing. Reason to run this late is that too1515 // early versioning may prevent further inlining due to increase of code1516 // size. Other optimizations which runs later might get benefit of no-alias1517 // assumption in clone loop.1518 if (UseLoopVersioningLICM) {1519 OptimizePM.addPass(1520 createFunctionToLoopPassAdaptor(LoopVersioningLICMPass()));1521 // LoopVersioningLICM pass might increase new LICM opportunities.1522 OptimizePM.addPass(createFunctionToLoopPassAdaptor(1523 LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,1524 /*AllowSpeculation=*/true),1525 /*USeMemorySSA=*/true));1526 }1527 1528 OptimizePM.addPass(Float2IntPass());1529 OptimizePM.addPass(LowerConstantIntrinsicsPass());1530 1531 if (EnableMatrix) {1532 OptimizePM.addPass(LowerMatrixIntrinsicsPass());1533 OptimizePM.addPass(EarlyCSEPass());1534 }1535 1536 // CHR pass should only be applied with the profile information.1537 // The check is to check the profile summary information in CHR.1538 if (EnableCHR && Level == OptimizationLevel::O3)1539 OptimizePM.addPass(ControlHeightReductionPass());1540 1541 // FIXME: We need to run some loop optimizations to re-rotate loops after1542 // simplifycfg and others undo their rotation.1543 1544 // Optimize the loop execution. These passes operate on entire loop nests1545 // rather than on each loop in an inside-out manner, and so they are actually1546 // function passes.1547 1548 invokeVectorizerStartEPCallbacks(OptimizePM, Level);1549 1550 LoopPassManager LPM;1551 // First rotate loops that may have been un-rotated by prior passes.1552 // Disable header duplication at -Oz.1553 LPM.addPass(LoopRotatePass(EnableLoopHeaderDuplication ||1554 Level != OptimizationLevel::Oz,1555 LTOPreLink));1556 // Some loops may have become dead by now. Try to delete them.1557 // FIXME: see discussion in https://reviews.llvm.org/D112851,1558 // this may need to be revisited once we run GVN before loop deletion1559 // in the simplification pipeline.1560 LPM.addPass(LoopDeletionPass());1561 1562 if (PTO.LoopInterchange)1563 LPM.addPass(LoopInterchangePass());1564 1565 OptimizePM.addPass(1566 createFunctionToLoopPassAdaptor(std::move(LPM), /*UseMemorySSA=*/false));1567 1568 // FIXME: This may not be the right place in the pipeline.1569 // We need to have the data to support the right place.1570 if (PTO.LoopFusion)1571 OptimizePM.addPass(LoopFusePass());1572 1573 // Distribute loops to allow partial vectorization. I.e. isolate dependences1574 // into separate loop that would otherwise inhibit vectorization. This is1575 // currently only performed for loops marked with the metadata1576 // llvm.loop.distribute=true or when -enable-loop-distribute is specified.1577 OptimizePM.addPass(LoopDistributePass());1578 1579 // Populates the VFABI attribute with the scalar-to-vector mappings1580 // from the TargetLibraryInfo.1581 OptimizePM.addPass(InjectTLIMappings());1582 1583 addVectorPasses(Level, OptimizePM, LTOPhase);1584 1585 invokeVectorizerEndEPCallbacks(OptimizePM, Level);1586 1587 // LoopSink pass sinks instructions hoisted by LICM, which serves as a1588 // canonicalization pass that enables other optimizations. As a result,1589 // LoopSink pass needs to be a very late IR pass to avoid undoing LICM1590 // result too early.1591 OptimizePM.addPass(LoopSinkPass());1592 1593 // And finally clean up LCSSA form before generating code.1594 OptimizePM.addPass(InstSimplifyPass());1595 1596 // This hoists/decomposes div/rem ops. It should run after other sink/hoist1597 // passes to avoid re-sinking, but before SimplifyCFG because it can allow1598 // flattening of blocks.1599 OptimizePM.addPass(DivRemPairsPass());1600 1601 // Try to annotate calls that were created during optimization.1602 OptimizePM.addPass(1603 TailCallElimPass(/*UpdateFunctionEntryCount=*/isInstrumentedPGOUse()));1604 1605 // LoopSink (and other loop passes since the last simplifyCFG) might have1606 // resulted in single-entry-single-exit or empty blocks. Clean up the CFG.1607 OptimizePM.addPass(1608 SimplifyCFGPass(SimplifyCFGOptions()1609 .convertSwitchRangeToICmp(true)1610 .convertSwitchToArithmetic(true)1611 .speculateUnpredictables(true)1612 .hoistLoadsStoresWithCondFaulting(true)));1613 1614 // Add the core optimizing pipeline.1615 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(OptimizePM),1616 PTO.EagerlyInvalidateAnalyses));1617 1618 invokeOptimizerLastEPCallbacks(MPM, Level, LTOPhase);1619 1620 // Split out cold code. Splitting is done late to avoid hiding context from1621 // other optimizations and inadvertently regressing performance. The tradeoff1622 // is that this has a higher code size cost than splitting early.1623 if (EnableHotColdSplit && !LTOPreLink)1624 MPM.addPass(HotColdSplittingPass());1625 1626 // Search the code for similar regions of code. If enough similar regions can1627 // be found where extracting the regions into their own function will decrease1628 // the size of the program, we extract the regions, a deduplicate the1629 // structurally similar regions.1630 if (EnableIROutliner)1631 MPM.addPass(IROutlinerPass());1632 1633 // Now we need to do some global optimization transforms.1634 // FIXME: It would seem like these should come first in the optimization1635 // pipeline and maybe be the bottom of the canonicalization pipeline? Weird1636 // ordering here.1637 MPM.addPass(GlobalDCEPass());1638 MPM.addPass(ConstantMergePass());1639 1640 // Merge functions if requested. It has a better chance to merge functions1641 // after ConstantMerge folded jump tables.1642 if (PTO.MergeFunctions)1643 MPM.addPass(MergeFunctionsPass());1644 1645 if (PTO.CallGraphProfile && !LTOPreLink)1646 MPM.addPass(CGProfilePass(isLTOPostLink(LTOPhase)));1647 1648 // RelLookupTableConverterPass runs later in LTO post-link pipeline.1649 if (!LTOPreLink)1650 MPM.addPass(RelLookupTableConverterPass());1651 1652 return MPM;1653}1654 1655ModulePassManager1656PassBuilder::buildPerModuleDefaultPipeline(OptimizationLevel Level,1657 ThinOrFullLTOPhase Phase) {1658 if (Level == OptimizationLevel::O0)1659 return buildO0DefaultPipeline(Level, Phase);1660 1661 ModulePassManager MPM;1662 1663 // Currently this pipeline is only invoked in an LTO pre link pass or when we1664 // are not running LTO. If that changes the below checks may need updating.1665 assert(isLTOPreLink(Phase) || Phase == ThinOrFullLTOPhase::None);1666 1667 // If we are invoking this in non-LTO mode, remove any MemProf related1668 // attributes and metadata, as we don't know whether we are linking with1669 // a library containing the necessary interfaces.1670 if (Phase == ThinOrFullLTOPhase::None)1671 MPM.addPass(MemProfRemoveInfo());1672 1673 // Convert @llvm.global.annotations to !annotation metadata.1674 MPM.addPass(Annotation2MetadataPass());1675 1676 // Force any function attributes we want the rest of the pipeline to observe.1677 MPM.addPass(ForceFunctionAttrsPass());1678 1679 if (PGOOpt && PGOOpt->DebugInfoForProfiling)1680 MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));1681 1682 // Apply module pipeline start EP callback.1683 invokePipelineStartEPCallbacks(MPM, Level);1684 1685 // Add the core simplification pipeline.1686 MPM.addPass(buildModuleSimplificationPipeline(Level, Phase));1687 1688 // Now add the optimization pipeline.1689 MPM.addPass(buildModuleOptimizationPipeline(Level, Phase));1690 1691 if (PGOOpt && PGOOpt->PseudoProbeForProfiling &&1692 PGOOpt->Action == PGOOptions::SampleUse)1693 MPM.addPass(PseudoProbeUpdatePass());1694 1695 // Emit annotation remarks.1696 addAnnotationRemarksPass(MPM);1697 1698 if (isLTOPreLink(Phase))1699 addRequiredLTOPreLinkPasses(MPM);1700 return MPM;1701}1702 1703ModulePassManager1704PassBuilder::buildFatLTODefaultPipeline(OptimizationLevel Level, bool ThinLTO,1705 bool EmitSummary) {1706 ModulePassManager MPM;1707 if (ThinLTO)1708 MPM.addPass(buildThinLTOPreLinkDefaultPipeline(Level));1709 else1710 MPM.addPass(buildLTOPreLinkDefaultPipeline(Level));1711 MPM.addPass(EmbedBitcodePass(ThinLTO, EmitSummary));1712 1713 // Perform any cleanups to the IR that aren't suitable for per TU compilation,1714 // like removing CFI/WPD related instructions. Note, we reuse1715 // LowerTypeTestsPass to clean up type tests rather than duplicate that logic1716 // in FatLtoCleanup.1717 MPM.addPass(FatLtoCleanup());1718 1719 // If we're doing FatLTO w/ CFI enabled, we don't want the type tests in the1720 // object code, only in the bitcode section, so drop it before we run1721 // module optimization and generate machine code. If llvm.type.test() isn't in1722 // the IR, this won't do anything.1723 MPM.addPass(1724 LowerTypeTestsPass(nullptr, nullptr, lowertypetests::DropTestKind::All));1725 1726 // Use the ThinLTO post-link pipeline with sample profiling1727 if (ThinLTO && PGOOpt && PGOOpt->Action == PGOOptions::SampleUse)1728 MPM.addPass(buildThinLTODefaultPipeline(Level, /*ImportSummary=*/nullptr));1729 else {1730 // ModuleSimplification does not run the coroutine passes for1731 // ThinLTOPreLink, so we need the coroutine passes to run for ThinLTO1732 // builds, otherwise they will miscompile.1733 if (ThinLTO) {1734 // TODO: replace w/ buildCoroWrapper() when it takes phase and level into1735 // consideration.1736 CGSCCPassManager CGPM;1737 CGPM.addPass(CoroSplitPass(Level != OptimizationLevel::O0));1738 CGPM.addPass(CoroAnnotationElidePass());1739 MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));1740 MPM.addPass(CoroCleanupPass());1741 }1742 1743 // otherwise, just use module optimization1744 MPM.addPass(1745 buildModuleOptimizationPipeline(Level, ThinOrFullLTOPhase::None));1746 // Emit annotation remarks.1747 addAnnotationRemarksPass(MPM);1748 }1749 return MPM;1750}1751 1752ModulePassManager1753PassBuilder::buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level) {1754 if (Level == OptimizationLevel::O0)1755 return buildO0DefaultPipeline(Level, ThinOrFullLTOPhase::ThinLTOPreLink);1756 1757 ModulePassManager MPM;1758 1759 // Convert @llvm.global.annotations to !annotation metadata.1760 MPM.addPass(Annotation2MetadataPass());1761 1762 // Force any function attributes we want the rest of the pipeline to observe.1763 MPM.addPass(ForceFunctionAttrsPass());1764 1765 if (PGOOpt && PGOOpt->DebugInfoForProfiling)1766 MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));1767 1768 // Apply module pipeline start EP callback.1769 invokePipelineStartEPCallbacks(MPM, Level);1770 1771 // If we are planning to perform ThinLTO later, we don't bloat the code with1772 // unrolling/vectorization/... now. Just simplify the module as much as we1773 // can.1774 MPM.addPass(buildModuleSimplificationPipeline(1775 Level, ThinOrFullLTOPhase::ThinLTOPreLink));1776 // In pre-link, for ctx prof use, we stop here with an instrumented IR. We let1777 // thinlto use the contextual info to perform imports; then use the contextual1778 // profile in the post-thinlink phase.1779 if (!UseCtxProfile.empty()) {1780 addRequiredLTOPreLinkPasses(MPM);1781 return MPM;1782 }1783 1784 // Run partial inlining pass to partially inline functions that have1785 // large bodies.1786 // FIXME: It isn't clear whether this is really the right place to run this1787 // in ThinLTO. Because there is another canonicalization and simplification1788 // phase that will run after the thin link, running this here ends up with1789 // less information than will be available later and it may grow functions in1790 // ways that aren't beneficial.1791 if (RunPartialInlining)1792 MPM.addPass(PartialInlinerPass());1793 1794 if (PGOOpt && PGOOpt->PseudoProbeForProfiling &&1795 PGOOpt->Action == PGOOptions::SampleUse)1796 MPM.addPass(PseudoProbeUpdatePass());1797 1798 // Handle Optimizer{Early,Last}EPCallbacks added by clang on PreLink. Actual1799 // optimization is going to be done in PostLink stage, but clang can't add1800 // callbacks there in case of in-process ThinLTO called by linker.1801 invokeOptimizerEarlyEPCallbacks(MPM, Level,1802 /*Phase=*/ThinOrFullLTOPhase::ThinLTOPreLink);1803 invokeOptimizerLastEPCallbacks(MPM, Level,1804 /*Phase=*/ThinOrFullLTOPhase::ThinLTOPreLink);1805 1806 // Emit annotation remarks.1807 addAnnotationRemarksPass(MPM);1808 1809 addRequiredLTOPreLinkPasses(MPM);1810 1811 return MPM;1812}1813 1814ModulePassManager PassBuilder::buildThinLTODefaultPipeline(1815 OptimizationLevel Level, const ModuleSummaryIndex *ImportSummary) {1816 ModulePassManager MPM;1817 1818 // If we are invoking this without a summary index noting that we are linking1819 // with a library containing the necessary APIs, remove any MemProf related1820 // attributes and metadata.1821 if (!ImportSummary || !ImportSummary->withSupportsHotColdNew())1822 MPM.addPass(MemProfRemoveInfo());1823 1824 if (ImportSummary) {1825 // For ThinLTO we must apply the context disambiguation decisions early, to1826 // ensure we can correctly match the callsites to summary data.1827 if (EnableMemProfContextDisambiguation)1828 MPM.addPass(MemProfContextDisambiguation(1829 ImportSummary, PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));1830 1831 // These passes import type identifier resolutions for whole-program1832 // devirtualization and CFI. They must run early because other passes may1833 // disturb the specific instruction patterns that these passes look for,1834 // creating dependencies on resolutions that may not appear in the summary.1835 //1836 // For example, GVN may transform the pattern assume(type.test) appearing in1837 // two basic blocks into assume(phi(type.test, type.test)), which would1838 // transform a dependency on a WPD resolution into a dependency on a type1839 // identifier resolution for CFI.1840 //1841 // Also, WPD has access to more precise information than ICP and can1842 // devirtualize more effectively, so it should operate on the IR first.1843 //1844 // The WPD and LowerTypeTest passes need to run at -O0 to lower type1845 // metadata and intrinsics.1846 MPM.addPass(WholeProgramDevirtPass(nullptr, ImportSummary));1847 MPM.addPass(LowerTypeTestsPass(nullptr, ImportSummary));1848 }1849 1850 if (Level == OptimizationLevel::O0) {1851 // Run a second time to clean up any type tests left behind by WPD for use1852 // in ICP.1853 MPM.addPass(LowerTypeTestsPass(nullptr, nullptr,1854 lowertypetests::DropTestKind::Assume));1855 MPM.addPass(buildCoroWrapper(ThinOrFullLTOPhase::ThinLTOPostLink));1856 // Drop available_externally and unreferenced globals. This is necessary1857 // with ThinLTO in order to avoid leaving undefined references to dead1858 // globals in the object file.1859 MPM.addPass(EliminateAvailableExternallyPass());1860 MPM.addPass(GlobalDCEPass());1861 return MPM;1862 }1863 if (!UseCtxProfile.empty()) {1864 MPM.addPass(1865 buildModuleInlinerPipeline(Level, ThinOrFullLTOPhase::ThinLTOPostLink));1866 } else {1867 // Add the core simplification pipeline.1868 MPM.addPass(buildModuleSimplificationPipeline(1869 Level, ThinOrFullLTOPhase::ThinLTOPostLink));1870 }1871 // Now add the optimization pipeline.1872 MPM.addPass(buildModuleOptimizationPipeline(1873 Level, ThinOrFullLTOPhase::ThinLTOPostLink));1874 1875 // Emit annotation remarks.1876 addAnnotationRemarksPass(MPM);1877 1878 return MPM;1879}1880 1881ModulePassManager1882PassBuilder::buildLTOPreLinkDefaultPipeline(OptimizationLevel Level) {1883 // FIXME: We should use a customized pre-link pipeline!1884 return buildPerModuleDefaultPipeline(Level,1885 ThinOrFullLTOPhase::FullLTOPreLink);1886}1887 1888ModulePassManager1889PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level,1890 ModuleSummaryIndex *ExportSummary) {1891 ModulePassManager MPM;1892 1893 invokeFullLinkTimeOptimizationEarlyEPCallbacks(MPM, Level);1894 1895 // If we are invoking this without a summary index noting that we are linking1896 // with a library containing the necessary APIs, remove any MemProf related1897 // attributes and metadata.1898 if (!ExportSummary || !ExportSummary->withSupportsHotColdNew())1899 MPM.addPass(MemProfRemoveInfo());1900 1901 // Create a function that performs CFI checks for cross-DSO calls with targets1902 // in the current module.1903 MPM.addPass(CrossDSOCFIPass());1904 1905 if (Level == OptimizationLevel::O0) {1906 // The WPD and LowerTypeTest passes need to run at -O0 to lower type1907 // metadata and intrinsics.1908 MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));1909 MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));1910 // Run a second time to clean up any type tests left behind by WPD for use1911 // in ICP.1912 MPM.addPass(LowerTypeTestsPass(nullptr, nullptr,1913 lowertypetests::DropTestKind::Assume));1914 1915 MPM.addPass(buildCoroWrapper(ThinOrFullLTOPhase::FullLTOPostLink));1916 1917 invokeFullLinkTimeOptimizationLastEPCallbacks(MPM, Level);1918 1919 // Emit annotation remarks.1920 addAnnotationRemarksPass(MPM);1921 1922 return MPM;1923 }1924 1925 if (PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) {1926 // Load sample profile before running the LTO optimization pipeline.1927 MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,1928 PGOOpt->ProfileRemappingFile,1929 ThinOrFullLTOPhase::FullLTOPostLink));1930 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert1931 // RequireAnalysisPass for PSI before subsequent non-module passes.1932 MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());1933 }1934 1935 // Try to run OpenMP optimizations, quick no-op if no OpenMP metadata present.1936 MPM.addPass(OpenMPOptPass(ThinOrFullLTOPhase::FullLTOPostLink));1937 1938 // Remove unused virtual tables to improve the quality of code generated by1939 // whole-program devirtualization and bitset lowering.1940 MPM.addPass(GlobalDCEPass(/*InLTOPostLink=*/true));1941 1942 // Do basic inference of function attributes from known properties of system1943 // libraries and other oracles.1944 MPM.addPass(InferFunctionAttrsPass());1945 1946 if (Level.getSpeedupLevel() > 1) {1947 MPM.addPass(createModuleToFunctionPassAdaptor(1948 CallSiteSplittingPass(), PTO.EagerlyInvalidateAnalyses));1949 1950 // Indirect call promotion. This should promote all the targets that are1951 // left by the earlier promotion pass that promotes intra-module targets.1952 // This two-step promotion is to save the compile time. For LTO, it should1953 // produce the same result as if we only do promotion here.1954 MPM.addPass(PGOIndirectCallPromotion(1955 true /* InLTO */, PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));1956 1957 // Promoting by-reference arguments to by-value exposes more constants to1958 // IPSCCP.1959 CGSCCPassManager CGPM;1960 CGPM.addPass(PostOrderFunctionAttrsPass());1961 CGPM.addPass(ArgumentPromotionPass());1962 CGPM.addPass(1963 createCGSCCToFunctionPassAdaptor(SROAPass(SROAOptions::ModifyCFG)));1964 MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));1965 1966 // Propagate constants at call sites into the functions they call. This1967 // opens opportunities for globalopt (and inlining) by substituting function1968 // pointers passed as arguments to direct uses of functions.1969 MPM.addPass(IPSCCPPass(IPSCCPOptions(/*AllowFuncSpec=*/1970 Level != OptimizationLevel::Os &&1971 Level != OptimizationLevel::Oz)));1972 1973 // Attach metadata to indirect call sites indicating the set of functions1974 // they may target at run-time. This should follow IPSCCP.1975 MPM.addPass(CalledValuePropagationPass());1976 }1977 1978 // Do RPO function attribute inference across the module to forward-propagate1979 // attributes where applicable.1980 // FIXME: Is this really an optimization rather than a canonicalization?1981 MPM.addPass(ReversePostOrderFunctionAttrsPass());1982 1983 // Use in-range annotations on GEP indices to split globals where beneficial.1984 MPM.addPass(GlobalSplitPass());1985 1986 // Run whole program optimization of virtual call when the list of callees1987 // is fixed.1988 MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));1989 1990 MPM.addPass(NoRecurseLTOInferencePass());1991 // Stop here at -O1.1992 if (Level == OptimizationLevel::O1) {1993 // The LowerTypeTestsPass needs to run to lower type metadata and the1994 // type.test intrinsics. The pass does nothing if CFI is disabled.1995 MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));1996 // Run a second time to clean up any type tests left behind by WPD for use1997 // in ICP (which is performed earlier than this in the regular LTO1998 // pipeline).1999 MPM.addPass(LowerTypeTestsPass(nullptr, nullptr,2000 lowertypetests::DropTestKind::Assume));2001 2002 MPM.addPass(buildCoroWrapper(ThinOrFullLTOPhase::FullLTOPostLink));2003 2004 invokeFullLinkTimeOptimizationLastEPCallbacks(MPM, Level);2005 2006 // Emit annotation remarks.2007 addAnnotationRemarksPass(MPM);2008 2009 return MPM;2010 }2011 2012 // TODO: Skip to match buildCoroWrapper.2013 MPM.addPass(CoroEarlyPass());2014 2015 // Optimize globals to try and fold them into constants.2016 MPM.addPass(GlobalOptPass());2017 2018 // Promote any localized globals to SSA registers.2019 MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass()));2020 2021 // Linking modules together can lead to duplicate global constant, only2022 // keep one copy of each constant.2023 MPM.addPass(ConstantMergePass());2024 2025 // Remove unused arguments from functions.2026 MPM.addPass(DeadArgumentEliminationPass());2027 2028 // Reduce the code after globalopt and ipsccp. Both can open up significant2029 // simplification opportunities, and both can propagate functions through2030 // function pointers. When this happens, we often have to resolve varargs2031 // calls, etc, so let instcombine do this.2032 FunctionPassManager PeepholeFPM;2033 PeepholeFPM.addPass(InstCombinePass());2034 if (Level.getSpeedupLevel() > 1)2035 PeepholeFPM.addPass(AggressiveInstCombinePass());2036 invokePeepholeEPCallbacks(PeepholeFPM, Level);2037 2038 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(PeepholeFPM),2039 PTO.EagerlyInvalidateAnalyses));2040 2041 // Lower variadic functions for supported targets prior to inlining.2042 MPM.addPass(ExpandVariadicsPass(ExpandVariadicsMode::Optimize));2043 2044 // Note: historically, the PruneEH pass was run first to deduce nounwind and2045 // generally clean up exception handling overhead. It isn't clear this is2046 // valuable as the inliner doesn't currently care whether it is inlining an2047 // invoke or a call.2048 // Run the inliner now.2049 if (EnableModuleInliner) {2050 MPM.addPass(ModuleInlinerPass(getInlineParamsFromOptLevel(Level),2051 UseInlineAdvisor,2052 ThinOrFullLTOPhase::FullLTOPostLink));2053 } else {2054 MPM.addPass(ModuleInlinerWrapperPass(2055 getInlineParamsFromOptLevel(Level),2056 /* MandatoryFirst */ true,2057 InlineContext{ThinOrFullLTOPhase::FullLTOPostLink,2058 InlinePass::CGSCCInliner}));2059 }2060 2061 // Perform context disambiguation after inlining, since that would reduce the2062 // amount of additional cloning required to distinguish the allocation2063 // contexts.2064 if (EnableMemProfContextDisambiguation)2065 MPM.addPass(MemProfContextDisambiguation(2066 /*Summary=*/nullptr,2067 PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));2068 2069 // Optimize globals again after we ran the inliner.2070 MPM.addPass(GlobalOptPass());2071 2072 // Run the OpenMPOpt pass again after global optimizations.2073 MPM.addPass(OpenMPOptPass(ThinOrFullLTOPhase::FullLTOPostLink));2074 2075 // Garbage collect dead functions.2076 MPM.addPass(GlobalDCEPass(/*InLTOPostLink=*/true));2077 2078 // If we didn't decide to inline a function, check to see if we can2079 // transform it to pass arguments by value instead of by reference.2080 CGSCCPassManager CGPM;2081 CGPM.addPass(ArgumentPromotionPass());2082 CGPM.addPass(CoroSplitPass(Level != OptimizationLevel::O0));2083 CGPM.addPass(CoroAnnotationElidePass());2084 MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));2085 2086 FunctionPassManager FPM;2087 // The IPO Passes may leave cruft around. Clean up after them.2088 FPM.addPass(InstCombinePass());2089 invokePeepholeEPCallbacks(FPM, Level);2090 2091 if (EnableConstraintElimination)2092 FPM.addPass(ConstraintEliminationPass());2093 2094 FPM.addPass(JumpThreadingPass());2095 2096 // Do a post inline PGO instrumentation and use pass. This is a context2097 // sensitive PGO pass.2098 if (PGOOpt) {2099 if (PGOOpt->CSAction == PGOOptions::CSIRInstr)2100 addPGOInstrPasses(MPM, Level, /*RunProfileGen=*/true,2101 /*IsCS=*/true, PGOOpt->AtomicCounterUpdate,2102 PGOOpt->CSProfileGenFile, PGOOpt->ProfileRemappingFile);2103 else if (PGOOpt->CSAction == PGOOptions::CSIRUse)2104 addPGOInstrPasses(MPM, Level, /*RunProfileGen=*/false,2105 /*IsCS=*/true, PGOOpt->AtomicCounterUpdate,2106 PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile);2107 }2108 2109 // Break up allocas2110 FPM.addPass(SROAPass(SROAOptions::ModifyCFG));2111 2112 // LTO provides additional opportunities for tailcall elimination due to2113 // link-time inlining, and visibility of nocapture attribute.2114 FPM.addPass(2115 TailCallElimPass(/*UpdateFunctionEntryCount=*/isInstrumentedPGOUse()));2116 2117 // Run a few AA driver optimizations here and now to cleanup the code.2118 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM),2119 PTO.EagerlyInvalidateAnalyses));2120 2121 MPM.addPass(2122 createModuleToPostOrderCGSCCPassAdaptor(PostOrderFunctionAttrsPass()));2123 2124 // Require the GlobalsAA analysis for the module so we can query it within2125 // MainFPM.2126 if (EnableGlobalAnalyses) {2127 MPM.addPass(RequireAnalysisPass<GlobalsAA, Module>());2128 // Invalidate AAManager so it can be recreated and pick up the newly2129 // available GlobalsAA.2130 MPM.addPass(2131 createModuleToFunctionPassAdaptor(InvalidateAnalysisPass<AAManager>()));2132 }2133 2134 FunctionPassManager MainFPM;2135 MainFPM.addPass(createFunctionToLoopPassAdaptor(2136 LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,2137 /*AllowSpeculation=*/true),2138 /*USeMemorySSA=*/true));2139 2140 if (RunNewGVN)2141 MainFPM.addPass(NewGVNPass());2142 else2143 MainFPM.addPass(GVNPass());2144 2145 // Remove dead memcpy()'s.2146 MainFPM.addPass(MemCpyOptPass());2147 2148 // Nuke dead stores.2149 MainFPM.addPass(DSEPass());2150 MainFPM.addPass(MoveAutoInitPass());2151 MainFPM.addPass(MergedLoadStoreMotionPass());2152 2153 invokeVectorizerStartEPCallbacks(MainFPM, Level);2154 2155 LoopPassManager LPM;2156 if (EnableLoopFlatten && Level.getSpeedupLevel() > 1)2157 LPM.addPass(LoopFlattenPass());2158 LPM.addPass(IndVarSimplifyPass());2159 LPM.addPass(LoopDeletionPass());2160 // FIXME: Add loop interchange.2161 2162 // Unroll small loops and perform peeling.2163 LPM.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(),2164 /* OnlyWhenForced= */ !PTO.LoopUnrolling,2165 PTO.ForgetAllSCEVInLoopUnroll));2166 // The loop passes in LPM (LoopFullUnrollPass) do not preserve MemorySSA.2167 // *All* loop passes must preserve it, in order to be able to use it.2168 MainFPM.addPass(2169 createFunctionToLoopPassAdaptor(std::move(LPM), /*UseMemorySSA=*/false));2170 2171 MainFPM.addPass(LoopDistributePass());2172 2173 addVectorPasses(Level, MainFPM, ThinOrFullLTOPhase::FullLTOPostLink);2174 2175 invokeVectorizerEndEPCallbacks(MainFPM, Level);2176 2177 // Run the OpenMPOpt CGSCC pass again late.2178 MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(2179 OpenMPOptCGSCCPass(ThinOrFullLTOPhase::FullLTOPostLink)));2180 2181 invokePeepholeEPCallbacks(MainFPM, Level);2182 MainFPM.addPass(JumpThreadingPass());2183 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(MainFPM),2184 PTO.EagerlyInvalidateAnalyses));2185 2186 // Lower type metadata and the type.test intrinsic. This pass supports2187 // clang's control flow integrity mechanisms (-fsanitize=cfi*) and needs2188 // to be run at link time if CFI is enabled. This pass does nothing if2189 // CFI is disabled.2190 MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));2191 // Run a second time to clean up any type tests left behind by WPD for use2192 // in ICP (which is performed earlier than this in the regular LTO pipeline).2193 MPM.addPass(LowerTypeTestsPass(nullptr, nullptr,2194 lowertypetests::DropTestKind::Assume));2195 2196 // Enable splitting late in the FullLTO post-link pipeline.2197 if (EnableHotColdSplit)2198 MPM.addPass(HotColdSplittingPass());2199 2200 // Add late LTO optimization passes.2201 FunctionPassManager LateFPM;2202 2203 // LoopSink pass sinks instructions hoisted by LICM, which serves as a2204 // canonicalization pass that enables other optimizations. As a result,2205 // LoopSink pass needs to be a very late IR pass to avoid undoing LICM2206 // result too early.2207 LateFPM.addPass(LoopSinkPass());2208 2209 // This hoists/decomposes div/rem ops. It should run after other sink/hoist2210 // passes to avoid re-sinking, but before SimplifyCFG because it can allow2211 // flattening of blocks.2212 LateFPM.addPass(DivRemPairsPass());2213 2214 // Delete basic blocks, which optimization passes may have killed.2215 LateFPM.addPass(SimplifyCFGPass(SimplifyCFGOptions()2216 .convertSwitchRangeToICmp(true)2217 .convertSwitchToArithmetic(true)2218 .hoistCommonInsts(true)2219 .speculateUnpredictables(true)));2220 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(LateFPM)));2221 2222 // Drop bodies of available eternally objects to improve GlobalDCE.2223 MPM.addPass(EliminateAvailableExternallyPass());2224 2225 // Now that we have optimized the program, discard unreachable functions.2226 MPM.addPass(GlobalDCEPass(/*InLTOPostLink=*/true));2227 2228 if (PTO.MergeFunctions)2229 MPM.addPass(MergeFunctionsPass());2230 2231 MPM.addPass(RelLookupTableConverterPass());2232 2233 if (PTO.CallGraphProfile)2234 MPM.addPass(CGProfilePass(/*InLTOPostLink=*/true));2235 2236 MPM.addPass(CoroCleanupPass());2237 2238 invokeFullLinkTimeOptimizationLastEPCallbacks(MPM, Level);2239 2240 // Emit annotation remarks.2241 addAnnotationRemarksPass(MPM);2242 2243 return MPM;2244}2245 2246ModulePassManager2247PassBuilder::buildO0DefaultPipeline(OptimizationLevel Level,2248 ThinOrFullLTOPhase Phase) {2249 assert(Level == OptimizationLevel::O0 &&2250 "buildO0DefaultPipeline should only be used with O0");2251 2252 ModulePassManager MPM;2253 2254 // Perform pseudo probe instrumentation in O0 mode. This is for the2255 // consistency between different build modes. For example, a LTO build can be2256 // mixed with an O0 prelink and an O2 postlink. Loading a sample profile in2257 // the postlink will require pseudo probe instrumentation in the prelink.2258 if (PGOOpt && PGOOpt->PseudoProbeForProfiling)2259 MPM.addPass(SampleProfileProbePass(TM));2260 2261 if (PGOOpt && (PGOOpt->Action == PGOOptions::IRInstr ||2262 PGOOpt->Action == PGOOptions::IRUse))2263 addPGOInstrPassesForO0(2264 MPM,2265 /*RunProfileGen=*/(PGOOpt->Action == PGOOptions::IRInstr),2266 /*IsCS=*/false, PGOOpt->AtomicCounterUpdate, PGOOpt->ProfileFile,2267 PGOOpt->ProfileRemappingFile);2268 2269 // Instrument function entry and exit before all inlining.2270 MPM.addPass(createModuleToFunctionPassAdaptor(2271 EntryExitInstrumenterPass(/*PostInlining=*/false)));2272 2273 invokePipelineStartEPCallbacks(MPM, Level);2274 2275 if (PGOOpt && PGOOpt->DebugInfoForProfiling)2276 MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));2277 2278 if (PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) {2279 // Explicitly disable sample loader inlining and use flattened profile in O02280 // pipeline.2281 MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,2282 PGOOpt->ProfileRemappingFile,2283 ThinOrFullLTOPhase::None, nullptr,2284 /*DisableSampleProfileInlining=*/true,2285 /*UseFlattenedProfile=*/true));2286 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert2287 // RequireAnalysisPass for PSI before subsequent non-module passes.2288 MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());2289 }2290 2291 invokePipelineEarlySimplificationEPCallbacks(MPM, Level, Phase);2292 2293 // Build a minimal pipeline based on the semantics required by LLVM,2294 // which is just that always inlining occurs. Further, disable generating2295 // lifetime intrinsics to avoid enabling further optimizations during2296 // code generation.2297 MPM.addPass(AlwaysInlinerPass(2298 /*InsertLifetimeIntrinsics=*/false));2299 2300 if (PTO.MergeFunctions)2301 MPM.addPass(MergeFunctionsPass());2302 2303 if (EnableMatrix)2304 MPM.addPass(2305 createModuleToFunctionPassAdaptor(LowerMatrixIntrinsicsPass(true)));2306 2307 if (!CGSCCOptimizerLateEPCallbacks.empty()) {2308 CGSCCPassManager CGPM;2309 invokeCGSCCOptimizerLateEPCallbacks(CGPM, Level);2310 if (!CGPM.isEmpty())2311 MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));2312 }2313 if (!LateLoopOptimizationsEPCallbacks.empty()) {2314 LoopPassManager LPM;2315 invokeLateLoopOptimizationsEPCallbacks(LPM, Level);2316 if (!LPM.isEmpty()) {2317 MPM.addPass(createModuleToFunctionPassAdaptor(2318 createFunctionToLoopPassAdaptor(std::move(LPM))));2319 }2320 }2321 if (!LoopOptimizerEndEPCallbacks.empty()) {2322 LoopPassManager LPM;2323 invokeLoopOptimizerEndEPCallbacks(LPM, Level);2324 if (!LPM.isEmpty()) {2325 MPM.addPass(createModuleToFunctionPassAdaptor(2326 createFunctionToLoopPassAdaptor(std::move(LPM))));2327 }2328 }2329 if (!ScalarOptimizerLateEPCallbacks.empty()) {2330 FunctionPassManager FPM;2331 invokeScalarOptimizerLateEPCallbacks(FPM, Level);2332 if (!FPM.isEmpty())2333 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));2334 }2335 2336 invokeOptimizerEarlyEPCallbacks(MPM, Level, Phase);2337 2338 if (!VectorizerStartEPCallbacks.empty()) {2339 FunctionPassManager FPM;2340 invokeVectorizerStartEPCallbacks(FPM, Level);2341 if (!FPM.isEmpty())2342 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));2343 }2344 2345 if (!VectorizerEndEPCallbacks.empty()) {2346 FunctionPassManager FPM;2347 invokeVectorizerEndEPCallbacks(FPM, Level);2348 if (!FPM.isEmpty())2349 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));2350 }2351 2352 MPM.addPass(buildCoroWrapper(Phase));2353 2354 invokeOptimizerLastEPCallbacks(MPM, Level, Phase);2355 2356 if (isLTOPreLink(Phase))2357 addRequiredLTOPreLinkPasses(MPM);2358 2359 MPM.addPass(createModuleToFunctionPassAdaptor(AnnotationRemarksPass()));2360 2361 return MPM;2362}2363 2364AAManager PassBuilder::buildDefaultAAPipeline() {2365 AAManager AA;2366 2367 // The order in which these are registered determines their priority when2368 // being queried.2369 2370 // Add any target-specific alias analyses that should be run early.2371 if (TM)2372 TM->registerEarlyDefaultAliasAnalyses(AA);2373 2374 // First we register the basic alias analysis that provides the majority of2375 // per-function local AA logic. This is a stateless, on-demand local set of2376 // AA techniques.2377 AA.registerFunctionAnalysis<BasicAA>();2378 2379 // Next we query fast, specialized alias analyses that wrap IR-embedded2380 // information about aliasing.2381 AA.registerFunctionAnalysis<ScopedNoAliasAA>();2382 AA.registerFunctionAnalysis<TypeBasedAA>();2383 2384 // Add support for querying global aliasing information when available.2385 // Because the `AAManager` is a function analysis and `GlobalsAA` is a module2386 // analysis, all that the `AAManager` can do is query for any *cached*2387 // results from `GlobalsAA` through a readonly proxy.2388 if (EnableGlobalAnalyses)2389 AA.registerModuleAnalysis<GlobalsAA>();2390 2391 // Add target-specific alias analyses.2392 if (TM)2393 TM->registerDefaultAliasAnalyses(AA);2394 2395 return AA;2396}2397 2398bool PassBuilder::isInstrumentedPGOUse() const {2399 return (PGOOpt && PGOOpt->Action == PGOOptions::IRUse) ||2400 !UseCtxProfile.empty();2401}2402