1591 lines · cpp
1//===- TargetPassConfig.cpp - Target independent code generation passes ---===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file defines interfaces to access the target independent code10// generation passes provided by the LLVM backend.11//12//===---------------------------------------------------------------------===//13 14#include "llvm/CodeGen/TargetPassConfig.h"15#include "llvm/ADT/DenseMap.h"16#include "llvm/ADT/SmallVector.h"17#include "llvm/ADT/StringRef.h"18#include "llvm/Analysis/BasicAliasAnalysis.h"19#include "llvm/Analysis/CallGraphSCCPass.h"20#include "llvm/Analysis/ScopedNoAliasAA.h"21#include "llvm/Analysis/TargetTransformInfo.h"22#include "llvm/Analysis/TypeBasedAliasAnalysis.h"23#include "llvm/CodeGen/BasicBlockSectionsProfileReader.h"24#include "llvm/CodeGen/CSEConfigBase.h"25#include "llvm/CodeGen/CodeGenTargetMachineImpl.h"26#include "llvm/CodeGen/MachineFunctionPass.h"27#include "llvm/CodeGen/MachinePassRegistry.h"28#include "llvm/CodeGen/Passes.h"29#include "llvm/CodeGen/RegAllocRegistry.h"30#include "llvm/IR/IRPrintingPasses.h"31#include "llvm/IR/LegacyPassManager.h"32#include "llvm/IR/PassInstrumentation.h"33#include "llvm/IR/Verifier.h"34#include "llvm/InitializePasses.h"35#include "llvm/MC/MCAsmInfo.h"36#include "llvm/MC/MCTargetOptions.h"37#include "llvm/Pass.h"38#include "llvm/Support/CodeGen.h"39#include "llvm/Support/CommandLine.h"40#include "llvm/Support/Compiler.h"41#include "llvm/Support/Debug.h"42#include "llvm/Support/Discriminator.h"43#include "llvm/Support/ErrorHandling.h"44#include "llvm/Support/SaveAndRestore.h"45#include "llvm/Support/Threading.h"46#include "llvm/Support/VirtualFileSystem.h"47#include "llvm/Support/WithColor.h"48#include "llvm/Target/CGPassBuilderOption.h"49#include "llvm/Target/TargetMachine.h"50#include "llvm/Transforms/ObjCARC.h"51#include "llvm/Transforms/Scalar.h"52#include "llvm/Transforms/Utils.h"53#include <cassert>54#include <optional>55#include <string>56 57using namespace llvm;58 59static cl::opt<bool>60 EnableIPRA("enable-ipra", cl::init(false), cl::Hidden,61 cl::desc("Enable interprocedural register allocation "62 "to reduce load/store at procedure calls."));63static cl::opt<bool> DisablePostRASched("disable-post-ra", cl::Hidden,64 cl::desc("Disable Post Regalloc Scheduler"));65static cl::opt<bool> DisableBranchFold("disable-branch-fold", cl::Hidden,66 cl::desc("Disable branch folding"));67static cl::opt<bool> DisableTailDuplicate("disable-tail-duplicate", cl::Hidden,68 cl::desc("Disable tail duplication"));69static cl::opt<bool> DisableEarlyTailDup("disable-early-taildup", cl::Hidden,70 cl::desc("Disable pre-register allocation tail duplication"));71static cl::opt<bool> DisableBlockPlacement("disable-block-placement",72 cl::Hidden, cl::desc("Disable probability-driven block placement"));73static cl::opt<bool> EnableBlockPlacementStats("enable-block-placement-stats",74 cl::Hidden, cl::desc("Collect probability-driven block placement stats"));75static cl::opt<bool> DisableSSC("disable-ssc", cl::Hidden,76 cl::desc("Disable Stack Slot Coloring"));77static cl::opt<bool> DisableMachineDCE("disable-machine-dce", cl::Hidden,78 cl::desc("Disable Machine Dead Code Elimination"));79static cl::opt<bool> DisableEarlyIfConversion("disable-early-ifcvt", cl::Hidden,80 cl::desc("Disable Early If-conversion"));81static cl::opt<bool> DisableMachineLICM("disable-machine-licm", cl::Hidden,82 cl::desc("Disable Machine LICM"));83static cl::opt<bool> DisableMachineCSE("disable-machine-cse", cl::Hidden,84 cl::desc("Disable Machine Common Subexpression Elimination"));85static cl::opt<cl::boolOrDefault> OptimizeRegAlloc(86 "optimize-regalloc", cl::Hidden,87 cl::desc("Enable optimized register allocation compilation path."));88static cl::opt<bool> DisablePostRAMachineLICM("disable-postra-machine-licm",89 cl::Hidden,90 cl::desc("Disable Machine LICM"));91static cl::opt<bool> DisableMachineSink("disable-machine-sink", cl::Hidden,92 cl::desc("Disable Machine Sinking"));93static cl::opt<bool> DisablePostRAMachineSink("disable-postra-machine-sink",94 cl::Hidden,95 cl::desc("Disable PostRA Machine Sinking"));96static cl::opt<bool> DisableLSR("disable-lsr", cl::Hidden,97 cl::desc("Disable Loop Strength Reduction Pass"));98static cl::opt<bool> DisableConstantHoisting("disable-constant-hoisting",99 cl::Hidden, cl::desc("Disable ConstantHoisting"));100static cl::opt<bool> DisableCGP("disable-cgp", cl::Hidden,101 cl::desc("Disable Codegen Prepare"));102static cl::opt<bool> DisableCopyProp("disable-copyprop", cl::Hidden,103 cl::desc("Disable Copy Propagation pass"));104static cl::opt<bool> DisablePartialLibcallInlining("disable-partial-libcall-inlining",105 cl::Hidden, cl::desc("Disable Partial Libcall Inlining"));106static cl::opt<bool> DisableAtExitBasedGlobalDtorLowering(107 "disable-atexit-based-global-dtor-lowering", cl::Hidden,108 cl::desc("For MachO, disable atexit()-based global destructor lowering"));109static cl::opt<bool> EnableImplicitNullChecks(110 "enable-implicit-null-checks",111 cl::desc("Fold null checks into faulting memory operations"),112 cl::init(false), cl::Hidden);113static cl::opt<bool> DisableMergeICmps("disable-mergeicmps",114 cl::desc("Disable MergeICmps Pass"),115 cl::init(false), cl::Hidden);116static cl::opt<bool>117 PrintISelInput("print-isel-input", cl::Hidden,118 cl::desc("Print LLVM IR input to isel pass"));119static cl::opt<cl::boolOrDefault>120 VerifyMachineCode("verify-machineinstrs", cl::Hidden,121 cl::desc("Verify generated machine code"));122static cl::opt<cl::boolOrDefault>123 DebugifyAndStripAll("debugify-and-strip-all-safe", cl::Hidden,124 cl::desc("Debugify MIR before and Strip debug after "125 "each pass except those known to be unsafe "126 "when debug info is present"));127static cl::opt<cl::boolOrDefault> DebugifyCheckAndStripAll(128 "debugify-check-and-strip-all-safe", cl::Hidden,129 cl::desc(130 "Debugify MIR before, by checking and stripping the debug info after, "131 "each pass except those known to be unsafe when debug info is "132 "present"));133// Enable or disable the MachineOutliner.134static cl::opt<RunOutliner> EnableMachineOutliner(135 "enable-machine-outliner", cl::desc("Enable the machine outliner"),136 cl::Hidden, cl::ValueOptional, cl::init(RunOutliner::TargetDefault),137 cl::values(138 clEnumValN(RunOutliner::AlwaysOutline, "always",139 "Run on all functions guaranteed to be beneficial"),140 clEnumValN(RunOutliner::OptimisticPGO, "optimistic-pgo",141 "Outline cold code only. If a code block does not have "142 "profile data, optimistically assume it is cold."),143 clEnumValN(RunOutliner::ConservativePGO, "conservative-pgo",144 "Outline cold code only. If a code block does not have "145 "profile, data, conservatively assume it is hot."),146 clEnumValN(RunOutliner::NeverOutline, "never", "Disable all outlining"),147 // Sentinel value for unspecified option.148 clEnumValN(RunOutliner::AlwaysOutline, "", "")));149static cl::opt<bool> EnableGlobalMergeFunc(150 "enable-global-merge-func", cl::Hidden,151 cl::desc("Enable global merge functions that are based on hash function"));152// Disable the pass to fix unwind information. Whether the pass is included in153// the pipeline is controlled via the target options, this option serves as154// manual override.155static cl::opt<bool> DisableCFIFixup("disable-cfi-fixup", cl::Hidden,156 cl::desc("Disable the CFI fixup pass"));157// Enable or disable FastISel. Both options are needed, because158// FastISel is enabled by default with -fast, and we wish to be159// able to enable or disable fast-isel independently from -O0.160static cl::opt<cl::boolOrDefault>161EnableFastISelOption("fast-isel", cl::Hidden,162 cl::desc("Enable the \"fast\" instruction selector"));163 164static cl::opt<cl::boolOrDefault> EnableGlobalISelOption(165 "global-isel", cl::Hidden,166 cl::desc("Enable the \"global\" instruction selector"));167 168// FIXME: remove this after switching to NPM or GlobalISel, whichever gets there169// first...170static cl::opt<bool>171 PrintAfterISel("print-after-isel", cl::init(false), cl::Hidden,172 cl::desc("Print machine instrs after ISel"));173 174static cl::opt<GlobalISelAbortMode> EnableGlobalISelAbort(175 "global-isel-abort", cl::Hidden,176 cl::desc("Enable abort calls when \"global\" instruction selection "177 "fails to lower/select an instruction"),178 cl::values(179 clEnumValN(GlobalISelAbortMode::Disable, "0", "Disable the abort"),180 clEnumValN(GlobalISelAbortMode::Enable, "1", "Enable the abort"),181 clEnumValN(GlobalISelAbortMode::DisableWithDiag, "2",182 "Disable the abort but emit a diagnostic on failure")));183 184// Disable MIRProfileLoader before RegAlloc. This is for for debugging and185// tuning purpose.186static cl::opt<bool> DisableRAFSProfileLoader(187 "disable-ra-fsprofile-loader", cl::init(false), cl::Hidden,188 cl::desc("Disable MIRProfileLoader before RegAlloc"));189// Disable MIRProfileLoader before BloackPlacement. This is for for debugging190// and tuning purpose.191static cl::opt<bool> DisableLayoutFSProfileLoader(192 "disable-layout-fsprofile-loader", cl::init(false), cl::Hidden,193 cl::desc("Disable MIRProfileLoader before BlockPlacement"));194// Specify FSProfile file name.195static cl::opt<std::string>196 FSProfileFile("fs-profile-file", cl::init(""), cl::value_desc("filename"),197 cl::desc("Flow Sensitive profile file name."), cl::Hidden);198// Specify Remapping file for FSProfile.199static cl::opt<std::string> FSRemappingFile(200 "fs-remapping-file", cl::init(""), cl::value_desc("filename"),201 cl::desc("Flow Sensitive profile remapping file name."), cl::Hidden);202 203// Temporary option to allow experimenting with MachineScheduler as a post-RA204// scheduler. Targets can "properly" enable this with205// substitutePass(&PostRASchedulerID, &PostMachineSchedulerID).206// Targets can return true in targetSchedulesPostRAScheduling() and207// insert a PostRA scheduling pass wherever it wants.208static cl::opt<bool> MISchedPostRA(209 "misched-postra", cl::Hidden,210 cl::desc(211 "Run MachineScheduler post regalloc (independent of preRA sched)"));212 213// Experimental option to run live interval analysis early.214static cl::opt<bool> EarlyLiveIntervals("early-live-intervals", cl::Hidden,215 cl::desc("Run live interval analysis earlier in the pipeline"));216 217static cl::opt<bool> DisableReplaceWithVecLib(218 "disable-replace-with-vec-lib", cl::Hidden,219 cl::desc("Disable replace with vector math call pass"));220 221/// Option names for limiting the codegen pipeline.222/// Those are used in error reporting and we didn't want223/// to duplicate their names all over the place.224static const char StartAfterOptName[] = "start-after";225static const char StartBeforeOptName[] = "start-before";226static const char StopAfterOptName[] = "stop-after";227static const char StopBeforeOptName[] = "stop-before";228 229static cl::opt<std::string>230 StartAfterOpt(StringRef(StartAfterOptName),231 cl::desc("Resume compilation after a specific pass"),232 cl::value_desc("pass-name"), cl::init(""), cl::Hidden);233 234static cl::opt<std::string>235 StartBeforeOpt(StringRef(StartBeforeOptName),236 cl::desc("Resume compilation before a specific pass"),237 cl::value_desc("pass-name"), cl::init(""), cl::Hidden);238 239static cl::opt<std::string>240 StopAfterOpt(StringRef(StopAfterOptName),241 cl::desc("Stop compilation after a specific pass"),242 cl::value_desc("pass-name"), cl::init(""), cl::Hidden);243 244static cl::opt<std::string>245 StopBeforeOpt(StringRef(StopBeforeOptName),246 cl::desc("Stop compilation before a specific pass"),247 cl::value_desc("pass-name"), cl::init(""), cl::Hidden);248 249/// Enable the machine function splitter pass.250static cl::opt<bool> EnableMachineFunctionSplitter(251 "enable-split-machine-functions", cl::Hidden,252 cl::desc("Split out cold blocks from machine functions based on profile "253 "information."));254 255/// Disable the expand reductions pass for testing.256static cl::opt<bool> DisableExpandReductions(257 "disable-expand-reductions", cl::init(false), cl::Hidden,258 cl::desc("Disable the expand reduction intrinsics pass from running"));259 260/// Disable the select optimization pass.261static cl::opt<bool> DisableSelectOptimize(262 "disable-select-optimize", cl::init(true), cl::Hidden,263 cl::desc("Disable the select-optimization pass from running"));264 265/// Enable garbage-collecting empty basic blocks.266static cl::opt<bool>267 GCEmptyBlocks("gc-empty-basic-blocks", cl::init(false), cl::Hidden,268 cl::desc("Enable garbage-collecting empty basic blocks"));269 270static cl::opt<bool>271 SplitStaticData("split-static-data", cl::Hidden, cl::init(false),272 cl::desc("Split static data sections into hot and cold "273 "sections using profile information"));274 275cl::opt<bool> EmitBBHash(276 "emit-bb-hash",277 cl::desc(278 "Emit the hash of basic block in the SHT_LLVM_BB_ADDR_MAP section."),279 cl::init(false), cl::Optional);280 281/// Allow standard passes to be disabled by command line options. This supports282/// simple binary flags that either suppress the pass or do nothing.283/// i.e. -disable-mypass=false has no effect.284/// These should be converted to boolOrDefault in order to use applyOverride.285static IdentifyingPassPtr applyDisable(IdentifyingPassPtr PassID,286 bool Override) {287 if (Override)288 return IdentifyingPassPtr();289 return PassID;290}291 292/// Allow standard passes to be disabled by the command line, regardless of who293/// is adding the pass.294///295/// StandardID is the pass identified in the standard pass pipeline and provided296/// to addPass(). It may be a target-specific ID in the case that the target297/// directly adds its own pass, but in that case we harmlessly fall through.298///299/// TargetID is the pass that the target has configured to override StandardID.300///301/// StandardID may be a pseudo ID. In that case TargetID is the name of the real302/// pass to run. This allows multiple options to control a single pass depending303/// on where in the pipeline that pass is added.304static IdentifyingPassPtr overridePass(AnalysisID StandardID,305 IdentifyingPassPtr TargetID) {306 if (StandardID == &PostRASchedulerID)307 return applyDisable(TargetID, DisablePostRASched);308 309 if (StandardID == &BranchFolderPassID)310 return applyDisable(TargetID, DisableBranchFold);311 312 if (StandardID == &TailDuplicateLegacyID)313 return applyDisable(TargetID, DisableTailDuplicate);314 315 if (StandardID == &EarlyTailDuplicateLegacyID)316 return applyDisable(TargetID, DisableEarlyTailDup);317 318 if (StandardID == &MachineBlockPlacementID)319 return applyDisable(TargetID, DisableBlockPlacement);320 321 if (StandardID == &StackSlotColoringID)322 return applyDisable(TargetID, DisableSSC);323 324 if (StandardID == &DeadMachineInstructionElimID)325 return applyDisable(TargetID, DisableMachineDCE);326 327 if (StandardID == &EarlyIfConverterLegacyID)328 return applyDisable(TargetID, DisableEarlyIfConversion);329 330 if (StandardID == &EarlyMachineLICMID)331 return applyDisable(TargetID, DisableMachineLICM);332 333 if (StandardID == &MachineCSELegacyID)334 return applyDisable(TargetID, DisableMachineCSE);335 336 if (StandardID == &MachineLICMID)337 return applyDisable(TargetID, DisablePostRAMachineLICM);338 339 if (StandardID == &MachineSinkingLegacyID)340 return applyDisable(TargetID, DisableMachineSink);341 342 if (StandardID == &PostRAMachineSinkingID)343 return applyDisable(TargetID, DisablePostRAMachineSink);344 345 if (StandardID == &MachineCopyPropagationID)346 return applyDisable(TargetID, DisableCopyProp);347 348 return TargetID;349}350 351// Find the FSProfile file name. The internal option takes the precedence352// before getting from TargetMachine.353static std::string getFSProfileFile(const TargetMachine *TM) {354 if (!FSProfileFile.empty())355 return FSProfileFile.getValue();356 const std::optional<PGOOptions> &PGOOpt = TM->getPGOOption();357 if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse)358 return std::string();359 return PGOOpt->ProfileFile;360}361 362// Find the Profile remapping file name. The internal option takes the363// precedence before getting from TargetMachine.364static std::string getFSRemappingFile(const TargetMachine *TM) {365 if (!FSRemappingFile.empty())366 return FSRemappingFile.getValue();367 const std::optional<PGOOptions> &PGOOpt = TM->getPGOOption();368 if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse)369 return std::string();370 return PGOOpt->ProfileRemappingFile;371}372 373//===---------------------------------------------------------------------===//374/// TargetPassConfig375//===---------------------------------------------------------------------===//376 377INITIALIZE_PASS(TargetPassConfig, "targetpassconfig",378 "Target Pass Configuration", false, false)379char TargetPassConfig::ID = 0;380 381namespace {382 383struct InsertedPass {384 AnalysisID TargetPassID;385 IdentifyingPassPtr InsertedPassID;386 387 InsertedPass(AnalysisID TargetPassID, IdentifyingPassPtr InsertedPassID)388 : TargetPassID(TargetPassID), InsertedPassID(InsertedPassID) {}389 390 Pass *getInsertedPass() const {391 assert(InsertedPassID.isValid() && "Illegal Pass ID!");392 if (InsertedPassID.isInstance())393 return InsertedPassID.getInstance();394 Pass *NP = Pass::createPass(InsertedPassID.getID());395 assert(NP && "Pass ID not registered");396 return NP;397 }398};399 400} // end anonymous namespace401 402namespace llvm {403 404class PassConfigImpl {405public:406 // List of passes explicitly substituted by this target. Normally this is407 // empty, but it is a convenient way to suppress or replace specific passes408 // that are part of a standard pass pipeline without overridding the entire409 // pipeline. This mechanism allows target options to inherit a standard pass's410 // user interface. For example, a target may disable a standard pass by411 // default by substituting a pass ID of zero, and the user may still enable412 // that standard pass with an explicit command line option.413 DenseMap<AnalysisID,IdentifyingPassPtr> TargetPasses;414 415 /// Store the pairs of <AnalysisID, AnalysisID> of which the second pass416 /// is inserted after each instance of the first one.417 SmallVector<InsertedPass, 4> InsertedPasses;418};419 420} // end namespace llvm421 422// Out of line virtual method.423TargetPassConfig::~TargetPassConfig() {424 delete Impl;425}426 427static const PassInfo *getPassInfo(StringRef PassName) {428 if (PassName.empty())429 return nullptr;430 431 const PassRegistry &PR = *PassRegistry::getPassRegistry();432 const PassInfo *PI = PR.getPassInfo(PassName);433 if (!PI)434 reportFatalUsageError(Twine('\"') + Twine(PassName) +435 Twine("\" pass is not registered."));436 return PI;437}438 439static AnalysisID getPassIDFromName(StringRef PassName) {440 const PassInfo *PI = getPassInfo(PassName);441 return PI ? PI->getTypeInfo() : nullptr;442}443 444static std::pair<StringRef, unsigned>445getPassNameAndInstanceNum(StringRef PassName) {446 StringRef Name, InstanceNumStr;447 std::tie(Name, InstanceNumStr) = PassName.split(',');448 449 unsigned InstanceNum = 0;450 if (!InstanceNumStr.empty() && InstanceNumStr.getAsInteger(10, InstanceNum))451 reportFatalUsageError("invalid pass instance specifier " + PassName);452 453 return std::make_pair(Name, InstanceNum);454}455 456void TargetPassConfig::setStartStopPasses() {457 StringRef StartBeforeName;458 std::tie(StartBeforeName, StartBeforeInstanceNum) =459 getPassNameAndInstanceNum(StartBeforeOpt);460 461 StringRef StartAfterName;462 std::tie(StartAfterName, StartAfterInstanceNum) =463 getPassNameAndInstanceNum(StartAfterOpt);464 465 StringRef StopBeforeName;466 std::tie(StopBeforeName, StopBeforeInstanceNum)467 = getPassNameAndInstanceNum(StopBeforeOpt);468 469 StringRef StopAfterName;470 std::tie(StopAfterName, StopAfterInstanceNum)471 = getPassNameAndInstanceNum(StopAfterOpt);472 473 StartBefore = getPassIDFromName(StartBeforeName);474 StartAfter = getPassIDFromName(StartAfterName);475 StopBefore = getPassIDFromName(StopBeforeName);476 StopAfter = getPassIDFromName(StopAfterName);477 if (StartBefore && StartAfter)478 reportFatalUsageError(Twine(StartBeforeOptName) + Twine(" and ") +479 Twine(StartAfterOptName) + Twine(" specified!"));480 if (StopBefore && StopAfter)481 reportFatalUsageError(Twine(StopBeforeOptName) + Twine(" and ") +482 Twine(StopAfterOptName) + Twine(" specified!"));483 Started = (StartAfter == nullptr) && (StartBefore == nullptr);484}485 486CGPassBuilderOption llvm::getCGPassBuilderOption() {487 CGPassBuilderOption Opt;488 489#define SET_OPTION(Option) \490 if (Option.getNumOccurrences()) \491 Opt.Option = Option;492 493 SET_OPTION(EnableFastISelOption)494 SET_OPTION(EnableGlobalISelAbort)495 SET_OPTION(EnableGlobalISelOption)496 SET_OPTION(EnableIPRA)497 SET_OPTION(OptimizeRegAlloc)498 SET_OPTION(VerifyMachineCode)499 SET_OPTION(DisableAtExitBasedGlobalDtorLowering)500 SET_OPTION(DisableExpandReductions)501 SET_OPTION(PrintAfterISel)502 SET_OPTION(FSProfileFile)503 SET_OPTION(GCEmptyBlocks)504 505#define SET_BOOLEAN_OPTION(Option) Opt.Option = Option;506 507 SET_BOOLEAN_OPTION(EarlyLiveIntervals)508 SET_BOOLEAN_OPTION(EnableBlockPlacementStats)509 SET_BOOLEAN_OPTION(EnableGlobalMergeFunc)510 SET_BOOLEAN_OPTION(EnableImplicitNullChecks)511 SET_BOOLEAN_OPTION(EnableMachineOutliner)512 SET_BOOLEAN_OPTION(MISchedPostRA)513 SET_BOOLEAN_OPTION(DisableMergeICmps)514 SET_BOOLEAN_OPTION(DisableLSR)515 SET_BOOLEAN_OPTION(DisableConstantHoisting)516 SET_BOOLEAN_OPTION(DisableCGP)517 SET_BOOLEAN_OPTION(DisablePartialLibcallInlining)518 SET_BOOLEAN_OPTION(DisableSelectOptimize)519 SET_BOOLEAN_OPTION(PrintISelInput)520 SET_BOOLEAN_OPTION(DebugifyAndStripAll)521 SET_BOOLEAN_OPTION(DebugifyCheckAndStripAll)522 SET_BOOLEAN_OPTION(DisableRAFSProfileLoader)523 SET_BOOLEAN_OPTION(DisableCFIFixup)524 SET_BOOLEAN_OPTION(EnableMachineFunctionSplitter)525 526 return Opt;527}528 529void llvm::registerCodeGenCallback(PassInstrumentationCallbacks &PIC,530 TargetMachine &TM) {531 532 // Register a callback for disabling passes.533 PIC.registerShouldRunOptionalPassCallback([](StringRef P, Any) {534 535#define DISABLE_PASS(Option, Name) \536 if (Option && P.contains(#Name)) \537 return false;538 DISABLE_PASS(DisableBlockPlacement, MachineBlockPlacementPass)539 DISABLE_PASS(DisableBranchFold, BranchFolderPass)540 DISABLE_PASS(DisableCopyProp, MachineCopyPropagationPass)541 DISABLE_PASS(DisableEarlyIfConversion, EarlyIfConverterLegacyPass)542 DISABLE_PASS(DisableEarlyTailDup, EarlyTailDuplicatePass)543 DISABLE_PASS(DisableMachineCSE, MachineCSELegacyPass)544 DISABLE_PASS(DisableMachineDCE, DeadMachineInstructionElimPass)545 DISABLE_PASS(DisableMachineLICM, EarlyMachineLICMPass)546 DISABLE_PASS(DisableMachineSink, MachineSinkingPass)547 DISABLE_PASS(DisablePostRAMachineLICM, MachineLICMPass)548 DISABLE_PASS(DisablePostRAMachineSink, PostRAMachineSinkingPass)549 DISABLE_PASS(DisablePostRASched, PostRASchedulerPass)550 DISABLE_PASS(DisableSSC, StackSlotColoringPass)551 DISABLE_PASS(DisableTailDuplicate, TailDuplicatePass)552 553 return true;554 });555}556 557Expected<TargetPassConfig::StartStopInfo>558TargetPassConfig::getStartStopInfo(PassInstrumentationCallbacks &PIC) {559 auto [StartBefore, StartBeforeInstanceNum] =560 getPassNameAndInstanceNum(StartBeforeOpt);561 auto [StartAfter, StartAfterInstanceNum] =562 getPassNameAndInstanceNum(StartAfterOpt);563 auto [StopBefore, StopBeforeInstanceNum] =564 getPassNameAndInstanceNum(StopBeforeOpt);565 auto [StopAfter, StopAfterInstanceNum] =566 getPassNameAndInstanceNum(StopAfterOpt);567 568 if (!StartBefore.empty() && !StartAfter.empty())569 return make_error<StringError>(570 Twine(StartBeforeOptName) + " and " + StartAfterOptName + " specified!",571 std::make_error_code(std::errc::invalid_argument));572 if (!StopBefore.empty() && !StopAfter.empty())573 return make_error<StringError>(574 Twine(StopBeforeOptName) + " and " + StopAfterOptName + " specified!",575 std::make_error_code(std::errc::invalid_argument));576 577 StartStopInfo Result;578 Result.StartPass = StartBefore.empty() ? StartAfter : StartBefore;579 Result.StopPass = StopBefore.empty() ? StopAfter : StopBefore;580 Result.StartInstanceNum =581 StartBefore.empty() ? StartAfterInstanceNum : StartBeforeInstanceNum;582 Result.StopInstanceNum =583 StopBefore.empty() ? StopAfterInstanceNum : StopBeforeInstanceNum;584 Result.StartAfter = !StartAfter.empty();585 Result.StopAfter = !StopAfter.empty();586 Result.StartInstanceNum += Result.StartInstanceNum == 0;587 Result.StopInstanceNum += Result.StopInstanceNum == 0;588 return Result;589}590 591// Out of line constructor provides default values for pass options and592// registers all common codegen passes.593TargetPassConfig::TargetPassConfig(TargetMachine &TM, PassManagerBase &PM)594 : ImmutablePass(ID), PM(&PM), TM(&TM) {595 Impl = new PassConfigImpl();596 597 PassRegistry &PR = *PassRegistry::getPassRegistry();598 // Register all target independent codegen passes to activate their PassIDs,599 // including this pass itself.600 initializeCodeGen(PR);601 602 // Also register alias analysis passes required by codegen passes.603 initializeBasicAAWrapperPassPass(PR);604 initializeAAResultsWrapperPassPass(PR);605 606 if (EnableIPRA.getNumOccurrences()) {607 TM.Options.EnableIPRA = EnableIPRA;608 } else {609 // If not explicitly specified, use target default.610 TM.Options.EnableIPRA |= TM.useIPRA();611 }612 613 if (TM.Options.EnableIPRA)614 setRequiresCodeGenSCCOrder();615 616 if (EnableGlobalISelAbort.getNumOccurrences())617 TM.Options.GlobalISelAbort = EnableGlobalISelAbort;618 619 setStartStopPasses();620}621 622CodeGenOptLevel TargetPassConfig::getOptLevel() const {623 return TM->getOptLevel();624}625 626/// Insert InsertedPassID pass after TargetPassID.627void TargetPassConfig::insertPass(AnalysisID TargetPassID,628 IdentifyingPassPtr InsertedPassID) {629 assert(((!InsertedPassID.isInstance() &&630 TargetPassID != InsertedPassID.getID()) ||631 (InsertedPassID.isInstance() &&632 TargetPassID != InsertedPassID.getInstance()->getPassID())) &&633 "Insert a pass after itself!");634 Impl->InsertedPasses.emplace_back(TargetPassID, InsertedPassID);635}636 637/// createPassConfig - Create a pass configuration object to be used by638/// addPassToEmitX methods for generating a pipeline of CodeGen passes.639///640/// Targets may override this to extend TargetPassConfig.641TargetPassConfig *642CodeGenTargetMachineImpl::createPassConfig(PassManagerBase &PM) {643 return new TargetPassConfig(*this, PM);644}645 646TargetPassConfig::TargetPassConfig()647 : ImmutablePass(ID) {648 reportFatalUsageError("trying to construct TargetPassConfig without a target "649 "machine. Scheduling a CodeGen pass without a target "650 "triple set?");651}652 653bool TargetPassConfig::willCompleteCodeGenPipeline() {654 return StopBeforeOpt.empty() && StopAfterOpt.empty();655}656 657bool TargetPassConfig::hasLimitedCodeGenPipeline() {658 return !StartBeforeOpt.empty() || !StartAfterOpt.empty() ||659 !willCompleteCodeGenPipeline();660}661 662std::string TargetPassConfig::getLimitedCodeGenPipelineReason() {663 if (!hasLimitedCodeGenPipeline())664 return std::string();665 std::string Res;666 static cl::opt<std::string> *PassNames[] = {&StartAfterOpt, &StartBeforeOpt,667 &StopAfterOpt, &StopBeforeOpt};668 static const char *OptNames[] = {StartAfterOptName, StartBeforeOptName,669 StopAfterOptName, StopBeforeOptName};670 bool IsFirst = true;671 for (int Idx = 0; Idx < 4; ++Idx)672 if (!PassNames[Idx]->empty()) {673 if (!IsFirst)674 Res += " and ";675 IsFirst = false;676 Res += OptNames[Idx];677 }678 return Res;679}680 681// Helper to verify the analysis is really immutable.682void TargetPassConfig::setOpt(bool &Opt, bool Val) {683 assert(!Initialized && "PassConfig is immutable");684 Opt = Val;685}686 687void TargetPassConfig::substitutePass(AnalysisID StandardID,688 IdentifyingPassPtr TargetID) {689 Impl->TargetPasses[StandardID] = TargetID;690}691 692IdentifyingPassPtr TargetPassConfig::getPassSubstitution(AnalysisID ID) const {693 DenseMap<AnalysisID, IdentifyingPassPtr>::const_iterator694 I = Impl->TargetPasses.find(ID);695 if (I == Impl->TargetPasses.end())696 return ID;697 return I->second;698}699 700bool TargetPassConfig::isPassSubstitutedOrOverridden(AnalysisID ID) const {701 IdentifyingPassPtr TargetID = getPassSubstitution(ID);702 IdentifyingPassPtr FinalPtr = overridePass(ID, TargetID);703 return !FinalPtr.isValid() || FinalPtr.isInstance() ||704 FinalPtr.getID() != ID;705}706 707/// Add a pass to the PassManager if that pass is supposed to be run. If the708/// Started/Stopped flags indicate either that the compilation should start at709/// a later pass or that it should stop after an earlier pass, then do not add710/// the pass. Finally, compare the current pass against the StartAfter711/// and StopAfter options and change the Started/Stopped flags accordingly.712void TargetPassConfig::addPass(Pass *P) {713 assert(!Initialized && "PassConfig is immutable");714 715 // Cache the Pass ID here in case the pass manager finds this pass is716 // redundant with ones already scheduled / available, and deletes it.717 // Fundamentally, once we add the pass to the manager, we no longer own it718 // and shouldn't reference it.719 AnalysisID PassID = P->getPassID();720 721 if (StartBefore == PassID && StartBeforeCount++ == StartBeforeInstanceNum)722 Started = true;723 if (StopBefore == PassID && StopBeforeCount++ == StopBeforeInstanceNum)724 Stopped = true;725 if (Started && !Stopped) {726 if (AddingMachinePasses) {727 // Construct banner message before PM->add() as that may delete the pass.728 std::string Banner =729 std::string("After ") + std::string(P->getPassName());730 addMachinePrePasses();731 PM->add(P);732 addMachinePostPasses(Banner);733 } else {734 PM->add(P);735 }736 737 // Add the passes after the pass P if there is any.738 for (const auto &IP : Impl->InsertedPasses)739 if (IP.TargetPassID == PassID)740 addPass(IP.getInsertedPass());741 } else {742 delete P;743 }744 745 if (StopAfter == PassID && StopAfterCount++ == StopAfterInstanceNum)746 Stopped = true;747 748 if (StartAfter == PassID && StartAfterCount++ == StartAfterInstanceNum)749 Started = true;750 if (Stopped && !Started)751 reportFatalUsageError("Cannot stop compilation after pass that is not run");752}753 754/// Add a CodeGen pass at this point in the pipeline after checking for target755/// and command line overrides.756///757/// addPass cannot return a pointer to the pass instance because is internal the758/// PassManager and the instance we create here may already be freed.759AnalysisID TargetPassConfig::addPass(AnalysisID PassID) {760 IdentifyingPassPtr TargetID = getPassSubstitution(PassID);761 IdentifyingPassPtr FinalPtr = overridePass(PassID, TargetID);762 if (!FinalPtr.isValid())763 return nullptr;764 765 Pass *P;766 if (FinalPtr.isInstance())767 P = FinalPtr.getInstance();768 else {769 P = Pass::createPass(FinalPtr.getID());770 if (!P)771 llvm_unreachable("Pass ID not registered");772 }773 AnalysisID FinalID = P->getPassID();774 addPass(P); // Ends the lifetime of P.775 776 return FinalID;777}778 779void TargetPassConfig::printAndVerify(const std::string &Banner) {780 addPrintPass(Banner);781 addVerifyPass(Banner);782}783 784void TargetPassConfig::addPrintPass(const std::string &Banner) {785 if (PrintAfterISel)786 PM->add(createMachineFunctionPrinterPass(dbgs(), Banner));787}788 789void TargetPassConfig::addVerifyPass(const std::string &Banner) {790 bool Verify = VerifyMachineCode == cl::BOU_TRUE;791#ifdef EXPENSIVE_CHECKS792 if (VerifyMachineCode == cl::BOU_UNSET)793 Verify = TM->isMachineVerifierClean();794#endif795 if (Verify)796 PM->add(createMachineVerifierPass(Banner));797}798 799void TargetPassConfig::addDebugifyPass() {800 PM->add(createDebugifyMachineModulePass());801}802 803void TargetPassConfig::addStripDebugPass() {804 PM->add(createStripDebugMachineModulePass(/*OnlyDebugified=*/true));805}806 807void TargetPassConfig::addCheckDebugPass() {808 PM->add(createCheckDebugMachineModulePass());809}810 811void TargetPassConfig::addMachinePrePasses(bool AllowDebugify) {812 if (AllowDebugify && DebugifyIsSafe &&813 (DebugifyAndStripAll == cl::BOU_TRUE ||814 DebugifyCheckAndStripAll == cl::BOU_TRUE))815 addDebugifyPass();816}817 818void TargetPassConfig::addMachinePostPasses(const std::string &Banner) {819 if (DebugifyIsSafe) {820 if (DebugifyCheckAndStripAll == cl::BOU_TRUE) {821 addCheckDebugPass();822 addStripDebugPass();823 } else if (DebugifyAndStripAll == cl::BOU_TRUE)824 addStripDebugPass();825 }826 addVerifyPass(Banner);827}828 829/// Add common target configurable passes that perform LLVM IR to IR transforms830/// following machine independent optimization.831void TargetPassConfig::addIRPasses() {832 // Before running any passes, run the verifier to determine if the input833 // coming from the front-end and/or optimizer is valid.834 if (!DisableVerify)835 addPass(createVerifierPass());836 837 if (getOptLevel() != CodeGenOptLevel::None) {838 // Basic AliasAnalysis support.839 // Add TypeBasedAliasAnalysis before BasicAliasAnalysis so that840 // BasicAliasAnalysis wins if they disagree. This is intended to help841 // support "obvious" type-punning idioms.842 addPass(createTypeBasedAAWrapperPass());843 addPass(createScopedNoAliasAAWrapperPass());844 addPass(createBasicAAWrapperPass());845 846 // Run loop strength reduction before anything else.847 if (!DisableLSR) {848 addPass(createCanonicalizeFreezeInLoopsPass());849 addPass(createLoopStrengthReducePass());850 if (EnableLoopTermFold)851 addPass(createLoopTermFoldPass());852 }853 854 // The MergeICmpsPass tries to create memcmp calls by grouping sequences of855 // loads and compares. ExpandMemCmpPass then tries to expand those calls856 // into optimally-sized loads and compares. The transforms are enabled by a857 // target lowering hook.858 if (!DisableMergeICmps)859 addPass(createMergeICmpsLegacyPass());860 addPass(createExpandMemCmpLegacyPass());861 }862 863 // Run GC lowering passes for builtin collectors864 // TODO: add a pass insertion point here865 addPass(&GCLoweringID);866 addPass(&ShadowStackGCLoweringID);867 868 // For MachO, lower @llvm.global_dtors into @llvm.global_ctors with869 // __cxa_atexit() calls to avoid emitting the deprecated __mod_term_func.870 if (TM->getTargetTriple().isOSBinFormatMachO() &&871 !DisableAtExitBasedGlobalDtorLowering)872 addPass(createLowerGlobalDtorsLegacyPass());873 874 // Make sure that no unreachable blocks are instruction selected.875 addPass(createUnreachableBlockEliminationPass());876 877 // Prepare expensive constants for SelectionDAG.878 if (getOptLevel() != CodeGenOptLevel::None && !DisableConstantHoisting)879 addPass(createConstantHoistingPass());880 881 if (getOptLevel() != CodeGenOptLevel::None && !DisableReplaceWithVecLib)882 addPass(createReplaceWithVeclibLegacyPass());883 884 if (getOptLevel() != CodeGenOptLevel::None && !DisablePartialLibcallInlining)885 addPass(createPartiallyInlineLibCallsPass());886 887 // Instrument function entry after all inlining.888 addPass(createPostInlineEntryExitInstrumenterPass());889 890 // Add scalarization of target's unsupported masked memory intrinsics pass.891 // the unsupported intrinsic will be replaced with a chain of basic blocks,892 // that stores/loads element one-by-one if the appropriate mask bit is set.893 addPass(createScalarizeMaskedMemIntrinLegacyPass());894 895 // Expand reduction intrinsics into shuffle sequences if the target wants to.896 // Allow disabling it for testing purposes.897 if (!DisableExpandReductions)898 addPass(createExpandReductionsPass());899 900 // Convert conditional moves to conditional jumps when profitable.901 if (getOptLevel() != CodeGenOptLevel::None && !DisableSelectOptimize)902 addPass(createSelectOptimizePass());903 904 if (EnableGlobalMergeFunc)905 addPass(createGlobalMergeFuncPass());906 907 if (TM->getTargetTriple().isOSWindows())908 addPass(createWindowsSecureHotPatchingPass());909}910 911/// Turn exception handling constructs into something the code generators can912/// handle.913void TargetPassConfig::addPassesToHandleExceptions() {914 const MCAsmInfo *MCAI = TM->getMCAsmInfo();915 assert(MCAI && "No MCAsmInfo");916 switch (MCAI->getExceptionHandlingType()) {917 case ExceptionHandling::SjLj:918 // SjLj piggy-backs on dwarf for this bit. The cleanups done apply to both919 // Dwarf EH prepare needs to be run after SjLj prepare. Otherwise,920 // catch info can get misplaced when a selector ends up more than one block921 // removed from the parent invoke(s). This could happen when a landing922 // pad is shared by multiple invokes and is also a target of a normal923 // edge from elsewhere.924 addPass(createSjLjEHPreparePass(TM));925 [[fallthrough]];926 case ExceptionHandling::DwarfCFI:927 case ExceptionHandling::ARM:928 case ExceptionHandling::AIX:929 case ExceptionHandling::ZOS:930 addPass(createDwarfEHPass(getOptLevel()));931 break;932 case ExceptionHandling::WinEH:933 // We support using both GCC-style and MSVC-style exceptions on Windows, so934 // add both preparation passes. Each pass will only actually run if it935 // recognizes the personality function.936 addPass(createWinEHPass());937 addPass(createDwarfEHPass(getOptLevel()));938 break;939 case ExceptionHandling::Wasm:940 // Wasm EH uses Windows EH instructions, but it does not need to demote PHIs941 // on catchpads and cleanuppads because it does not outline them into942 // funclets. Catchswitch blocks are not lowered in SelectionDAG, so we943 // should remove PHIs there.944 addPass(createWinEHPass(/*DemoteCatchSwitchPHIOnly=*/true));945 addPass(createWasmEHPass());946 break;947 case ExceptionHandling::None:948 addPass(createLowerInvokePass());949 950 // The lower invoke pass may create unreachable code. Remove it.951 addPass(createUnreachableBlockEliminationPass());952 break;953 }954}955 956/// Add pass to prepare the LLVM IR for code generation. This should be done957/// before exception handling preparation passes.958void TargetPassConfig::addCodeGenPrepare() {959 if (getOptLevel() != CodeGenOptLevel::None && !DisableCGP)960 addPass(createCodeGenPrepareLegacyPass());961}962 963/// Add common passes that perform LLVM IR to IR transforms in preparation for964/// instruction selection.965void TargetPassConfig::addISelPrepare() {966 addPreISel();967 968 // Force codegen to run according to the callgraph.969 if (requiresCodeGenSCCOrder())970 addPass(new DummyCGSCCPass);971 972 if (getOptLevel() != CodeGenOptLevel::None)973 addPass(createObjCARCContractPass());974 975 addPass(createCallBrPass());976 977 // Add both the safe stack and the stack protection passes: each of them will978 // only protect functions that have corresponding attributes.979 addPass(createSafeStackPass());980 addPass(createStackProtectorPass());981 982 if (PrintISelInput)983 addPass(createPrintFunctionPass(984 dbgs(), "\n\n*** Final LLVM Code input to ISel ***\n"));985 986 // All passes which modify the LLVM IR are now complete; run the verifier987 // to ensure that the IR is valid.988 if (!DisableVerify)989 addPass(createVerifierPass());990}991 992bool TargetPassConfig::addCoreISelPasses() {993 // Enable FastISel with -fast-isel, but allow that to be overridden.994 TM->setO0WantsFastISel(EnableFastISelOption != cl::BOU_FALSE);995 996 // Determine an instruction selector.997 enum class SelectorType { SelectionDAG, FastISel, GlobalISel };998 SelectorType Selector;999 1000 if (EnableFastISelOption == cl::BOU_TRUE)1001 Selector = SelectorType::FastISel;1002 else if (EnableGlobalISelOption == cl::BOU_TRUE ||1003 (TM->Options.EnableGlobalISel &&1004 EnableGlobalISelOption != cl::BOU_FALSE))1005 Selector = SelectorType::GlobalISel;1006 else if (TM->getOptLevel() == CodeGenOptLevel::None &&1007 TM->getO0WantsFastISel())1008 Selector = SelectorType::FastISel;1009 else1010 Selector = SelectorType::SelectionDAG;1011 1012 // Set consistently TM->Options.EnableFastISel and EnableGlobalISel.1013 if (Selector == SelectorType::FastISel) {1014 TM->setFastISel(true);1015 TM->setGlobalISel(false);1016 } else if (Selector == SelectorType::GlobalISel) {1017 TM->setFastISel(false);1018 TM->setGlobalISel(true);1019 }1020 1021 // FIXME: Injecting into the DAGISel pipeline seems to cause issues with1022 // analyses needing to be re-run. This can result in being unable to1023 // schedule passes (particularly with 'Function Alias Analysis1024 // Results'). It's not entirely clear why but AFAICT this seems to be1025 // due to one FunctionPassManager not being able to use analyses from a1026 // previous one. As we're injecting a ModulePass we break the usual1027 // pass manager into two. GlobalISel with the fallback path disabled1028 // and -run-pass seem to be unaffected. The majority of GlobalISel1029 // testing uses -run-pass so this probably isn't too bad.1030 SaveAndRestore SavedDebugifyIsSafe(DebugifyIsSafe);1031 if (Selector != SelectorType::GlobalISel || !isGlobalISelAbortEnabled())1032 DebugifyIsSafe = false;1033 1034 // Add instruction selector passes for global isel if enabled.1035 if (Selector == SelectorType::GlobalISel) {1036 SaveAndRestore SavedAddingMachinePasses(AddingMachinePasses, true);1037 if (addIRTranslator())1038 return true;1039 1040 addPreLegalizeMachineIR();1041 1042 if (addLegalizeMachineIR())1043 return true;1044 1045 // Before running the register bank selector, ask the target if it1046 // wants to run some passes.1047 addPreRegBankSelect();1048 1049 if (addRegBankSelect())1050 return true;1051 1052 addPreGlobalInstructionSelect();1053 1054 if (addGlobalInstructionSelect())1055 return true;1056 }1057 1058 // Pass to reset the MachineFunction if the ISel failed. Outside of the above1059 // if so that the verifier is not added to it.1060 if (Selector == SelectorType::GlobalISel)1061 addPass(createResetMachineFunctionPass(1062 reportDiagnosticWhenGlobalISelFallback(), isGlobalISelAbortEnabled()));1063 1064 // Run the SDAG InstSelector, providing a fallback path when we do not want to1065 // abort on not-yet-supported input.1066 if (Selector != SelectorType::GlobalISel || !isGlobalISelAbortEnabled())1067 if (addInstSelector())1068 return true;1069 1070 // Expand pseudo-instructions emitted by ISel. Don't run the verifier before1071 // FinalizeISel.1072 addPass(&FinalizeISelID);1073 1074 // Print the instruction selected machine code...1075 printAndVerify("After Instruction Selection");1076 1077 return false;1078}1079 1080bool TargetPassConfig::addISelPasses() {1081 if (TM->useEmulatedTLS())1082 addPass(createLowerEmuTLSPass());1083 1084 PM->add(createTargetTransformInfoWrapperPass(TM->getTargetIRAnalysis()));1085 addPass(createPreISelIntrinsicLoweringPass());1086 addPass(createExpandLargeDivRemPass());1087 addPass(createExpandFpPass(getOptLevel()));1088 addIRPasses();1089 addCodeGenPrepare();1090 addPassesToHandleExceptions();1091 addISelPrepare();1092 1093 return addCoreISelPasses();1094}1095 1096/// -regalloc=... command line option.1097static FunctionPass *useDefaultRegisterAllocator() { return nullptr; }1098static cl::opt<RegisterRegAlloc::FunctionPassCtor, false,1099 RegisterPassParser<RegisterRegAlloc>>1100 RegAlloc("regalloc", cl::Hidden, cl::init(&useDefaultRegisterAllocator),1101 cl::desc("Register allocator to use"));1102 1103/// Add the complete set of target-independent postISel code generator passes.1104///1105/// This can be read as the standard order of major LLVM CodeGen stages. Stages1106/// with nontrivial configuration or multiple passes are broken out below in1107/// add%Stage routines.1108///1109/// Any TargetPassConfig::addXX routine may be overriden by the Target. The1110/// addPre/Post methods with empty header implementations allow injecting1111/// target-specific fixups just before or after major stages. Additionally,1112/// targets have the flexibility to change pass order within a stage by1113/// overriding default implementation of add%Stage routines below. Each1114/// technique has maintainability tradeoffs because alternate pass orders are1115/// not well supported. addPre/Post works better if the target pass is easily1116/// tied to a common pass. But if it has subtle dependencies on multiple passes,1117/// the target should override the stage instead.1118///1119/// TODO: We could use a single addPre/Post(ID) hook to allow pass injection1120/// before/after any target-independent pass. But it's currently overkill.1121void TargetPassConfig::addMachinePasses() {1122 AddingMachinePasses = true;1123 1124 // Add passes that optimize machine instructions in SSA form.1125 if (getOptLevel() != CodeGenOptLevel::None) {1126 addMachineSSAOptimization();1127 } else {1128 // If the target requests it, assign local variables to stack slots relative1129 // to one another and simplify frame index references where possible.1130 addPass(&LocalStackSlotAllocationID);1131 }1132 1133 if (TM->Options.EnableIPRA)1134 addPass(createRegUsageInfoPropPass());1135 1136 // Run pre-ra passes.1137 addPreRegAlloc();1138 1139 // Debugifying the register allocator passes seems to provoke some1140 // non-determinism that affects CodeGen and there doesn't seem to be a point1141 // where it becomes safe again so stop debugifying here.1142 DebugifyIsSafe = false;1143 1144 // Add a FSDiscriminator pass right before RA, so that we could get1145 // more precise SampleFDO profile for RA.1146 if (EnableFSDiscriminator) {1147 addPass(createMIRAddFSDiscriminatorsPass(1148 sampleprof::FSDiscriminatorPass::Pass1));1149 const std::string ProfileFile = getFSProfileFile(TM);1150 if (!ProfileFile.empty() && !DisableRAFSProfileLoader)1151 addPass(createMIRProfileLoaderPass(ProfileFile, getFSRemappingFile(TM),1152 sampleprof::FSDiscriminatorPass::Pass1,1153 nullptr));1154 }1155 1156 // Run register allocation and passes that are tightly coupled with it,1157 // including phi elimination and scheduling.1158 if (getOptimizeRegAlloc())1159 addOptimizedRegAlloc();1160 else1161 addFastRegAlloc();1162 1163 // Run post-ra passes.1164 addPostRegAlloc();1165 1166 addPass(&RemoveRedundantDebugValuesID);1167 1168 addPass(&FixupStatepointCallerSavedID);1169 1170 // Insert prolog/epilog code. Eliminate abstract frame index references...1171 if (getOptLevel() != CodeGenOptLevel::None) {1172 addPass(&PostRAMachineSinkingID);1173 addPass(&ShrinkWrapID);1174 }1175 1176 // Prolog/Epilog inserter needs a TargetMachine to instantiate. But only1177 // do so if it hasn't been disabled, substituted, or overridden.1178 if (!isPassSubstitutedOrOverridden(&PrologEpilogCodeInserterID))1179 addPass(createPrologEpilogInserterPass());1180 1181 /// Add passes that optimize machine instructions after register allocation.1182 if (getOptLevel() != CodeGenOptLevel::None)1183 addMachineLateOptimization();1184 1185 // Expand pseudo instructions before second scheduling pass.1186 addPass(&ExpandPostRAPseudosID);1187 1188 // Run pre-sched2 passes.1189 addPreSched2();1190 1191 if (EnableImplicitNullChecks)1192 addPass(&ImplicitNullChecksID);1193 1194 // Second pass scheduler.1195 // Let Target optionally insert this pass by itself at some other1196 // point.1197 if (getOptLevel() != CodeGenOptLevel::None &&1198 !TM->targetSchedulesPostRAScheduling()) {1199 if (MISchedPostRA)1200 addPass(&PostMachineSchedulerID);1201 else1202 addPass(&PostRASchedulerID);1203 }1204 1205 // GC1206 addGCPasses();1207 1208 // Basic block placement.1209 if (getOptLevel() != CodeGenOptLevel::None)1210 addBlockPlacement();1211 1212 // Insert before XRay Instrumentation.1213 addPass(&FEntryInserterID);1214 1215 addPass(&XRayInstrumentationID);1216 addPass(&PatchableFunctionID);1217 1218 addPreEmitPass();1219 1220 if (TM->Options.EnableIPRA)1221 // Collect register usage information and produce a register mask of1222 // clobbered registers, to be used to optimize call sites.1223 addPass(createRegUsageInfoCollector());1224 1225 // FIXME: Some backends are incompatible with running the verifier after1226 // addPreEmitPass. Maybe only pass "false" here for those targets?1227 addPass(&FuncletLayoutID);1228 1229 addPass(&RemoveLoadsIntoFakeUsesID);1230 addPass(&StackMapLivenessID);1231 addPass(&LiveDebugValuesID);1232 addPass(&MachineSanitizerBinaryMetadataID);1233 1234 if (TM->Options.EnableMachineOutliner &&1235 getOptLevel() != CodeGenOptLevel::None &&1236 EnableMachineOutliner != RunOutliner::NeverOutline) {1237 if (EnableMachineOutliner != RunOutliner::TargetDefault ||1238 TM->Options.SupportsDefaultOutlining)1239 addPass(createMachineOutlinerPass(EnableMachineOutliner));1240 }1241 1242 if (GCEmptyBlocks)1243 addPass(llvm::createGCEmptyBasicBlocksPass());1244 1245 if (EnableFSDiscriminator)1246 addPass(createMIRAddFSDiscriminatorsPass(1247 sampleprof::FSDiscriminatorPass::PassLast));1248 1249 if (TM->Options.EnableMachineFunctionSplitter ||1250 EnableMachineFunctionSplitter || SplitStaticData ||1251 TM->Options.EnableStaticDataPartitioning) {1252 const std::string ProfileFile = getFSProfileFile(TM);1253 if (!ProfileFile.empty()) {1254 if (EnableFSDiscriminator) {1255 addPass(createMIRProfileLoaderPass(1256 ProfileFile, getFSRemappingFile(TM),1257 sampleprof::FSDiscriminatorPass::PassLast, nullptr));1258 } else {1259 // Sample profile is given, but FSDiscriminator is not1260 // enabled, this may result in performance regression.1261 WithColor::warning()1262 << "Using AutoFDO without FSDiscriminator for MFS may regress "1263 "performance.\n";1264 }1265 }1266 }1267 1268 // Machine function splitter uses the basic block sections feature.1269 // When used along with `-basic-block-sections=`, the basic-block-sections1270 // feature takes precedence. This means functions eligible for1271 // basic-block-sections optimizations (`=all`, or `=list=` with function1272 // included in the list profile) will get that optimization instead.1273 if (TM->Options.EnableMachineFunctionSplitter ||1274 EnableMachineFunctionSplitter)1275 addPass(createMachineFunctionSplitterPass());1276 1277 if (SplitStaticData || TM->Options.EnableStaticDataPartitioning) {1278 // The static data splitter pass is a machine function pass. and1279 // static data annotator pass is a module-wide pass. See the file comment1280 // in StaticDataAnnotator.cpp for the motivation.1281 addPass(createStaticDataSplitterPass());1282 addPass(createStaticDataAnnotatorPass());1283 }1284 // We run the BasicBlockSections pass if either we need BB sections or BB1285 // address map (or both).1286 if (TM->getBBSectionsType() != llvm::BasicBlockSection::None ||1287 TM->Options.BBAddrMap) {1288 if (EmitBBHash)1289 addPass(llvm::createMachineBlockHashInfoPass());1290 if (TM->getBBSectionsType() == llvm::BasicBlockSection::List) {1291 addPass(llvm::createBasicBlockSectionsProfileReaderWrapperPass(1292 TM->getBBSectionsFuncListBuf()));1293 addPass(llvm::createBasicBlockPathCloningPass());1294 }1295 addPass(llvm::createBasicBlockSectionsPass());1296 }1297 1298 addPostBBSections();1299 1300 if (!DisableCFIFixup && TM->Options.EnableCFIFixup)1301 addPass(createCFIFixup());1302 1303 PM->add(createStackFrameLayoutAnalysisPass());1304 1305 // Add passes that directly emit MI after all other MI passes.1306 addPreEmitPass2();1307 1308 AddingMachinePasses = false;1309}1310 1311/// Add passes that optimize machine instructions in SSA form.1312void TargetPassConfig::addMachineSSAOptimization() {1313 // Pre-ra tail duplication.1314 addPass(&EarlyTailDuplicateLegacyID);1315 1316 // Optimize PHIs before DCE: removing dead PHI cycles may make more1317 // instructions dead.1318 addPass(&OptimizePHIsLegacyID);1319 1320 // This pass merges large allocas. StackSlotColoring is a different pass1321 // which merges spill slots.1322 addPass(&StackColoringLegacyID);1323 1324 // If the target requests it, assign local variables to stack slots relative1325 // to one another and simplify frame index references where possible.1326 addPass(&LocalStackSlotAllocationID);1327 1328 // With optimization, dead code should already be eliminated. However1329 // there is one known exception: lowered code for arguments that are only1330 // used by tail calls, where the tail calls reuse the incoming stack1331 // arguments directly (see t11 in test/CodeGen/X86/sibcall.ll).1332 addPass(&DeadMachineInstructionElimID);1333 1334 // Allow targets to insert passes that improve instruction level parallelism,1335 // like if-conversion. Such passes will typically need dominator trees and1336 // loop info, just like LICM and CSE below.1337 addILPOpts();1338 1339 addPass(&EarlyMachineLICMID);1340 addPass(&MachineCSELegacyID);1341 1342 addPass(&MachineSinkingLegacyID);1343 1344 addPass(&PeepholeOptimizerLegacyID);1345 // Clean-up the dead code that may have been generated by peephole1346 // rewriting.1347 addPass(&DeadMachineInstructionElimID);1348}1349 1350//===---------------------------------------------------------------------===//1351/// Register Allocation Pass Configuration1352//===---------------------------------------------------------------------===//1353 1354bool TargetPassConfig::getOptimizeRegAlloc() const {1355 switch (OptimizeRegAlloc) {1356 case cl::BOU_UNSET:1357 return getOptLevel() != CodeGenOptLevel::None;1358 case cl::BOU_TRUE: return true;1359 case cl::BOU_FALSE: return false;1360 }1361 llvm_unreachable("Invalid optimize-regalloc state");1362}1363 1364/// A dummy default pass factory indicates whether the register allocator is1365/// overridden on the command line.1366static llvm::once_flag InitializeDefaultRegisterAllocatorFlag;1367 1368static RegisterRegAlloc1369defaultRegAlloc("default",1370 "pick register allocator based on -O option",1371 useDefaultRegisterAllocator);1372 1373static void initializeDefaultRegisterAllocatorOnce() {1374 if (!RegisterRegAlloc::getDefault())1375 RegisterRegAlloc::setDefault(RegAlloc);1376}1377 1378/// Instantiate the default register allocator pass for this target for either1379/// the optimized or unoptimized allocation path. This will be added to the pass1380/// manager by addFastRegAlloc in the unoptimized case or addOptimizedRegAlloc1381/// in the optimized case.1382///1383/// A target that uses the standard regalloc pass order for fast or optimized1384/// allocation may still override this for per-target regalloc1385/// selection. But -regalloc=... always takes precedence.1386FunctionPass *TargetPassConfig::createTargetRegisterAllocator(bool Optimized) {1387 if (Optimized)1388 return createGreedyRegisterAllocator();1389 else1390 return createFastRegisterAllocator();1391}1392 1393/// Find and instantiate the register allocation pass requested by this target1394/// at the current optimization level. Different register allocators are1395/// defined as separate passes because they may require different analysis.1396///1397/// This helper ensures that the regalloc= option is always available,1398/// even for targets that override the default allocator.1399///1400/// FIXME: When MachinePassRegistry register pass IDs instead of function ptrs,1401/// this can be folded into addPass.1402FunctionPass *TargetPassConfig::createRegAllocPass(bool Optimized) {1403 // Initialize the global default.1404 llvm::call_once(InitializeDefaultRegisterAllocatorFlag,1405 initializeDefaultRegisterAllocatorOnce);1406 1407 RegisterRegAlloc::FunctionPassCtor Ctor = RegisterRegAlloc::getDefault();1408 if (Ctor != useDefaultRegisterAllocator)1409 return Ctor();1410 1411 // With no -regalloc= override, ask the target for a regalloc pass.1412 return createTargetRegisterAllocator(Optimized);1413}1414 1415bool TargetPassConfig::isCustomizedRegAlloc() {1416 return RegAlloc !=1417 (RegisterRegAlloc::FunctionPassCtor)&useDefaultRegisterAllocator;1418}1419 1420bool TargetPassConfig::addRegAssignAndRewriteFast() {1421 if (RegAlloc != (RegisterRegAlloc::FunctionPassCtor)&useDefaultRegisterAllocator &&1422 RegAlloc != (RegisterRegAlloc::FunctionPassCtor)&createFastRegisterAllocator)1423 reportFatalUsageError(1424 "Must use fast (default) register allocator for unoptimized regalloc.");1425 1426 addPass(createRegAllocPass(false));1427 1428 // Allow targets to change the register assignments after1429 // fast register allocation.1430 addPostFastRegAllocRewrite();1431 return true;1432}1433 1434bool TargetPassConfig::addRegAssignAndRewriteOptimized() {1435 // Add the selected register allocation pass.1436 addPass(createRegAllocPass(true));1437 1438 // Allow targets to change the register assignments before rewriting.1439 addPreRewrite();1440 1441 // Finally rewrite virtual registers.1442 addPass(&VirtRegRewriterID);1443 1444 // Regalloc scoring for ML-driven eviction - noop except when learning a new1445 // eviction policy.1446 addPass(createRegAllocScoringPass());1447 return true;1448}1449 1450/// Return true if the default global register allocator is in use and1451/// has not be overriden on the command line with '-regalloc=...'1452bool TargetPassConfig::usingDefaultRegAlloc() const {1453 return RegAlloc.getNumOccurrences() == 0;1454}1455 1456/// Add the minimum set of target-independent passes that are required for1457/// register allocation. No coalescing or scheduling.1458void TargetPassConfig::addFastRegAlloc() {1459 addPass(&PHIEliminationID);1460 addPass(&TwoAddressInstructionPassID);1461 1462 addRegAssignAndRewriteFast();1463}1464 1465/// Add standard target-independent passes that are tightly coupled with1466/// optimized register allocation, including coalescing, machine instruction1467/// scheduling, and register allocation itself.1468void TargetPassConfig::addOptimizedRegAlloc() {1469 addPass(&DetectDeadLanesID);1470 1471 addPass(&InitUndefID);1472 1473 addPass(&ProcessImplicitDefsID);1474 1475 // LiveVariables currently requires pure SSA form.1476 //1477 // FIXME: Once TwoAddressInstruction pass no longer uses kill flags,1478 // LiveVariables can be removed completely, and LiveIntervals can be directly1479 // computed. (We still either need to regenerate kill flags after regalloc, or1480 // preferably fix the scavenger to not depend on them).1481 // FIXME: UnreachableMachineBlockElim is a dependant pass of LiveVariables.1482 // When LiveVariables is removed this has to be removed/moved either.1483 // Explicit addition of UnreachableMachineBlockElim allows stopping before or1484 // after it with -stop-before/-stop-after.1485 addPass(&UnreachableMachineBlockElimID);1486 addPass(&LiveVariablesID);1487 1488 // Edge splitting is smarter with machine loop info.1489 addPass(&MachineLoopInfoID);1490 addPass(&PHIEliminationID);1491 1492 // Eventually, we want to run LiveIntervals before PHI elimination.1493 if (EarlyLiveIntervals)1494 addPass(&LiveIntervalsID);1495 1496 addPass(&TwoAddressInstructionPassID);1497 addPass(&RegisterCoalescerID);1498 1499 // The machine scheduler may accidentally create disconnected components1500 // when moving subregister definitions around, avoid this by splitting them to1501 // separate vregs before. Splitting can also improve reg. allocation quality.1502 addPass(&RenameIndependentSubregsID);1503 1504 // PreRA instruction scheduling.1505 addPass(&MachineSchedulerID);1506 1507 if (addRegAssignAndRewriteOptimized()) {1508 // Perform stack slot coloring and post-ra machine LICM.1509 addPass(&StackSlotColoringID);1510 1511 // Allow targets to expand pseudo instructions depending on the choice of1512 // registers before MachineCopyPropagation.1513 addPostRewrite();1514 1515 // Copy propagate to forward register uses and try to eliminate COPYs that1516 // were not coalesced.1517 addPass(&MachineCopyPropagationID);1518 1519 // Run post-ra machine LICM to hoist reloads / remats.1520 //1521 // FIXME: can this move into MachineLateOptimization?1522 addPass(&MachineLICMID);1523 }1524}1525 1526//===---------------------------------------------------------------------===//1527/// Post RegAlloc Pass Configuration1528//===---------------------------------------------------------------------===//1529 1530/// Add passes that optimize machine instructions after register allocation.1531void TargetPassConfig::addMachineLateOptimization() {1532 // Cleanup of redundant immediate/address loads.1533 addPass(&MachineLateInstrsCleanupID);1534 1535 // Branch folding must be run after regalloc and prolog/epilog insertion.1536 addPass(&BranchFolderPassID);1537 1538 // Tail duplication.1539 // Note that duplicating tail just increases code size and degrades1540 // performance for targets that require Structured Control Flow.1541 // In addition it can also make CFG irreducible. Thus we disable it.1542 if (!TM->requiresStructuredCFG())1543 addPass(&TailDuplicateLegacyID);1544 1545 // Copy propagation.1546 addPass(&MachineCopyPropagationID);1547}1548 1549/// Add standard GC passes.1550bool TargetPassConfig::addGCPasses() {1551 addPass(&GCMachineCodeAnalysisID);1552 return true;1553}1554 1555/// Add standard basic block placement passes.1556void TargetPassConfig::addBlockPlacement() {1557 if (EnableFSDiscriminator) {1558 addPass(createMIRAddFSDiscriminatorsPass(1559 sampleprof::FSDiscriminatorPass::Pass2));1560 const std::string ProfileFile = getFSProfileFile(TM);1561 if (!ProfileFile.empty() && !DisableLayoutFSProfileLoader)1562 addPass(createMIRProfileLoaderPass(ProfileFile, getFSRemappingFile(TM),1563 sampleprof::FSDiscriminatorPass::Pass2,1564 nullptr));1565 }1566 if (addPass(&MachineBlockPlacementID)) {1567 // Run a separate pass to collect block placement statistics.1568 if (EnableBlockPlacementStats)1569 addPass(&MachineBlockPlacementStatsID);1570 }1571}1572 1573//===---------------------------------------------------------------------===//1574/// GlobalISel Configuration1575//===---------------------------------------------------------------------===//1576bool TargetPassConfig::isGlobalISelAbortEnabled() const {1577 return TM->Options.GlobalISelAbort == GlobalISelAbortMode::Enable;1578}1579 1580bool TargetPassConfig::reportDiagnosticWhenGlobalISelFallback() const {1581 return TM->Options.GlobalISelAbort == GlobalISelAbortMode::DisableWithDiag;1582}1583 1584bool TargetPassConfig::isGISelCSEEnabled() const {1585 return true;1586}1587 1588std::unique_ptr<CSEConfigBase> TargetPassConfig::getCSEConfig() const {1589 return std::make_unique<CSEConfigBase>();1590}1591