671 lines · cpp
1//===-- RISCVTargetMachine.cpp - Define TargetMachine for RISC-V ----------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// Implements the info about RISC-V target spec.10//11//===----------------------------------------------------------------------===//12 13#include "RISCVTargetMachine.h"14#include "MCTargetDesc/RISCVBaseInfo.h"15#include "RISCV.h"16#include "RISCVMachineFunctionInfo.h"17#include "RISCVTargetObjectFile.h"18#include "RISCVTargetTransformInfo.h"19#include "TargetInfo/RISCVTargetInfo.h"20#include "llvm/Analysis/TargetTransformInfo.h"21#include "llvm/CodeGen/GlobalISel/CSEInfo.h"22#include "llvm/CodeGen/GlobalISel/IRTranslator.h"23#include "llvm/CodeGen/GlobalISel/InstructionSelect.h"24#include "llvm/CodeGen/GlobalISel/Legalizer.h"25#include "llvm/CodeGen/GlobalISel/RegBankSelect.h"26#include "llvm/CodeGen/MIRParser/MIParser.h"27#include "llvm/CodeGen/MIRYamlMapping.h"28#include "llvm/CodeGen/MachineScheduler.h"29#include "llvm/CodeGen/MacroFusion.h"30#include "llvm/CodeGen/Passes.h"31#include "llvm/CodeGen/RegAllocRegistry.h"32#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"33#include "llvm/CodeGen/TargetPassConfig.h"34#include "llvm/InitializePasses.h"35#include "llvm/MC/TargetRegistry.h"36#include "llvm/Passes/PassBuilder.h"37#include "llvm/Support/Compiler.h"38#include "llvm/Target/TargetOptions.h"39#include "llvm/Transforms/IPO.h"40#include "llvm/Transforms/Scalar.h"41#include "llvm/Transforms/Vectorize/LoopIdiomVectorize.h"42#include <optional>43using namespace llvm;44 45static cl::opt<bool> EnableRedundantCopyElimination(46 "riscv-enable-copyelim",47 cl::desc("Enable the redundant copy elimination pass"), cl::init(true),48 cl::Hidden);49 50// FIXME: Unify control over GlobalMerge.51static cl::opt<cl::boolOrDefault>52 EnableGlobalMerge("riscv-enable-global-merge", cl::Hidden,53 cl::desc("Enable the global merge pass"));54 55static cl::opt<bool>56 EnableMachineCombiner("riscv-enable-machine-combiner",57 cl::desc("Enable the machine combiner pass"),58 cl::init(true), cl::Hidden);59 60static cl::opt<unsigned> RVVVectorBitsMaxOpt(61 "riscv-v-vector-bits-max",62 cl::desc("Assume V extension vector registers are at most this big, "63 "with zero meaning no maximum size is assumed."),64 cl::init(0), cl::Hidden);65 66static cl::opt<int> RVVVectorBitsMinOpt(67 "riscv-v-vector-bits-min",68 cl::desc("Assume V extension vector registers are at least this big, "69 "with zero meaning no minimum size is assumed. A value of -1 "70 "means use Zvl*b extension. This is primarily used to enable "71 "autovectorization with fixed width vectors."),72 cl::init(-1), cl::Hidden);73 74static cl::opt<bool> EnableRISCVCopyPropagation(75 "riscv-enable-copy-propagation",76 cl::desc("Enable the copy propagation with RISC-V copy instr"),77 cl::init(true), cl::Hidden);78 79static cl::opt<bool> EnableRISCVDeadRegisterElimination(80 "riscv-enable-dead-defs", cl::Hidden,81 cl::desc("Enable the pass that removes dead"82 " definitions and replaces stores to"83 " them with stores to x0"),84 cl::init(true));85 86static cl::opt<bool>87 EnableSinkFold("riscv-enable-sink-fold",88 cl::desc("Enable sinking and folding of instruction copies"),89 cl::init(true), cl::Hidden);90 91static cl::opt<bool>92 EnableLoopDataPrefetch("riscv-enable-loop-data-prefetch", cl::Hidden,93 cl::desc("Enable the loop data prefetch pass"),94 cl::init(true));95 96static cl::opt<bool> DisableVectorMaskMutation(97 "riscv-disable-vector-mask-mutation",98 cl::desc("Disable the vector mask scheduling mutation"), cl::init(false),99 cl::Hidden);100 101static cl::opt<bool>102 EnableMachinePipeliner("riscv-enable-pipeliner",103 cl::desc("Enable Machine Pipeliner for RISC-V"),104 cl::init(false), cl::Hidden);105 106static cl::opt<bool> EnableCFIInstrInserter(107 "riscv-enable-cfi-instr-inserter",108 cl::desc("Enable CFI Instruction Inserter for RISC-V"), cl::init(false),109 cl::Hidden);110 111extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVTarget() {112 RegisterTargetMachine<RISCVTargetMachine> X(getTheRISCV32Target());113 RegisterTargetMachine<RISCVTargetMachine> Y(getTheRISCV64Target());114 RegisterTargetMachine<RISCVTargetMachine> A(getTheRISCV32beTarget());115 RegisterTargetMachine<RISCVTargetMachine> B(getTheRISCV64beTarget());116 auto *PR = PassRegistry::getPassRegistry();117 initializeGlobalISel(*PR);118 initializeRISCVO0PreLegalizerCombinerPass(*PR);119 initializeRISCVPreLegalizerCombinerPass(*PR);120 initializeRISCVPostLegalizerCombinerPass(*PR);121 initializeKCFIPass(*PR);122 initializeRISCVDeadRegisterDefinitionsPass(*PR);123 initializeRISCVLateBranchOptPass(*PR);124 initializeRISCVMakeCompressibleOptPass(*PR);125 initializeRISCVGatherScatterLoweringPass(*PR);126 initializeRISCVCodeGenPrepareLegacyPassPass(*PR);127 initializeRISCVPostRAExpandPseudoPass(*PR);128 initializeRISCVMergeBaseOffsetOptPass(*PR);129 initializeRISCVOptWInstrsPass(*PR);130 initializeRISCVFoldMemOffsetPass(*PR);131 initializeRISCVPreRAExpandPseudoPass(*PR);132 initializeRISCVExpandPseudoPass(*PR);133 initializeRISCVVectorPeepholePass(*PR);134 initializeRISCVVLOptimizerPass(*PR);135 initializeRISCVVMV0EliminationPass(*PR);136 initializeRISCVInsertVSETVLIPass(*PR);137 initializeRISCVInsertReadWriteCSRPass(*PR);138 initializeRISCVInsertWriteVXRMPass(*PR);139 initializeRISCVDAGToDAGISelLegacyPass(*PR);140 initializeRISCVMoveMergePass(*PR);141 initializeRISCVPushPopOptPass(*PR);142 initializeRISCVIndirectBranchTrackingPass(*PR);143 initializeRISCVLoadStoreOptPass(*PR);144 initializeRISCVPreAllocZilsdOptPass(*PR);145 initializeRISCVExpandAtomicPseudoPass(*PR);146 initializeRISCVRedundantCopyEliminationPass(*PR);147 initializeRISCVAsmPrinterPass(*PR);148 initializeRISCVPromoteConstantPass(*PR);149}150 151static Reloc::Model getEffectiveRelocModel(std::optional<Reloc::Model> RM) {152 return RM.value_or(Reloc::Static);153}154 155RISCVTargetMachine::RISCVTargetMachine(const Target &T, const Triple &TT,156 StringRef CPU, StringRef FS,157 const TargetOptions &Options,158 std::optional<Reloc::Model> RM,159 std::optional<CodeModel::Model> CM,160 CodeGenOptLevel OL, bool JIT)161 : CodeGenTargetMachineImpl(162 T, TT.computeDataLayout(Options.MCOptions.getABIName()), TT, CPU, FS,163 Options, getEffectiveRelocModel(RM),164 getEffectiveCodeModel(CM, CodeModel::Small), OL),165 TLOF(std::make_unique<RISCVELFTargetObjectFile>()) {166 initAsmInfo();167 168 // RISC-V supports the MachineOutliner.169 setMachineOutliner(true);170 setSupportsDefaultOutlining(true);171 172 // RISC-V supports the debug entry values.173 setSupportsDebugEntryValues(true);174 175 if (TT.isOSFuchsia() && !TT.isArch64Bit())176 report_fatal_error("Fuchsia is only supported for 64-bit");177 178 setCFIFixup(!EnableCFIInstrInserter);179}180 181const RISCVSubtarget *182RISCVTargetMachine::getSubtargetImpl(const Function &F) const {183 Attribute CPUAttr = F.getFnAttribute("target-cpu");184 Attribute TuneAttr = F.getFnAttribute("tune-cpu");185 Attribute FSAttr = F.getFnAttribute("target-features");186 187 std::string CPU =188 CPUAttr.isValid() ? CPUAttr.getValueAsString().str() : TargetCPU;189 std::string TuneCPU =190 TuneAttr.isValid() ? TuneAttr.getValueAsString().str() : CPU;191 std::string FS =192 FSAttr.isValid() ? FSAttr.getValueAsString().str() : TargetFS;193 194 unsigned RVVBitsMin = RVVVectorBitsMinOpt;195 unsigned RVVBitsMax = RVVVectorBitsMaxOpt;196 197 Attribute VScaleRangeAttr = F.getFnAttribute(Attribute::VScaleRange);198 if (VScaleRangeAttr.isValid()) {199 if (!RVVVectorBitsMinOpt.getNumOccurrences())200 RVVBitsMin = VScaleRangeAttr.getVScaleRangeMin() * RISCV::RVVBitsPerBlock;201 std::optional<unsigned> VScaleMax = VScaleRangeAttr.getVScaleRangeMax();202 if (VScaleMax.has_value() && !RVVVectorBitsMaxOpt.getNumOccurrences())203 RVVBitsMax = *VScaleMax * RISCV::RVVBitsPerBlock;204 }205 206 if (RVVBitsMin != -1U) {207 // FIXME: Change to >= 32 when VLEN = 32 is supported.208 assert((RVVBitsMin == 0 || (RVVBitsMin >= 64 && RVVBitsMin <= 65536 &&209 isPowerOf2_32(RVVBitsMin))) &&210 "V or Zve* extension requires vector length to be in the range of "211 "64 to 65536 and a power 2!");212 assert((RVVBitsMax >= RVVBitsMin || RVVBitsMax == 0) &&213 "Minimum V extension vector length should not be larger than its "214 "maximum!");215 }216 assert((RVVBitsMax == 0 || (RVVBitsMax >= 64 && RVVBitsMax <= 65536 &&217 isPowerOf2_32(RVVBitsMax))) &&218 "V or Zve* extension requires vector length to be in the range of "219 "64 to 65536 and a power 2!");220 221 if (RVVBitsMin != -1U) {222 if (RVVBitsMax != 0) {223 RVVBitsMin = std::min(RVVBitsMin, RVVBitsMax);224 RVVBitsMax = std::max(RVVBitsMin, RVVBitsMax);225 }226 227 RVVBitsMin = llvm::bit_floor(228 (RVVBitsMin < 64 || RVVBitsMin > 65536) ? 0 : RVVBitsMin);229 }230 RVVBitsMax =231 llvm::bit_floor((RVVBitsMax < 64 || RVVBitsMax > 65536) ? 0 : RVVBitsMax);232 233 SmallString<512> Key;234 raw_svector_ostream(Key) << "RVVMin" << RVVBitsMin << "RVVMax" << RVVBitsMax235 << CPU << TuneCPU << FS;236 auto &I = SubtargetMap[Key];237 if (!I) {238 // This needs to be done before we create a new subtarget since any239 // creation will depend on the TM and the code generation flags on the240 // function that reside in TargetOptions.241 resetTargetOptions(F);242 auto ABIName = Options.MCOptions.getABIName();243 if (const MDString *ModuleTargetABI = dyn_cast_or_null<MDString>(244 F.getParent()->getModuleFlag("target-abi"))) {245 auto TargetABI = RISCVABI::getTargetABI(ABIName);246 if (TargetABI != RISCVABI::ABI_Unknown &&247 ModuleTargetABI->getString() != ABIName) {248 report_fatal_error("-target-abi option != target-abi module flag");249 }250 ABIName = ModuleTargetABI->getString();251 }252 I = std::make_unique<RISCVSubtarget>(253 TargetTriple, CPU, TuneCPU, FS, ABIName, RVVBitsMin, RVVBitsMax, *this);254 }255 return I.get();256}257 258MachineFunctionInfo *RISCVTargetMachine::createMachineFunctionInfo(259 BumpPtrAllocator &Allocator, const Function &F,260 const TargetSubtargetInfo *STI) const {261 return RISCVMachineFunctionInfo::create<RISCVMachineFunctionInfo>(262 Allocator, F, static_cast<const RISCVSubtarget *>(STI));263}264 265TargetTransformInfo266RISCVTargetMachine::getTargetTransformInfo(const Function &F) const {267 return TargetTransformInfo(std::make_unique<RISCVTTIImpl>(this, F));268}269 270// A RISC-V hart has a single byte-addressable address space of 2^XLEN bytes271// for all memory accesses, so it is reasonable to assume that an272// implementation has no-op address space casts. If an implementation makes a273// change to this, they can override it here.274bool RISCVTargetMachine::isNoopAddrSpaceCast(unsigned SrcAS,275 unsigned DstAS) const {276 return true;277}278 279ScheduleDAGInstrs *280RISCVTargetMachine::createMachineScheduler(MachineSchedContext *C) const {281 const RISCVSubtarget &ST = C->MF->getSubtarget<RISCVSubtarget>();282 ScheduleDAGMILive *DAG = createSchedLive(C);283 284 if (ST.enableMISchedLoadClustering())285 DAG->addMutation(createLoadClusterDAGMutation(286 DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));287 288 if (ST.enableMISchedStoreClustering())289 DAG->addMutation(createStoreClusterDAGMutation(290 DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));291 292 if (!DisableVectorMaskMutation && ST.hasVInstructions())293 DAG->addMutation(createRISCVVectorMaskDAGMutation(DAG->TRI));294 295 return DAG;296}297 298ScheduleDAGInstrs *299RISCVTargetMachine::createPostMachineScheduler(MachineSchedContext *C) const {300 const RISCVSubtarget &ST = C->MF->getSubtarget<RISCVSubtarget>();301 ScheduleDAGMI *DAG = createSchedPostRA(C);302 303 if (ST.enablePostMISchedLoadClustering())304 DAG->addMutation(createLoadClusterDAGMutation(305 DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));306 307 if (ST.enablePostMISchedStoreClustering())308 DAG->addMutation(createStoreClusterDAGMutation(309 DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));310 311 return DAG;312}313 314namespace {315 316class RVVRegisterRegAlloc : public RegisterRegAllocBase<RVVRegisterRegAlloc> {317public:318 RVVRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)319 : RegisterRegAllocBase(N, D, C) {}320};321 322static bool onlyAllocateRVVReg(const TargetRegisterInfo &TRI,323 const MachineRegisterInfo &MRI,324 const Register Reg) {325 const TargetRegisterClass *RC = MRI.getRegClass(Reg);326 return RISCVRegisterInfo::isRVVRegClass(RC);327}328 329static FunctionPass *useDefaultRegisterAllocator() { return nullptr; }330 331static llvm::once_flag InitializeDefaultRVVRegisterAllocatorFlag;332 333/// -riscv-rvv-regalloc=<fast|basic|greedy> command line option.334/// This option could designate the rvv register allocator only.335/// For example: -riscv-rvv-regalloc=basic336static cl::opt<RVVRegisterRegAlloc::FunctionPassCtor, false,337 RegisterPassParser<RVVRegisterRegAlloc>>338 RVVRegAlloc("riscv-rvv-regalloc", cl::Hidden,339 cl::init(&useDefaultRegisterAllocator),340 cl::desc("Register allocator to use for RVV register."));341 342static void initializeDefaultRVVRegisterAllocatorOnce() {343 RegisterRegAlloc::FunctionPassCtor Ctor = RVVRegisterRegAlloc::getDefault();344 345 if (!Ctor) {346 Ctor = RVVRegAlloc;347 RVVRegisterRegAlloc::setDefault(RVVRegAlloc);348 }349}350 351static FunctionPass *createBasicRVVRegisterAllocator() {352 return createBasicRegisterAllocator(onlyAllocateRVVReg);353}354 355static FunctionPass *createGreedyRVVRegisterAllocator() {356 return createGreedyRegisterAllocator(onlyAllocateRVVReg);357}358 359static FunctionPass *createFastRVVRegisterAllocator() {360 return createFastRegisterAllocator(onlyAllocateRVVReg, false);361}362 363static RVVRegisterRegAlloc basicRegAllocRVVReg("basic",364 "basic register allocator",365 createBasicRVVRegisterAllocator);366static RVVRegisterRegAlloc367 greedyRegAllocRVVReg("greedy", "greedy register allocator",368 createGreedyRVVRegisterAllocator);369 370static RVVRegisterRegAlloc fastRegAllocRVVReg("fast", "fast register allocator",371 createFastRVVRegisterAllocator);372 373class RISCVPassConfig : public TargetPassConfig {374public:375 RISCVPassConfig(RISCVTargetMachine &TM, PassManagerBase &PM)376 : TargetPassConfig(TM, PM) {377 if (TM.getOptLevel() != CodeGenOptLevel::None)378 substitutePass(&PostRASchedulerID, &PostMachineSchedulerID);379 setEnableSinkAndFold(EnableSinkFold);380 EnableLoopTermFold = true;381 }382 383 RISCVTargetMachine &getRISCVTargetMachine() const {384 return getTM<RISCVTargetMachine>();385 }386 387 void addIRPasses() override;388 bool addPreISel() override;389 void addCodeGenPrepare() override;390 bool addInstSelector() override;391 bool addIRTranslator() override;392 void addPreLegalizeMachineIR() override;393 bool addLegalizeMachineIR() override;394 void addPreRegBankSelect() override;395 bool addRegBankSelect() override;396 bool addGlobalInstructionSelect() override;397 void addPreEmitPass() override;398 void addPreEmitPass2() override;399 void addPreSched2() override;400 void addMachineSSAOptimization() override;401 FunctionPass *createRVVRegAllocPass(bool Optimized);402 bool addRegAssignAndRewriteFast() override;403 bool addRegAssignAndRewriteOptimized() override;404 void addPreRegAlloc() override;405 void addPostRegAlloc() override;406 void addFastRegAlloc() override;407 bool addILPOpts() override;408 409 std::unique_ptr<CSEConfigBase> getCSEConfig() const override;410};411} // namespace412 413TargetPassConfig *RISCVTargetMachine::createPassConfig(PassManagerBase &PM) {414 return new RISCVPassConfig(*this, PM);415}416 417std::unique_ptr<CSEConfigBase> RISCVPassConfig::getCSEConfig() const {418 return getStandardCSEConfigForOpt(TM->getOptLevel());419}420 421FunctionPass *RISCVPassConfig::createRVVRegAllocPass(bool Optimized) {422 // Initialize the global default.423 llvm::call_once(InitializeDefaultRVVRegisterAllocatorFlag,424 initializeDefaultRVVRegisterAllocatorOnce);425 426 RegisterRegAlloc::FunctionPassCtor Ctor = RVVRegisterRegAlloc::getDefault();427 if (Ctor != useDefaultRegisterAllocator)428 return Ctor();429 430 if (Optimized)431 return createGreedyRVVRegisterAllocator();432 433 return createFastRVVRegisterAllocator();434}435 436bool RISCVPassConfig::addRegAssignAndRewriteFast() {437 addPass(createRVVRegAllocPass(false));438 addPass(createRISCVInsertVSETVLIPass());439 if (TM->getOptLevel() != CodeGenOptLevel::None &&440 EnableRISCVDeadRegisterElimination)441 addPass(createRISCVDeadRegisterDefinitionsPass());442 return TargetPassConfig::addRegAssignAndRewriteFast();443}444 445bool RISCVPassConfig::addRegAssignAndRewriteOptimized() {446 addPass(createRVVRegAllocPass(true));447 addPass(createVirtRegRewriter(false));448 addPass(createRISCVInsertVSETVLIPass());449 if (TM->getOptLevel() != CodeGenOptLevel::None &&450 EnableRISCVDeadRegisterElimination)451 addPass(createRISCVDeadRegisterDefinitionsPass());452 return TargetPassConfig::addRegAssignAndRewriteOptimized();453}454 455void RISCVPassConfig::addIRPasses() {456 addPass(createAtomicExpandLegacyPass());457 addPass(createRISCVZacasABIFixPass());458 459 if (getOptLevel() != CodeGenOptLevel::None) {460 if (EnableLoopDataPrefetch)461 addPass(createLoopDataPrefetchPass());462 463 addPass(createRISCVGatherScatterLoweringPass());464 addPass(createInterleavedAccessPass());465 addPass(createRISCVCodeGenPrepareLegacyPass());466 }467 468 TargetPassConfig::addIRPasses();469}470 471bool RISCVPassConfig::addPreISel() {472 if (TM->getOptLevel() != CodeGenOptLevel::None)473 addPass(createRISCVPromoteConstantPass());474 if (TM->getOptLevel() != CodeGenOptLevel::None) {475 // Add a barrier before instruction selection so that we will not get476 // deleted block address after enabling default outlining. See D99707 for477 // more details.478 addPass(createBarrierNoopPass());479 }480 481 if ((TM->getOptLevel() != CodeGenOptLevel::None &&482 EnableGlobalMerge == cl::BOU_UNSET) ||483 EnableGlobalMerge == cl::BOU_TRUE) {484 // FIXME: Like AArch64, we disable extern global merging by default due to485 // concerns it might regress some workloads. Unlike AArch64, we don't486 // currently support enabling the pass in an "OnlyOptimizeForSize" mode.487 // Investigating and addressing both items are TODO.488 addPass(createGlobalMergePass(TM, /* MaxOffset */ 2047,489 /* OnlyOptimizeForSize */ false,490 /* MergeExternalByDefault */ true));491 }492 493 return false;494}495 496void RISCVPassConfig::addCodeGenPrepare() {497 if (getOptLevel() != CodeGenOptLevel::None)498 addPass(createTypePromotionLegacyPass());499 TargetPassConfig::addCodeGenPrepare();500}501 502bool RISCVPassConfig::addInstSelector() {503 addPass(createRISCVISelDag(getRISCVTargetMachine(), getOptLevel()));504 505 return false;506}507 508bool RISCVPassConfig::addIRTranslator() {509 addPass(new IRTranslator(getOptLevel()));510 return false;511}512 513void RISCVPassConfig::addPreLegalizeMachineIR() {514 if (getOptLevel() == CodeGenOptLevel::None) {515 addPass(createRISCVO0PreLegalizerCombiner());516 } else {517 addPass(createRISCVPreLegalizerCombiner());518 }519}520 521bool RISCVPassConfig::addLegalizeMachineIR() {522 addPass(new Legalizer());523 return false;524}525 526void RISCVPassConfig::addPreRegBankSelect() {527 if (getOptLevel() != CodeGenOptLevel::None)528 addPass(createRISCVPostLegalizerCombiner());529}530 531bool RISCVPassConfig::addRegBankSelect() {532 addPass(new RegBankSelect());533 return false;534}535 536bool RISCVPassConfig::addGlobalInstructionSelect() {537 addPass(new InstructionSelect(getOptLevel()));538 return false;539}540 541void RISCVPassConfig::addPreSched2() {542 addPass(createRISCVPostRAExpandPseudoPass());543 544 // Emit KCFI checks for indirect calls.545 addPass(createKCFIPass());546 if (TM->getOptLevel() != CodeGenOptLevel::None)547 addPass(createRISCVLoadStoreOptPass());548}549 550void RISCVPassConfig::addPreEmitPass() {551 // TODO: It would potentially be better to schedule copy propagation after552 // expanding pseudos (in addPreEmitPass2). However, performing copy553 // propagation after the machine outliner (which runs after addPreEmitPass)554 // currently leads to incorrect code-gen, where copies to registers within555 // outlined functions are removed erroneously.556 if (TM->getOptLevel() >= CodeGenOptLevel::Default &&557 EnableRISCVCopyPropagation)558 addPass(createMachineCopyPropagationPass(true));559 if (TM->getOptLevel() >= CodeGenOptLevel::Default)560 addPass(createRISCVLateBranchOptPass());561 // The IndirectBranchTrackingPass inserts lpad and could have changed the562 // basic block alignment. It must be done before Branch Relaxation to563 // prevent the adjusted offset exceeding the branch range.564 addPass(createRISCVIndirectBranchTrackingPass());565 addPass(&BranchRelaxationPassID);566 addPass(createRISCVMakeCompressibleOptPass());567}568 569void RISCVPassConfig::addPreEmitPass2() {570 if (TM->getOptLevel() != CodeGenOptLevel::None) {571 addPass(createRISCVMoveMergePass());572 // Schedule PushPop Optimization before expansion of Pseudo instruction,573 // ensuring return instruction is detected correctly.574 addPass(createRISCVPushPopOptimizationPass());575 }576 addPass(createRISCVExpandPseudoPass());577 578 // Schedule the expansion of AMOs at the last possible moment, avoiding the579 // possibility for other passes to break the requirements for forward580 // progress in the LR/SC block.581 addPass(createRISCVExpandAtomicPseudoPass());582 583 // KCFI indirect call checks are lowered to a bundle.584 addPass(createUnpackMachineBundles([&](const MachineFunction &MF) {585 return MF.getFunction().getParent()->getModuleFlag("kcfi");586 }));587 588 if (EnableCFIInstrInserter)589 addPass(createCFIInstrInserter());590}591 592void RISCVPassConfig::addMachineSSAOptimization() {593 addPass(createRISCVVectorPeepholePass());594 addPass(createRISCVFoldMemOffsetPass());595 596 TargetPassConfig::addMachineSSAOptimization();597 598 if (TM->getTargetTriple().isRISCV64()) {599 addPass(createRISCVOptWInstrsPass());600 }601}602 603void RISCVPassConfig::addPreRegAlloc() {604 addPass(createRISCVPreRAExpandPseudoPass());605 if (TM->getOptLevel() != CodeGenOptLevel::None) {606 addPass(createRISCVMergeBaseOffsetOptPass());607 addPass(createRISCVVLOptimizerPass());608 // Add Zilsd pre-allocation load/store optimization609 addPass(createRISCVPreAllocZilsdOptPass());610 }611 612 addPass(createRISCVInsertReadWriteCSRPass());613 addPass(createRISCVInsertWriteVXRMPass());614 addPass(createRISCVLandingPadSetupPass());615 616 if (TM->getOptLevel() != CodeGenOptLevel::None && EnableMachinePipeliner)617 addPass(&MachinePipelinerID);618 619 addPass(createRISCVVMV0EliminationPass());620}621 622void RISCVPassConfig::addFastRegAlloc() {623 addPass(&InitUndefID);624 TargetPassConfig::addFastRegAlloc();625}626 627 628void RISCVPassConfig::addPostRegAlloc() {629 if (TM->getOptLevel() != CodeGenOptLevel::None &&630 EnableRedundantCopyElimination)631 addPass(createRISCVRedundantCopyEliminationPass());632}633 634bool RISCVPassConfig::addILPOpts() {635 if (EnableMachineCombiner)636 addPass(&MachineCombinerID);637 638 return true;639}640 641void RISCVTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {642#define GET_PASS_REGISTRY "RISCVPassRegistry.def"643#include "llvm/Passes/TargetPassRegistry.inc"644 645 PB.registerLateLoopOptimizationsEPCallback([=](LoopPassManager &LPM,646 OptimizationLevel Level) {647 if (Level != OptimizationLevel::O0)648 LPM.addPass(LoopIdiomVectorizePass(LoopIdiomVectorizeStyle::Predicated));649 });650}651 652yaml::MachineFunctionInfo *653RISCVTargetMachine::createDefaultFuncInfoYAML() const {654 return new yaml::RISCVMachineFunctionInfo();655}656 657yaml::MachineFunctionInfo *658RISCVTargetMachine::convertFuncInfoToYAML(const MachineFunction &MF) const {659 const auto *MFI = MF.getInfo<RISCVMachineFunctionInfo>();660 return new yaml::RISCVMachineFunctionInfo(*MFI);661}662 663bool RISCVTargetMachine::parseMachineFunctionInfo(664 const yaml::MachineFunctionInfo &MFI, PerFunctionMIParsingState &PFS,665 SMDiagnostic &Error, SMRange &SourceRange) const {666 const auto &YamlMFI =667 static_cast<const yaml::RISCVMachineFunctionInfo &>(MFI);668 PFS.MF.getInfo<RISCVMachineFunctionInfo>()->initializeBaseYamlFields(YamlMFI);669 return false;670}671