1014 lines · cpp
1//===- AArch64MIPeepholeOpt.cpp - AArch64 MI peephole optimization pass ---===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This pass performs below peephole optimizations on MIR level.10//11// 1. MOVi32imm + (ANDS?|EOR|ORR)Wrr ==> (AND|EOR|ORR)Wri + (ANDS?|EOR|ORR)Wri12// MOVi64imm + (ANDS?|EOR|ORR)Xrr ==> (AND|EOR|ORR)Xri + (ANDS?|EOR|ORR)Xri13//14// 2. MOVi32imm + ADDWrr ==> ADDWRi + ADDWRi15// MOVi64imm + ADDXrr ==> ADDXri + ADDXri16//17// 3. MOVi32imm + SUBWrr ==> SUBWRi + SUBWRi18// MOVi64imm + SUBXrr ==> SUBXri + SUBXri19//20// The mov pseudo instruction could be expanded to multiple mov instructions21// later. In this case, we could try to split the constant operand of mov22// instruction into two immediates which can be directly encoded into23// *Wri/*Xri instructions. It makes two AND/ADD/SUB instructions instead of24// multiple `mov` + `and/add/sub` instructions.25//26// 4. Remove redundant ORRWrs which is generated by zero-extend.27//28// %3:gpr32 = ORRWrs $wzr, %2, 029// %4:gpr64 = SUBREG_TO_REG 0, %3, %subreg.sub_3230//31// If AArch64's 32-bit form of instruction defines the source operand of32// ORRWrs, we can remove the ORRWrs because the upper 32 bits of the source33// operand are set to zero.34//35// 5. %reg = INSERT_SUBREG %reg(tied-def 0), %subreg, subidx36// ==> %reg:subidx = SUBREG_TO_REG 0, %subreg, subidx37//38// 6. %intermediate:gpr32 = COPY %src:fpr12839// %dst:fpr128 = INSvi32gpr %dst_vec:fpr128, dst_index, %intermediate:gpr3240// ==> %dst:fpr128 = INSvi32lane %dst_vec:fpr128, dst_index, %src:fpr128, 041//42// In cases where a source FPR is copied to a GPR in order to be copied43// to a destination FPR, we can directly copy the values between the FPRs,44// eliminating the use of the Integer unit. When we match a pattern of45// INSvi[X]gpr that is preceded by a chain of COPY instructions from a FPR46// source, we use the INSvi[X]lane to replace the COPY & INSvi[X]gpr47// instructions.48//49// 7. If MI sets zero for high 64-bits implicitly, remove `mov 0` for high50// 64-bits. For example,51//52// %1:fpr64 = nofpexcept FCVTNv4i16 %0:fpr128, implicit $fpcr53// %2:fpr64 = MOVID 054// %4:fpr128 = IMPLICIT_DEF55// %3:fpr128 = INSERT_SUBREG %4:fpr128(tied-def 0), %2:fpr64, %subreg.dsub56// %6:fpr128 = IMPLICIT_DEF57// %5:fpr128 = INSERT_SUBREG %6:fpr128(tied-def 0), %1:fpr64, %subreg.dsub58// %7:fpr128 = INSvi64lane %5:fpr128(tied-def 0), 1, %3:fpr128, 059// ==>60// %1:fpr64 = nofpexcept FCVTNv4i16 %0:fpr128, implicit $fpcr61// %6:fpr128 = IMPLICIT_DEF62// %7:fpr128 = INSERT_SUBREG %6:fpr128(tied-def 0), %1:fpr64, %subreg.dsub63//64// 8. Remove redundant CSELs that select between identical registers, by65// replacing them with unconditional moves.66//67// 9. Replace UBFMXri with UBFMWri if the instruction is equivalent to a 32 bit68// LSR or LSL alias of UBFM.69//70//===----------------------------------------------------------------------===//71 72#include "AArch64ExpandImm.h"73#include "AArch64InstrInfo.h"74#include "MCTargetDesc/AArch64AddressingModes.h"75#include "llvm/CodeGen/MachineDominators.h"76#include "llvm/CodeGen/MachineLoopInfo.h"77 78using namespace llvm;79 80#define DEBUG_TYPE "aarch64-mi-peephole-opt"81 82namespace {83 84struct AArch64MIPeepholeOpt : public MachineFunctionPass {85 static char ID;86 87 AArch64MIPeepholeOpt() : MachineFunctionPass(ID) {}88 89 const AArch64InstrInfo *TII;90 const AArch64RegisterInfo *TRI;91 MachineLoopInfo *MLI;92 MachineRegisterInfo *MRI;93 94 using OpcodePair = std::pair<unsigned, unsigned>;95 template <typename T>96 using SplitAndOpcFunc =97 std::function<std::optional<OpcodePair>(T, unsigned, T &, T &)>;98 using BuildMIFunc =99 std::function<void(MachineInstr &, OpcodePair, unsigned, unsigned,100 Register, Register, Register)>;101 102 /// For instructions where an immediate operand could be split into two103 /// separate immediate instructions, use the splitTwoPartImm two handle the104 /// optimization.105 ///106 /// To implement, the following function types must be passed to107 /// splitTwoPartImm. A SplitAndOpcFunc must be implemented that determines if108 /// splitting the immediate is valid and returns the associated new opcode. A109 /// BuildMIFunc must be implemented to build the two immediate instructions.110 ///111 /// Example Pattern (where IMM would require 2+ MOV instructions):112 /// %dst = <Instr>rr %src IMM [...]113 /// becomes:114 /// %tmp = <Instr>ri %src (encode half IMM) [...]115 /// %dst = <Instr>ri %tmp (encode half IMM) [...]116 template <typename T>117 bool splitTwoPartImm(MachineInstr &MI,118 SplitAndOpcFunc<T> SplitAndOpc, BuildMIFunc BuildInstr);119 120 bool checkMovImmInstr(MachineInstr &MI, MachineInstr *&MovMI,121 MachineInstr *&SubregToRegMI);122 123 template <typename T>124 bool visitADDSUB(unsigned PosOpc, unsigned NegOpc, MachineInstr &MI);125 template <typename T>126 bool visitADDSSUBS(OpcodePair PosOpcs, OpcodePair NegOpcs, MachineInstr &MI);127 128 // Strategy used to split logical immediate bitmasks.129 enum class SplitStrategy {130 Intersect,131 Disjoint,132 };133 template <typename T>134 bool trySplitLogicalImm(unsigned Opc, MachineInstr &MI,135 SplitStrategy Strategy, unsigned OtherOpc = 0);136 bool visitORR(MachineInstr &MI);137 bool visitCSEL(MachineInstr &MI);138 bool visitINSERT(MachineInstr &MI);139 bool visitINSviGPR(MachineInstr &MI, unsigned Opc);140 bool visitINSvi64lane(MachineInstr &MI);141 bool visitFMOVDr(MachineInstr &MI);142 bool visitUBFMXri(MachineInstr &MI);143 bool visitCopy(MachineInstr &MI);144 bool runOnMachineFunction(MachineFunction &MF) override;145 146 StringRef getPassName() const override {147 return "AArch64 MI Peephole Optimization pass";148 }149 150 void getAnalysisUsage(AnalysisUsage &AU) const override {151 AU.setPreservesCFG();152 AU.addRequired<MachineLoopInfoWrapperPass>();153 MachineFunctionPass::getAnalysisUsage(AU);154 }155};156 157char AArch64MIPeepholeOpt::ID = 0;158 159} // end anonymous namespace160 161INITIALIZE_PASS(AArch64MIPeepholeOpt, "aarch64-mi-peephole-opt",162 "AArch64 MI Peephole Optimization", false, false)163 164template <typename T>165static bool splitBitmaskImm(T Imm, unsigned RegSize, T &Imm1Enc, T &Imm2Enc) {166 T UImm = static_cast<T>(Imm);167 assert(UImm && (UImm != ~static_cast<T>(0)) && "Invalid immediate!");168 169 // The bitmask immediate consists of consecutive ones. Let's say there is170 // constant 0b00000000001000000000010000000000 which does not consist of171 // consecutive ones. We can split it in to two bitmask immediate like172 // 0b00000000001111111111110000000000 and 0b11111111111000000000011111111111.173 // If we do AND with these two bitmask immediate, we can see original one.174 unsigned LowestBitSet = llvm::countr_zero(UImm);175 unsigned HighestBitSet = Log2_64(UImm);176 177 // Create a mask which is filled with one from the position of lowest bit set178 // to the position of highest bit set.179 T NewImm1 = (static_cast<T>(2) << HighestBitSet) -180 (static_cast<T>(1) << LowestBitSet);181 // Create a mask which is filled with one outside the position of lowest bit182 // set and the position of highest bit set.183 T NewImm2 = UImm | ~NewImm1;184 185 // If the split value is not valid bitmask immediate, do not split this186 // constant.187 if (!AArch64_AM::isLogicalImmediate(NewImm2, RegSize))188 return false;189 190 Imm1Enc = AArch64_AM::encodeLogicalImmediate(NewImm1, RegSize);191 Imm2Enc = AArch64_AM::encodeLogicalImmediate(NewImm2, RegSize);192 return true;193}194 195template <typename T>196static bool splitDisjointBitmaskImm(T Imm, unsigned RegSize, T &Imm1Enc,197 T &Imm2Enc) {198 assert(Imm && (Imm != ~static_cast<T>(0)) && "Invalid immediate!");199 200 // Try to split a bitmask of the form 0b00000000011000000000011110000000 into201 // two disjoint masks such as 0b00000000011000000000000000000000 and202 // 0b00000000000000000000011110000000 where the inclusive/exclusive OR of the203 // new masks match the original mask.204 unsigned LowestBitSet = llvm::countr_zero(Imm);205 unsigned LowestGapBitUnset =206 LowestBitSet + llvm::countr_one(Imm >> LowestBitSet);207 208 // Create a mask for the least significant group of consecutive ones.209 assert(LowestGapBitUnset < sizeof(T) * CHAR_BIT && "Undefined behaviour!");210 T NewImm1 = (static_cast<T>(1) << LowestGapBitUnset) -211 (static_cast<T>(1) << LowestBitSet);212 // Create a disjoint mask for the remaining ones.213 T NewImm2 = Imm & ~NewImm1;214 215 // Do not split if NewImm2 is not a valid bitmask immediate.216 if (!AArch64_AM::isLogicalImmediate(NewImm2, RegSize))217 return false;218 219 Imm1Enc = AArch64_AM::encodeLogicalImmediate(NewImm1, RegSize);220 Imm2Enc = AArch64_AM::encodeLogicalImmediate(NewImm2, RegSize);221 return true;222}223 224template <typename T>225bool AArch64MIPeepholeOpt::trySplitLogicalImm(unsigned Opc, MachineInstr &MI,226 SplitStrategy Strategy,227 unsigned OtherOpc) {228 // Try below transformations.229 //230 // MOVi32imm + (ANDS?|EOR|ORR)Wrr ==> (AND|EOR|ORR)Wri + (ANDS?|EOR|ORR)Wri231 // MOVi64imm + (ANDS?|EOR|ORR)Xrr ==> (AND|EOR|ORR)Xri + (ANDS?|EOR|ORR)Xri232 //233 // The mov pseudo instruction could be expanded to multiple mov instructions234 // later. Let's try to split the constant operand of mov instruction into two235 // bitmask immediates based on the given split strategy. It makes only two236 // logical instructions instead of multiple mov + logic instructions.237 238 return splitTwoPartImm<T>(239 MI,240 [Opc, Strategy, OtherOpc](T Imm, unsigned RegSize, T &Imm0,241 T &Imm1) -> std::optional<OpcodePair> {242 // If this immediate is already a suitable bitmask, don't split it.243 // TODO: Should we just combine the two instructions in this case?244 if (AArch64_AM::isLogicalImmediate(Imm, RegSize))245 return std::nullopt;246 247 // If this immediate can be handled by one instruction, don't split it.248 SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn;249 AArch64_IMM::expandMOVImm(Imm, RegSize, Insn);250 if (Insn.size() == 1)251 return std::nullopt;252 253 bool SplitSucc = false;254 switch (Strategy) {255 case SplitStrategy::Intersect:256 SplitSucc = splitBitmaskImm(Imm, RegSize, Imm0, Imm1);257 break;258 case SplitStrategy::Disjoint:259 SplitSucc = splitDisjointBitmaskImm(Imm, RegSize, Imm0, Imm1);260 break;261 }262 if (SplitSucc)263 return std::make_pair(Opc, !OtherOpc ? Opc : OtherOpc);264 return std::nullopt;265 },266 [&TII = TII](MachineInstr &MI, OpcodePair Opcode, unsigned Imm0,267 unsigned Imm1, Register SrcReg, Register NewTmpReg,268 Register NewDstReg) {269 DebugLoc DL = MI.getDebugLoc();270 MachineBasicBlock *MBB = MI.getParent();271 BuildMI(*MBB, MI, DL, TII->get(Opcode.first), NewTmpReg)272 .addReg(SrcReg)273 .addImm(Imm0);274 BuildMI(*MBB, MI, DL, TII->get(Opcode.second), NewDstReg)275 .addReg(NewTmpReg)276 .addImm(Imm1);277 });278}279 280bool AArch64MIPeepholeOpt::visitORR(MachineInstr &MI) {281 // Check this ORR comes from below zero-extend pattern.282 //283 // def : Pat<(i64 (zext GPR32:$src)),284 // (SUBREG_TO_REG (i32 0), (ORRWrs WZR, GPR32:$src, 0), sub_32)>;285 if (MI.getOperand(3).getImm() != 0)286 return false;287 288 if (MI.getOperand(1).getReg() != AArch64::WZR)289 return false;290 291 MachineInstr *SrcMI = MRI->getUniqueVRegDef(MI.getOperand(2).getReg());292 if (!SrcMI)293 return false;294 295 // From https://developer.arm.com/documentation/dui0801/b/BABBGCAC296 //297 // When you use the 32-bit form of an instruction, the upper 32 bits of the298 // source registers are ignored and the upper 32 bits of the destination299 // register are set to zero.300 //301 // If AArch64's 32-bit form of instruction defines the source operand of302 // zero-extend, we do not need the zero-extend. Let's check the MI's opcode is303 // real AArch64 instruction and if it is not, do not process the opcode304 // conservatively.305 if (SrcMI->getOpcode() == TargetOpcode::COPY &&306 SrcMI->getOperand(1).getReg().isVirtual()) {307 const TargetRegisterClass *RC =308 MRI->getRegClass(SrcMI->getOperand(1).getReg());309 310 // A COPY from an FPR will become a FMOVSWr, so do so now so that we know311 // that the upper bits are zero.312 if (RC != &AArch64::FPR32RegClass &&313 ((RC != &AArch64::FPR64RegClass && RC != &AArch64::FPR128RegClass &&314 RC != &AArch64::ZPRRegClass) ||315 SrcMI->getOperand(1).getSubReg() != AArch64::ssub))316 return false;317 Register CpySrc;318 if (SrcMI->getOperand(1).getSubReg() == AArch64::ssub) {319 CpySrc = MRI->createVirtualRegister(&AArch64::FPR32RegClass);320 BuildMI(*SrcMI->getParent(), SrcMI, SrcMI->getDebugLoc(),321 TII->get(TargetOpcode::COPY), CpySrc)322 .add(SrcMI->getOperand(1));323 } else {324 CpySrc = SrcMI->getOperand(1).getReg();325 }326 BuildMI(*SrcMI->getParent(), SrcMI, SrcMI->getDebugLoc(),327 TII->get(AArch64::FMOVSWr), SrcMI->getOperand(0).getReg())328 .addReg(CpySrc);329 SrcMI->eraseFromParent();330 }331 else if (SrcMI->getOpcode() <= TargetOpcode::GENERIC_OP_END)332 return false;333 334 Register DefReg = MI.getOperand(0).getReg();335 Register SrcReg = MI.getOperand(2).getReg();336 MRI->replaceRegWith(DefReg, SrcReg);337 MRI->clearKillFlags(SrcReg);338 LLVM_DEBUG(dbgs() << "Removed: " << MI << "\n");339 MI.eraseFromParent();340 341 return true;342}343 344bool AArch64MIPeepholeOpt::visitCSEL(MachineInstr &MI) {345 // Replace CSEL with MOV when both inputs are the same register.346 if (MI.getOperand(1).getReg() != MI.getOperand(2).getReg())347 return false;348 349 auto ZeroReg =350 MI.getOpcode() == AArch64::CSELXr ? AArch64::XZR : AArch64::WZR;351 auto OrOpcode =352 MI.getOpcode() == AArch64::CSELXr ? AArch64::ORRXrs : AArch64::ORRWrs;353 354 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), TII->get(OrOpcode))355 .addReg(MI.getOperand(0).getReg(), RegState::Define)356 .addReg(ZeroReg)357 .addReg(MI.getOperand(1).getReg())358 .addImm(0);359 360 MI.eraseFromParent();361 return true;362}363 364bool AArch64MIPeepholeOpt::visitINSERT(MachineInstr &MI) {365 // Check this INSERT_SUBREG comes from below zero-extend pattern.366 //367 // From %reg = INSERT_SUBREG %reg(tied-def 0), %subreg, subidx368 // To %reg:subidx = SUBREG_TO_REG 0, %subreg, subidx369 //370 // We're assuming the first operand to INSERT_SUBREG is irrelevant because a371 // COPY would destroy the upper part of the register anyway372 if (!MI.isRegTiedToDefOperand(1))373 return false;374 375 Register DstReg = MI.getOperand(0).getReg();376 const TargetRegisterClass *RC = MRI->getRegClass(DstReg);377 MachineInstr *SrcMI = MRI->getUniqueVRegDef(MI.getOperand(2).getReg());378 if (!SrcMI)379 return false;380 381 // From https://developer.arm.com/documentation/dui0801/b/BABBGCAC382 //383 // When you use the 32-bit form of an instruction, the upper 32 bits of the384 // source registers are ignored and the upper 32 bits of the destination385 // register are set to zero.386 //387 // If AArch64's 32-bit form of instruction defines the source operand of388 // zero-extend, we do not need the zero-extend. Let's check the MI's opcode is389 // real AArch64 instruction and if it is not, do not process the opcode390 // conservatively.391 if ((SrcMI->getOpcode() <= TargetOpcode::GENERIC_OP_END) ||392 !AArch64::GPR64allRegClass.hasSubClassEq(RC))393 return false;394 395 // Build a SUBREG_TO_REG instruction396 MachineInstr *SubregMI =397 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),398 TII->get(TargetOpcode::SUBREG_TO_REG), DstReg)399 .addImm(0)400 .add(MI.getOperand(2))401 .add(MI.getOperand(3));402 LLVM_DEBUG(dbgs() << MI << " replace by:\n: " << *SubregMI << "\n");403 (void)SubregMI;404 MI.eraseFromParent();405 406 return true;407}408 409template <typename T>410static bool splitAddSubImm(T Imm, unsigned RegSize, T &Imm0, T &Imm1) {411 // The immediate must be in the form of ((imm0 << 12) + imm1), in which both412 // imm0 and imm1 are non-zero 12-bit unsigned int.413 if ((Imm & 0xfff000) == 0 || (Imm & 0xfff) == 0 ||414 (Imm & ~static_cast<T>(0xffffff)) != 0)415 return false;416 417 // The immediate can not be composed via a single instruction.418 SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn;419 AArch64_IMM::expandMOVImm(Imm, RegSize, Insn);420 if (Insn.size() == 1)421 return false;422 423 // Split Imm into (Imm0 << 12) + Imm1;424 Imm0 = (Imm >> 12) & 0xfff;425 Imm1 = Imm & 0xfff;426 return true;427}428 429template <typename T>430bool AArch64MIPeepholeOpt::visitADDSUB(431 unsigned PosOpc, unsigned NegOpc, MachineInstr &MI) {432 // Try below transformation.433 //434 // ADDWrr X, MOVi32imm ==> ADDWri + ADDWri435 // ADDXrr X, MOVi64imm ==> ADDXri + ADDXri436 //437 // SUBWrr X, MOVi32imm ==> SUBWri + SUBWri438 // SUBXrr X, MOVi64imm ==> SUBXri + SUBXri439 //440 // The mov pseudo instruction could be expanded to multiple mov instructions441 // later. Let's try to split the constant operand of mov instruction into two442 // legal add/sub immediates. It makes only two ADD/SUB instructions instead of443 // multiple `mov` + `and/sub` instructions.444 445 // We can sometimes have ADDWrr WZR, MULi32imm that have not been constant446 // folded. Make sure that we don't generate invalid instructions that use XZR447 // in those cases.448 if (MI.getOperand(1).getReg() == AArch64::XZR ||449 MI.getOperand(1).getReg() == AArch64::WZR)450 return false;451 452 return splitTwoPartImm<T>(453 MI,454 [PosOpc, NegOpc](T Imm, unsigned RegSize, T &Imm0,455 T &Imm1) -> std::optional<OpcodePair> {456 if (splitAddSubImm(Imm, RegSize, Imm0, Imm1))457 return std::make_pair(PosOpc, PosOpc);458 if (splitAddSubImm(-Imm, RegSize, Imm0, Imm1))459 return std::make_pair(NegOpc, NegOpc);460 return std::nullopt;461 },462 [&TII = TII](MachineInstr &MI, OpcodePair Opcode, unsigned Imm0,463 unsigned Imm1, Register SrcReg, Register NewTmpReg,464 Register NewDstReg) {465 DebugLoc DL = MI.getDebugLoc();466 MachineBasicBlock *MBB = MI.getParent();467 BuildMI(*MBB, MI, DL, TII->get(Opcode.first), NewTmpReg)468 .addReg(SrcReg)469 .addImm(Imm0)470 .addImm(12);471 BuildMI(*MBB, MI, DL, TII->get(Opcode.second), NewDstReg)472 .addReg(NewTmpReg)473 .addImm(Imm1)474 .addImm(0);475 });476}477 478template <typename T>479bool AArch64MIPeepholeOpt::visitADDSSUBS(480 OpcodePair PosOpcs, OpcodePair NegOpcs, MachineInstr &MI) {481 // Try the same transformation as ADDSUB but with additional requirement482 // that the condition code usages are only for Equal and Not Equal483 484 if (MI.getOperand(1).getReg() == AArch64::XZR ||485 MI.getOperand(1).getReg() == AArch64::WZR)486 return false;487 488 return splitTwoPartImm<T>(489 MI,490 [PosOpcs, NegOpcs, &MI, &TRI = TRI,491 &MRI = MRI](T Imm, unsigned RegSize, T &Imm0,492 T &Imm1) -> std::optional<OpcodePair> {493 OpcodePair OP;494 if (splitAddSubImm(Imm, RegSize, Imm0, Imm1))495 OP = PosOpcs;496 else if (splitAddSubImm(-Imm, RegSize, Imm0, Imm1))497 OP = NegOpcs;498 else499 return std::nullopt;500 // Check conditional uses last since it is expensive for scanning501 // proceeding instructions502 MachineInstr &SrcMI = *MRI->getUniqueVRegDef(MI.getOperand(1).getReg());503 std::optional<UsedNZCV> NZCVUsed = examineCFlagsUse(SrcMI, MI, *TRI);504 if (!NZCVUsed || NZCVUsed->C || NZCVUsed->V)505 return std::nullopt;506 return OP;507 },508 [&TII = TII](MachineInstr &MI, OpcodePair Opcode, unsigned Imm0,509 unsigned Imm1, Register SrcReg, Register NewTmpReg,510 Register NewDstReg) {511 DebugLoc DL = MI.getDebugLoc();512 MachineBasicBlock *MBB = MI.getParent();513 BuildMI(*MBB, MI, DL, TII->get(Opcode.first), NewTmpReg)514 .addReg(SrcReg)515 .addImm(Imm0)516 .addImm(12);517 BuildMI(*MBB, MI, DL, TII->get(Opcode.second), NewDstReg)518 .addReg(NewTmpReg)519 .addImm(Imm1)520 .addImm(0);521 });522}523 524// Checks if the corresponding MOV immediate instruction is applicable for525// this peephole optimization.526bool AArch64MIPeepholeOpt::checkMovImmInstr(MachineInstr &MI,527 MachineInstr *&MovMI,528 MachineInstr *&SubregToRegMI) {529 // Check whether current MBB is in loop and the AND is loop invariant.530 MachineBasicBlock *MBB = MI.getParent();531 MachineLoop *L = MLI->getLoopFor(MBB);532 if (L && !L->isLoopInvariant(MI))533 return false;534 535 // Check whether current MI's operand is MOV with immediate.536 MovMI = MRI->getUniqueVRegDef(MI.getOperand(2).getReg());537 if (!MovMI)538 return false;539 540 // If it is SUBREG_TO_REG, check its operand.541 SubregToRegMI = nullptr;542 if (MovMI->getOpcode() == TargetOpcode::SUBREG_TO_REG) {543 SubregToRegMI = MovMI;544 MovMI = MRI->getUniqueVRegDef(MovMI->getOperand(2).getReg());545 if (!MovMI)546 return false;547 }548 549 if (MovMI->getOpcode() != AArch64::MOVi32imm &&550 MovMI->getOpcode() != AArch64::MOVi64imm)551 return false;552 553 // If the MOV has multiple uses, do not split the immediate because it causes554 // more instructions.555 if (!MRI->hasOneUse(MovMI->getOperand(0).getReg()))556 return false;557 if (SubregToRegMI && !MRI->hasOneUse(SubregToRegMI->getOperand(0).getReg()))558 return false;559 560 // It is OK to perform this peephole optimization.561 return true;562}563 564template <typename T>565bool AArch64MIPeepholeOpt::splitTwoPartImm(566 MachineInstr &MI,567 SplitAndOpcFunc<T> SplitAndOpc, BuildMIFunc BuildInstr) {568 unsigned RegSize = sizeof(T) * 8;569 assert((RegSize == 32 || RegSize == 64) &&570 "Invalid RegSize for legal immediate peephole optimization");571 572 // Perform several essential checks against current MI.573 MachineInstr *MovMI, *SubregToRegMI;574 if (!checkMovImmInstr(MI, MovMI, SubregToRegMI))575 return false;576 577 // Split the immediate to Imm0 and Imm1, and calculate the Opcode.578 T Imm = static_cast<T>(MovMI->getOperand(1).getImm()), Imm0, Imm1;579 // For the 32 bit form of instruction, the upper 32 bits of the destination580 // register are set to zero. If there is SUBREG_TO_REG, set the upper 32 bits581 // of Imm to zero. This is essential if the Immediate value was a negative582 // number since it was sign extended when we assign to the 64-bit Imm.583 if (SubregToRegMI)584 Imm &= 0xFFFFFFFF;585 OpcodePair Opcode;586 if (auto R = SplitAndOpc(Imm, RegSize, Imm0, Imm1))587 Opcode = *R;588 else589 return false;590 591 // Create new MIs using the first and second opcodes. Opcodes might differ for592 // flag setting operations that should only set flags on second instruction.593 // NewTmpReg = Opcode.first SrcReg Imm0594 // NewDstReg = Opcode.second NewTmpReg Imm1595 596 // Determine register classes for destinations and register operands597 const TargetRegisterClass *FirstInstrDstRC =598 TII->getRegClass(TII->get(Opcode.first), 0);599 const TargetRegisterClass *FirstInstrOperandRC =600 TII->getRegClass(TII->get(Opcode.first), 1);601 const TargetRegisterClass *SecondInstrDstRC =602 (Opcode.first == Opcode.second)603 ? FirstInstrDstRC604 : TII->getRegClass(TII->get(Opcode.second), 0);605 const TargetRegisterClass *SecondInstrOperandRC =606 (Opcode.first == Opcode.second)607 ? FirstInstrOperandRC608 : TII->getRegClass(TII->get(Opcode.second), 1);609 610 // Get old registers destinations and new register destinations611 Register DstReg = MI.getOperand(0).getReg();612 Register SrcReg = MI.getOperand(1).getReg();613 Register NewTmpReg = MRI->createVirtualRegister(FirstInstrDstRC);614 // In the situation that DstReg is not Virtual (likely WZR or XZR), we want to615 // reuse that same destination register.616 Register NewDstReg = DstReg.isVirtual()617 ? MRI->createVirtualRegister(SecondInstrDstRC)618 : DstReg;619 620 // Constrain registers based on their new uses621 MRI->constrainRegClass(SrcReg, FirstInstrOperandRC);622 MRI->constrainRegClass(NewTmpReg, SecondInstrOperandRC);623 if (DstReg != NewDstReg)624 MRI->constrainRegClass(NewDstReg, MRI->getRegClass(DstReg));625 626 // Call the delegating operation to build the instruction627 BuildInstr(MI, Opcode, Imm0, Imm1, SrcReg, NewTmpReg, NewDstReg);628 629 // replaceRegWith changes MI's definition register. Keep it for SSA form until630 // deleting MI. Only if we made a new destination register.631 if (DstReg != NewDstReg) {632 MRI->replaceRegWith(DstReg, NewDstReg);633 MI.getOperand(0).setReg(DstReg);634 }635 636 // Record the MIs need to be removed.637 MI.eraseFromParent();638 if (SubregToRegMI)639 SubregToRegMI->eraseFromParent();640 MovMI->eraseFromParent();641 642 return true;643}644 645bool AArch64MIPeepholeOpt::visitINSviGPR(MachineInstr &MI, unsigned Opc) {646 // Check if this INSvi[X]gpr comes from COPY of a source FPR128647 //648 // From649 // %intermediate1:gpr64 = COPY %src:fpr128650 // %intermediate2:gpr32 = COPY %intermediate1:gpr64651 // %dst:fpr128 = INSvi[X]gpr %dst_vec:fpr128, dst_index, %intermediate2:gpr32652 // To653 // %dst:fpr128 = INSvi[X]lane %dst_vec:fpr128, dst_index, %src:fpr128,654 // src_index655 // where src_index = 0, X = [8|16|32|64]656 657 MachineInstr *SrcMI = MRI->getUniqueVRegDef(MI.getOperand(3).getReg());658 659 // For a chain of COPY instructions, find the initial source register660 // and check if it's an FPR128661 while (true) {662 if (!SrcMI || SrcMI->getOpcode() != TargetOpcode::COPY)663 return false;664 665 if (!SrcMI->getOperand(1).getReg().isVirtual())666 return false;667 668 if (MRI->getRegClass(SrcMI->getOperand(1).getReg()) ==669 &AArch64::FPR128RegClass) {670 break;671 }672 SrcMI = MRI->getUniqueVRegDef(SrcMI->getOperand(1).getReg());673 }674 675 Register DstReg = MI.getOperand(0).getReg();676 Register SrcReg = SrcMI->getOperand(1).getReg();677 MachineInstr *INSvilaneMI =678 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), TII->get(Opc), DstReg)679 .add(MI.getOperand(1))680 .add(MI.getOperand(2))681 .addUse(SrcReg, getRegState(SrcMI->getOperand(1)))682 .addImm(0);683 684 LLVM_DEBUG(dbgs() << MI << " replace by:\n: " << *INSvilaneMI << "\n");685 (void)INSvilaneMI;686 MI.eraseFromParent();687 return true;688}689 690// All instructions that set a FPR64 will implicitly zero the top bits of the691// register.692static bool is64bitDefwithZeroHigh64bit(MachineInstr *MI,693 MachineRegisterInfo *MRI) {694 if (!MI->getOperand(0).isReg() || !MI->getOperand(0).isDef())695 return false;696 const TargetRegisterClass *RC = MRI->getRegClass(MI->getOperand(0).getReg());697 if (RC != &AArch64::FPR64RegClass)698 return false;699 return MI->getOpcode() > TargetOpcode::GENERIC_OP_END;700}701 702bool AArch64MIPeepholeOpt::visitINSvi64lane(MachineInstr &MI) {703 // Check the MI for low 64-bits sets zero for high 64-bits implicitly.704 // We are expecting below case.705 //706 // %1:fpr64 = nofpexcept FCVTNv4i16 %0:fpr128, implicit $fpcr707 // %6:fpr128 = IMPLICIT_DEF708 // %5:fpr128 = INSERT_SUBREG %6:fpr128(tied-def 0), killed %1:fpr64, %subreg.dsub709 // %7:fpr128 = INSvi64lane %5:fpr128(tied-def 0), 1, killed %3:fpr128, 0710 MachineInstr *Low64MI = MRI->getUniqueVRegDef(MI.getOperand(1).getReg());711 if (Low64MI->getOpcode() != AArch64::INSERT_SUBREG)712 return false;713 Low64MI = MRI->getUniqueVRegDef(Low64MI->getOperand(2).getReg());714 if (!Low64MI || !is64bitDefwithZeroHigh64bit(Low64MI, MRI))715 return false;716 717 // Check there is `mov 0` MI for high 64-bits.718 // We are expecting below cases.719 //720 // %2:fpr64 = MOVID 0721 // %4:fpr128 = IMPLICIT_DEF722 // %3:fpr128 = INSERT_SUBREG %4:fpr128(tied-def 0), killed %2:fpr64, %subreg.dsub723 // %7:fpr128 = INSvi64lane %5:fpr128(tied-def 0), 1, killed %3:fpr128, 0724 // or725 // %5:fpr128 = MOVIv2d_ns 0726 // %6:fpr64 = COPY %5.dsub:fpr128727 // %8:fpr128 = IMPLICIT_DEF728 // %7:fpr128 = INSERT_SUBREG %8:fpr128(tied-def 0), killed %6:fpr64, %subreg.dsub729 // %11:fpr128 = INSvi64lane %9:fpr128(tied-def 0), 1, killed %7:fpr128, 0730 MachineInstr *High64MI = MRI->getUniqueVRegDef(MI.getOperand(3).getReg());731 if (!High64MI || High64MI->getOpcode() != AArch64::INSERT_SUBREG)732 return false;733 High64MI = MRI->getUniqueVRegDef(High64MI->getOperand(2).getReg());734 if (High64MI && High64MI->getOpcode() == TargetOpcode::COPY)735 High64MI = MRI->getUniqueVRegDef(High64MI->getOperand(1).getReg());736 if (!High64MI || (High64MI->getOpcode() != AArch64::MOVID &&737 High64MI->getOpcode() != AArch64::MOVIv2d_ns))738 return false;739 if (High64MI->getOperand(1).getImm() != 0)740 return false;741 742 // Let's remove MIs for high 64-bits.743 Register OldDef = MI.getOperand(0).getReg();744 Register NewDef = MI.getOperand(1).getReg();745 MRI->constrainRegClass(NewDef, MRI->getRegClass(OldDef));746 MRI->replaceRegWith(OldDef, NewDef);747 MI.eraseFromParent();748 749 return true;750}751 752bool AArch64MIPeepholeOpt::visitFMOVDr(MachineInstr &MI) {753 // An FMOVDr sets the high 64-bits to zero implicitly, similar to ORR for GPR.754 MachineInstr *Low64MI = MRI->getUniqueVRegDef(MI.getOperand(1).getReg());755 if (!Low64MI || !is64bitDefwithZeroHigh64bit(Low64MI, MRI))756 return false;757 758 // Let's remove MIs for high 64-bits.759 Register OldDef = MI.getOperand(0).getReg();760 Register NewDef = MI.getOperand(1).getReg();761 LLVM_DEBUG(dbgs() << "Removing: " << MI << "\n");762 MRI->clearKillFlags(OldDef);763 MRI->clearKillFlags(NewDef);764 MRI->constrainRegClass(NewDef, MRI->getRegClass(OldDef));765 MRI->replaceRegWith(OldDef, NewDef);766 MI.eraseFromParent();767 768 return true;769}770 771bool AArch64MIPeepholeOpt::visitUBFMXri(MachineInstr &MI) {772 // Check if the instruction is equivalent to a 32 bit LSR or LSL alias of773 // UBFM, and replace the UBFMXri instruction with its 32 bit variant, UBFMWri.774 int64_t Immr = MI.getOperand(2).getImm();775 int64_t Imms = MI.getOperand(3).getImm();776 777 bool IsLSR = Imms == 31 && Immr <= Imms;778 bool IsLSL = Immr == Imms + 33;779 if (!IsLSR && !IsLSL)780 return false;781 782 if (IsLSL) {783 Immr -= 32;784 }785 786 const TargetRegisterClass *DstRC64 =787 TII->getRegClass(TII->get(MI.getOpcode()), 0);788 const TargetRegisterClass *DstRC32 =789 TRI->getSubRegisterClass(DstRC64, AArch64::sub_32);790 assert(DstRC32 && "Destination register class of UBFMXri doesn't have a "791 "sub_32 subregister class");792 793 const TargetRegisterClass *SrcRC64 =794 TII->getRegClass(TII->get(MI.getOpcode()), 1);795 const TargetRegisterClass *SrcRC32 =796 TRI->getSubRegisterClass(SrcRC64, AArch64::sub_32);797 assert(SrcRC32 && "Source register class of UBFMXri doesn't have a sub_32 "798 "subregister class");799 800 Register DstReg64 = MI.getOperand(0).getReg();801 Register DstReg32 = MRI->createVirtualRegister(DstRC32);802 Register SrcReg64 = MI.getOperand(1).getReg();803 Register SrcReg32 = MRI->createVirtualRegister(SrcRC32);804 805 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), TII->get(AArch64::COPY),806 SrcReg32)807 .addReg(SrcReg64, 0, AArch64::sub_32);808 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), TII->get(AArch64::UBFMWri),809 DstReg32)810 .addReg(SrcReg32)811 .addImm(Immr)812 .addImm(Imms);813 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),814 TII->get(AArch64::SUBREG_TO_REG), DstReg64)815 .addImm(0)816 .addReg(DstReg32)817 .addImm(AArch64::sub_32);818 MI.eraseFromParent();819 return true;820}821 822// Across a basic-block we might have in i32 extract from a value that only823// operates on upper bits (for example a sxtw). We can replace the COPY with a824// new version skipping the sxtw.825bool AArch64MIPeepholeOpt::visitCopy(MachineInstr &MI) {826 Register InputReg = MI.getOperand(1).getReg();827 if (MI.getOperand(1).getSubReg() != AArch64::sub_32 ||828 !MRI->hasOneNonDBGUse(InputReg))829 return false;830 831 MachineInstr *SrcMI = MRI->getUniqueVRegDef(InputReg);832 SmallPtrSet<MachineInstr *, 4> DeadInstrs;833 DeadInstrs.insert(SrcMI);834 while (SrcMI && SrcMI->isFullCopy() &&835 MRI->hasOneNonDBGUse(SrcMI->getOperand(1).getReg())) {836 SrcMI = MRI->getUniqueVRegDef(SrcMI->getOperand(1).getReg());837 DeadInstrs.insert(SrcMI);838 }839 840 if (!SrcMI)841 return false;842 843 // Look for SXTW(X) and return Reg.844 auto getSXTWSrcReg = [](MachineInstr *SrcMI) -> Register {845 if (SrcMI->getOpcode() != AArch64::SBFMXri ||846 SrcMI->getOperand(2).getImm() != 0 ||847 SrcMI->getOperand(3).getImm() != 31)848 return AArch64::NoRegister;849 return SrcMI->getOperand(1).getReg();850 };851 // Look for SUBREG_TO_REG(ORRWrr(WZR, COPY(X.sub_32)))852 auto getUXTWSrcReg = [&](MachineInstr *SrcMI) -> Register {853 if (SrcMI->getOpcode() != AArch64::SUBREG_TO_REG ||854 SrcMI->getOperand(3).getImm() != AArch64::sub_32 ||855 !MRI->hasOneNonDBGUse(SrcMI->getOperand(2).getReg()))856 return AArch64::NoRegister;857 MachineInstr *Orr = MRI->getUniqueVRegDef(SrcMI->getOperand(2).getReg());858 if (!Orr || Orr->getOpcode() != AArch64::ORRWrr ||859 Orr->getOperand(1).getReg() != AArch64::WZR ||860 !MRI->hasOneNonDBGUse(Orr->getOperand(2).getReg()))861 return AArch64::NoRegister;862 MachineInstr *Cpy = MRI->getUniqueVRegDef(Orr->getOperand(2).getReg());863 if (!Cpy || Cpy->getOpcode() != AArch64::COPY ||864 Cpy->getOperand(1).getSubReg() != AArch64::sub_32)865 return AArch64::NoRegister;866 DeadInstrs.insert(Orr);867 return Cpy->getOperand(1).getReg();868 };869 870 Register SrcReg = getSXTWSrcReg(SrcMI);871 if (!SrcReg)872 SrcReg = getUXTWSrcReg(SrcMI);873 if (!SrcReg)874 return false;875 876 MRI->constrainRegClass(SrcReg, MRI->getRegClass(InputReg));877 LLVM_DEBUG(dbgs() << "Optimizing: " << MI);878 MI.getOperand(1).setReg(SrcReg);879 LLVM_DEBUG(dbgs() << " to: " << MI);880 for (auto *DeadMI : DeadInstrs) {881 LLVM_DEBUG(dbgs() << " Removing: " << *DeadMI);882 DeadMI->eraseFromParent();883 }884 return true;885}886 887bool AArch64MIPeepholeOpt::runOnMachineFunction(MachineFunction &MF) {888 if (skipFunction(MF.getFunction()))889 return false;890 891 TII = static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());892 TRI = static_cast<const AArch64RegisterInfo *>(893 MF.getSubtarget().getRegisterInfo());894 MLI = &getAnalysis<MachineLoopInfoWrapperPass>().getLI();895 MRI = &MF.getRegInfo();896 897 assert(MRI->isSSA() && "Expected to be run on SSA form!");898 899 bool Changed = false;900 901 for (MachineBasicBlock &MBB : MF) {902 for (MachineInstr &MI : make_early_inc_range(MBB)) {903 switch (MI.getOpcode()) {904 default:905 break;906 case AArch64::INSERT_SUBREG:907 Changed |= visitINSERT(MI);908 break;909 case AArch64::ANDWrr:910 Changed |= trySplitLogicalImm<uint32_t>(AArch64::ANDWri, MI,911 SplitStrategy::Intersect);912 break;913 case AArch64::ANDXrr:914 Changed |= trySplitLogicalImm<uint64_t>(AArch64::ANDXri, MI,915 SplitStrategy::Intersect);916 break;917 case AArch64::ANDSWrr:918 Changed |= trySplitLogicalImm<uint32_t>(919 AArch64::ANDWri, MI, SplitStrategy::Intersect, AArch64::ANDSWri);920 break;921 case AArch64::ANDSXrr:922 Changed |= trySplitLogicalImm<uint64_t>(923 AArch64::ANDXri, MI, SplitStrategy::Intersect, AArch64::ANDSXri);924 break;925 case AArch64::EORWrr:926 Changed |= trySplitLogicalImm<uint32_t>(AArch64::EORWri, MI,927 SplitStrategy::Disjoint);928 break;929 case AArch64::EORXrr:930 Changed |= trySplitLogicalImm<uint64_t>(AArch64::EORXri, MI,931 SplitStrategy::Disjoint);932 break;933 case AArch64::ORRWrr:934 Changed |= trySplitLogicalImm<uint32_t>(AArch64::ORRWri, MI,935 SplitStrategy::Disjoint);936 break;937 case AArch64::ORRXrr:938 Changed |= trySplitLogicalImm<uint64_t>(AArch64::ORRXri, MI,939 SplitStrategy::Disjoint);940 break;941 case AArch64::ORRWrs:942 Changed |= visitORR(MI);943 break;944 case AArch64::ADDWrr:945 Changed |= visitADDSUB<uint32_t>(AArch64::ADDWri, AArch64::SUBWri, MI);946 break;947 case AArch64::SUBWrr:948 Changed |= visitADDSUB<uint32_t>(AArch64::SUBWri, AArch64::ADDWri, MI);949 break;950 case AArch64::ADDXrr:951 Changed |= visitADDSUB<uint64_t>(AArch64::ADDXri, AArch64::SUBXri, MI);952 break;953 case AArch64::SUBXrr:954 Changed |= visitADDSUB<uint64_t>(AArch64::SUBXri, AArch64::ADDXri, MI);955 break;956 case AArch64::ADDSWrr:957 Changed |=958 visitADDSSUBS<uint32_t>({AArch64::ADDWri, AArch64::ADDSWri},959 {AArch64::SUBWri, AArch64::SUBSWri}, MI);960 break;961 case AArch64::SUBSWrr:962 Changed |=963 visitADDSSUBS<uint32_t>({AArch64::SUBWri, AArch64::SUBSWri},964 {AArch64::ADDWri, AArch64::ADDSWri}, MI);965 break;966 case AArch64::ADDSXrr:967 Changed |=968 visitADDSSUBS<uint64_t>({AArch64::ADDXri, AArch64::ADDSXri},969 {AArch64::SUBXri, AArch64::SUBSXri}, MI);970 break;971 case AArch64::SUBSXrr:972 Changed |=973 visitADDSSUBS<uint64_t>({AArch64::SUBXri, AArch64::SUBSXri},974 {AArch64::ADDXri, AArch64::ADDSXri}, MI);975 break;976 case AArch64::CSELWr:977 case AArch64::CSELXr:978 Changed |= visitCSEL(MI);979 break;980 case AArch64::INSvi64gpr:981 Changed |= visitINSviGPR(MI, AArch64::INSvi64lane);982 break;983 case AArch64::INSvi32gpr:984 Changed |= visitINSviGPR(MI, AArch64::INSvi32lane);985 break;986 case AArch64::INSvi16gpr:987 Changed |= visitINSviGPR(MI, AArch64::INSvi16lane);988 break;989 case AArch64::INSvi8gpr:990 Changed |= visitINSviGPR(MI, AArch64::INSvi8lane);991 break;992 case AArch64::INSvi64lane:993 Changed |= visitINSvi64lane(MI);994 break;995 case AArch64::FMOVDr:996 Changed |= visitFMOVDr(MI);997 break;998 case AArch64::UBFMXri:999 Changed |= visitUBFMXri(MI);1000 break;1001 case AArch64::COPY:1002 Changed |= visitCopy(MI);1003 break;1004 }1005 }1006 }1007 1008 return Changed;1009}1010 1011FunctionPass *llvm::createAArch64MIPeepholeOptPass() {1012 return new AArch64MIPeepholeOpt();1013}1014